@iris-eval/mcp-server 0.4.6 → 0.5.1

package/dist/tools/log-trace.d.ts

@@ -1,3 +1,54 @@
+import { z } from 'zod';
 import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
 import type { IStorageAdapter } from '../types/query.js';
+export declare const logTraceInputShape: {
+    agent_name: z.ZodString;
+    framework: z.ZodOptional<z.ZodString>;
+    input: z.ZodOptional<z.ZodString>;
+    output: z.ZodOptional<z.ZodString>;
+    tool_calls: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        tool_name: z.ZodString;
+        input: z.ZodOptional<z.ZodUnknown>;
+        output: z.ZodOptional<z.ZodUnknown>;
+        latency_ms: z.ZodOptional<z.ZodNumber>;
+        error: z.ZodOptional<z.ZodString>;
+    }, z.core.$strip>>>;
+    latency_ms: z.ZodOptional<z.ZodNumber>;
+    token_usage: z.ZodOptional<z.ZodObject<{
+        prompt_tokens: z.ZodOptional<z.ZodNumber>;
+        completion_tokens: z.ZodOptional<z.ZodNumber>;
+        total_tokens: z.ZodOptional<z.ZodNumber>;
+    }, z.core.$strip>>;
+    cost_usd: z.ZodOptional<z.ZodNumber>;
+    metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+    spans: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        span_id: z.ZodOptional<z.ZodString>;
+        parent_span_id: z.ZodOptional<z.ZodString>;
+        name: z.ZodString;
+        kind: z.ZodDefault<z.ZodEnum<{
+            INTERNAL: "INTERNAL";
+            SERVER: "SERVER";
+            CLIENT: "CLIENT";
+            PRODUCER: "PRODUCER";
+            CONSUMER: "CONSUMER";
+            LLM: "LLM";
+            TOOL: "TOOL";
+        }>>;
+        status_code: z.ZodDefault<z.ZodEnum<{
+            UNSET: "UNSET";
+            OK: "OK";
+            ERROR: "ERROR";
+        }>>;
+        status_message: z.ZodOptional<z.ZodString>;
+        start_time: z.ZodString;
+        end_time: z.ZodOptional<z.ZodString>;
+        attributes: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+        events: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            name: z.ZodString;
+            timestamp: z.ZodString;
+            attributes: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+        }, z.core.$strip>>>;
+    }, z.core.$strip>>>;
+    timestamp: z.ZodOptional<z.ZodString>;
+};
 export declare function registerLogTraceTool(server: McpServer, storage: IStorageAdapter): void;

package/dist/tools/log-trace.js

@@ -2,6 +2,7 @@ import { z } from 'zod';
 import { generateTraceId, generateSpanId } from '../utils/ids.js';
 import { LOCAL_TENANT } from '../types/tenant.js';
 import { bestEffortExport } from '../otel/lazy.js';
+import { strictInput } from './strict-input.js';
 const ToolCallSchema = z.object({
     tool_name: z.string(),
     input: z.unknown().optional(),
@@ -30,7 +31,14 @@ const TokenUsageSchema = z.object({
     completion_tokens: z.number().optional(),
     total_tokens: z.number().optional(),
 });
-const inputSchema = {
+/*
+ * The log_trace input contract. Exported because POST /api/v1/traces
+ * (src/dashboard/routes/traces.ts) accepts the SAME body — one schema,
+ * two capture paths. Duplicating it there would let the tool and the
+ * HTTP endpoint drift apart silently; importing it means a field added
+ * here is accepted (and validated identically) on both.
+ */
+export const logTraceInputShape = {
     agent_name: z.string().describe('Agent name — used for filtering in get_traces (e.g., "customer-support-bot")'),
     framework: z.string().optional().describe('Agent framework identifier (e.g., langchain, autogen, custom)'),
     input: z.string().optional().describe('Agent input text — the user prompt or upstream input that produced this output'),
@@ -51,7 +59,7 @@ export function registerLogTraceTool(server, storage) {
             '',
             'Sibling tools — evaluate_output runs heuristic scoring on the trace; evaluate_with_llm_judge runs semantic LLM-based scoring; verify_citations checks citation grounding; get_traces queries stored traces; delete_trace removes a single trace; list_rules / deploy_rule / delete_rule manage custom evaluation rules. log_trace is the WRITE path that records executions; everything else reads, scores, or manages around it.',
             '',
-            'Behavior. Writes one row to Iris storage (SQLite by default; Postgres in Cloud tier). When IRIS_OTEL_ENDPOINT is set, ALSO fires a best-effort async export to the configured OTLP/HTTP collector (Jaeger, Tempo, Datadog OTLP, OTEL Collector). The OTel export is fire-and-forget — its success does not affect the tool response; failures are logged but the trace is still stored locally. No authentication in stdio mode; HTTP mode requires Bearer token. Rate-limited to 20 req/min on HTTP MCP, unlimited on stdio. Not idempotent: each call mints a fresh trace_id, so resubmitting the same payload creates a duplicate trace.',
+            'Behavior. Writes one row to Iris storage (SQLite by default; Postgres in Cloud tier). When IRIS_OTEL_ENDPOINT is set, ALSO fires a best-effort async export to the configured OTLP/HTTP collector (Jaeger, Tempo, Datadog OTLP, OTEL Collector). The OTel export is fire-and-forget — its success does not affect the tool response; failures are logged but the trace is still stored locally. No authentication in stdio mode. HTTP mode requires a Bearer token ONLY when --api-key / IRIS_API_KEY is set (recommended); with no key configured the auth middleware is a pass-through and writes are unauthenticated — a default HTTP server is protected by its loopback bind (127.0.0.1) and Origin validation, not by a credential. Rate-limited to 20 req/min on HTTP MCP, unlimited on stdio. Not idempotent: each call mints a fresh trace_id, so resubmitting the same payload creates a duplicate trace.',
             '',
             'Output shape. Returns a JSON string: `{ "trace_id": "<32-hex>", "status": "stored" }`. The trace_id is the key you pass to evaluate_output or get_traces afterwards.',
             '',
@@ -63,7 +71,11 @@ export function registerLogTraceTool(server, storage) {
             '',
             'Error modes. Throws on missing agent_name. Throws on malformed span or tool_call objects (Zod rejects). Returns 500 on storage failure (disk full, DB locked). Never blocks on the agent — returns within ~50ms for typical payloads.',
         ].join('\n'),
-        inputSchema,
+        // Strict at the MCP boundary (unknown args rejected, not stripped).
+        // The dashboard's HTTP ingest builds its own schema FROM this shape
+        // (dashboard/validation.ts) and keeps default stripping there on
+        // purpose — it relies on it to discard a client-supplied trace_id.
+        inputSchema: strictInput(logTraceInputShape),
         annotations: {
             readOnlyHint: false, // Writes a row to storage
             destructiveHint: false, // Creates new data; doesn't overwrite or delete

package/dist/tools/strict-input.d.ts

@@ -0,0 +1,2 @@
+import { z } from 'zod';
+export declare function strictInput<T extends z.ZodRawShape>(shape: T): z.ZodObject<{ -readonly [P in keyof T]: T[P]; }, z.core.$strict>;

package/dist/tools/strict-input.js

@@ -0,0 +1,35 @@
+import { z } from 'zod';
+/*
+ * Wraps a tool's input shape in a STRICT object schema so unknown argument
+ * names are REJECTED with an error that names the offending key(s) and
+ * lists the valid ones.
+ *
+ * Why this exists: a bare shape (or z.object()) silently STRIPS unknown
+ * keys. At the MCP tool boundary that is dangerous, not lenient — an LLM
+ * guessing an argument name is the normal case, not an edge case. Before
+ * this wrapper, `evaluate_output({ criteria: ["safety"], ... })` (a
+ * plausible guess) and `eval_typ: "safety"` (a one-character typo) both
+ * "succeeded": the arguments were dropped, the DEFAULT completeness bundle
+ * ran instead of the safety rules, and the response said passed:true on
+ * text containing real PII — with nothing indicating the arguments were
+ * ignored. Meanwhile a missing REQUIRED field produced a precise Zod
+ * error, so the failure mode was inconsistent as well as unsafe.
+ *
+ * The MCP SDK accepts a schema object (not just a raw shape) for
+ * inputSchema and validates tool calls through it, so the custom
+ * unrecognized-keys message below is exactly what the caller sees.
+ * Strictness also reaches tools/list: the generated JSON Schema carries
+ * additionalProperties:false, telling well-behaved clients up front.
+ */
+export function strictInput(shape) {
+    const validKeys = Object.keys(shape).join(', ');
+    return z.strictObject(shape, {
+        error: (issue) => issue.code === 'unrecognized_keys'
+            ? `Unknown argument(s): ${issue.keys.map((k) => `"${k}"`).join(', ')}. ` +
+                `Valid arguments: ${validKeys}. ` +
+                'Unknown arguments are rejected rather than silently ignored, so a misspelled ' +
+                'argument name cannot change what gets evaluated — check the spelling against ' +
+                "the tool's input schema and retry."
+            : undefined,
+    });
+}

package/dist/tools/verify-citations.js

@@ -3,6 +3,7 @@ import { LOCAL_TENANT } from '../types/tenant.js';
 import { verifyCitations } from '../eval/citation-verify/verifier.js';
 import { findPricing } from '../eval/llm-judge/pricing.js';
 import { generateEvalId } from '../utils/ids.js';
+import { strictInput } from './strict-input.js';
 const inputSchema = {
     output: z.string().min(1).describe('The agent output containing citations to verify'),
     model: z
@@ -57,17 +58,17 @@ export function registerVerifyCitationsTool(server, storage) {
             '',
             'Behavior. Three-phase pipeline: (1) regex extraction of [N] numbered refs, (Author, Year) parentheticals, bare URLs, and DOIs (in-process, no network); (2) SSRF-guarded fetch of URL + DOI citations, with scheme allowlist, private/link-local/cloud-metadata IP blocking, optional domain allowlist (IRIS_CITATION_DOMAINS), 10s timeout, 5MB body cap, manual redirect chase (max 3, re-checked), in-process LRU cache; (3) per-citation LLM judge call asking "does this source support this claim?" with a 256-token verdict. Opt-in via allow_fetch=true or IRIS_CITATION_ALLOW_FETCH=1 — Iris refuses outbound HTTP by default. Cost-capped across the entire call by max_cost_usd_total (default $1.00) — the pipeline stops when the cap would be exceeded. Rate-limited to 20 req/min on HTTP MCP. Writes one eval_result row tagged with per-citation provenance.',
             '',
-            'Output shape. Returns JSON: `{ "id": "<uuid>", "overall_score": 0..1|null, "passed": boolean, "total_citations_found": number, "total_resolved": number, "total_supported": number, "total_cost_usd": number, "citations": [{ "citation": { "raw", "kind", "identifier", "offset_start", "offset_end" }, "resolve_status": "ok"|"skipped"|"error", "resolve_error"?, "source"?: { "url", "status", "content_type", "bytes_fetched", "truncated" }, "judge"?: { "supported", "confidence", "rationale", "cost_usd", "latency_ms", "input_tokens", "output_tokens" } }] }`. `overall_score = supported / resolved`; `null` when nothing resolvable was found.',
+            'Output shape. Returns JSON: `{ "id": "<uuid>", "overall_score": 0..1|null, "passed": boolean, "total_citations_found": number, "total_resolved": number, "total_judged": number, "total_supported": number, "total_cost_usd": number, "citations": [{ "citation": { "raw", "kind", "identifier", "offset_start", "offset_end" }, "resolve_status": "ok"|"skipped"|"error", "resolve_error"?, "source"?: { "url", "status", "content_type", "bytes_fetched", "truncated" }, "judge"?: { "supported", "confidence", "rationale", "cost_usd", "latency_ms", "input_tokens", "output_tokens" } }] }`. `overall_score = supported / judged`; `null` when nothing was judged (no resolvable citations, or every judge call failed). Infrastructure failures (cost cap, judge timeout/error, malformed verdict) leave a citation resolved-but-unjudged — reported per-citation via resolve_error, never scored as unsupported.',
             '',
             'Use when the output makes factual claims backed by [1]-style references, DOIs, or URLs and you want to separate "cited correctly" from "cited and wrong" from "cited but unresolvable". Particularly useful for research/legal/medical agents where fabricated citations are the dominant failure mode.',
             "",
             "Don't use when the agent output has no citations at all (overall_score will be null; the tool degrades gracefully but a heuristic rule is cheaper). Don't use without allow_fetch=true or IRIS_CITATION_ALLOW_FETCH=1 — the tool refuses outbound HTTP unless explicitly enabled. Don't use with an open allowlist + untrusted output on the public internet; you are effectively running a user-directed fetcher. For stricter safety set IRIS_CITATION_DOMAINS to a curated list.",
             '',
-            'Parameters. model is required; provider auto-detected from model name (override only for ambiguous IDs). allow_fetch=false by default — outbound HTTP is REFUSED unless explicitly true OR IRIS_CITATION_ALLOW_FETCH=1 env. domain_allowlist suffix-matches hostnames (e.g., "wikipedia.org" allows en.wikipedia.org); merged with IRIS_CITATION_DOMAINS env (UNION — either source permits). max_citations defaults 20, hard cap 50 (extras are skipped silently, NOT errored — check total_citations_found in the response if precise). max_cost_usd_total defaults $1.00 — the pipeline stops mid-citation when the next judge call would exceed the cap (returns partial verdicts). per_source_timeout_ms defaults 10000 (10s); per_source_max_bytes defaults 5MB (truncates at boundary, judges still run on truncated content). trace_id optional but recommended. Defaults: max_citations=20, max_cost_usd_total=$1.00, per_source_timeout_ms=10000, per_source_max_bytes=5242880, allow_fetch=false.',
+            'Parameters. model is required; provider auto-detected from model name (override only for ambiguous IDs). allow_fetch=false by default — outbound HTTP is REFUSED unless explicitly true OR IRIS_CITATION_ALLOW_FETCH=1 env. domain_allowlist suffix-matches hostnames (e.g., "wikipedia.org" allows en.wikipedia.org); merged with IRIS_CITATION_DOMAINS env (UNION — either source permits). max_citations defaults 20, hard cap 50 (extras are skipped silently, NOT errored — check total_citations_found in the response if precise). max_cost_usd_total defaults $1.00 — the pipeline stops mid-citation when the next judge call would exceed the cap (returns partial verdicts). per_source_timeout_ms defaults 10000 (10s); per_source_max_bytes defaults 5MB (truncates at boundary, judges still run on truncated content); independently of that, the judge reads at most the first 12,000 characters of each fetched source, and the per-citation cost estimate is taken on that truncated prompt, not on the full body. trace_id optional but recommended. Defaults: max_citations=20, max_cost_usd_total=$1.00, per_source_timeout_ms=10000, per_source_max_bytes=5242880, allow_fetch=false.',
             '',
             'Error modes. Throws when the API key env var is missing. Throws "Unknown model" on unsupported model IDs. Per-citation errors are collected (resolve_error.kind = bad_scheme / ssrf / not_allowed_domain / timeout / too_large / bad_status / redirect_loop / not_text / fetch_disabled / malformed_judge_response / cost_cap_reached / unresolvable_kind) and returned in the response rather than thrown. An empty output or output with zero extractable citations returns overall_score=null + passed=true (nothing to fail).',
         ].join('\n'),
-        inputSchema,
+        inputSchema: strictInput(inputSchema),
         annotations: {
             readOnlyHint: false, // Writes eval_result + spends money
             destructiveHint: false, // Creates data; doesn't overwrite/delete
@@ -109,11 +110,11 @@ export function registerVerifyCitationsTool(server, storage) {
                     passed: result.passed,
                     score,
                     message: result.overallScore === null
-                        ? `No resolvable citations (found ${result.totalCitationsFound}, resolved ${result.totalResolved})`
-                        : `${result.totalSupported}/${result.totalResolved} cited sources supported the output`,
+                        ? `No citations judged (found ${result.totalCitationsFound}, resolved ${result.totalResolved}, judged 0)`
+                        : `${result.totalSupported}/${result.totalJudged} judged sources supported the output`,
                 },
             ],
-            suggestions: result.passed ? [] : [`Only ${result.totalSupported}/${result.totalResolved} cited sources actually supported the claim.`],
+            suggestions: result.passed ? [] : [`Only ${result.totalSupported}/${result.totalJudged} judged sources actually supported the claim.`],
             rules_evaluated: 1,
             rules_skipped: 0,
             insufficient_data: result.overallScore === null,
@@ -128,6 +129,7 @@ export function registerVerifyCitationsTool(server, storage) {
                         passed: result.passed,
                         total_citations_found: result.totalCitationsFound,
                         total_resolved: result.totalResolved,
+                        total_judged: result.totalJudged,
                         total_supported: result.totalSupported,
                         total_cost_usd: result.totalCostUsd,
                         citations: result.citations.map((c) => ({

package/dist/transport/http.js

@@ -56,9 +56,31 @@ export async function createHttpTransport(mcpServer, config, logger) {
      * `127.0.0.1:0` and reject every real request with a 403 that looks
      * exactly like an attack. Routes are registered immediately after, and
      * the port is not discoverable by any client until this function returns.
+     *
+     * The callback MUST inspect its error argument. Express 5 wires the
+     * listen callback as `server.once('error', done)` as well as the
+     * listening callback — so on EADDRINUSE it is invoked WITH the error.
+     * Ignoring that argument resolved this promise on a server that never
+     * bound: the caller then logged "HTTP transport listening on <port>"
+     * while another process owned the port, and the process idled forever.
+     * A CI health poll got 200 from the OTHER instance and shipped
+     * evaluations to a stranger's database. A bind failure must reject,
+     * name the port, and take the process down nonzero.
      */
-    const httpServer = await new Promise((resolve) => {
-        const server = app.listen(config.transport.port, config.transport.host, () => resolve(server));
+    const httpServer = await new Promise((resolve, reject) => {
+        const server = app.listen(config.transport.port, config.transport.host, (err) => {
+            if (err) {
+                const bind = `${config.transport.host}:${config.transport.port}`;
+                const code = err.code;
+                reject(code === 'EADDRINUSE'
+                    ? new Error(`HTTP transport failed to start: port ${config.transport.port} is already in use ` +
+                        `(EADDRINUSE on ${bind}). Another process — possibly another iris instance — owns it. ` +
+                        `Pass --port <other> (or set IRIS_PORT) or stop the other process.`)
+                    : new Error(`HTTP transport failed to bind ${bind}: ${err.message}`));
+                return;
+            }
+            resolve(server);
+        });
     });
     const address = httpServer.address();
     const port = typeof address === 'object' && address ? address.port : config.transport.port;

package/dist/types/decision-moment.d.ts

@@ -69,6 +69,14 @@ export interface DecisionMomentDetail extends DecisionMoment {
             skipReason?: string;
         }>;
         suggestions: string[];
+        /**
+         * Rules that HARD-FAILED this evaluation — a critical safety rule
+         * (no_pii / no_injection_patterns / no_blocklist_words) or a deployed
+         * rule with severity high/critical. Present only when the veto fired;
+         * absent means nothing vetoed, or the row predates migration 006. This
+         * is what lets a surface distinguish "vetoed" from "scored low".
+         */
+        criticalFailures?: string[];
         createdAt?: string;
     }>;
     /** Full input (uncompressed). */
@@ -120,3 +128,15 @@ export interface MomentQueryResult {
     limit: number;
     offset: number;
 }
+export interface RankedFailure extends DecisionMoment {
+    /** Severity × recency-decay blend, 0-1. Higher = shown first. */
+    rankScore: number;
+}
+export interface FailureQueryResult {
+    failures: RankedFailure[];
+    /** How many recent traces were scanned to build the list. */
+    scanned: number;
+    /** Total traces matching the filter (pre-scan-cap). */
+    total: number;
+    limit: number;
+}

package/dist/types/eval.d.ts

@@ -4,6 +4,18 @@ export interface EvalRule {
     description: string;
     evalType: EvalType;
     weight: number;
+    /**
+     * Hard-fail marker. When a critical rule FAILS (and was not skipped), the
+     * overall eval reports passed=false regardless of the weighted score.
+     *
+     * Exists because the weighted average routinely outvotes a genuine
+     * violation: an output leaking a real SSN failed no_pii while the other
+     * safety rules passed, scoring ~0.765 — above the 0.7 threshold — so the
+     * one field every CI gate reads said passed:true about the product's
+     * flagship failure scenario. The score stays a quality gradient; `passed`
+     * is the verdict, and a critical violation must never be averaged away.
+     */
+    critical?: boolean;
     evaluate(context: EvalContext): EvalRuleResult;
 }
 export interface EvalContext {
@@ -23,6 +35,17 @@ export interface EvalContext {
     costUsd?: number;
     metadata?: Record<string, unknown>;
     customConfig?: Record<string, unknown>;
+    /**
+     * Per-evaluation regex circuit breaker, initialized by the engine (never
+     * by callers). Each sandbox budget breach increments `breaches`; once it
+     * reaches the cap, remaining regex rules in the SAME evaluation skip
+     * without running. Bounds how long a single hostile output can stall a
+     * request: without it, N regex rules × (budget + worker respawn) of
+     * main-thread stall scale linearly with N.
+     */
+    regexBudget?: {
+        breaches: number;
+    };
 }
 export interface EvalRuleResult {
     ruleName: string;
@@ -31,6 +54,8 @@ export interface EvalRuleResult {
     message: string;
     skipped?: boolean;
     skipReason?: string;
+    configInvalid?: boolean;
+    budgetExceeded?: boolean;
 }
 export interface EvalResult {
     id: string;
@@ -46,6 +71,28 @@ export interface EvalResult {
     rules_evaluated?: number;
     rules_skipped?: number;
     insufficient_data?: boolean;
+    /**
+     * Names of critical rules that failed (present only when non-empty).
+     * Any entry here forces passed=false regardless of the weighted score —
+     * this field is how a caller tells "failed the quality bar" apart from
+     * "committed a hard violation".
+     */
+    critical_failures?: string[];
+    /**
+     * Names of critical rules that were SKIPPED and therefore did not judge
+     * this output (present only when non-empty). Almost always a sandbox
+     * budget breach — a regex killed mid-backtrack, which an adversary can
+     * provoke deliberately by crafting output that stalls a known pattern.
+     *
+     * This is the fail-open seam between the release's two headline features:
+     * a budget-killed critical rule does NOT veto, so the evaluation can
+     * return passed=true with no `critical_failures` at all. That is
+     * deliberate (failing closed would let the same adversary force false
+     * violations on benign output), but a consumer that must fail closed
+     * needs to see it WITHOUT walking rule_results[].budgetExceeded. Treat a
+     * non-empty `critical_skipped` as "unknown", not as "clean".
+     */
+    critical_skipped?: string[];
 }
 export type CustomRuleType = 'regex_match' | 'regex_no_match' | 'min_length' | 'max_length' | 'contains_keywords' | 'excludes_keywords' | 'json_schema' | 'cost_threshold';
 export interface CustomRuleDefinition {

package/dist/types/query.d.ts

@@ -23,7 +23,7 @@ export interface TraceQueryResult {
     limit: number;
     offset: number;
 }
-export type EvalStatsPeriod = '24h' | '7d' | '30d' | 'all';
+export type EvalStatsPeriod = '24h' | '2d' | '7d' | '14d' | '30d' | '60d' | '90d' | '180d' | 'all';
 export interface EvalStats {
     passRate: number;
     avgScore: number;

package/dist/utils/write-atomic.d.ts

@@ -1 +1,3 @@
+export declare const OWNER_ONLY_FILE_MODE = 384;
+export declare function ensureOwnerOnly(...paths: string[]): void;
 export declare function writeAtomic(targetPath: string, contents: string): void;

package/dist/utils/write-atomic.js

@@ -1,4 +1,4 @@
-import { mkdirSync, writeFileSync, renameSync, unlinkSync } from 'node:fs';
+import { mkdirSync, writeFileSync, renameSync, unlinkSync, chmodSync, existsSync } from 'node:fs';
 import { dirname } from 'node:path';
 import { randomBytes } from 'node:crypto';
 /*
@@ -28,6 +28,38 @@ import { randomBytes } from 'node:crypto';
  */
 const TRANSIENT_RENAME_ERRORS = new Set(['EPERM', 'EACCES', 'EBUSY']);
 const MAX_ATTEMPTS = 5;
+/*
+ * Owner-only (0600). These files hold agent inputs and outputs verbatim —
+ * and a tool whose job is detecting PII necessarily stores the PII it found.
+ * Node's default is 0666 before umask, so a typical umask leaves them 0644:
+ * world-readable to every local account on a shared POSIX host. The mode is
+ * a no-op on Windows (ACL inheritance governs there), which is exactly why
+ * local testing never surfaces it.
+ *
+ * Set at creation on the temp file, so the bytes are never briefly readable
+ * between write and chmod; rename preserves the mode.
+ */
+export const OWNER_ONLY_FILE_MODE = 0o600;
+/*
+ * Narrow an EXISTING file to owner-only. Two cases need this, because a mode
+ * passed at write time only applies when the write creates the file:
+ *   - files written before this change (every install that predates it),
+ *   - files a library creates for us (better-sqlite3 opens iris.db, and WAL
+ *     mode adds iris.db-wal / iris.db-shm on first write).
+ * Best-effort by design: a missing file, a read-only mount, or a
+ * non-POSIX filesystem must never take the server down over permissions.
+ */
+export function ensureOwnerOnly(...paths) {
+    for (const p of paths) {
+        try {
+            if (existsSync(p))
+                chmodSync(p, OWNER_ONLY_FILE_MODE);
+        }
+        catch {
+            // Best effort — see above.
+        }
+    }
+}
 function sleepSync(ms) {
     // Synchronous by necessity — writeAtomic is sync, and making it async
     // would ripple through every caller for a Windows-only edge case.
@@ -36,7 +68,7 @@ function sleepSync(ms) {
 export function writeAtomic(targetPath, contents) {
     mkdirSync(dirname(targetPath), { recursive: true });
     const tmp = `${targetPath}.tmp.${process.pid}.${randomBytes(6).toString('hex')}`;
-    writeFileSync(tmp, contents, 'utf-8');
+    writeFileSync(tmp, contents, { encoding: 'utf-8', mode: OWNER_ONLY_FILE_MODE });
     let lastError;
     for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
         try {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@iris-eval/mcp-server",
-  "version": "0.4.6",
-  "description": "The agent eval standard for MCP. Score every agent output for quality, safety, and cost.",
+  "version": "0.5.1",
+  "description": "Stop shipping agents on vibes. Score every agent output for quality, safety, and cost.",
   "mcpName": "io.github.iris-eval/mcp-server",
   "type": "module",
   "main": "dist/index.js",
@@ -22,6 +22,7 @@
     "test:coverage": "vitest run --coverage",
     "test:integration": "vitest run tests/integration/",
     "test:e2e": "playwright test",
+    "test:uat": "node tests/uat/run-uat.mjs",
     "test:e2e:ui": "playwright test --ui",
     "version:check": "bash scripts/check-version.sh",
     "version:sync": "node scripts/sync-versions.mjs",

package/server.json

@@ -1,17 +1,17 @@
 {
   "$schema": "https://static.modelcontextprotocol.io/schemas/2025-12-11/server.schema.json",
   "name": "io.github.iris-eval/mcp-server",
-  "description": "The agent eval standard for MCP. Score every agent output for quality, safety, and cost.",
+  "description": "Stop shipping agents on vibes. Score every agent output for quality, safety, and cost.",
   "repository": {
     "url": "https://github.com/iris-eval/mcp-server",
     "source": "github"
   },
-  "version": "0.4.6",
+  "version": "0.5.1",
   "packages": [
     {
       "registryType": "npm",
       "identifier": "@iris-eval/mcp-server",
-      "version": "0.4.6",
+      "version": "0.5.1",
       "transport": {
         "type": "stdio"
       },

package/dist/dashboard/assets/index-B4Aw6ozt.css

@@ -1 +0,0 @@
-@import "https://fonts.googleapis.com/css2?family=Space+Grotesk:wght@500..700&family=Manrope:wght@400..700&family=JetBrains+Mono:wght@400..700&display=swap";:root{--lightningcss-light:initial;--lightningcss-dark: ;color-scheme:light dark;--iris-50:#f0fdfa;--iris-100:#ccfbf1;--iris-200:#99f6e4;--iris-300:#5eead4;--iris-400:#2dd4bf;--iris-500:#14b8a6;--iris-600:#0d9488;--iris-700:#0f766e;--iris-800:#115e59;--iris-900:#134e4a;--iris-950:#042f2e;--eval-pass:#22c55e;--eval-warn:#eab308;--eval-fail:#ef4444;--eval-tool:#3b82f6;--eval-llm:#a855f7;--eval-skipped:#71717a}@media (prefers-color-scheme:dark){:root{--lightningcss-light: ;--lightningcss-dark:initial}}:root,[data-theme=dark]{--bg-base:#050508;--bg-raised:#08080e;--bg-surface:#0d0d15;--bg-card:#101018;--bg-card-hover:#16161f;--border-subtle:#ffffff0d;--border-default:#ffffff14;--border-strong:#ffffff24;--border-glow:#14b8a680;--text-primary:#f0f0f5;--text-secondary:#9494a8;--text-muted:#5e5e72;--text-accent:var(--iris-400);--glow-primary:#14b8a61f;--glow-strong:#14b8a640;--shadow-sm:0 1px 2px #0000004d;--shadow-md:0 4px 6px #0006;--shadow-lg:0 10px 15px #00000080}[data-theme=light]{--bg-base:#fafcfc;--bg-raised:#f1f5f5;--bg-surface:#e8eded;--bg-card:#fff;--bg-card-hover:#f4f8f8;--border-subtle:#0000000a;--border-default:#00000014;--border-strong:#00000024;--border-glow:#0d948859;--text-primary:#0a0f0e;--text-secondary:#3d5250;--text-muted:#7a908e;--text-accent:var(--iris-700);--glow-primary:#0d94880f;--glow-strong:#0d94881f;--shadow-sm:0 1px 2px #0000000f;--shadow-md:0 4px 6px #00000014;--shadow-lg:0 10px 15px #0000001a}:root{--font-display:"Space Grotesk", -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;--font-body:"Manrope", -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;--font-mono:"JetBrains Mono", "Fira Code", ui-monospace, monospace;--font-sans:var(--font-body);--text-caption-xs:11px;--text-caption:12px;--text-body-sm:13px;--text-body:14px;--text-body-lg:15px;--text-heading-sm:16px;--text-heading:20px;--text-display-sm:28px;--text-display:40px;--font-size-xs:var(--text-caption);--font-size-sm:var(--text-body-sm);--font-size-base:var(--text-body);--font-size-lg:var(--text-body-lg);--font-size-xl:var(--text-heading-sm);--font-size-2xl:var(--text-heading);--font-size-3xl:var(--text-display-sm);--leading-body:1.5;--leading-heading:1.2;--leading-display:1.1;--leading-mono:1.4;--space-0_5:2px;--space-1:4px;--space-1_5:6px;--space-2:8px;--space-2_5:10px;--space-3:12px;--space-4:16px;--space-5:20px;--space-6:24px;--space-8:32px;--space-10:40px;--space-12:48px;--space-16:64px;--space-20:80px;--space-24:96px}:root,[data-density=compact]{--density-row:32px;--density-padding:var(--space-3);--density-body:var(--text-body-sm)}[data-density=comfortable]{--density-row:44px;--density-padding:var(--space-4);--density-body:var(--text-body)}:root{--sidebar-width-expanded:256px;--sidebar-width-collapsed:64px;--header-height:56px;--page-toolbar-height:40px;--radius-xs:4px;--radius-sm:6px;--radius:8px;--radius-lg:12px;--radius-xl:16px;--radius-pill:999px;--border-radius:var(--radius);--border-radius-sm:var(--radius-xs);--border-radius-lg:var(--radius-lg);--transition-instant:.1s ease;--transition-fast:.15s ease;--transition-base:.2s ease;--transition-slow:.3s ease;--ease-iris:cubic-bezier(.25, .4, .25, 1);--bg-primary:var(--bg-base);--bg-secondary:var(--bg-raised);--bg-tertiary:var(--bg-surface);--bg-hover:var(--bg-card-hover);--border-color:var(--border-default);--accent-primary:var(--iris-500);--accent-primary-hover:var(--iris-400);--accent-success:var(--eval-pass);--accent-error:var(--eval-fail);--accent-warning:var(--eval-warn);--accent-tool:var(--eval-tool);--accent-llm:var(--eval-llm)}html{transition:background-color var(--transition-base), color var(--transition-base)}*,:before,:after{box-sizing:border-box;margin:0;padding:0}html,body,#root{width:100%;height:100%}body{font-family:var(--font-body);font-size:var(--text-body);color:var(--text-primary);background-color:var(--bg-base);line-height:var(--leading-body);-webkit-font-smoothing:antialiased;font-feature-settings:"cv11", "ss01"}h1,h2,h3,h4,.display{font-family:var(--font-display);letter-spacing:-.01em;font-weight:600}a{color:var(--text-accent);text-decoration:none}a:hover{color:var(--iris-300)}button{cursor:pointer;font-family:inherit}input,select{font-family:inherit;font-size:inherit}code,pre{font-family:var(--font-mono)}a:focus-visible,button:focus-visible,input:focus-visible,select:focus-visible,[role=button]:focus-visible{outline:2px solid var(--iris-500);outline-offset:2px;border-radius:var(--radius-xs)}.iris-sr-reveal{clip:rect(0, 0, 0, 0);white-space:nowrap;border:0;width:1px;height:1px;margin:-1px;padding:0;position:absolute;overflow:hidden}.iris-sr-reveal:focus-within{width:auto;height:auto;margin:var(--space-3) 0 0 0;padding:var(--space-3) var(--space-4);clip:auto;white-space:normal;background:var(--bg-card);color:var(--text-primary);border:1px solid var(--iris-500);border-radius:var(--radius-sm);position:static;overflow:visible}.iris-sr-reveal:focus-within>li{padding:var(--space-1) 0;list-style:none}::selection{background:var(--iris-600);color:#fff}::-webkit-scrollbar{width:8px;height:8px}::-webkit-scrollbar-track{background:var(--bg-base)}::-webkit-scrollbar-thumb{background:var(--border-strong);border-radius:var(--radius-xs)}::-webkit-scrollbar-thumb:hover{background:var(--text-muted)}html{scrollbar-color:var(--border-strong) transparent;scrollbar-width:thin}@keyframes pulse-ring{0%{opacity:.5;transform:scale(1)}to{opacity:0;transform:scale(2.5)}}.pulse-dot{position:relative}.pulse-dot:after{content:"";background:var(--iris-500);border-radius:50%;animation:2s ease-out infinite pulse-ring;position:absolute;inset:-2px}@media (prefers-reduced-motion:reduce){*,:before,:after{scroll-behavior:auto!important;transition-duration:.01ms!important;animation-duration:.01ms!important;animation-iteration-count:1!important}}@media (width<=767px){aside[aria-label=Main\ navigation]{width:160px}main{overflow-x:auto}}@media print{body{color:#000!important;background:#fff!important}aside[aria-label=Main\ navigation],header,[role=region][aria-label=Welcome],[role=region][aria-label=Bulk\ actions],[role=status],[role=dialog]{display:none!important}body,#root,main{height:auto!important;overflow:visible!important}main{padding:0!important}tr,pre,code{page-break-inside:avoid}h1,h2,h3{page-break-after:avoid}[aria-label*=violation],[aria-label*=spike],[aria-label*=collision],[aria-label*=Pass],[aria-label*=Fail]{border:1px solid #000!important}a{color:#000!important;text-decoration:underline!important}a[href^=http]:after{content:" (" attr(href) ")";color:#555;font-size:80%}}