@iris-eval/mcp-server 0.5.0 → 0.5.1

package/dist/storage/migrations/006-eval-critical-failures.d.ts

@@ -0,0 +1,3 @@
+import type Database from 'better-sqlite3';
+export declare const id = "006-eval-critical-failures";
+export declare function up(db: Database.Database): void;

package/dist/storage/migrations/006-eval-critical-failures.js

@@ -0,0 +1,23 @@
+export const id = '006-eval-critical-failures';
+/*
+ * v0.5.0's headline feature — the critical-rule veto — was response-only.
+ * `critical_failures` was returned to the caller and then dropped on the
+ * floor: `insertEvalResult` never wrote it, so once an evaluation was
+ * stored, a vetoed eval was indistinguishable from one that simply scored
+ * below the threshold. Nothing downstream could filter, count or badge the
+ * flagship behaviour, and the dashboard showed "safety · fail  score 0.92"
+ * with no way to say WHY it failed.
+ *
+ * JSON text rather than a join table: it mirrors how rule_results and
+ * suggestions are already stored, keeps the read path a single row, and the
+ * array is small and read-only after write.
+ *
+ * NULL for every row written before this migration, which is honest — those
+ * evaluations predate the veto, so "no recorded veto" is the truth rather
+ * than an empty array asserting there was none.
+ */
+export function up(db) {
+    db.exec(`
+    ALTER TABLE eval_results ADD COLUMN critical_failures TEXT;
+  `);
+}

package/dist/storage/migrations/index.js

@@ -3,12 +3,14 @@ import * as migration002 from './002-eval-skip-fields.js';
 import * as migration003 from './003-eval-passed-index.js';
 import * as migration004 from './004-tenant-id.js';
 import * as migration005 from './005-normalize-created-at.js';
+import * as migration006 from './006-eval-critical-failures.js';
 const migrations = [
     migration001,
     migration002,
     migration003,
     migration004,
     migration005,
+    migration006,
 ];
 export function runMigrations(db) {
     db.exec(`

package/dist/storage/sqlite-adapter.js

@@ -185,10 +185,18 @@ export class SqliteAdapter {
          * calendar date matched the boundary's date was dropped from the
          * window. Migration 005 rewrites rows written before this line existed.
          */
+        /*
+         * critical_failures is PERSISTED (migration 006) because the veto is a
+         * verdict, not a presentation detail. It used to live only in the live
+         * tool response, so the moment an evaluation was stored a vetoed eval
+         * became indistinguishable from one that merely scored below threshold —
+         * no surface could filter, count, or explain the release's flagship
+         * behaviour. NULL when nothing vetoed.
+         */
         this.db.prepare(`
-      INSERT INTO eval_results (tenant_id, id, trace_id, eval_type, output_text, expected_text, score, passed, rule_results, suggestions, rules_evaluated, rules_skipped, insufficient_data, created_at)
-      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-    `).run(tenantId, result.id, result.trace_id ?? null, result.eval_type, result.output_text, result.expected_text ?? null, result.score, result.passed ? 1 : 0, JSON.stringify(result.rule_results), JSON.stringify(result.suggestions), result.rules_evaluated ?? null, result.rules_skipped ?? null, result.insufficient_data ? 1 : 0, new Date().toISOString());
+      INSERT INTO eval_results (tenant_id, id, trace_id, eval_type, output_text, expected_text, score, passed, rule_results, suggestions, rules_evaluated, rules_skipped, insufficient_data, critical_failures, created_at)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `).run(tenantId, result.id, result.trace_id ?? null, result.eval_type, result.output_text, result.expected_text ?? null, result.score, result.passed ? 1 : 0, JSON.stringify(result.rule_results), JSON.stringify(result.suggestions), result.rules_evaluated ?? null, result.rules_skipped ?? null, result.insufficient_data ? 1 : 0, result.critical_failures?.length ? JSON.stringify(result.critical_failures) : null, new Date().toISOString());
     }
     async getEvalsByTraceId(tenantId, traceId) {
         assertTenant(tenantId);
@@ -553,6 +561,14 @@ export class SqliteAdapter {
             rules_evaluated: row.rules_evaluated,
             rules_skipped: row.rules_skipped,
             insufficient_data: row.insufficient_data != null ? row.insufficient_data === 1 : undefined,
+            /*
+             * Absent, not [], when NULL. Rows written before migration 006 never
+             * captured the field, and returning an empty array would assert "no
+             * critical rule failed" about an evaluation that never recorded one.
+             */
+            ...(row.critical_failures != null
+                ? { critical_failures: JSON.parse(row.critical_failures) }
+                : {}),
         };
     }
 }

package/dist/tools/evaluate-output.js

@@ -17,8 +17,8 @@ const inputSchema = {
     // case gets a note in the response saying safety rules did not run. The
     // effective default is still completeness.
     eval_type: z.enum(['completeness', 'relevance', 'safety', 'cost', 'custom']).optional().describe('Rule bundle to apply: completeness | relevance | safety | cost | custom — picks which built-in rules fire. Defaults to "completeness" when omitted (the response then carries a note that safety rules did not run)'),
-    expected: z.string().optional().describe('Expected output for comparison — REQUIRED when eval_type="relevance" (used as keyword-overlap target)'),
-    input: z.string().optional().describe('Original input for context (the ask + any source material the agent was given) — improves relevance scoring and grounds the safety bundle\'s hallucination signals'),
+    expected: z.string().optional().describe('Expected output for comparison — consulted only by the completeness bundle\'s expected_coverage rule; NOT used by relevance (the relevance rules compare the output against `input`)'),
+    input: z.string().optional().describe('Original input for context (the ask + any source material the agent was given) — REQUIRED when eval_type="relevance" (keyword_overlap and topic_consistency compare the output against it and skip without it); also grounds the safety bundle\'s hallucination signals'),
     trace_id: z.string().optional().describe('Link evaluation to a trace — surfaces this eval in the dashboard\'s trace drill-through'),
     // .max(10): inline rules skip the deploy-time probe, and the engine runs
     // rules synchronously — without a cap, one request carrying N sandbox-
@@ -26,7 +26,7 @@ const inputSchema = {
     // N=50). Ten is ample for per-call rules; persistent sets belong in
     // deploy_rule, where deploy-time validation probes each pattern.
     custom_rules: z.array(CustomRuleSchema).max(10).optional().describe('Custom evaluation rules, max 10 per call (deploy persistent rule sets via deploy_rule instead) — fires REGARDLESS of eval_type; pass eval_type="custom" if you want ONLY these'),
-    cost_usd: z.number().optional().describe('Cost in USD — only consulted when eval_type="cost" (compared against cost_threshold rules)'),
+    cost_usd: z.number().optional().describe('Cost in USD — consulted by the cost bundle (eval_type="cost") AND by any cost_threshold custom rule regardless of eval_type; omit it and such a rule skips rather than passes (a critical one is listed in critical_skipped)'),
     token_usage: z.object({
         prompt_tokens: z.number().optional(),
         completion_tokens: z.number().optional(),
@@ -43,17 +43,17 @@ export function registerEvaluateOutputTool(server, storage, evalEngine) {
             '',
             'Behavior. Deterministic, in-process scoring — same inputs always produce the same result. Writes one eval_result row to Iris storage (linked to trace_id if provided; unlinked otherwise). No external network calls in heuristic mode (v0.4 adds an llm_as_judge eval_type that DOES call LLM APIs; see the separate evaluate_with_llm_judge tool for that). Rate-limited to 20 req/min on HTTP MCP, unlimited on stdio. Runs in ~5-50ms for rule-based evaluation.',
             '',
-            'Output shape. Returns JSON: `{ "id": "<uuid>", "eval_type": "<bundle that ran>", "score": 0..1, "passed": boolean, "critical_failures?": string[], "rule_results": [{ "ruleName", "passed", "score", "message", "skipped?" }], "suggestions": string[], "rules_evaluated": number, "rules_skipped": number, "insufficient_data": boolean, "note?": string }`. `insufficient_data=true` means no applicable rules fired (e.g., safety eval with only cost data). `note` appears only when eval_type was omitted, naming the defaulted bundle and that safety rules did not run.',
+            'Output shape. Returns JSON: `{ "id": "<uuid>", "eval_type": "<bundle that ran>", "score": 0..1, "passed": boolean, "critical_failures?": string[], "critical_skipped?": string[], "rule_results": [{ "ruleName", "passed", "score", "message", "skipped?", "skipReason?", "budgetExceeded?", "configInvalid?" }], "suggestions": string[], "rules_evaluated": number, "rules_skipped": number, "insufficient_data": boolean, "note?": string }`. `insufficient_data=true` means no applicable rules fired (e.g., safety eval with only cost data). `note` appears only when eval_type was omitted, naming the defaulted bundle and that safety rules did not run.',
             '',
-            'What `passed` means. `score` and `passed` answer different questions. `score` is the weighted average across the rules that ran — a 0..1 quality gradient. `passed` is the ship/no-ship verdict: true only when the score clears the pass threshold (default 0.7, configurable via config `eval.defaultThreshold`) AND no critical rule failed. Critical rules HARD-FAIL: if one fails, `passed` is false regardless of the weighted score, and the culprits are listed in `critical_failures`. The critical rules are the genuine safety violations — `no_pii`, `no_injection_patterns`, `no_blocklist_words` — plus any deployed custom rule with severity high/critical. A leaked SSN can never be averaged away by other rules passing.',
+            'What `passed` means. `score` and `passed` answer different questions. `score` is the weighted average across the rules that ran — a 0..1 quality gradient. `passed` is the ship/no-ship verdict: true only when the score clears the pass threshold (default 0.7, configurable via config `eval.defaultThreshold`) AND no critical rule failed. Critical rules HARD-FAIL: if one fails, `passed` is false regardless of the weighted score, and the culprits are listed in `critical_failures`. The critical rules are the genuine safety violations — `no_pii`, `no_injection_patterns`, `no_blocklist_words` — plus any deployed custom rule with severity high/critical. A leaked SSN can never be averaged away by other rules passing. One caveat, stated because it is reachable on purpose: a critical rule that SKIPPED did not judge the output and therefore cannot veto — a regex rule whose match blew the 100ms sandbox budget on crafted output skips, so `passed` can be true with no `critical_failures`. Every such rule is named in `critical_skipped`. If your gate must fail closed, treat a non-empty `critical_skipped` as UNKNOWN, not clean.',
             '',
-            'Use when you want a quality score on a specific output — typically after log_trace records the execution. Pass `eval_type` to route to the right rule bundle: `completeness` (length, sentence count, relevance to input), `relevance` (keyword overlap, topic consistency), `safety` (PII leak, prompt injection, hallucination markers, stub-output detection — pass `input` so the hallucination signals can cross-check the output against the material the agent was given), `cost` (budget threshold), or `custom` (bring your own rules via `custom_rules`).',
+            'Use when you want a quality score on a specific output — typically after log_trace records the execution. Pass `eval_type` to route to the right rule bundle: `completeness` (length, non-empty output, sentence count, coverage of `expected`), `relevance` (keyword overlap and topic consistency against `input`), `safety` (PII leak, prompt injection, hallucination markers, stub-output detection — pass `input` so the hallucination signals can cross-check the output against the material the agent was given), `cost` (budget threshold), or `custom` (bring your own rules via `custom_rules`).',
             '',
             'Don\'t use when the output is empty or has no applicable rules — the eval_type decides which rules apply, and invalid combinations return score=0 + insufficient_data=true (not an error, but not actionable). Don\'t use to VALIDATE JSON schemas directly (use your language\'s JSON Schema validator — Iris\'s `json_schema` custom rule type is for output-shape assertions, not arbitrary validation).',
             '',
-            'Parameters. expected is REQUIRED when eval_type="relevance" (used as the comparison target for keyword overlap + topic consistency); ignored for other eval_types. cost_usd + token_usage are ONLY consulted when eval_type="cost" (ignored otherwise). custom_rules ALWAYS fires regardless of eval_type — pass eval_type="custom" if you want ONLY your rules to run (otherwise both your rules AND the eval_type bundle run together). trace_id is optional but recommended (linking the eval to its trace surfaces it in the dashboard\'s drill-through). input adds context to keyword-overlap relevance checks AND grounds the safety bundle\'s hallucination signals (without it those signals stay silent rather than guess); ignored otherwise. Defaults: eval_type="completeness" — and when you rely on that default, the response carries a `note` reminding you that the safety bundle did not run.',
+            'Parameters. input is REQUIRED when eval_type="relevance" (keyword_overlap and topic_consistency compare the output against it; without it both rules skip and the response reports insufficient_data=true) AND grounds the safety bundle\'s hallucination signals (without it those signals stay silent rather than guess); ignored otherwise. expected is consulted only by the completeness bundle\'s expected_coverage rule; ignored for other eval_types — it is NOT the relevance target. cost_usd is consulted by the cost bundle AND by any cost_threshold custom rule regardless of eval_type — omit it and such a rule skips rather than passes (a critical one is listed in critical_skipped); token_usage is ONLY consulted when eval_type="cost". custom_rules ALWAYS fires regardless of eval_type — pass eval_type="custom" if you want ONLY your rules to run (otherwise both your rules AND the eval_type bundle run together). trace_id is optional but recommended (linking the eval to its trace surfaces it in the dashboard\'s drill-through). Defaults: eval_type="completeness" — and when you rely on that default, the response carries a `note` reminding you that the safety bundle did not run.',
             '',
-            'Error modes. Throws on unknown argument names (strict schema — a misspelled argument is rejected with the valid argument list, never silently dropped). Throws on malformed custom_rules (Zod rejects) and on more than 10 custom_rules in one call (use deploy_rule for persistent rule sets). Returns 400 on regex patterns that fail safe-regex2 ReDoS check or exceed 1000-char limit. Returns 429 when HTTP rate limit exceeded. Storage failures propagate as 500. The eval itself never throws — failing rules report `passed: false` with a message, they don\'t bubble exceptions. A regex that exceeds the 100ms sandbox matching budget on a given output reports skipped with budgetExceeded=true instead of hanging the server (fail-open per rule — gate on that flag if you must fail closed).',
+            'Error modes. Throws on unknown argument names (strict schema — a misspelled argument is rejected with the valid argument list, never silently dropped). Throws on malformed custom_rules (Zod rejects the shape: missing name/type, unknown type, non-object config) and on more than 10 custom_rules in one call (use deploy_rule for persistent rule sets). An inline rule whose CONFIG is unusable — a regex that fails the safe-regex2 ReDoS check or exceeds the 1000-char limit, a missing or non-string config.pattern, non-string keywords — does NOT error: that rule reports skipped with configInvalid=true and a skipReason naming the field, and the other rules still run (deploy_rule rejects the same configs with a 400 at deploy time). Returns 429 when HTTP rate limit exceeded. Storage failures propagate as 500. The eval itself never throws — failing rules report `passed: false` with a message, they don\'t bubble exceptions. A regex that exceeds the 100ms sandbox matching budget on a given output reports skipped with budgetExceeded=true instead of hanging the server (fail-open per rule — gate on that flag if you must fail closed).',
         ].join('\n'),
         inputSchema: strictInput(inputSchema),
         annotations: {
@@ -95,6 +95,13 @@ export function registerEvaluateOutputTool(server, storage, evalEngine) {
                         score: result.score,
                         passed: result.passed,
                         ...(result.critical_failures ? { critical_failures: result.critical_failures } : {}),
+                        // The other half of the veto contract. The engine names every
+                        // critical rule that SKIPPED (budget-killed regex, missing cost
+                        // data) so a fail-closed gate can treat the eval as unknown;
+                        // this response used to drop the field, so the gate the
+                        // description tells users to write keyed on something that
+                        // never arrived and read passed:true as clean.
+                        ...(result.critical_skipped ? { critical_skipped: result.critical_skipped } : {}),
                         rule_results: result.rule_results,
                         suggestions: result.suggestions,
                         rules_evaluated: result.rules_evaluated,

package/dist/tools/evaluate-with-llm-judge.js

@@ -70,7 +70,7 @@ export function registerEvaluateWithLLMJudgeTool(server, storage) {
             '',
             "Don't use for simple regex/length/keyword checks (use evaluate_output with heuristic rules — they're free, deterministic, 1000x faster). Don't use without an API key set (IRIS_ANTHROPIC_API_KEY or IRIS_OPENAI_API_KEY). Don't use on very large outputs (>8K tokens) without raising max_cost_usd — the pre-check will refuse the call.",
             '',
-            'Parameters. model is required (no default — pick consciously since cost varies 100x across models). provider is auto-detected from the model name; override only for ambiguous IDs. expected is REQUIRED when template="correctness" (the reference answer to compare against); ignored for other templates. source_material is REQUIRED when template="faithfulness" (the RAG sources to ground against); ignored otherwise. input is optional but improves scoring on helpfulness/safety templates (gives the judge the user prompt that produced the output). max_cost_usd defaults to env var IRIS_LLM_JUDGE_MAX_COST_USD_PER_EVAL or $0.25 — the worst-case cost is computed BEFORE the call (input_tokens × prompt_price + max_output_tokens × completion_price); call refused upfront if it would exceed. max_output_tokens caps the judge response (default 512, max 4096); higher = more rationale detail + more cost. temperature default 0 (deterministic). timeout_ms default 60000. trace_id optional but recommended (links eval to trace in dashboard). Defaults: temperature=0, max_output_tokens=512, max_cost_usd=$0.25, timeout_ms=60000.',
+            'Parameters. model is required (no default — pick consciously since cost varies 100x across models). provider is auto-detected from the model name; override only for ambiguous IDs. expected is REQUIRED when template="correctness" (the reference answer to compare against); ignored for other templates. source_material is REQUIRED when template="faithfulness" (the RAG sources to ground against); ignored otherwise. input is optional but improves scoring on helpfulness/safety templates (gives the judge the user prompt that produced the output). max_cost_usd defaults to env var IRIS_LLM_JUDGE_MAX_COST_USD_PER_EVAL or $0.25 — the worst-case cost is computed BEFORE the call (input_tokens × prompt_price + max_output_tokens × completion_price, PLUS the same for the one retry that fires if the judge\'s first reply is not valid JSON); call refused upfront if that two-attempt worst case would exceed. When a retry does run, the reported input_tokens / output_tokens / cost_usd / latency_ms are totals across both attempts. max_output_tokens caps the judge response (default 512, max 4096); higher = more rationale detail + more cost. temperature default 0 (deterministic). timeout_ms default 60000. trace_id optional but recommended (links eval to trace in dashboard). Defaults: temperature=0, max_output_tokens=512, max_cost_usd=$0.25, timeout_ms=60000.',
             '',
             'Error modes. Throws when the required API key env var is missing. Throws when the estimated worst-case cost exceeds max_cost_usd (raise the cap or trim prompts). Throws LLMJudgeError on provider errors — kind=`auth` on 401/403, `rate_limit` on 429 (auto-retried once), `server_error` on 5xx, `timeout` on abort, `malformed_response` when the judge fails to emit valid JSON on both attempts. Throws "Unknown model" for unsupported model IDs — update src/eval/llm-judge/pricing.ts first.',
         ].join('\n'),

package/dist/tools/log-trace.js

@@ -59,7 +59,7 @@ export function registerLogTraceTool(server, storage) {
             '',
             'Sibling tools — evaluate_output runs heuristic scoring on the trace; evaluate_with_llm_judge runs semantic LLM-based scoring; verify_citations checks citation grounding; get_traces queries stored traces; delete_trace removes a single trace; list_rules / deploy_rule / delete_rule manage custom evaluation rules. log_trace is the WRITE path that records executions; everything else reads, scores, or manages around it.',
             '',
-            'Behavior. Writes one row to Iris storage (SQLite by default; Postgres in Cloud tier). When IRIS_OTEL_ENDPOINT is set, ALSO fires a best-effort async export to the configured OTLP/HTTP collector (Jaeger, Tempo, Datadog OTLP, OTEL Collector). The OTel export is fire-and-forget — its success does not affect the tool response; failures are logged but the trace is still stored locally. No authentication in stdio mode; HTTP mode requires Bearer token. Rate-limited to 20 req/min on HTTP MCP, unlimited on stdio. Not idempotent: each call mints a fresh trace_id, so resubmitting the same payload creates a duplicate trace.',
+            'Behavior. Writes one row to Iris storage (SQLite by default; Postgres in Cloud tier). When IRIS_OTEL_ENDPOINT is set, ALSO fires a best-effort async export to the configured OTLP/HTTP collector (Jaeger, Tempo, Datadog OTLP, OTEL Collector). The OTel export is fire-and-forget — its success does not affect the tool response; failures are logged but the trace is still stored locally. No authentication in stdio mode. HTTP mode requires a Bearer token ONLY when --api-key / IRIS_API_KEY is set (recommended); with no key configured the auth middleware is a pass-through and writes are unauthenticated — a default HTTP server is protected by its loopback bind (127.0.0.1) and Origin validation, not by a credential. Rate-limited to 20 req/min on HTTP MCP, unlimited on stdio. Not idempotent: each call mints a fresh trace_id, so resubmitting the same payload creates a duplicate trace.',
             '',
             'Output shape. Returns a JSON string: `{ "trace_id": "<32-hex>", "status": "stored" }`. The trace_id is the key you pass to evaluate_output or get_traces afterwards.',
             '',

package/dist/tools/verify-citations.js

@@ -64,7 +64,7 @@ export function registerVerifyCitationsTool(server, storage) {
             "",
             "Don't use when the agent output has no citations at all (overall_score will be null; the tool degrades gracefully but a heuristic rule is cheaper). Don't use without allow_fetch=true or IRIS_CITATION_ALLOW_FETCH=1 — the tool refuses outbound HTTP unless explicitly enabled. Don't use with an open allowlist + untrusted output on the public internet; you are effectively running a user-directed fetcher. For stricter safety set IRIS_CITATION_DOMAINS to a curated list.",
             '',
-            'Parameters. model is required; provider auto-detected from model name (override only for ambiguous IDs). allow_fetch=false by default — outbound HTTP is REFUSED unless explicitly true OR IRIS_CITATION_ALLOW_FETCH=1 env. domain_allowlist suffix-matches hostnames (e.g., "wikipedia.org" allows en.wikipedia.org); merged with IRIS_CITATION_DOMAINS env (UNION — either source permits). max_citations defaults 20, hard cap 50 (extras are skipped silently, NOT errored — check total_citations_found in the response if precise). max_cost_usd_total defaults $1.00 — the pipeline stops mid-citation when the next judge call would exceed the cap (returns partial verdicts). per_source_timeout_ms defaults 10000 (10s); per_source_max_bytes defaults 5MB (truncates at boundary, judges still run on truncated content). trace_id optional but recommended. Defaults: max_citations=20, max_cost_usd_total=$1.00, per_source_timeout_ms=10000, per_source_max_bytes=5242880, allow_fetch=false.',
+            'Parameters. model is required; provider auto-detected from model name (override only for ambiguous IDs). allow_fetch=false by default — outbound HTTP is REFUSED unless explicitly true OR IRIS_CITATION_ALLOW_FETCH=1 env. domain_allowlist suffix-matches hostnames (e.g., "wikipedia.org" allows en.wikipedia.org); merged with IRIS_CITATION_DOMAINS env (UNION — either source permits). max_citations defaults 20, hard cap 50 (extras are skipped silently, NOT errored — check total_citations_found in the response if precise). max_cost_usd_total defaults $1.00 — the pipeline stops mid-citation when the next judge call would exceed the cap (returns partial verdicts). per_source_timeout_ms defaults 10000 (10s); per_source_max_bytes defaults 5MB (truncates at boundary, judges still run on truncated content); independently of that, the judge reads at most the first 12,000 characters of each fetched source, and the per-citation cost estimate is taken on that truncated prompt, not on the full body. trace_id optional but recommended. Defaults: max_citations=20, max_cost_usd_total=$1.00, per_source_timeout_ms=10000, per_source_max_bytes=5242880, allow_fetch=false.',
             '',
             'Error modes. Throws when the API key env var is missing. Throws "Unknown model" on unsupported model IDs. Per-citation errors are collected (resolve_error.kind = bad_scheme / ssrf / not_allowed_domain / timeout / too_large / bad_status / redirect_loop / not_text / fetch_disabled / malformed_judge_response / cost_cap_reached / unresolvable_kind) and returned in the response rather than thrown. An empty output or output with zero extractable citations returns overall_score=null + passed=true (nothing to fail).',
         ].join('\n'),

package/dist/types/decision-moment.d.ts

@@ -69,6 +69,14 @@ export interface DecisionMomentDetail extends DecisionMoment {
             skipReason?: string;
         }>;
         suggestions: string[];
+        /**
+         * Rules that HARD-FAILED this evaluation — a critical safety rule
+         * (no_pii / no_injection_patterns / no_blocklist_words) or a deployed
+         * rule with severity high/critical. Present only when the veto fired;
+         * absent means nothing vetoed, or the row predates migration 006. This
+         * is what lets a surface distinguish "vetoed" from "scored low".
+         */
+        criticalFailures?: string[];
         createdAt?: string;
     }>;
     /** Full input (uncompressed). */

package/dist/types/eval.d.ts

@@ -78,6 +78,21 @@ export interface EvalResult {
      * "committed a hard violation".
      */
     critical_failures?: string[];
+    /**
+     * Names of critical rules that were SKIPPED and therefore did not judge
+     * this output (present only when non-empty). Almost always a sandbox
+     * budget breach — a regex killed mid-backtrack, which an adversary can
+     * provoke deliberately by crafting output that stalls a known pattern.
+     *
+     * This is the fail-open seam between the release's two headline features:
+     * a budget-killed critical rule does NOT veto, so the evaluation can
+     * return passed=true with no `critical_failures` at all. That is
+     * deliberate (failing closed would let the same adversary force false
+     * violations on benign output), but a consumer that must fail closed
+     * needs to see it WITHOUT walking rule_results[].budgetExceeded. Treat a
+     * non-empty `critical_skipped` as "unknown", not as "clean".
+     */
+    critical_skipped?: string[];
 }
 export type CustomRuleType = 'regex_match' | 'regex_no_match' | 'min_length' | 'max_length' | 'contains_keywords' | 'excludes_keywords' | 'json_schema' | 'cost_threshold';
 export interface CustomRuleDefinition {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@iris-eval/mcp-server",
-  "version": "0.5.0",
+  "version": "0.5.1",
   "description": "Stop shipping agents on vibes. Score every agent output for quality, safety, and cost.",
   "mcpName": "io.github.iris-eval/mcp-server",
   "type": "module",

package/server.json

@@ -6,12 +6,12 @@
     "url": "https://github.com/iris-eval/mcp-server",
     "source": "github"
   },
-  "version": "0.5.0",
+  "version": "0.5.1",
   "packages": [
     {
       "registryType": "npm",
       "identifier": "@iris-eval/mcp-server",
-      "version": "0.5.0",
+      "version": "0.5.1",
       "transport": {
         "type": "stdio"
       },