@iris-eval/mcp-server 0.4.6 → 0.5.0

package/dist/dashboard/server.js

@@ -2,7 +2,8 @@ import express from 'express';
 import helmet from 'helmet';
 import { fileURLToPath } from 'node:url';
 import { dirname, join } from 'node:path';
-import { existsSync } from 'node:fs';
+import { existsSync, mkdirSync, writeFileSync } from 'node:fs';
+import { irisHome } from '../utils/iris-home.js';
 import { createAuthMiddleware } from '../middleware/auth.js';
 import { createCorsMiddleware } from '../middleware/cors.js';
 import { createErrorHandler } from '../middleware/error-handler.js';
@@ -16,6 +17,7 @@ import { registerFilterRoutes } from './routes/filters.js';
 import { registerEvalStatsRoutes } from './routes/eval-stats.js';
 import { registerHealthRoutes } from './routes/health.js';
 import { registerMomentRoutes } from './routes/moments.js';
+import { registerFailureRoutes } from './routes/failures.js';
 import { registerRuleRoutes } from './routes/rules.js';
 import { registerPreferencesRoutes } from './routes/preferences.js';
 import { registerAuditRoutes } from './routes/audit.js';
@@ -27,15 +29,13 @@ export function createDashboardServer(storage, config, logger, options) {
             directives: {
                 defaultSrc: ["'self'"],
                 scriptSrc: ["'self'"],
-                // 'self' covers our bundled CSS. fonts.googleapis.com hosts the
-                // brand fonts (Space Grotesk + Manrope + JetBrains Mono) loaded
-                // via @import in tokens.css. Without this, the @import gets
-                // blocked and the entire stylesheet is dropped by the browser.
-                // v0.4.1 will self-host these fonts and let us tighten this back
-                // to 'self' only.
-                styleSrc: ["'self'", "'unsafe-inline'", "https://fonts.googleapis.com"],
-                // The fontFaces in those stylesheets resolve to fonts.gstatic.com.
-                fontSrc: ["'self'", "https://fonts.gstatic.com", "data:"],
+                // 'self' covers our bundled CSS. The brand fonts (Space Grotesk +
+                // Manrope + JetBrains Mono) are self-hosted from /fonts as of
+                // #334, so no Google Fonts origins are needed. 'unsafe-inline'
+                // stays: the React components set style={} inline throughout.
+                styleSrc: ["'self'", "'unsafe-inline'"],
+                // Self-hosted woff2 under /fonts resolves via 'self'.
+                fontSrc: ["'self'", "data:"],
                 connectSrc: ["'self'"],
             },
         },
@@ -65,13 +65,14 @@ export function createDashboardServer(storage, config, logger, options) {
     // API routes with rate limiting
     const router = express.Router();
     router.use(createApiRateLimiter(config));
-    registerTraceRoutes(router, storage);
+    registerTraceRoutes(router, storage, { evalEngine: options?.evalEngine });
     registerSummaryRoutes(router, storage);
     registerEvaluationRoutes(router, storage);
     registerEvalStatsRoutes(router, storage);
     registerFilterRoutes(router, storage);
     registerHealthRoutes(router, storage, config.server.version);
     registerMomentRoutes(router, storage);
+    registerFailureRoutes(router, storage);
     if (options?.customRuleStore && options?.evalEngine) {
         registerRuleRoutes(router, storage, {
             customRuleStore: options.customRuleStore,
@@ -101,6 +102,24 @@ export function createDashboardServer(storage, config, logger, options) {
     const currentDir = dirname(fileURLToPath(import.meta.url));
     const staticDir = join(currentDir, '..', '..', 'dist', 'dashboard');
     const indexHtml = join(staticDir, 'index.html');
+    /*
+     * An unmatched /api/ path must answer as an API, not as the app.
+     *
+     * The SPA fallback below is deliberately a blanket catch-all so deep links
+     * like /traces/<id> survive a reload. Without this guard it also swallowed
+     * mistyped API routes: `GET /api/v1/tracez` returned 200 with index.html,
+     * so a client saw SUCCESS and then threw "Unexpected token '<'" from
+     * res.json() — sending the developer to debug their payload instead of
+     * their URL. A liveness check asserting only status === 200 would call a
+     * nonexistent endpoint healthy. POST to an unknown /api/ route reached
+     * Express's HTML error page, which is the same problem in a smaller hat.
+     *
+     * Mounted before the static handler so it wins regardless of method, and
+     * scoped to /api/ so nothing else changes.
+     */
+    app.use('/api', (_req, res) => {
+        res.status(404).json({ error: 'Unknown API route' });
+    });
     if (existsSync(indexHtml)) {
         app.use(createApiRateLimiter(config));
         app.use(express.static(staticDir));
@@ -131,12 +150,46 @@ export function createDashboardServer(storage, config, logger, options) {
              * http` started the dashboard implicitly, so binding the MCP
              * transport to loopback still left a wide-open second server.
              */
-            const server = app.listen(config.dashboard.port, config.dashboard.host, () => {
+            // Distinguishes "never bound" from "failed after startup" so the
+            // error handler below can say which one actually happened.
+            let bound = false;
+            const server = app.listen(config.dashboard.port, config.dashboard.host, (err) => {
+                /*
+                 * Express 5 also invokes this callback on a bind ERROR (it wires it
+                 * via `server.once('error', done)`). Before this guard, a port
+                 * collision ran the success path anyway: it logged "Dashboard
+                 * available at http://localhost:<port>" — a URL owned by a DIFFERENT
+                 * process — and overwrote runtime.json to point capture clients at
+                 * that stranger. Failures belong to the 'error' handler below.
+                 */
+                if (err)
+                    return;
+                bound = true;
                 // Record the port actually bound so the rebinding guard builds its
                 // allowlist from it rather than from a configured 0.
                 const addr = server.address();
                 if (typeof addr === 'object' && addr)
                     boundPort = addr.port;
+                /*
+                 * Port-discovery handshake for capture clients (the
+                 * @iris-eval/capture design pins this contract): write the port
+                 * actually bound to ${IRIS_HOME}/runtime.json so an SDK can find
+                 * the ingest endpoint without configuration. Best-effort — a
+                 * failed write must never take the dashboard down. The file may
+                 * go stale after an unclean exit; clients are expected to verify
+                 * with GET /api/v1/health before trusting it.
+                 */
+                try {
+                    mkdirSync(irisHome(), { recursive: true });
+                    writeFileSync(join(irisHome(), 'runtime.json'), JSON.stringify({
+                        dashboardPort: boundPort ?? config.dashboard.port,
+                        pid: process.pid,
+                        startedAt: new Date().toISOString(),
+                    }, null, 2));
+                }
+                catch (err) {
+                    logger.warn(`Could not write runtime.json: ${err.message}`);
+                }
                 const shown = isLoopbackHost(config.dashboard.host) ? 'localhost' : config.dashboard.host;
                 logger.info(`Dashboard available at http://${shown}:${boundPort ?? config.dashboard.port}`);
                 if (!isLoopbackHost(config.dashboard.host) && !config.security.apiKey) {
@@ -152,14 +205,27 @@ export function createDashboardServer(storage, config, logger, options) {
              * handler which emits a warning but doesn't crash — so the process
              * keeps running in a broken state. We log the specific cause then
              * exit(1) so the user sees the actual problem.
+             *
+             * Exiting nonzero is correct here because the dashboard only starts
+             * when EXPLICITLY requested (--dashboard / IRIS_DASHBOARD / --demo —
+             * see src/index.ts): the user asked for a surface they will not get,
+             * and running on while a health gate reports "ready" would send them
+             * to a port owned by a different process.
              */
             server.on('error', (err) => {
                 if (err.code === 'EADDRINUSE') {
-                    logger.error(`Dashboard failed to start: port ${config.dashboard.port} is already in use. ` +
-                        `If running HTTP transport on the same port, use --dashboard-port <other>.`);
+                    logger.error(`Dashboard failed to start: port ${config.dashboard.port} is already in use ` +
+                        `(EADDRINUSE on ${config.dashboard.host}:${config.dashboard.port}). The dashboard was ` +
+                        `explicitly requested, so iris is exiting. Pass --dashboard-port <other> (or set ` +
+                        `IRIS_DASHBOARD_PORT) or stop the process that owns the port.`);
+                }
+                else if (!bound) {
+                    logger.error(`Dashboard failed to start on ${config.dashboard.host}:${config.dashboard.port}: ${err.message}`);
                 }
                 else {
-                    logger.error(`Dashboard server error: ${err.message}`);
+                    // Post-bind failure (e.g. EMFILE on accept) — "failed to start"
+                    // would misdescribe a server that had been up and serving.
+                    logger.error(`Dashboard server error after startup: ${err.message}`);
                 }
                 process.exit(1);
             });

package/dist/dashboard/validation.d.ts

@@ -1,4 +1,62 @@
 import { z } from 'zod';
+export declare const ingestTraceSchema: z.ZodObject<{
+    evaluate: z.ZodDefault<z.ZodBoolean>;
+    eval_type: z.ZodDefault<z.ZodEnum<{
+        completeness: "completeness";
+        relevance: "relevance";
+        safety: "safety";
+        cost: "cost";
+        custom: "custom";
+    }>>;
+    agent_name: z.ZodString;
+    framework: z.ZodOptional<z.ZodString>;
+    input: z.ZodOptional<z.ZodString>;
+    output: z.ZodOptional<z.ZodString>;
+    tool_calls: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        tool_name: z.ZodString;
+        input: z.ZodOptional<z.ZodUnknown>;
+        output: z.ZodOptional<z.ZodUnknown>;
+        latency_ms: z.ZodOptional<z.ZodNumber>;
+        error: z.ZodOptional<z.ZodString>;
+    }, z.core.$strip>>>;
+    latency_ms: z.ZodOptional<z.ZodNumber>;
+    token_usage: z.ZodOptional<z.ZodObject<{
+        prompt_tokens: z.ZodOptional<z.ZodNumber>;
+        completion_tokens: z.ZodOptional<z.ZodNumber>;
+        total_tokens: z.ZodOptional<z.ZodNumber>;
+    }, z.core.$strip>>;
+    cost_usd: z.ZodOptional<z.ZodNumber>;
+    metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+    spans: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        span_id: z.ZodOptional<z.ZodString>;
+        parent_span_id: z.ZodOptional<z.ZodString>;
+        name: z.ZodString;
+        kind: z.ZodDefault<z.ZodEnum<{
+            INTERNAL: "INTERNAL";
+            SERVER: "SERVER";
+            CLIENT: "CLIENT";
+            PRODUCER: "PRODUCER";
+            CONSUMER: "CONSUMER";
+            LLM: "LLM";
+            TOOL: "TOOL";
+        }>>;
+        status_code: z.ZodDefault<z.ZodEnum<{
+            UNSET: "UNSET";
+            OK: "OK";
+            ERROR: "ERROR";
+        }>>;
+        status_message: z.ZodOptional<z.ZodString>;
+        start_time: z.ZodString;
+        end_time: z.ZodOptional<z.ZodString>;
+        attributes: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+        events: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            name: z.ZodString;
+            timestamp: z.ZodString;
+            attributes: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+        }, z.core.$strip>>>;
+    }, z.core.$strip>>>;
+    timestamp: z.ZodOptional<z.ZodString>;
+}, z.core.$strip>;
 export declare const traceQuerySchema: z.ZodObject<{
     agent_name: z.ZodOptional<z.ZodString>;
     framework: z.ZodOptional<z.ZodString>;
@@ -33,17 +91,33 @@ export declare const summaryQuerySchema: z.ZodObject<{
 export declare const evalStatsPeriodSchema: z.ZodObject<{
     period: z.ZodDefault<z.ZodEnum<{
         "24h": "24h";
+        "2d": "2d";
         "7d": "7d";
+        "14d": "14d";
         "30d": "30d";
+        "60d": "60d";
+        "90d": "90d";
+        "180d": "180d";
         all: "all";
     }>>;
 }, z.core.$strip>;
 export declare const evalStatsFailuresSchema: z.ZodObject<{
     period: z.ZodDefault<z.ZodEnum<{
         "24h": "24h";
+        "2d": "2d";
         "7d": "7d";
+        "14d": "14d";
         "30d": "30d";
+        "60d": "60d";
+        "90d": "90d";
+        "180d": "180d";
         all: "all";
     }>>;
     limit: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
 }, z.core.$strip>;
+export declare const failuresQuerySchema: z.ZodObject<{
+    agent_name: z.ZodOptional<z.ZodString>;
+    since: z.ZodOptional<z.ZodString>;
+    until: z.ZodOptional<z.ZodString>;
+    limit: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+}, z.core.$strip>;

package/dist/dashboard/validation.js

@@ -1,4 +1,27 @@
 import { z } from 'zod';
+import { logTraceInputShape } from '../tools/log-trace.js';
+/*
+ * POST /api/v1/traces body — the log_trace tool contract plus the
+ * HTTP-only evaluation opt-in. Built FROM logTraceInputShape rather than
+ * restating it so the two capture paths (MCP tool, HTTP ingest) cannot
+ * drift. `trace_id` is deliberately absent: the server mints it, and
+ * zod's default unknown-key stripping discards any client-supplied one.
+ */
+export const ingestTraceSchema = z
+    .object({
+    ...logTraceInputShape,
+    evaluate: z.boolean().default(false),
+    eval_type: z.enum(['completeness', 'relevance', 'safety', 'cost', 'custom']).default('completeness'),
+})
+    .superRefine((body, ctx) => {
+    if (body.evaluate && body.output === undefined) {
+        ctx.addIssue({
+            code: z.ZodIssueCode.custom,
+            path: ['output'],
+            message: '"output" is required when "evaluate" is true — the eval engine scores the output text',
+        });
+    }
+});
 export const traceQuerySchema = z.object({
     agent_name: z.string().optional(),
     framework: z.string().optional(),
@@ -21,9 +44,15 @@ export const summaryQuerySchema = z.object({
     hours: z.coerce.number().int().min(1).max(8760).default(24),
 });
 export const evalStatsPeriodSchema = z.object({
-    period: z.enum(['24h', '7d', '30d', 'all']).default('24h'),
+    period: z.enum(['24h', '2d', '7d', '14d', '30d', '60d', '90d', '180d', 'all']).default('24h'),
 });
 export const evalStatsFailuresSchema = z.object({
-    period: z.enum(['24h', '7d', '30d', 'all']).default('24h'),
+    period: z.enum(['24h', '2d', '7d', '14d', '30d', '60d', '90d', '180d', 'all']).default('24h'),
     limit: z.coerce.number().int().min(1).max(100).default(10),
 });
+export const failuresQuerySchema = z.object({
+    agent_name: z.string().min(1).max(200).optional(),
+    since: z.string().datetime({ offset: true }).optional(),
+    until: z.string().datetime({ offset: true }).optional(),
+    limit: z.coerce.number().int().min(1).max(100).default(50),
+});

package/dist/eval/citation-verify/resolve.js

@@ -55,6 +55,17 @@ const BLOCKED_IPV4 = [
     /^255\.255\.255\.255$/,
     // This-network
     /^0\./,
+    // Carrier-grade NAT (RFC 6598). Routable inside an ISP or a corporate
+    // overlay — Tailscale hands out 100.64/10 addresses, so this range reaches
+    // real internal hosts on a very common setup.
+    /^100\.(6[4-9]|[7-9]\d|1[01]\d|12[0-7])\./,
+    // IETF protocol assignments (RFC 6890) incl. 192.0.0.0/24
+    /^192\.0\.0\./,
+    // Benchmarking (RFC 2544) — routed to internal test networks in practice
+    /^198\.(1[89])\./,
+    // Multicast and reserved/future space
+    /^(22[4-9]|23\d)\./,
+    /^(24\d|25[0-5])\./,
 ];
 const BLOCKED_HOST_SUBSTRINGS = ['localhost', 'internal', '.local', 'metadata.google', 'metadata.azure'];
 function isIpv4(host) {
@@ -144,6 +155,24 @@ function isBlockedIpv6(addr) {
     // fc00::/7 unique-local (fc.. / fd..)
     if (first.startsWith('fc') || first.startsWith('fd'))
         return true;
+    /*
+     * Transition mechanisms tunnel an IPv4 destination inside an IPv6 literal,
+     * so the v4 blocklist has to be applied to the embedded address or the
+     * whole v4 ruleset is bypassable by re-encoding the target.
+     *
+     * 6to4 (2002::/16, RFC 3056): the destination v4 is hextets 1-2, plain.
+     * Teredo (2001:0000::/32, RFC 4380): the client v4 is hextets 6-7, stored
+     * one's-complemented, so it must be un-obfuscated before classification.
+     */
+    if (first === '2002') {
+        return BLOCKED_IPV4.some((re) => re.test(ipv4FromHextets(g[1], g[2])));
+    }
+    if (first === '2001' && g[1] === '0000') {
+        const deobfuscate = (h) => (parseInt(h, 16) ^ 0xffff).toString(16).padStart(4, '0');
+        const client = ipv4FromHextets(deobfuscate(g[6]), deobfuscate(g[7]));
+        const server = ipv4FromHextets(g[2], g[3]);
+        return BLOCKED_IPV4.some((re) => re.test(client) || re.test(server));
+    }
     // IPv4-mapped ::ffff:a.b.c.d  and  IPv4-compatible ::a.b.c.d (deprecated)
     const mapped = g.slice(0, 5).every((h) => h === '0000') && g[5] === 'ffff';
     const compat = g.slice(0, 6).every((h) => h === '0000') && !(g[6] === '0000' && g[7] === '0000');

package/dist/eval/citation-verify/verifier.d.ts

@@ -38,6 +38,7 @@ export interface VerifyCitationsResult {
     totalCostUsd: number;
     totalCitationsFound: number;
     totalResolved: number;
+    totalJudged: number;
     totalSupported: number;
 }
 export declare function verifyCitations(params: VerifyCitationsParams): Promise<VerifyCitationsResult>;

package/dist/eval/citation-verify/verifier.js

@@ -56,6 +56,7 @@ export async function verifyCitations(params) {
     const out = [];
     let totalCost = 0;
     let totalResolved = 0;
+    let totalJudged = 0;
     let totalSupported = 0;
     for (const citation of selected) {
         // Only URL/DOI can be resolved. Numbered citations without
@@ -163,6 +164,7 @@ export async function verifyCitations(params) {
             });
             continue;
         }
+        totalJudged++;
         if (parsed.supported)
             totalSupported++;
         out.push({
@@ -186,10 +188,15 @@ export async function verifyCitations(params) {
             },
         });
     }
-    const overallScore = totalResolved > 0 ? Math.round((totalSupported / totalResolved) * 100) / 100 : null;
-    // Fail if >= 50% of resolved sources don't support the claim. When
-    // no citations or none resolved, we don't fail — there's nothing to
-    // score, we just report that.
+    // Denominator = citations the judge actually ruled on. A resolved
+    // citation whose judge call hit the cost cap, timed out, errored, or
+    // emitted unparseable JSON was never verified — counting it as
+    // unsupported would make a judge outage on 5 of 10 supported citations
+    // score 0.5, indistinguishable from fabrication.
+    const overallScore = totalJudged > 0 ? Math.round((totalSupported / totalJudged) * 100) / 100 : null;
+    // Fail if >= 50% of judged sources don't support the claim. When no
+    // citations, none resolved, or none judged, we don't fail — there's
+    // nothing to score, we just report that.
     const passed = overallScore === null ? true : overallScore >= 0.5;
     return {
         overallScore,
@@ -198,6 +205,7 @@ export async function verifyCitations(params) {
         totalCostUsd: Math.round(totalCost * 1_000_000) / 1_000_000,
         totalCitationsFound: totalFound,
         totalResolved,
+        totalJudged,
         totalSupported,
     };
 }

package/dist/eval/engine.d.ts

@@ -1,9 +1,23 @@
 import type { EvalRule, EvalContext, EvalResult, EvalType, CustomRuleDefinition } from '../types/eval.js';
 export declare class EvalEngine {
     private additionalRules;
+    /**
+     * Registered-rule handles keyed by deployed rule id, so delete paths can
+     * hot-remove exactly the instance they registered. Keyed by id (not name)
+     * because deploy_rule doesn't enforce name uniqueness — two rules can
+     * share a name with different definitions.
+     */
+    private rulesById;
     private threshold;
     private ruleThresholds?;
     constructor(threshold?: number, ruleThresholds?: Record<string, unknown>);
-    registerRule(evalType: EvalType, rule: EvalRule): void;
+    registerRule(evalType: EvalType, rule: EvalRule, ruleId?: string): void;
+    /**
+     * Hot-remove a rule registered under `ruleId` so it stops firing on the
+     * live process — what delete_rule's description promises (#332). Returns
+     * false when the id was never registered (already removed, or registered
+     * without an id); callers treat that as a no-op, not an error.
+     */
+    unregisterRule(ruleId: string): boolean;
     evaluate(evalType: EvalType, context: EvalContext, customRules?: CustomRuleDefinition[]): EvalResult;
 }

package/dist/eval/engine.js

@@ -2,16 +2,45 @@ import { getRulesForType, createCustomRule } from './rules/index.js';
 import { generateEvalId } from '../utils/ids.js';
 export class EvalEngine {
     additionalRules = new Map();
+    /**
+     * Registered-rule handles keyed by deployed rule id, so delete paths can
+     * hot-remove exactly the instance they registered. Keyed by id (not name)
+     * because deploy_rule doesn't enforce name uniqueness — two rules can
+     * share a name with different definitions.
+     */
+    rulesById = new Map();
     threshold;
     ruleThresholds;
     constructor(threshold = 0.7, ruleThresholds) {
         this.threshold = threshold;
         this.ruleThresholds = ruleThresholds;
     }
-    registerRule(evalType, rule) {
+    registerRule(evalType, rule, ruleId) {
         const existing = this.additionalRules.get(evalType) ?? [];
         existing.push(rule);
         this.additionalRules.set(evalType, existing);
+        if (ruleId !== undefined) {
+            this.rulesById.set(ruleId, { evalType, rule });
+        }
+    }
+    /**
+     * Hot-remove a rule registered under `ruleId` so it stops firing on the
+     * live process — what delete_rule's description promises (#332). Returns
+     * false when the id was never registered (already removed, or registered
+     * without an id); callers treat that as a no-op, not an error.
+     */
+    unregisterRule(ruleId) {
+        const entry = this.rulesById.get(ruleId);
+        if (!entry)
+            return false;
+        this.rulesById.delete(ruleId);
+        const rules = this.additionalRules.get(entry.evalType);
+        if (rules) {
+            const idx = rules.indexOf(entry.rule);
+            if (idx !== -1)
+                rules.splice(idx, 1);
+        }
+        return true;
     }
     evaluate(evalType, context, customRules) {
         // Merge system-level thresholds into customConfig (user-provided values take precedence)
@@ -61,7 +90,16 @@ export class EvalEngine {
                 insufficient_data: true,
             };
         }
-        const ruleResults = rules.map((rule) => rule.evaluate(context));
+        /*
+         * Shallow copy so the regex circuit breaker is scoped to THIS evaluation
+         * and never leaks into a caller-held context object. All rules in one
+         * evaluation share the breaker: after MAX_REGEX_BREACHES_PER_EVAL sandbox
+         * budget breaches (see rules/custom.ts), remaining regex rules skip
+         * without running — one hostile output cannot stall the request once per
+         * rule it carries.
+         */
+        const evalContext = { ...context, regexBudget: { breaches: 0 } };
+        const ruleResults = rules.map((rule) => rule.evaluate(evalContext));
         // Partition into evaluated vs skipped
         const evaluatedIndices = [];
         const skippedIndices = [];
@@ -105,16 +143,46 @@ export class EvalEngine {
         }, 0);
         const rawScore = totalWeight > 0 ? weightedScore / totalWeight : 0;
         const score = Number.isFinite(rawScore) ? rawScore : 0;
-        const passed = score >= this.threshold;
+        /*
+         * Critical rules hard-fail. Before this existed, the weighted average
+         * routinely outvoted a genuine violation: an output containing a real
+         * SSN failed no_pii while the other safety rules passed, landing at
+         * ~0.765 — over the 0.7 threshold — so `passed`, the one field every
+         * automated gate keys on, said true about the product's flagship
+         * failure scenario. A detection that reports an all-clear is worse
+         * than no detection.
+         *
+         * Only EVALUATED failures count: a critical rule that skipped (missing
+         * context, broken config) has not judged the output and must not veto
+         * it. The score is left as-is — it stays a quality gradient; `passed`
+         * is the verdict, and the two answer different questions.
+         */
+        const criticalFailures = evaluatedIndices
+            .filter((i) => rules[i].critical === true && !ruleResults[i].passed)
+            .map((i) => ruleResults[i].ruleName);
+        const passed = score >= this.threshold && criticalFailures.length === 0;
         const suggestions = [];
         for (const result of ruleResults) {
             if (!result.passed && !result.skipped) {
                 suggestions.push(`[${result.ruleName}] ${result.message}`);
             }
         }
+        if (criticalFailures.length > 0 && score >= this.threshold) {
+            suggestions.push(`Critical rule(s) failed (${criticalFailures.join(', ')}) — passed=false regardless of the weighted score`);
+        }
         if (rulesSkipped > 0) {
-            const skippedNames = ruleResults.filter((r) => r.skipped).map((r) => r.ruleName);
-            suggestions.push(`${rulesSkipped} rule(s) skipped (missing context): ${skippedNames.join(', ')}`);
+            /*
+             * Say WHY each rule skipped. The old line hardcoded "(missing
+             * context)" — but a rule whose regex was killed at the sandbox budget
+             * did not lack context, it was DEFEATED by this output, and labeling
+             * that "missing context" hid the one signal a fail-closed consumer
+             * needs. Each rule's own skipReason is the truth; missing context is
+             * only the default for rules that skip without stating a reason.
+             */
+            const skippedParts = ruleResults
+                .filter((r) => r.skipped)
+                .map((r) => `${r.ruleName} (${r.skipReason ?? 'missing context'})`);
+            suggestions.push(`${rulesSkipped} rule(s) skipped — excluded from the weighted score: ${skippedParts.join('; ')}`);
         }
         return {
             id: generateEvalId(),
@@ -128,6 +196,7 @@ export class EvalEngine {
             rules_evaluated: rulesEvaluated,
             rules_skipped: rulesSkipped,
             insufficient_data: false,
+            ...(criticalFailures.length > 0 ? { critical_failures: criticalFailures } : {}),
         };
     }
 }

package/dist/eval/failure-rank.d.ts

@@ -0,0 +1,14 @@
+import type { DecisionMoment } from '../types/decision-moment.js';
+/** Recency half-life: a failure loses half its rank weight every 24h. */
+export declare const FAILURE_RANK_HALF_LIFE_MS: number;
+/**
+ * Is this moment a failure (verdict fail/partial) or flagged
+ * (safety/cost significance regardless of verdict)?
+ */
+export declare function isFailureMoment(moment: DecisionMoment): boolean;
+/**
+ * Rank score for a failure moment: significance × recency decay.
+ * Higher = shown first. Future timestamps (clock skew) clamp to age 0
+ * rather than inflating the score.
+ */
+export declare function rankFailureScore(moment: DecisionMoment, nowMs: number): number;

package/dist/eval/failure-rank.js

@@ -0,0 +1,44 @@
+/*
+ * failure-rank — pure ranking logic for the failure-first landing list.
+ *
+ * The dashboard's default screen is a ranked list of recent failures
+ * ("what's new and bad"), not an aggregate. Ranking blends two signals:
+ *
+ *   severity — the significance classifier's 0-1 score (safety-violation
+ *              1.0 > cost-spike 0.9 > rule-collision 0.7 > normal-fail
+ *              0.5/0.4). See classifySignificance in decision-moment.ts.
+ *   recency  — exponential decay with a 24h half-life. A safety violation
+ *              from three days ago ranks below a plain fail from an hour
+ *              ago, which is the right call for a "since you last looked"
+ *              surface — old severity is history, not news.
+ *
+ * Kept as a pure module (no storage, no clock reads — `nowMs` is a
+ * parameter) so tests can pin time and assert exact orderings.
+ */
+/** Recency half-life: a failure loses half its rank weight every 24h. */
+export const FAILURE_RANK_HALF_LIFE_MS = 24 * 60 * 60 * 1000;
+/*
+ * Significance kinds that flag a moment for the failure list even when
+ * its verdict is not fail/partial. A cost spike on a passing trace is
+ * still something the builder should see on the landing screen.
+ */
+const FLAGGED_KINDS = new Set(['safety-violation', 'cost-spike']);
+/**
+ * Is this moment a failure (verdict fail/partial) or flagged
+ * (safety/cost significance regardless of verdict)?
+ */
+export function isFailureMoment(moment) {
+    if (moment.verdict === 'fail' || moment.verdict === 'partial')
+        return true;
+    return FLAGGED_KINDS.has(moment.significance.kind);
+}
+/**
+ * Rank score for a failure moment: significance × recency decay.
+ * Higher = shown first. Future timestamps (clock skew) clamp to age 0
+ * rather than inflating the score.
+ */
+export function rankFailureScore(moment, nowMs) {
+    const ageMs = Math.max(0, nowMs - new Date(moment.timestamp).getTime());
+    const recency = Math.pow(0.5, ageMs / FAILURE_RANK_HALF_LIFE_MS);
+    return moment.significance.score * recency;
+}

package/dist/eval/rules/custom.d.ts

@@ -1,2 +1,30 @@
 import type { EvalRule, CustomRuleDefinition } from '../../types/eval.js';
-export declare function createCustomRule(definition: CustomRuleDefinition): EvalRule;
+import type { RuleSeverity } from '../../types/custom-rule.js';
+/**
+ * Converts a leading inline flag group like `(?i)` or `(?im)` into a real
+ * flags argument. Node's RegExp engine does not support inline flag groups,
+ * and a user pasting `(?i)foo` from a regex tutorial would otherwise hit
+ * "Invalid group" with no clear recovery.
+ *
+ * Exported so deploy-time validation (custom-rule-store) probes the SAME
+ * pattern+flags pair the evaluator will actually run — the store used to
+ * strip the inline group but not merge its flags, probing `(?i)…` under
+ * different flags than evaluation used.
+ */
+export declare function normalizeRegexSource(patternStr: string, flags: string): {
+    pattern: string;
+    flags: string;
+};
+/**
+ * Builds a runnable EvalRule from a persisted/inline definition.
+ *
+ * `severity` comes from the DEPLOYED rule's metadata (deploy_rule / the
+ * dashboard composer). high/critical severities make the rule CRITICAL:
+ * a failing evaluation forces the overall eval to passed=false regardless
+ * of the weighted score. Before this, a rule-author could deploy a
+ * severity="critical" policy rule, watch it FAIL on a violating output,
+ * and still get passed:true (score 0.895) — severity affected nothing but
+ * dashboard sorting. Inline custom_rules (evaluate_output's per-call
+ * definitions) carry no severity and stay weight-only.
+ */
+export declare function createCustomRule(definition: CustomRuleDefinition, severity?: RuleSeverity): EvalRule;