@iris-eval/mcp-server 0.5.1 → 0.7.0

package/dist/dashboard/session-auth.js

@@ -0,0 +1,237 @@
+/*
+ * session-auth — a browser can sign in to an --api-key dashboard.
+ *
+ * The auth middleware (middleware/auth.ts) is Bearer-only, which is right
+ * for MCP clients and capture SDKs and useless for a browser: with
+ * `--api-key` set — the README's own "production deployment" command — the
+ * dashboard UI 401'd every page load, because nothing lets a browser
+ * present the key (#373 item 6).
+ *
+ * This layer sits in FRONT of the Bearer middleware and adds one thing: a
+ * session cookie, obtained by presenting the key once.
+ *
+ *   GET  /?key=<api key>      exchange the key for a session, then redirect
+ *                             to the same URL with the key stripped
+ *   POST /session  key=<…>    same exchange from the sign-in form
+ *   GET  <any page>           without a session → a 401 sign-in page
+ *                             (HTML, only for requests that ask for HTML)
+ *
+ * A request carrying a valid session cookie skips the Bearer check. Every
+ * other request — API calls without a cookie, wrong keys, non-browser
+ * clients — falls through to the unchanged Bearer middleware, so nothing
+ * that worked before behaves differently.
+ *
+ * What the cookie is NOT: it is not the API key. It is a random 256-bit
+ * token that maps, in this process's memory, to "a request presented the
+ * key". HttpOnly (no script can read it), SameSite=Lax (never sent on a
+ * cross-site fetch, XHR or form POST — the CSRF vectors; still sent on a
+ * plain link into the dashboard so a shared URL opens without re-signing
+ * in), Path=/ (this origin only), Secure when the request arrived over
+ * HTTPS. Sessions die with the process; there is no persistence to leak.
+ *
+ * Brute force: the key exchange is capped at SIGN_IN_ATTEMPTS_PER_MINUTE
+ * per client address by its own limiter below, and the whole of this
+ * layer — cookie check, Bearer check and the exchange alike — sits behind
+ * the per-address auth-gate limiter server.ts mounts directly in front of
+ * it (middleware/rate-limit.ts, createAuthGateRateLimiter), so no
+ * authorization decision runs unthrottled.
+ */
+import { randomBytes, timingSafeEqual } from 'node:crypto';
+import express from 'express';
+import rateLimit from 'express-rate-limit';
+export const SESSION_COOKIE = 'iris_session';
+const SESSION_TTL_MS = 30 * 24 * 60 * 60 * 1000;
+const MAX_SESSIONS = 256;
+const SIGN_IN_ATTEMPTS_PER_MINUTE = 10;
+function keyMatches(candidateRaw, apiKey) {
+    // Same shape as middleware/auth.ts: pad to the key length and compare
+    // fixed-size buffers so the compare path does not depend on the
+    // candidate's length.
+    const keyBuffer = Buffer.from(apiKey);
+    const tokenBuffer = Buffer.from(candidateRaw);
+    const candidate = Buffer.alloc(keyBuffer.length);
+    tokenBuffer.copy(candidate, 0, 0, keyBuffer.length);
+    const cmpEq = timingSafeEqual(candidate, keyBuffer);
+    const lenEq = tokenBuffer.length === keyBuffer.length;
+    return cmpEq && lenEq;
+}
+function readCookie(req, name) {
+    const header = req.headers.cookie;
+    if (!header)
+        return undefined;
+    for (const part of header.split(';')) {
+        const eq = part.indexOf('=');
+        if (eq === -1)
+            continue;
+        if (part.slice(0, eq).trim() !== name)
+            continue;
+        try {
+            return decodeURIComponent(part.slice(eq + 1).trim());
+        }
+        catch {
+            return undefined;
+        }
+    }
+    return undefined;
+}
+function wantsHtml(req) {
+    return req.method === 'GET' && !req.path.startsWith('/api/') && /text\/html/i.test(req.headers.accept ?? '');
+}
+const PAGE_STYLE = `
+  :root { color-scheme: dark; }
+  body { margin: 0; min-height: 100vh; display: grid; place-items: center; background: #0b0f14; color: #e6edf3;
+         font: 15px/1.6 system-ui, -apple-system, "Segoe UI", sans-serif; }
+  main { width: min(92vw, 460px); padding: 32px; border: 1px solid #263140; border-radius: 12px; background: #111820; }
+  h1 { margin: 0 0 4px; font-size: 20px; letter-spacing: -0.01em; }
+  .tagline { margin: 0 0 20px; color: #8b97a6; font-size: 13px; }
+  p { margin: 0 0 14px; color: #c3ccd6; }
+  code { font-family: ui-monospace, "JetBrains Mono", Menlo, monospace; font-size: 13px; background: #1a2330; padding: 1px 5px; border-radius: 4px; }
+  label { display: block; font-size: 12px; text-transform: uppercase; letter-spacing: .06em; color: #8b97a6; margin-bottom: 6px; }
+  input { width: 100%; box-sizing: border-box; padding: 10px 12px; border: 1px solid #2c3949; border-radius: 8px; background: #0b0f14; color: #e6edf3; font: inherit; }
+  button { margin-top: 14px; width: 100%; padding: 10px 12px; border: 0; border-radius: 8px; background: #2dd4bf; color: #04201c; font: inherit; font-weight: 600; cursor: pointer; }
+  .error { color: #f87171; font-weight: 600; }
+  .fine { margin-top: 18px; font-size: 12px; color: #8b97a6; }
+`;
+function signInPage(error) {
+    return `<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>Iris — sign in</title>
+<style>${PAGE_STYLE}</style>
+</head>
+<body>
+<main>
+  <h1>Iris dashboard</h1>
+  <p class="tagline">This dashboard is protected by an API key.</p>
+  ${error ? `<p class="error" role="alert">${error}</p>` : ''}
+  <p>This server was started with <code>--api-key</code> (or <code>IRIS_API_KEY</code>), so the dashboard asks for that key once. Your browser then keeps an HttpOnly session cookie for this origin — the key itself is never stored in the browser.</p>
+  <form method="post" action="/session">
+    <label for="key">API key</label>
+    <input id="key" name="key" type="password" autocomplete="off" autofocus required>
+    <button type="submit">Open dashboard</button>
+  </form>
+  <p class="fine">Sharing a link with a teammate? Append <code>?key=&lt;api key&gt;</code> to any dashboard URL — it signs the browser in and redirects to the page with the key removed from the address bar. API clients keep using <code>Authorization: Bearer &lt;api key&gt;</code>.</p>
+</main>
+</body>
+</html>`;
+}
+export function createSessionAuth(opts) {
+    const { apiKey, bearerAuth } = opts;
+    if (!apiKey) {
+        // No key configured: the Bearer middleware is a pass-through and so is
+        // this. A `?key=` on the URL is left alone — nothing to exchange.
+        return bearerAuth;
+    }
+    /** token → expiry (epoch ms). Insertion order doubles as age order. */
+    const sessions = new Map();
+    function createSession() {
+        const token = randomBytes(32).toString('base64url');
+        sessions.set(token, Date.now() + SESSION_TTL_MS);
+        while (sessions.size > MAX_SESSIONS) {
+            const oldest = sessions.keys().next().value;
+            if (oldest === undefined)
+                break;
+            sessions.delete(oldest);
+        }
+        return token;
+    }
+    function hasSession(req) {
+        const token = readCookie(req, SESSION_COOKIE);
+        if (!token)
+            return false;
+        const expires = sessions.get(token);
+        if (expires === undefined)
+            return false;
+        if (expires <= Date.now()) {
+            sessions.delete(token);
+            return false;
+        }
+        return true;
+    }
+    function setSessionCookie(req, res) {
+        const token = createSession();
+        const attrs = [
+            `${SESSION_COOKIE}=${token}`,
+            'HttpOnly',
+            'SameSite=Lax',
+            'Path=/',
+            `Max-Age=${Math.floor(SESSION_TTL_MS / 1000)}`,
+        ];
+        if (req.protocol === 'https')
+            attrs.push('Secure');
+        res.append('Set-Cookie', attrs.join('; '));
+    }
+    function sendSignIn(res, status, error) {
+        res.status(status).type('html').send(signInPage(error));
+    }
+    const signInLimiter = rateLimit({
+        windowMs: 60_000,
+        limit: SIGN_IN_ATTEMPTS_PER_MINUTE,
+        standardHeaders: 'draft-7',
+        legacyHeaders: false,
+        message: { error: 'Too many sign-in attempts, please try again later' },
+    });
+    const formBody = express.urlencoded({ extended: false, limit: '4kb' });
+    const exchange = (req, res, next) => {
+        // 1. `?key=` on a page URL — the one-line team recipe.
+        if (req.method === 'GET' && !req.path.startsWith('/api/') && typeof req.query.key === 'string') {
+            if (!keyMatches(req.query.key, apiKey)) {
+                sendSignIn(res, 403, 'That API key did not match.');
+                return;
+            }
+            setSessionCookie(req, res);
+            const url = new URL(req.originalUrl, 'http://localhost');
+            url.searchParams.delete('key');
+            res.redirect(302, `${url.pathname}${url.search}`);
+            return;
+        }
+        // 2. The sign-in form (or a JSON body with the same shape).
+        if (req.method === 'POST' && req.path === '/session') {
+            formBody(req, res, (err) => {
+                if (err) {
+                    next(err);
+                    return;
+                }
+                const body = req.body;
+                const key = typeof body?.key === 'string' ? body.key : '';
+                if (!key || !keyMatches(key, apiKey)) {
+                    sendSignIn(res, 403, 'That API key did not match.');
+                    return;
+                }
+                setSessionCookie(req, res);
+                res.redirect(303, '/');
+            });
+            return;
+        }
+        next();
+    };
+    return (req, res, next) => {
+        if (req.path === '/health' || req.path === '/api/v1/health') {
+            next();
+            return;
+        }
+        /*
+         * The exchange runs BEFORE the session check on purpose: a browser
+         * that already has a session and opens a shared `?key=` link must
+         * still be redirected to the key-free URL, or the key sits in its
+         * address bar (and history) for the rest of the visit.
+         */
+        const isExchange = (req.method === 'GET' && !req.path.startsWith('/api/') && typeof req.query.key === 'string') ||
+            (req.method === 'POST' && req.path === '/session');
+        if (isExchange) {
+            signInLimiter(req, res, (err) => (err ? next(err) : exchange(req, res, next)));
+            return;
+        }
+        if (hasSession(req)) {
+            next();
+            return;
+        }
+        if (wantsHtml(req)) {
+            sendSignIn(res, 401);
+            return;
+        }
+        bearerAuth(req, res, next);
+    };
+}

package/dist/dashboard/validation.d.ts

@@ -1,12 +1,16 @@
 import { z } from 'zod';
+export declare function strictBody<T extends z.ZodRawShape>(shape: T, opts?: {
+    reserved?: Record<string, string>;
+}): z.ZodObject<{ -readonly [P in keyof T]: T[P]; }, z.core.$strict>;
 export declare const ingestTraceSchema: z.ZodObject<{
     evaluate: z.ZodDefault<z.ZodBoolean>;
-    eval_type: z.ZodDefault<z.ZodEnum<{
+    eval_type: z.ZodOptional<z.ZodEnum<{
         completeness: "completeness";
         relevance: "relevance";
         safety: "safety";
         cost: "cost";
         custom: "custom";
+        all: "all";
     }>>;
     agent_name: z.ZodString;
     framework: z.ZodOptional<z.ZodString>;
@@ -56,12 +60,14 @@ export declare const ingestTraceSchema: z.ZodObject<{
         }, z.core.$strip>>>;
     }, z.core.$strip>>>;
     timestamp: z.ZodOptional<z.ZodString>;
-}, z.core.$strip>;
+}, z.core.$strict>;
 export declare const traceQuerySchema: z.ZodObject<{
     agent_name: z.ZodOptional<z.ZodString>;
     framework: z.ZodOptional<z.ZodString>;
     since: z.ZodOptional<z.ZodString>;
     until: z.ZodOptional<z.ZodString>;
+    min_score: z.ZodOptional<z.ZodCoercedNumber<unknown>>;
+    max_score: z.ZodOptional<z.ZodCoercedNumber<unknown>>;
     limit: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     offset: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     sort_by: z.ZodDefault<z.ZodEnum<{
@@ -90,6 +96,7 @@ export declare const summaryQuerySchema: z.ZodObject<{
 }, z.core.$strip>;
 export declare const evalStatsPeriodSchema: z.ZodObject<{
     period: z.ZodDefault<z.ZodEnum<{
+        all: "all";
         "24h": "24h";
         "2d": "2d";
         "7d": "7d";
@@ -98,11 +105,11 @@ export declare const evalStatsPeriodSchema: z.ZodObject<{
         "60d": "60d";
         "90d": "90d";
         "180d": "180d";
-        all: "all";
     }>>;
 }, z.core.$strip>;
 export declare const evalStatsFailuresSchema: z.ZodObject<{
     period: z.ZodDefault<z.ZodEnum<{
+        all: "all";
         "24h": "24h";
         "2d": "2d";
         "7d": "7d";
@@ -111,7 +118,6 @@ export declare const evalStatsFailuresSchema: z.ZodObject<{
         "60d": "60d";
         "90d": "90d";
         "180d": "180d";
-        all: "all";
     }>>;
     limit: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
 }, z.core.$strip>;

package/dist/dashboard/validation.js

@@ -1,19 +1,69 @@
 import { z } from 'zod';
 import { logTraceInputShape } from '../tools/log-trace.js';
+import { isoTimestamp, addTraceRangeIssues } from '../tools/get-traces.js';
+/*
+ * Strict request-body schema for the dashboard's MUTATING routes.
+ *
+ * A bare z.object() silently STRIPS unknown keys. On a read route that is
+ * merely lenient; on a write route it is the exact defect v0.5.0 fixed at
+ * the MCP boundary (tools/strict-input.ts) and then left open on its HTTP
+ * twin: `POST /api/v1/traces {evaluate: true, eval_typ: "safety", output:
+ * "<PII>"}` dropped the misspelled key, ran the DEFAULT completeness
+ * bundle, and returned a green result with nothing saying an argument had
+ * been ignored. Same family, one transport over (#376 item 2).
+ *
+ * `reserved` names keys the server owns and a client must never send —
+ * each gets a pointed sentence appended to the rejection so the caller
+ * learns WHY (e.g. trace_id is server-minted) instead of just "unknown".
+ */
+export function strictBody(shape, opts) {
+    const validKeys = Object.keys(shape).join(', ');
+    return z.strictObject(shape, {
+        error: (issue) => {
+            if (issue.code !== 'unrecognized_keys')
+                return undefined;
+            const reservedNotes = issue.keys
+                .filter((k) => opts?.reserved?.[k] !== undefined)
+                .map((k) => ` ${opts.reserved[k]}`)
+                .join('');
+            return (`Unknown key(s): ${issue.keys.map((k) => `"${k}"`).join(', ')}. ` +
+                `Valid keys: ${validKeys}. ` +
+                'Unknown keys are rejected rather than silently dropped, so a misspelled field ' +
+                'cannot change what gets stored or evaluated — check the spelling and retry.' +
+                reservedNotes);
+        },
+    });
+}
 /*
  * POST /api/v1/traces body — the log_trace tool contract plus the
  * HTTP-only evaluation opt-in. Built FROM logTraceInputShape rather than
  * restating it so the two capture paths (MCP tool, HTTP ingest) cannot
- * drift. `trace_id` is deliberately absent: the server mints it, and
- * zod's default unknown-key stripping discards any client-supplied one.
+ * drift.
+ *
+ * `trace_id` is deliberately absent: the server mints it. This schema
+ * used to rely on default unknown-key stripping to discard a client-
+ * supplied one — which also discarded every misspelled field. It is now
+ * strict, and a client-supplied trace_id is REJECTED with a message that
+ * says the server owns it, rather than silently replaced.
  */
-export const ingestTraceSchema = z
-    .object({
+export const ingestTraceSchema = strictBody({
     ...logTraceInputShape,
     evaluate: z.boolean().default(false),
-    eval_type: z.enum(['completeness', 'relevance', 'safety', 'cost', 'custom']).default('completeness'),
-})
-    .superRefine((body, ctx) => {
+    // Same bundle list evaluate_output accepts, "all" included — the
+    // ingest path used to stop one short and run the single-bundle engine
+    // no matter what, so an HTTP caller could not get the per-category
+    // verdict the MCP tool returns. Optional rather than defaulted HERE so
+    // the route can tell "chose all" from "never chose": the effective
+    // default is every bundle (DEFAULT_EVAL_TYPE in eval/engine.ts, the
+    // same constant evaluate_output reads), and an omitted eval_type gets a
+    // note in the response saying the default ran.
+    eval_type: z.enum(['completeness', 'relevance', 'safety', 'cost', 'custom', 'all']).optional(),
+}, {
+    reserved: {
+        trace_id: 'trace_id is minted by the server on every ingest and cannot be supplied by the client — ' +
+            'read it from the 201 response instead.',
+    },
+}).superRefine((body, ctx) => {
     if (body.evaluate && body.output === undefined) {
         ctx.addIssue({
             code: z.ZodIssueCode.custom,
@@ -22,16 +72,28 @@ export const ingestTraceSchema = z
         });
     }
 });
-export const traceQuerySchema = z.object({
+/*
+ * GET /api/v1/traces — the HTTP twin of the get_traces tool's query. The
+ * bounds are validated by the SAME helpers the tool uses (isoTimestamp,
+ * addTraceRangeIssues), so a `since` of "yesterday", a `since` later than
+ * `until`, a score outside 0..1 or a `min_score` above `max_score` is
+ * refused here with both values named — exactly as the tool refuses it —
+ * instead of returning an empty page that reads as "no such traces".
+ */
+export const traceQuerySchema = z
+    .object({
     agent_name: z.string().optional(),
     framework: z.string().optional(),
-    since: z.string().optional(),
-    until: z.string().optional(),
+    since: isoTimestamp.optional(),
+    until: isoTimestamp.optional(),
+    min_score: z.coerce.number().min(0).max(1).optional(),
+    max_score: z.coerce.number().min(0).max(1).optional(),
     limit: z.coerce.number().int().min(1).max(1000).default(50),
     offset: z.coerce.number().int().min(0).default(0),
     sort_by: z.enum(['timestamp', 'latency_ms', 'cost_usd']).default('timestamp'),
     sort_order: z.enum(['asc', 'desc']).default('desc'),
-});
+})
+    .superRefine(addTraceRangeIssues);
 export const evalQuerySchema = z.object({
     eval_type: z.string().optional(),
     passed: z.enum(['true', 'false']).transform((v) => v === 'true').optional(),

package/dist/eval/engine.d.ts

@@ -1,4 +1,27 @@
-import type { EvalRule, EvalContext, EvalResult, EvalType, CustomRuleDefinition } from '../types/eval.js';
+import type { EvalRule, EvalContext, EvalResult, EvalResultType, EvalType, CustomRuleDefinition } from '../types/eval.js';
+/**
+ * Every bundle eval_type="all" walks, in the order their categories are
+ * reported. 'custom' is last: it holds only deployed rules registered under
+ * evalType "custom" plus the call's inline custom_rules, so it is absent
+ * from the breakdown when neither exists.
+ */
+export declare const ALL_EVAL_TYPES: readonly EvalType[];
+/**
+ * What runs when a caller never chose a bundle. It used to be
+ * 'completeness', so a CI gate keyed on `passed` skipped PII and injection
+ * unless the caller knew to set eval_type — six of seven UAT personas read
+ * passed:true on PII-laden text with nothing in the payload saying the
+ * safety bundle had not run. Every bundle is the only default under which
+ * an omitted argument cannot silently narrow the verdict. The MCP tool and
+ * the HTTP ingest route both read this constant, so the two surfaces
+ * cannot default differently.
+ */
+export declare const DEFAULT_EVAL_TYPE: EvalResultType;
+/**
+ * The one-line note both surfaces attach when the default ran, so a reader
+ * of the response knows the bundle was chosen for them and how to narrow it.
+ */
+export declare const DEFAULT_EVAL_TYPE_NOTE = "eval_type was omitted, so the default ran every bundle \u2014 completeness, relevance, safety, cost and any custom rules \u2014 the same as eval_type=\"all\"; pass a single bundle name to narrow the run.";
 export declare class EvalEngine {
     private additionalRules;
     /**
@@ -8,9 +31,21 @@ export declare class EvalEngine {
      * share a name with different definitions.
      */
     private rulesById;
+    /**
+     * Reverse index, so each rule's result can carry its deployed id
+     * (EvalRuleResult.ruleId) without mutating the rule object itself.
+     */
+    private idByRule;
     private threshold;
     private ruleThresholds?;
     constructor(threshold?: number, ruleThresholds?: Record<string, unknown>);
+    /**
+     * Register a rule under a bundle. When `ruleId` is given the registration
+     * is IDEMPOTENT by id: registering an id that is already live replaces the
+     * earlier instance instead of adding a second one. That is what re-enable
+     * (delete_rule enabled:true) and a reload after edit need — without it,
+     * every toggle stacked another copy that fired alongside the first.
+     */
     registerRule(evalType: EvalType, rule: EvalRule, ruleId?: string): void;
     /**
      * Hot-remove a rule registered under `ruleId` so it stops firing on the
@@ -19,5 +54,48 @@ export declare class EvalEngine {
      * without an id); callers treat that as a no-op, not an error.
      */
     unregisterRule(ruleId: string): boolean;
+    /** Whether a deployed rule id is currently registered (and therefore firing). */
+    hasRule(ruleId: string): boolean;
     evaluate(evalType: EvalType, context: EvalContext, customRules?: CustomRuleDefinition[]): EvalResult;
+    /**
+     * eval_type="all" (#370): every built-in bundle, each with the deployed
+     * rules registered under it, plus the rules deployed under "custom" and
+     * the call's inline custom_rules — in ONE pass, sharing one regex budget,
+     * so the whole call is bounded exactly like a single bundle. The overall
+     * verdict is the same arithmetic as a single bundle applied to every rule
+     * that ran (weighted score against the threshold, critical veto across
+     * all bundles); `categories` carries the same arithmetic per bundle.
+     */
+    evaluateAll(context: EvalContext, customRules?: CustomRuleDefinition[]): EvalResult;
+    private run;
+    /**
+     * Weighted average over the rules that ran, plus the critical veto.
+     *
+     * Critical rules hard-fail. Before this existed, the weighted average
+     * routinely outvoted a genuine violation: an output containing a real
+     * SSN failed no_pii while the other safety rules passed, landing at
+     * ~0.765 — over the 0.7 threshold — so `passed`, the one field every
+     * automated gate keys on, said true about the product's flagship
+     * failure scenario. A detection that reports an all-clear is worse
+     * than no detection.
+     *
+     * Only EVALUATED failures count: a critical rule that skipped (missing
+     * context, broken config) has not judged the output and must not veto
+     * it. The score is left as-is — it stays a quality gradient; `passed`
+     * is the verdict, and the two answer different questions.
+     *
+     * A critical rule that SKIPPED is the fail-open seam between the
+     * release's two headline features: an adversary who knows a deployed
+     * critical regex can craft output that stalls it past the sandbox
+     * budget, and the rule then neither judges nor vetoes — so the eval
+     * returns passed=true with an EMPTY critical_failures on output that
+     * nobody actually cleared. The trade-off is deliberate (failing closed
+     * would let the same adversary force false violations on benign
+     * output), but before `criticalSkipped` the only trace of it was a
+     * suggestions line — prose. A gate that must fail closed should not
+     * have to walk rule_results[].budgetExceeded to discover it was defeated.
+     */
+    private summarize;
+    /** The per-bundle breakdown for eval_type="all": summarize() over each bundle's slice. */
+    private categorize;
 }