@iris-eval/mcp-server 0.3.1 → 0.4.0

package/dist/tools/verify-citations.js

@@ -0,0 +1,157 @@
+import { z } from 'zod';
+import { LOCAL_TENANT } from '../types/tenant.js';
+import { verifyCitations } from '../eval/citation-verify/verifier.js';
+import { findPricing } from '../eval/llm-judge/pricing.js';
+import { generateEvalId } from '../utils/ids.js';
+const inputSchema = {
+    output: z.string().min(1).describe('The agent output containing citations to verify'),
+    model: z
+        .string()
+        .describe('Judge model for per-citation verification. Supported: anthropic = claude-opus-4-7 | claude-sonnet-4-6 | claude-haiku-4-5-20251001; openai = gpt-4o | gpt-4o-mini | o1-mini.'),
+    provider: z.enum(['anthropic', 'openai']).optional().describe('Auto-detected from model when omitted'),
+    allow_fetch: z.boolean().optional().describe('Permit outbound HTTP to resolve URLs/DOIs. Defaults to IRIS_CITATION_ALLOW_FETCH=1; false otherwise. SSRF-guarded regardless.'),
+    domain_allowlist: z
+        .array(z.string())
+        .optional()
+        .describe('Restrict fetches to hostnames in this list (suffix match allowed). Merged with IRIS_CITATION_DOMAINS env.'),
+    max_cost_usd_total: z.number().positive().optional().describe('Cap TOTAL judge cost across all citations in this call; default $1.00'),
+    max_citations: z.number().int().positive().max(50).optional().describe('Max citations to verify (extras skipped); default 20'),
+    per_source_timeout_ms: z.number().int().positive().optional().describe('Per-URL fetch timeout; default 10_000'),
+    per_source_max_bytes: z.number().int().positive().optional().describe('Per-URL body cap; default 5MB'),
+    trace_id: z.string().optional().describe('Link verification result to a trace'),
+};
+function inferProvider(model) {
+    const pricing = findPricing(model);
+    if (!pricing) {
+        throw new Error(`Unknown model "${model}". Provider cannot be inferred. Supported models: src/eval/llm-judge/pricing.ts.`);
+    }
+    return pricing.provider;
+}
+function resolveApiKey(provider) {
+    const key = provider === 'anthropic' ? process.env.IRIS_ANTHROPIC_API_KEY : process.env.IRIS_OPENAI_API_KEY;
+    if (!key) {
+        throw new Error(`${provider === 'anthropic' ? 'Anthropic' : 'OpenAI'} judge requires IRIS_${provider === 'anthropic' ? 'ANTHROPIC' : 'OPENAI'}_API_KEY for verify_citations.`);
+    }
+    return key;
+}
+function resolveAllowFetch(paramValue) {
+    if (paramValue !== undefined)
+        return paramValue;
+    return process.env.IRIS_CITATION_ALLOW_FETCH === '1';
+}
+function resolveDomainAllowlist(paramValue) {
+    const envRaw = process.env.IRIS_CITATION_DOMAINS;
+    const fromEnv = envRaw ? envRaw.split(',').map((s) => s.trim()).filter(Boolean) : [];
+    if (paramValue && paramValue.length > 0) {
+        return [...new Set([...fromEnv, ...paramValue])];
+    }
+    return fromEnv.length > 0 ? fromEnv : undefined;
+}
+export function registerVerifyCitationsTool(server, storage) {
+    server.registerTool('verify_citations', {
+        title: 'Verify Citations',
+        description: [
+            'Extract citations from agent output, fetch the cited sources, and use an LLM judge to check whether each source supports the claim in context. Returns per-citation verdicts + an overall support ratio.',
+            '',
+            'Behavior. Three-phase pipeline: (1) regex extraction of [N] numbered refs, (Author, Year) parentheticals, bare URLs, and DOIs (in-process, no network); (2) SSRF-guarded fetch of URL + DOI citations, with scheme allowlist, private/link-local/cloud-metadata IP blocking, optional domain allowlist (IRIS_CITATION_DOMAINS), 10s timeout, 5MB body cap, manual redirect chase (max 3, re-checked), in-process LRU cache; (3) per-citation LLM judge call asking "does this source support this claim?" with a 256-token verdict. Opt-in via allow_fetch=true or IRIS_CITATION_ALLOW_FETCH=1 — Iris refuses outbound HTTP by default. Cost-capped across the entire call by max_cost_usd_total (default $1.00) — the pipeline stops when the cap would be exceeded. Rate-limited to 20 req/min on HTTP MCP. Writes one eval_result row tagged with per-citation provenance.',
+            '',
+            'Output shape. Returns JSON: `{ "id": "<uuid>", "overall_score": 0..1|null, "passed": boolean, "total_citations_found": number, "total_resolved": number, "total_supported": number, "total_cost_usd": number, "citations": [{ "citation": { "raw", "kind", "identifier", "offset_start", "offset_end" }, "resolve_status": "ok"|"skipped"|"error", "resolve_error"?, "source"?: { "url", "status", "content_type", "bytes_fetched", "truncated" }, "judge"?: { "supported", "confidence", "rationale", "cost_usd", "latency_ms", "input_tokens", "output_tokens" } }] }`. `overall_score = supported / resolved`; `null` when nothing resolvable was found.',
+            '',
+            'Use when the output makes factual claims backed by [1]-style references, DOIs, or URLs and you want to separate "cited correctly" from "cited and wrong" from "cited but unresolvable". Particularly useful for research/legal/medical agents where fabricated citations are the dominant failure mode.',
+            "",
+            "Don't use when the agent output has no citations at all (overall_score will be null; the tool degrades gracefully but a heuristic rule is cheaper). Don't use without allow_fetch=true or IRIS_CITATION_ALLOW_FETCH=1 — the tool refuses outbound HTTP unless explicitly enabled. Don't use with an open allowlist + untrusted output on the public internet; you are effectively running a user-directed fetcher. For stricter safety set IRIS_CITATION_DOMAINS to a curated list.",
+            '',
+            'Error modes. Throws when the API key env var is missing. Throws "Unknown model" on unsupported model IDs. Per-citation errors are collected (resolve_error.kind = bad_scheme / ssrf / not_allowed_domain / timeout / too_large / bad_status / redirect_loop / not_text / fetch_disabled / malformed_judge_response / cost_cap_reached / unresolvable_kind) and returned in the response rather than thrown. An empty output or output with zero extractable citations returns overall_score=null + passed=true (nothing to fail).',
+        ].join('\n'),
+        inputSchema,
+        annotations: {
+            readOnlyHint: false, // Writes eval_result + spends money
+            destructiveHint: false, // Creates data; doesn't overwrite/delete
+            idempotentHint: false, // External fetches + provider non-determinism
+            openWorldHint: true, // Outbound HTTP to citation URLs + LLM provider API
+        },
+    }, async (args) => {
+        const provider = args.provider ?? inferProvider(args.model);
+        const apiKey = resolveApiKey(provider);
+        const allowFetch = resolveAllowFetch(args.allow_fetch);
+        const domainAllowlist = resolveDomainAllowlist(args.domain_allowlist);
+        const result = await verifyCitations({
+            output: args.output,
+            provider,
+            model: args.model,
+            apiKey,
+            allowFetch,
+            domainAllowlist,
+            maxCostUsdTotal: args.max_cost_usd_total,
+            maxCitations: args.max_citations,
+            perSourceTimeoutMs: args.per_source_timeout_ms,
+            perSourceMaxBytes: args.per_source_max_bytes,
+        });
+        const evalId = generateEvalId();
+        const score = result.overallScore ?? 0;
+        // Persist so dashboard can surface. eval_type='custom' — same
+        // rationale as evaluate_with_llm_judge (spans all 4 heuristic
+        // categories). rule_results[0] carries per-citation summary.
+        await storage.insertEvalResult(LOCAL_TENANT, {
+            id: evalId,
+            trace_id: args.trace_id,
+            eval_type: 'custom',
+            output_text: args.output,
+            score,
+            passed: result.passed,
+            rule_results: [
+                {
+                    ruleName: `semantic_citation_verify:${provider}/${args.model}`,
+                    passed: result.passed,
+                    score,
+                    message: result.overallScore === null
+                        ? `No resolvable citations (found ${result.totalCitationsFound}, resolved ${result.totalResolved})`
+                        : `${result.totalSupported}/${result.totalResolved} cited sources supported the output`,
+                },
+            ],
+            suggestions: result.passed ? [] : [`Only ${result.totalSupported}/${result.totalResolved} cited sources actually supported the claim.`],
+            rules_evaluated: 1,
+            rules_skipped: 0,
+            insufficient_data: result.overallScore === null,
+        });
+        return {
+            content: [
+                {
+                    type: 'text',
+                    text: JSON.stringify({
+                        id: evalId,
+                        overall_score: result.overallScore,
+                        passed: result.passed,
+                        total_citations_found: result.totalCitationsFound,
+                        total_resolved: result.totalResolved,
+                        total_supported: result.totalSupported,
+                        total_cost_usd: result.totalCostUsd,
+                        citations: result.citations.map((c) => ({
+                            citation: {
+                                raw: c.citation.raw,
+                                kind: c.citation.kind,
+                                identifier: c.citation.identifier,
+                                offset_start: c.citation.offsetStart,
+                                offset_end: c.citation.offsetEnd,
+                            },
+                            resolve_status: c.resolveStatus,
+                            resolve_error: c.resolveError,
+                            source: c.source,
+                            judge: c.judge
+                                ? {
+                                    supported: c.judge.supported,
+                                    confidence: c.judge.confidence,
+                                    rationale: c.judge.rationale,
+                                    cost_usd: c.judge.costUsd,
+                                    latency_ms: c.judge.latencyMs,
+                                    input_tokens: c.judge.inputTokens,
+                                    output_tokens: c.judge.outputTokens,
+                                }
+                                : undefined,
+                        })),
+                    }),
+                },
+            ],
+        };
+    });
+}

package/dist/types/custom-rule.d.ts

@@ -0,0 +1,70 @@
+import type { CustomRuleDefinition, EvalType } from './eval.js';
+export type RuleSeverity = 'low' | 'medium' | 'high' | 'critical';
+export interface DeployedCustomRule {
+    /** Stable id (e.g., "rule-<8-char-hex>"). Generated server-side on deploy. */
+    id: string;
+    /** User-readable name. Becomes the rule name in eval results. */
+    name: string;
+    /** Human-readable description (what the rule checks for, why it matters). */
+    description: string;
+    /** Eval category this rule belongs to. Determines when it fires. */
+    evalType: EvalType;
+    /** Severity used to sort rules in the dashboard + audit alerts. */
+    severity: RuleSeverity;
+    /** The check definition (regex pattern, length threshold, etc.). */
+    definition: CustomRuleDefinition;
+    /** Whether this rule is currently active. Disabled rules don't fire but are kept for audit. */
+    enabled: boolean;
+    /** ISO timestamp of deploy. */
+    createdAt: string;
+    /** ISO timestamp of most recent edit. */
+    updatedAt: string;
+    /** Optional moment ID the rule was extracted from (workflow inversion provenance). */
+    sourceMomentId?: string;
+    /** Version counter — incremented on edit. Starts at 1. */
+    version: number;
+}
+export interface CustomRulesFile {
+    /** File schema version — bump when shape changes. */
+    version: 1;
+    rules: DeployedCustomRule[];
+}
+export interface AuditLogEntry {
+    ts: string;
+    /**
+     * Which tenant the action belongs to. OSS installs always emit 'local'.
+     * Cloud installs emit the tenant resolved from the authenticated session.
+     *
+     * Optional for backward compatibility: entries written before v0.4.0
+     * don't have this field. Readers MUST treat missing `tenantId` as
+     * 'local' so old audit logs remain queryable on upgrade.
+     */
+    tenantId?: string;
+    /** Action taken — currently rule.deploy / rule.delete / rule.toggle. */
+    action: 'rule.deploy' | 'rule.delete' | 'rule.toggle' | 'rule.update';
+    /** Who initiated. v0.4 is single-user local — always "local". v0.5+ adds users. */
+    user: string;
+    ruleId: string;
+    ruleName?: string;
+    /** Optional detail: source moment id, prior version, etc. */
+    details?: Record<string, unknown>;
+}
+export interface RulePreviewResult {
+    /** Number of historical traces evaluated. */
+    tracesEvaluated: number;
+    /** How many traces would have FAILED this proposed rule. */
+    wouldFail: number;
+    /** How many would have PASSED. */
+    wouldPass: number;
+    /** How many would have skipped (rule not applicable to context). */
+    wouldSkip: number;
+    /** First 5 example traces that would fail (with brief output preview). */
+    examples: Array<{
+        traceId: string;
+        agentName: string;
+        timestamp: string;
+        outputPreview: string;
+    }>;
+    /** Time window covered. */
+    windowSinceIso: string;
+}

package/dist/types/custom-rule.js

@@ -0,0 +1 @@
+export {};

package/dist/types/decision-moment.d.ts

@@ -0,0 +1,122 @@
+export type MomentVerdict = 'pass' | 'fail' | 'partial' | 'unevaluated';
+export type MomentSignificanceKind = 'safety-violation' | 'cost-spike' | 'first-failure' | 'novel-pattern' | 'rule-collision' | 'normal-pass' | 'normal-fail';
+export interface MomentSignificance {
+    /** Classifier kind. */
+    kind: MomentSignificanceKind;
+    /** 0-1; how moment-worthy this trace is. Used for visual emphasis on the timeline. */
+    score: number;
+    /** Short human-readable label (≤30 chars) — shown on timeline dot tooltip. */
+    label: string;
+    /** Longer explanation — shown in the detail surface. */
+    reason: string;
+}
+export interface MomentRuleSnapshot {
+    /** Names of rules that failed. */
+    failed: string[];
+    /** Names of rules that were skipped (insufficient context). */
+    skipped: string[];
+    /** Count of rules that passed. */
+    passedCount: number;
+    /** Count of rules that fired total (across all eval_types). */
+    totalCount: number;
+}
+export interface DecisionMoment {
+    /** Stable id; equals the source trace_id. */
+    id: string;
+    /** Source trace_id (same as id; included for query convenience). */
+    traceId: string;
+    /** Agent that produced the output. */
+    agentName: string;
+    /** ISO timestamp of the trace. */
+    timestamp: string;
+    /** Input the agent received (may be omitted for storage-size reasons). */
+    input?: string;
+    /** Output the agent produced. */
+    output?: string;
+    /** Trace-level cost in USD. */
+    costUsd?: number;
+    /** Trace-level end-to-end latency. */
+    latencyMs?: number;
+    /** Aggregated eval verdict across all eval_types that ran on this trace. */
+    verdict: MomentVerdict;
+    /** Weighted average score across all eval rules that fired (excludes skipped). */
+    overallScore: number;
+    /** Count of distinct evaluations (one per eval_type) recorded for this trace. */
+    evalCount: number;
+    /** Per-rule pass/fail/skip breakdown. */
+    ruleSnapshot: MomentRuleSnapshot;
+    /** Why this trace is (or isn't) moment-worthy. */
+    significance: MomentSignificance;
+}
+/**
+ * Detailed view returned by the moment-detail endpoint. Includes the full
+ * eval result list (with messages + scores per rule) and the source trace
+ * for context.
+ */
+export interface DecisionMomentDetail extends DecisionMoment {
+    /** Every eval recorded for this trace, with full rule result detail. */
+    evals: Array<{
+        id: string;
+        evalType: string;
+        score: number;
+        passed: boolean;
+        ruleResults: Array<{
+            ruleName: string;
+            passed: boolean;
+            score: number;
+            message: string;
+            skipped?: boolean;
+            skipReason?: string;
+        }>;
+        suggestions: string[];
+        createdAt?: string;
+    }>;
+    /** Full input (uncompressed). */
+    input?: string;
+    /** Full output (uncompressed). */
+    output?: string;
+    /** Tool-call sequence from the trace, if any. */
+    toolCalls?: Array<{
+        tool_name: string;
+        input?: unknown;
+        output?: unknown;
+        latency_ms?: number;
+        error?: string;
+    }>;
+    /** Span tree from the trace, if any. */
+    spans?: Array<{
+        span_id: string;
+        parent_span_id?: string;
+        name: string;
+        kind: string;
+        start_time: string;
+        end_time?: string;
+    }>;
+}
+export interface MomentQueryFilter {
+    /** Filter by agent name. */
+    agentName?: string;
+    /** Filter by verdict. */
+    verdict?: MomentVerdict;
+    /** Filter to only show moments above a significance threshold. */
+    minSignificance?: number;
+    /** Filter by significance kind (e.g., only show safety-violations). */
+    significanceKind?: MomentSignificanceKind;
+    /** ISO timestamp; only moments at or after. */
+    since?: string;
+    /** ISO timestamp; only moments at or before. */
+    until?: string;
+}
+export interface MomentQueryOptions {
+    filter?: MomentQueryFilter;
+    limit?: number;
+    offset?: number;
+    /** Default 'desc' (most recent first). */
+    sortOrder?: 'asc' | 'desc';
+}
+export interface MomentQueryResult {
+    moments: DecisionMoment[];
+    total: number;
+    limit: number;
+    offset: number;
+}

package/dist/types/decision-moment.js

@@ -0,0 +1,17 @@
+/*
+ * Decision Moment — the new primary unit of the Iris dashboard.
+ *
+ * A Decision Moment is the point at which an agent's output was scored.
+ * It is derived from existing data: one trace + its aggregated eval results.
+ * No schema migration — the moment is computed on read.
+ *
+ * The system-design reframe: not every eval is a moment. The significance
+ * classifier separates moments worth a human's attention (safety violations,
+ * cost spikes, first-failures, novel patterns) from operational data
+ * (passing evals on the happy path).
+ *
+ * This is the workflow primitive that supports Make-This-A-Rule: a user
+ * looking at a moment can promote the observed behavior into a new rule
+ * without leaving the moment context.
+ */
+export {};

package/dist/types/index.d.ts

@@ -2,3 +2,5 @@ export type { SpanKind, SpanStatus, SpanEvent, ToolCallRecord, TokenUsage, Span,
 export type { EvalType, EvalRule, EvalContext, EvalRuleResult, EvalResult, CustomRuleType, CustomRuleDefinition, } from './eval.js';
 export type { TraceFilter, TraceQueryOptions, TraceQueryResult, DashboardSummary, EvalStatsPeriod, EvalStats, EvalStatsTrendBucket, EvalStatsRuleBreakdown, EvalStatsFailure, IStorageAdapter, } from './query.js';
 export type { IrisConfig } from './config.js';
+export type { TenantId } from './tenant.js';
+export { LOCAL_TENANT, asTenantId, TenantContextRequiredError } from './tenant.js';

package/dist/types/index.js

@@ -1 +1 @@
-export {};
+export { LOCAL_TENANT, asTenantId, TenantContextRequiredError } from './tenant.js';

package/dist/types/query.d.ts

@@ -1,5 +1,6 @@
 import type { Trace, Span } from './trace.js';
 import type { EvalResult } from './eval.js';
+import type { TenantId } from './tenant.js';
 export interface TraceFilter {
     agent_name?: string;
     framework?: string;
@@ -74,14 +75,14 @@ export interface DashboardSummary {
 export interface IStorageAdapter {
     initialize(): Promise<void>;
     close(): Promise<void>;
-    insertTrace(trace: Trace): Promise<void>;
-    getTrace(traceId: string): Promise<Trace | null>;
-    queryTraces(options: TraceQueryOptions): Promise<TraceQueryResult>;
-    insertSpan(span: Span): Promise<void>;
-    getSpansByTraceId(traceId: string): Promise<Span[]>;
-    insertEvalResult(result: EvalResult): Promise<void>;
-    getEvalsByTraceId(traceId: string): Promise<EvalResult[]>;
-    queryEvalResults(options: {
+    insertTrace(tenantId: TenantId, trace: Trace): Promise<void>;
+    getTrace(tenantId: TenantId, traceId: string): Promise<Trace | null>;
+    queryTraces(tenantId: TenantId, options: TraceQueryOptions): Promise<TraceQueryResult>;
+    insertSpan(tenantId: TenantId, span: Span): Promise<void>;
+    getSpansByTraceId(tenantId: TenantId, traceId: string): Promise<Span[]>;
+    insertEvalResult(tenantId: TenantId, result: EvalResult): Promise<void>;
+    getEvalsByTraceId(tenantId: TenantId, traceId: string): Promise<EvalResult[]>;
+    queryEvalResults(tenantId: TenantId, options: {
         eval_type?: string;
         passed?: boolean;
         since?: string;
@@ -92,11 +93,20 @@ export interface IStorageAdapter {
         results: EvalResult[];
         total: number;
     }>;
-    getDashboardSummary(sinceHours?: number): Promise<DashboardSummary>;
-    deleteTracesOlderThan(days: number): Promise<number>;
-    getDistinctValues(column: string): Promise<string[]>;
-    getEvalStats(period: EvalStatsPeriod): Promise<EvalStats>;
-    getEvalStatsTrend(period: EvalStatsPeriod): Promise<EvalStatsTrendBucket[]>;
-    getEvalStatsRules(period: EvalStatsPeriod): Promise<EvalStatsRuleBreakdown[]>;
-    getEvalStatsFailures(period: EvalStatsPeriod, limit: number): Promise<EvalStatsFailure[]>;
+    getDashboardSummary(tenantId: TenantId, sinceHours?: number): Promise<DashboardSummary>;
+    deleteTracesOlderThan(tenantId: TenantId, days: number): Promise<number>;
+    /**
+     * Delete a single trace by id. Cascades to spans via FK ON DELETE
+     * CASCADE; eval_results get their trace_id set to NULL (so score
+     * history survives even after the trace is deleted).
+     *
+     * Returns true if a row was deleted, false if the id didn't exist
+     * (or belonged to a different tenant).
+     */
+    deleteTrace(tenantId: TenantId, traceId: string): Promise<boolean>;
+    getDistinctValues(tenantId: TenantId, column: string): Promise<string[]>;
+    getEvalStats(tenantId: TenantId, period: EvalStatsPeriod): Promise<EvalStats>;
+    getEvalStatsTrend(tenantId: TenantId, period: EvalStatsPeriod): Promise<EvalStatsTrendBucket[]>;
+    getEvalStatsRules(tenantId: TenantId, period: EvalStatsPeriod): Promise<EvalStatsRuleBreakdown[]>;
+    getEvalStatsFailures(tenantId: TenantId, period: EvalStatsPeriod, limit: number): Promise<EvalStatsFailure[]>;
 }

package/dist/types/tenant.d.ts

@@ -0,0 +1,26 @@
+/** Nominal brand on string. A plain string cannot be used where TenantId is required. */
+export type TenantId = string & {
+    readonly __brand: 'TenantId';
+};
+/** The sentinel tenant used by OSS / single-user installs. */
+export declare const LOCAL_TENANT: TenantId;
+/**
+ * Coerce a raw string into a TenantId after validating non-empty.
+ * Throws TenantContextRequiredError when the input is empty/null/undefined.
+ *
+ * This is the only officially-sanctioned way to mint a TenantId from user
+ * input. It centralizes the non-empty invariant so every call site is
+ * safe by construction.
+ */
+export declare function asTenantId(value: string | null | undefined): TenantId;
+/**
+ * Thrown when a storage method is invoked without a valid tenant
+ * context. Fail-safe: we'd rather crash than return data that might
+ * cross a tenant boundary.
+ *
+ * Catch at the route-handler layer and translate to 500 + correlation
+ * ID; never surface the raw message to end users.
+ */
+export declare class TenantContextRequiredError extends Error {
+    constructor(message?: string);
+}

package/dist/types/tenant.js

@@ -0,0 +1,58 @@
+/*
+ * tenant — tenant identity primitives.
+ *
+ * Every storage read/write in Iris takes a TenantId. For OSS / single-
+ * user installs the value is always `LOCAL_TENANT` (the literal string
+ * `'local'`). For the future Cloud SKU, the value is resolved by the
+ * authentication middleware from the session and carried on the
+ * request context.
+ *
+ * Design constraints (from the 2026-04-23 threat model §5.1):
+ *   1. Tenant context is a REQUIRED parameter on every storage method;
+ *      no Optional<TenantId>. The type system enforces it.
+ *   2. The default for OSS is the literal string 'local' — never null,
+ *      never undefined, never auto-generated. No "no tenant" code path
+ *      exists that could become a "show me everything" path.
+ *   3. Tenant resolution happens in middleware, not storage. Storage
+ *      receives the resolved TenantId and uses it verbatim.
+ *   4. Default-deny: a storage method receiving an empty tenantId MUST
+ *      throw TenantContextRequiredError, not return all rows.
+ *   5. TenantId is an opaque non-empty string. No assumption beyond
+ *      "valid non-empty UTF-8"; Cloud will use UUID/KSUID, OSS uses
+ *      'local'.
+ *
+ * Branded string pattern: we wrap `string` in a nominal brand so passing
+ * a raw `string` where `TenantId` is expected is a compile error. The
+ * only path to a TenantId is through `asTenantId()` or the LOCAL_TENANT
+ * constant — both of which validate non-empty.
+ */
+/** The sentinel tenant used by OSS / single-user installs. */
+export const LOCAL_TENANT = 'local';
+/**
+ * Coerce a raw string into a TenantId after validating non-empty.
+ * Throws TenantContextRequiredError when the input is empty/null/undefined.
+ *
+ * This is the only officially-sanctioned way to mint a TenantId from user
+ * input. It centralizes the non-empty invariant so every call site is
+ * safe by construction.
+ */
+export function asTenantId(value) {
+    if (typeof value !== 'string' || value.length === 0) {
+        throw new TenantContextRequiredError(`TenantId must be a non-empty string; got ${value === null ? 'null' : typeof value}`);
+    }
+    return value;
+}
+/**
+ * Thrown when a storage method is invoked without a valid tenant
+ * context. Fail-safe: we'd rather crash than return data that might
+ * cross a tenant boundary.
+ *
+ * Catch at the route-handler layer and translate to 500 + correlation
+ * ID; never surface the raw message to end users.
+ */
+export class TenantContextRequiredError extends Error {
+    constructor(message = 'Tenant context required; storage cannot be called without a resolved TenantId') {
+        super(message);
+        this.name = 'TenantContextRequiredError';
+    }
+}

package/dist/utils/open-browser.d.ts

@@ -0,0 +1 @@
+export declare function openBrowser(url: string): void;

package/dist/utils/open-browser.js

@@ -0,0 +1,45 @@
+/*
+ * open-browser — minimal cross-platform browser launcher.
+ *
+ * Avoids the `open` npm dep so iris-mcp's install size stays small.
+ * Uses the platform's default URL handler:
+ *   Windows: `cmd /c start "" "<url>"`
+ *   macOS:   `open "<url>"`
+ *   Linux:   `xdg-open "<url>"`
+ *
+ * Spawns detached + ignores stdio so the iris-mcp process doesn't depend
+ * on the launched browser staying alive.
+ */
+import { spawn } from 'node:child_process';
+export function openBrowser(url) {
+    const platform = process.platform;
+    let command;
+    let args;
+    if (platform === 'win32') {
+        // The empty title argument is required so cmd treats the URL as the
+        // target rather than as the window title.
+        command = 'cmd';
+        args = ['/c', 'start', '', url];
+    }
+    else if (platform === 'darwin') {
+        command = 'open';
+        args = [url];
+    }
+    else {
+        // Linux + BSDs.
+        command = 'xdg-open';
+        args = [url];
+    }
+    try {
+        const child = spawn(command, args, {
+            detached: true,
+            stdio: 'ignore',
+        });
+        child.unref();
+    }
+    catch {
+        // Best-effort: if the launch fails (no browser, missing utility,
+        // sandboxed environment), the dashboard is still reachable manually
+        // at the URL we logged.
+    }
+}

package/dist/utils/validate-port-config.d.ts

@@ -0,0 +1,2 @@
+import type { IrisConfig } from '../types/index.js';
+export declare function validatePortConfig(config: IrisConfig): void;

package/dist/utils/validate-port-config.js

@@ -0,0 +1,9 @@
+export function validatePortConfig(config) {
+    if (config.transport.type === 'http' &&
+        config.dashboard.enabled &&
+        config.transport.port === config.dashboard.port) {
+        throw new Error(`Port collision: HTTP transport and dashboard are both configured for port ${config.transport.port}. ` +
+            `Pass --dashboard-port <other> or set IRIS_DASHBOARD_PORT to a different port. ` +
+            `Default dashboard port is 6920; a common pair is --port 6919 --dashboard-port 6920.`);
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@iris-eval/mcp-server",
-  "version": "0.3.1",
+  "version": "0.4.0",
   "description": "The agent eval standard for MCP. Score every agent output for quality, safety, and cost.",
   "mcpName": "io.github.iris-eval/mcp-server",
   "type": "module",
@@ -21,6 +21,8 @@
     "test:watch": "vitest",
     "test:coverage": "vitest run --coverage",
     "test:integration": "vitest run tests/integration/",
+    "test:e2e": "playwright test",
+    "test:e2e:ui": "playwright test --ui",
     "version:check": "bash scripts/check-version.sh",
     "version:sync": "node scripts/sync-versions.mjs",
     "clean": "rm -rf dist coverage",
@@ -81,6 +83,7 @@
     "zod": "^3.25.0"
   },
   "devDependencies": {
+    "@playwright/test": "^1.59.1",
     "@types/better-sqlite3": "^7.6.0",
     "@types/express": "^5.0.0",
     "@types/node": "^25.5.2",

package/server.json

@@ -6,12 +6,12 @@
     "url": "https://github.com/iris-eval/mcp-server",
     "source": "github"
   },
-  "version": "0.3.1",
+  "version": "0.4.0",
   "packages": [
     {
       "registryType": "npm",
       "identifier": "@iris-eval/mcp-server",
-      "version": "0.3.1",
+      "version": "0.4.0",
       "transport": {
         "type": "stdio"
       },