@iris-eval/mcp-server 0.3.0 → 0.4.0-rc.1

package/dist/types/decision-moment.d.ts

@@ -0,0 +1,122 @@
+export type MomentVerdict = 'pass' | 'fail' | 'partial' | 'unevaluated';
+export type MomentSignificanceKind = 'safety-violation' | 'cost-spike' | 'first-failure' | 'novel-pattern' | 'rule-collision' | 'normal-pass' | 'normal-fail';
+export interface MomentSignificance {
+    /** Classifier kind. */
+    kind: MomentSignificanceKind;
+    /** 0-1; how moment-worthy this trace is. Used for visual emphasis on the timeline. */
+    score: number;
+    /** Short human-readable label (≤30 chars) — shown on timeline dot tooltip. */
+    label: string;
+    /** Longer explanation — shown in the detail surface. */
+    reason: string;
+}
+export interface MomentRuleSnapshot {
+    /** Names of rules that failed. */
+    failed: string[];
+    /** Names of rules that were skipped (insufficient context). */
+    skipped: string[];
+    /** Count of rules that passed. */
+    passedCount: number;
+    /** Count of rules that fired total (across all eval_types). */
+    totalCount: number;
+}
+export interface DecisionMoment {
+    /** Stable id; equals the source trace_id. */
+    id: string;
+    /** Source trace_id (same as id; included for query convenience). */
+    traceId: string;
+    /** Agent that produced the output. */
+    agentName: string;
+    /** ISO timestamp of the trace. */
+    timestamp: string;
+    /** Input the agent received (may be omitted for storage-size reasons). */
+    input?: string;
+    /** Output the agent produced. */
+    output?: string;
+    /** Trace-level cost in USD. */
+    costUsd?: number;
+    /** Trace-level end-to-end latency. */
+    latencyMs?: number;
+    /** Aggregated eval verdict across all eval_types that ran on this trace. */
+    verdict: MomentVerdict;
+    /** Weighted average score across all eval rules that fired (excludes skipped). */
+    overallScore: number;
+    /** Count of distinct evaluations (one per eval_type) recorded for this trace. */
+    evalCount: number;
+    /** Per-rule pass/fail/skip breakdown. */
+    ruleSnapshot: MomentRuleSnapshot;
+    /** Why this trace is (or isn't) moment-worthy. */
+    significance: MomentSignificance;
+}
+/**
+ * Detailed view returned by the moment-detail endpoint. Includes the full
+ * eval result list (with messages + scores per rule) and the source trace
+ * for context.
+ */
+export interface DecisionMomentDetail extends DecisionMoment {
+    /** Every eval recorded for this trace, with full rule result detail. */
+    evals: Array<{
+        id: string;
+        evalType: string;
+        score: number;
+        passed: boolean;
+        ruleResults: Array<{
+            ruleName: string;
+            passed: boolean;
+            score: number;
+            message: string;
+            skipped?: boolean;
+            skipReason?: string;
+        }>;
+        suggestions: string[];
+        createdAt?: string;
+    }>;
+    /** Full input (uncompressed). */
+    input?: string;
+    /** Full output (uncompressed). */
+    output?: string;
+    /** Tool-call sequence from the trace, if any. */
+    toolCalls?: Array<{
+        tool_name: string;
+        input?: unknown;
+        output?: unknown;
+        latency_ms?: number;
+        error?: string;
+    }>;
+    /** Span tree from the trace, if any. */
+    spans?: Array<{
+        span_id: string;
+        parent_span_id?: string;
+        name: string;
+        kind: string;
+        start_time: string;
+        end_time?: string;
+    }>;
+}
+export interface MomentQueryFilter {
+    /** Filter by agent name. */
+    agentName?: string;
+    /** Filter by verdict. */
+    verdict?: MomentVerdict;
+    /** Filter to only show moments above a significance threshold. */
+    minSignificance?: number;
+    /** Filter by significance kind (e.g., only show safety-violations). */
+    significanceKind?: MomentSignificanceKind;
+    /** ISO timestamp; only moments at or after. */
+    since?: string;
+    /** ISO timestamp; only moments at or before. */
+    until?: string;
+}
+export interface MomentQueryOptions {
+    filter?: MomentQueryFilter;
+    limit?: number;
+    offset?: number;
+    /** Default 'desc' (most recent first). */
+    sortOrder?: 'asc' | 'desc';
+}
+export interface MomentQueryResult {
+    moments: DecisionMoment[];
+    total: number;
+    limit: number;
+    offset: number;
+}

package/dist/types/decision-moment.js

@@ -0,0 +1,17 @@
+/*
+ * Decision Moment — the new primary unit of the Iris dashboard.
+ *
+ * A Decision Moment is the point at which an agent's output was scored.
+ * It is derived from existing data: one trace + its aggregated eval results.
+ * No schema migration — the moment is computed on read.
+ *
+ * The system-design reframe: not every eval is a moment. The significance
+ * classifier separates moments worth a human's attention (safety violations,
+ * cost spikes, first-failures, novel patterns) from operational data
+ * (passing evals on the happy path).
+ *
+ * This is the workflow primitive that supports Make-This-A-Rule: a user
+ * looking at a moment can promote the observed behavior into a new rule
+ * without leaving the moment context.
+ */
+export {};

package/dist/types/index.d.ts

@@ -2,3 +2,5 @@ export type { SpanKind, SpanStatus, SpanEvent, ToolCallRecord, TokenUsage, Span,
 export type { EvalType, EvalRule, EvalContext, EvalRuleResult, EvalResult, CustomRuleType, CustomRuleDefinition, } from './eval.js';
 export type { TraceFilter, TraceQueryOptions, TraceQueryResult, DashboardSummary, EvalStatsPeriod, EvalStats, EvalStatsTrendBucket, EvalStatsRuleBreakdown, EvalStatsFailure, IStorageAdapter, } from './query.js';
 export type { IrisConfig } from './config.js';
+export type { TenantId } from './tenant.js';
+export { LOCAL_TENANT, asTenantId, TenantContextRequiredError } from './tenant.js';

package/dist/types/index.js

@@ -1 +1 @@
-export {};
+export { LOCAL_TENANT, asTenantId, TenantContextRequiredError } from './tenant.js';

package/dist/types/query.d.ts

@@ -1,5 +1,6 @@
 import type { Trace, Span } from './trace.js';
 import type { EvalResult } from './eval.js';
+import type { TenantId } from './tenant.js';
 export interface TraceFilter {
     agent_name?: string;
     framework?: string;
@@ -74,14 +75,14 @@ export interface DashboardSummary {
 export interface IStorageAdapter {
     initialize(): Promise<void>;
     close(): Promise<void>;
-    insertTrace(trace: Trace): Promise<void>;
-    getTrace(traceId: string): Promise<Trace | null>;
-    queryTraces(options: TraceQueryOptions): Promise<TraceQueryResult>;
-    insertSpan(span: Span): Promise<void>;
-    getSpansByTraceId(traceId: string): Promise<Span[]>;
-    insertEvalResult(result: EvalResult): Promise<void>;
-    getEvalsByTraceId(traceId: string): Promise<EvalResult[]>;
-    queryEvalResults(options: {
+    insertTrace(tenantId: TenantId, trace: Trace): Promise<void>;
+    getTrace(tenantId: TenantId, traceId: string): Promise<Trace | null>;
+    queryTraces(tenantId: TenantId, options: TraceQueryOptions): Promise<TraceQueryResult>;
+    insertSpan(tenantId: TenantId, span: Span): Promise<void>;
+    getSpansByTraceId(tenantId: TenantId, traceId: string): Promise<Span[]>;
+    insertEvalResult(tenantId: TenantId, result: EvalResult): Promise<void>;
+    getEvalsByTraceId(tenantId: TenantId, traceId: string): Promise<EvalResult[]>;
+    queryEvalResults(tenantId: TenantId, options: {
         eval_type?: string;
         passed?: boolean;
         since?: string;
@@ -92,11 +93,20 @@ export interface IStorageAdapter {
         results: EvalResult[];
         total: number;
     }>;
-    getDashboardSummary(sinceHours?: number): Promise<DashboardSummary>;
-    deleteTracesOlderThan(days: number): Promise<number>;
-    getDistinctValues(column: string): Promise<string[]>;
-    getEvalStats(period: EvalStatsPeriod): Promise<EvalStats>;
-    getEvalStatsTrend(period: EvalStatsPeriod): Promise<EvalStatsTrendBucket[]>;
-    getEvalStatsRules(period: EvalStatsPeriod): Promise<EvalStatsRuleBreakdown[]>;
-    getEvalStatsFailures(period: EvalStatsPeriod, limit: number): Promise<EvalStatsFailure[]>;
+    getDashboardSummary(tenantId: TenantId, sinceHours?: number): Promise<DashboardSummary>;
+    deleteTracesOlderThan(tenantId: TenantId, days: number): Promise<number>;
+    /**
+     * Delete a single trace by id. Cascades to spans via FK ON DELETE
+     * CASCADE; eval_results get their trace_id set to NULL (so score
+     * history survives even after the trace is deleted).
+     *
+     * Returns true if a row was deleted, false if the id didn't exist
+     * (or belonged to a different tenant).
+     */
+    deleteTrace(tenantId: TenantId, traceId: string): Promise<boolean>;
+    getDistinctValues(tenantId: TenantId, column: string): Promise<string[]>;
+    getEvalStats(tenantId: TenantId, period: EvalStatsPeriod): Promise<EvalStats>;
+    getEvalStatsTrend(tenantId: TenantId, period: EvalStatsPeriod): Promise<EvalStatsTrendBucket[]>;
+    getEvalStatsRules(tenantId: TenantId, period: EvalStatsPeriod): Promise<EvalStatsRuleBreakdown[]>;
+    getEvalStatsFailures(tenantId: TenantId, period: EvalStatsPeriod, limit: number): Promise<EvalStatsFailure[]>;
 }

package/dist/types/tenant.d.ts

@@ -0,0 +1,26 @@
+/** Nominal brand on string. A plain string cannot be used where TenantId is required. */
+export type TenantId = string & {
+    readonly __brand: 'TenantId';
+};
+/** The sentinel tenant used by OSS / single-user installs. */
+export declare const LOCAL_TENANT: TenantId;
+/**
+ * Coerce a raw string into a TenantId after validating non-empty.
+ * Throws TenantContextRequiredError when the input is empty/null/undefined.
+ *
+ * This is the only officially-sanctioned way to mint a TenantId from user
+ * input. It centralizes the non-empty invariant so every call site is
+ * safe by construction.
+ */
+export declare function asTenantId(value: string | null | undefined): TenantId;
+/**
+ * Thrown when a storage method is invoked without a valid tenant
+ * context. Fail-safe: we'd rather crash than return data that might
+ * cross a tenant boundary.
+ *
+ * Catch at the route-handler layer and translate to 500 + correlation
+ * ID; never surface the raw message to end users.
+ */
+export declare class TenantContextRequiredError extends Error {
+    constructor(message?: string);
+}

package/dist/types/tenant.js

@@ -0,0 +1,58 @@
+/*
+ * tenant — tenant identity primitives.
+ *
+ * Every storage read/write in Iris takes a TenantId. For OSS / single-
+ * user installs the value is always `LOCAL_TENANT` (the literal string
+ * `'local'`). For the future Cloud SKU, the value is resolved by the
+ * authentication middleware from the session and carried on the
+ * request context.
+ *
+ * Design constraints (from the 2026-04-23 threat model §5.1):
+ *   1. Tenant context is a REQUIRED parameter on every storage method;
+ *      no Optional<TenantId>. The type system enforces it.
+ *   2. The default for OSS is the literal string 'local' — never null,
+ *      never undefined, never auto-generated. No "no tenant" code path
+ *      exists that could become a "show me everything" path.
+ *   3. Tenant resolution happens in middleware, not storage. Storage
+ *      receives the resolved TenantId and uses it verbatim.
+ *   4. Default-deny: a storage method receiving an empty tenantId MUST
+ *      throw TenantContextRequiredError, not return all rows.
+ *   5. TenantId is an opaque non-empty string. No assumption beyond
+ *      "valid non-empty UTF-8"; Cloud will use UUID/KSUID, OSS uses
+ *      'local'.
+ *
+ * Branded string pattern: we wrap `string` in a nominal brand so passing
+ * a raw `string` where `TenantId` is expected is a compile error. The
+ * only path to a TenantId is through `asTenantId()` or the LOCAL_TENANT
+ * constant — both of which validate non-empty.
+ */
+/** The sentinel tenant used by OSS / single-user installs. */
+export const LOCAL_TENANT = 'local';
+/**
+ * Coerce a raw string into a TenantId after validating non-empty.
+ * Throws TenantContextRequiredError when the input is empty/null/undefined.
+ *
+ * This is the only officially-sanctioned way to mint a TenantId from user
+ * input. It centralizes the non-empty invariant so every call site is
+ * safe by construction.
+ */
+export function asTenantId(value) {
+    if (typeof value !== 'string' || value.length === 0) {
+        throw new TenantContextRequiredError(`TenantId must be a non-empty string; got ${value === null ? 'null' : typeof value}`);
+    }
+    return value;
+}
+/**
+ * Thrown when a storage method is invoked without a valid tenant
+ * context. Fail-safe: we'd rather crash than return data that might
+ * cross a tenant boundary.
+ *
+ * Catch at the route-handler layer and translate to 500 + correlation
+ * ID; never surface the raw message to end users.
+ */
+export class TenantContextRequiredError extends Error {
+    constructor(message = 'Tenant context required; storage cannot be called without a resolved TenantId') {
+        super(message);
+        this.name = 'TenantContextRequiredError';
+    }
+}

package/dist/utils/open-browser.d.ts

@@ -0,0 +1 @@
+export declare function openBrowser(url: string): void;

package/dist/utils/open-browser.js

@@ -0,0 +1,45 @@
+/*
+ * open-browser — minimal cross-platform browser launcher.
+ *
+ * Avoids the `open` npm dep so iris-mcp's install size stays small.
+ * Uses the platform's default URL handler:
+ *   Windows: `cmd /c start "" "<url>"`
+ *   macOS:   `open "<url>"`
+ *   Linux:   `xdg-open "<url>"`
+ *
+ * Spawns detached + ignores stdio so the iris-mcp process doesn't depend
+ * on the launched browser staying alive.
+ */
+import { spawn } from 'node:child_process';
+export function openBrowser(url) {
+    const platform = process.platform;
+    let command;
+    let args;
+    if (platform === 'win32') {
+        // The empty title argument is required so cmd treats the URL as the
+        // target rather than as the window title.
+        command = 'cmd';
+        args = ['/c', 'start', '', url];
+    }
+    else if (platform === 'darwin') {
+        command = 'open';
+        args = [url];
+    }
+    else {
+        // Linux + BSDs.
+        command = 'xdg-open';
+        args = [url];
+    }
+    try {
+        const child = spawn(command, args, {
+            detached: true,
+            stdio: 'ignore',
+        });
+        child.unref();
+    }
+    catch {
+        // Best-effort: if the launch fails (no browser, missing utility,
+        // sandboxed environment), the dashboard is still reachable manually
+        // at the URL we logged.
+    }
+}

package/dist/utils/validate-port-config.d.ts

@@ -0,0 +1,2 @@
+import type { IrisConfig } from '../types/index.js';
+export declare function validatePortConfig(config: IrisConfig): void;

package/dist/utils/validate-port-config.js

@@ -0,0 +1,9 @@
+export function validatePortConfig(config) {
+    if (config.transport.type === 'http' &&
+        config.dashboard.enabled &&
+        config.transport.port === config.dashboard.port) {
+        throw new Error(`Port collision: HTTP transport and dashboard are both configured for port ${config.transport.port}. ` +
+            `Pass --dashboard-port <other> or set IRIS_DASHBOARD_PORT to a different port. ` +
+            `Default dashboard port is 6920; a common pair is --port 6919 --dashboard-port 6920.`);
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@iris-eval/mcp-server",
-  "version": "0.3.0",
+  "version": "0.4.0-rc.1",
   "description": "The agent eval standard for MCP. Score every agent output for quality, safety, and cost.",
   "mcpName": "io.github.iris-eval/mcp-server",
   "type": "module",
@@ -21,6 +21,8 @@
     "test:watch": "vitest",
     "test:coverage": "vitest run --coverage",
     "test:integration": "vitest run tests/integration/",
+    "test:e2e": "playwright test",
+    "test:e2e:ui": "playwright test --ui",
     "version:check": "bash scripts/check-version.sh",
     "version:sync": "node scripts/sync-versions.mjs",
     "clean": "rm -rf dist coverage",
@@ -81,6 +83,7 @@
     "zod": "^3.25.0"
   },
   "devDependencies": {
+    "@playwright/test": "^1.59.1",
     "@types/better-sqlite3": "^7.6.0",
     "@types/express": "^5.0.0",
     "@types/node": "^25.5.2",

package/server.json

@@ -6,12 +6,12 @@
     "url": "https://github.com/iris-eval/mcp-server",
     "source": "github"
   },
-  "version": "0.3.0",
+  "version": "0.4.0-rc.1",
   "packages": [
     {
       "registryType": "npm",
       "identifier": "@iris-eval/mcp-server",
-      "version": "0.3.0",
+      "version": "0.4.0-rc.1",
       "transport": {
         "type": "stdio"
       },