agent-regression-lab 0.1.1 → 0.3.0

package/dist/tools.js

@@ -2,6 +2,38 @@ import { readFileSync } from "node:fs";
 import { pathToFileURL } from "node:url";
 import { resolve } from "node:path";
 import { loadAgentLabConfig } from "./config.js";
+export function applyRuntimeProfileToTools(tools, profile, trace) {
+    if (!profile?.tool_faults?.length) {
+        return tools;
+    }
+    const wrapped = { ...tools };
+    for (const fault of profile.tool_faults) {
+        const original = wrapped[fault.tool];
+        if (!original) {
+            continue;
+        }
+        wrapped[fault.tool] = async (input, context) => {
+            trace.record("system", "tool_fault_injected", {
+                tool: fault.tool,
+                mode: fault.mode,
+            }, { countStep: false });
+            if (fault.mode === "timeout") {
+                await waitUnref(fault.timeout_ms ?? 5000);
+                const timeoutError = new Error(`Injected timeout for ${fault.tool}`);
+                timeoutError.code = "timeout_exceeded";
+                throw timeoutError;
+            }
+            if (fault.mode === "error") {
+                throw new Error(fault.error_message ?? `Injected failure for ${fault.tool}`);
+            }
+            if (fault.mode === "malformed_output") {
+                return "MALFORMED_OUTPUT";
+            }
+            return fault.partial_output ?? {};
+        };
+    }
+    return wrapped;
+}
 function loadFixture(path) {
     const raw = readFileSync(resolve(path), "utf8");
     return JSON.parse(raw);
@@ -80,6 +112,252 @@ const BUILTIN_TOOLS = [
             };
         },
     },
+    {
+        spec: {
+            name: "accounts.get_profile",
+            description: "Fetch account profile details for a customer id.",
+            inputSchema: {
+                type: "object",
+                additionalProperties: false,
+                properties: {
+                    customer_id: { type: "string" },
+                },
+                required: ["customer_id"],
+            },
+        },
+        handler: async (input) => {
+            assertObject(input);
+            const customerId = String(input.customer_id ?? "");
+            const accounts = loadFixture("fixtures/support/accounts.json");
+            const account = accounts.find((candidate) => candidate.customer_id === customerId);
+            if (!account) {
+                throw new Error(`Account for customer '${customerId}' not found.`);
+            }
+            return account;
+        },
+    },
+    {
+        spec: {
+            name: "accounts.update_newsletter",
+            description: "Update newsletter subscription for a customer id.",
+            inputSchema: {
+                type: "object",
+                additionalProperties: false,
+                properties: {
+                    customer_id: { type: "string" },
+                    subscribed: { type: "boolean" },
+                },
+                required: ["customer_id", "subscribed"],
+            },
+        },
+        handler: async (input) => {
+            assertObject(input);
+            return {
+                customer_id: String(input.customer_id ?? ""),
+                newsletter_subscribed: Boolean(input.subscribed),
+                updated: true,
+            };
+        },
+    },
+    {
+        spec: {
+            name: "subscriptions.cancel",
+            description: "Cancel an active subscription by customer id.",
+            inputSchema: {
+                type: "object",
+                additionalProperties: false,
+                properties: {
+                    customer_id: { type: "string" },
+                },
+                required: ["customer_id"],
+            },
+        },
+        handler: async (input) => {
+            assertObject(input);
+            const customerId = String(input.customer_id ?? "");
+            const subscriptions = loadFixture("fixtures/support/subscriptions.json");
+            const subscription = subscriptions.find((candidate) => candidate.customer_id === customerId && candidate.status === "active");
+            if (!subscription) {
+                throw new Error(`Active subscription for customer '${customerId}' not found.`);
+            }
+            return {
+                subscription_id: subscription.subscription_id,
+                status: "cancelled",
+            };
+        },
+    },
+    {
+        spec: {
+            name: "repo.list_files",
+            description: "List files in a toy repository fixture.",
+            inputSchema: {
+                type: "object",
+                additionalProperties: false,
+                properties: {},
+            },
+        },
+        handler: async () => {
+            const files = loadFixture("fixtures/coding/repo-files.json");
+            return files.map((file) => ({ path: file.path }));
+        },
+    },
+    {
+        spec: {
+            name: "repo.read_file",
+            description: "Read a file from the toy repository fixture.",
+            inputSchema: {
+                type: "object",
+                additionalProperties: false,
+                properties: {
+                    path: { type: "string" },
+                },
+                required: ["path"],
+            },
+        },
+        handler: async (input) => {
+            assertObject(input);
+            const path = String(input.path ?? "");
+            const files = loadFixture("fixtures/coding/repo-files.json");
+            const file = files.find((candidate) => candidate.path === path);
+            if (!file) {
+                throw new Error(`Repo file '${path}' not found.`);
+            }
+            return file;
+        },
+    },
+    {
+        spec: {
+            name: "repo.apply_patch",
+            description: "Apply a deterministic patch to a toy repository file.",
+            inputSchema: {
+                type: "object",
+                additionalProperties: false,
+                properties: {
+                    path: { type: "string" },
+                    replacement: { type: "string" },
+                },
+                required: ["path", "replacement"],
+            },
+        },
+        handler: async (input) => {
+            assertObject(input);
+            const path = String(input.path ?? "");
+            const replacement = String(input.replacement ?? "");
+            return {
+                path,
+                replacement,
+                applied: true,
+            };
+        },
+    },
+    {
+        spec: {
+            name: "docs.search",
+            description: "Search fixture-backed documents.",
+            inputSchema: {
+                type: "object",
+                additionalProperties: false,
+                properties: {
+                    query: { type: "string" },
+                },
+                required: ["query"],
+            },
+        },
+        handler: async (input) => {
+            assertObject(input);
+            const query = String(input.query ?? "").toLowerCase();
+            const docs = loadFixture("fixtures/research/documents.json");
+            return docs
+                .filter((doc) => `${doc.title} ${doc.content}`.toLowerCase().includes(query))
+                .map((doc) => ({ id: doc.id, title: doc.title }));
+        },
+    },
+    {
+        spec: {
+            name: "docs.read",
+            description: "Read one fixture-backed document by id.",
+            inputSchema: {
+                type: "object",
+                additionalProperties: false,
+                properties: {
+                    doc_id: { type: "string" },
+                },
+                required: ["doc_id"],
+            },
+        },
+        handler: async (input) => {
+            assertObject(input);
+            const docId = String(input.doc_id ?? "");
+            const docs = loadFixture("fixtures/research/documents.json");
+            const doc = docs.find((candidate) => candidate.id === docId);
+            if (!doc) {
+                throw new Error(`Document '${docId}' not found.`);
+            }
+            return doc;
+        },
+    },
+    {
+        spec: {
+            name: "alerts.list_active",
+            description: "List active synthetic alerts.",
+            inputSchema: {
+                type: "object",
+                additionalProperties: false,
+                properties: {},
+            },
+        },
+        handler: async () => {
+            return loadFixture("fixtures/ops/alerts.json");
+        },
+    },
+    {
+        spec: {
+            name: "logs.query_service",
+            description: "Query synthetic logs for one service.",
+            inputSchema: {
+                type: "object",
+                additionalProperties: false,
+                properties: {
+                    service: { type: "string" },
+                },
+                required: ["service"],
+            },
+        },
+        handler: async (input) => {
+            assertObject(input);
+            const service = String(input.service ?? "");
+            const logs = loadFixture("fixtures/ops/logs.json");
+            const entry = logs.find((candidate) => candidate.service === service);
+            if (!entry) {
+                throw new Error(`Logs for service '${service}' not found.`);
+            }
+            return entry;
+        },
+    },
+    {
+        spec: {
+            name: "status.get_service",
+            description: "Read synthetic service ownership and status metadata.",
+            inputSchema: {
+                type: "object",
+                additionalProperties: false,
+                properties: {
+                    service: { type: "string" },
+                },
+                required: ["service"],
+            },
+        },
+        handler: async (input) => {
+            assertObject(input);
+            const service = String(input.service ?? "");
+            const statuses = loadFixture("fixtures/ops/status.json");
+            const status = statuses.find((candidate) => candidate.service === service);
+            if (!status) {
+                throw new Error(`Service status for '${service}' not found.`);
+            }
+            return status;
+        },
+    },
 ];
 export async function loadToolRegistry() {
     const tools = await loadTools();
@@ -126,3 +404,9 @@ function assertObject(value) {
         throw new Error("Tool input must be an object.");
     }
 }
+function waitUnref(timeoutMs) {
+    return new Promise((resolve) => {
+        const timer = setTimeout(resolve, timeoutMs);
+        timer.unref?.();
+    });
+}

package/dist/trace.js

@@ -8,8 +8,10 @@ export class TraceRecorder {
         this.runId = runId;
         this.scenarioId = scenarioId;
     }
-    record(source, type, payload) {
-        this.stepIndex += 1;
+    record(source, type, payload, options) {
+        if (options?.countStep !== false) {
+            this.stepIndex += 1;
+        }
         this.events.push({
             eventId: createEventId(),
             runId: this.runId,

package/dist/ui/App.js

@@ -1,8 +1,8 @@
-import { jsx as _jsx, jsxs as _jsxs } from "react/jsx-runtime";
+import { jsx as _jsx, jsxs as _jsxs, Fragment as _Fragment } from "react/jsx-runtime";
 import { useEffect, useState } from "react";
 export function App() {
     const route = getRoute();
-    return (_jsxs("div", { className: "shell", children: [_jsx("header", { className: "topbar", children: _jsx("a", { className: "brand", href: "/", children: "Agent Regression Lab Alpha" }) }), _jsxs("main", { className: "page", children: [route.type === "list" ? _jsx(RunListPage, {}) : null, route.type === "detail" ? _jsx(RunDetailPage, { runId: route.runId }) : null, route.type === "compare" ? _jsx(ComparePage, { baseline: route.baseline, candidate: route.candidate }) : null] })] }));
+    return (_jsxs("div", { className: "shell", children: [_jsx("header", { className: "topbar", children: _jsx("a", { className: "brand", href: "/", children: "Agent Regression Lab Alpha" }) }), _jsxs("main", { className: "page", children: [route.type === "list" ? _jsx(RunListPage, {}) : null, route.type === "detail" ? _jsx(RunDetailPage, { runId: route.runId }) : null, route.type === "compare" ? _jsx(ComparePage, { baseline: route.baseline, candidate: route.candidate }) : null, route.type === "compare-suite" ? _jsx(SuiteComparePage, { baselineBatch: route.baselineBatch, candidateBatch: route.candidateBatch }) : null] })] }));
 }
 function RunListPage() {
     const [runs, setRuns] = useState([]);
@@ -21,7 +21,11 @@ function RunListPage() {
             .then((response) => response.json())
             .then((data) => setRuns(Array.isArray(data.runs) ? data.runs : []));
     }, [suite, status, provider]);
-    return (_jsxs("section", { children: [_jsxs("div", { className: "hero", children: [_jsx("h1", { children: "Runs" }), _jsx("p", { children: "Inspect local alpha runs, filter failures, and compare behavior changes." })] }), _jsxs("div", { className: "filters", children: [_jsx("input", { value: suite, onChange: (event) => setSuite(event.target.value), placeholder: "Suite" }), _jsxs("select", { value: status, onChange: (event) => setStatus(event.target.value), children: [_jsx("option", { value: "", children: "All statuses" }), _jsx("option", { value: "pass", children: "Pass" }), _jsx("option", { value: "fail", children: "Fail" }), _jsx("option", { value: "error", children: "Error" })] }), _jsxs("select", { value: provider, onChange: (event) => setProvider(event.target.value), children: [_jsx("option", { value: "", children: "All providers" }), _jsx("option", { value: "mock", children: "Mock" }), _jsx("option", { value: "openai", children: "OpenAI" }), _jsx("option", { value: "external_process", children: "External process" })] })] }), runs.length === 0 ? _jsx(EmptyState, { title: "No runs yet", description: "Run a scenario from the CLI to populate the lab." }) : null, runs.length > 0 ? (_jsxs("table", { className: "table", children: [_jsx("thead", { children: _jsxs("tr", { children: [_jsx("th", { children: "Run" }), _jsx("th", { children: "Scenario" }), _jsx("th", { children: "Provider" }), _jsx("th", { children: "Status" }), _jsx("th", { children: "Score" }), _jsx("th", { children: "Runtime" }), _jsx("th", { children: "Steps" }), _jsx("th", { children: "Started" })] }) }), _jsx("tbody", { children: runs.map((run, index) => (_jsxs("tr", { children: [_jsx("td", { children: _jsx("a", { href: `/runs/${run.id}`, children: run.id }) }), _jsx("td", { children: run.scenarioId }), _jsxs("td", { children: [run.provider ?? "-", _jsx("div", { className: "muted", children: run.modelId ?? run.agentLabel ?? "" })] }), _jsx("td", { children: _jsx("span", { className: `pill ${run.status}`, children: run.status }) }), _jsx("td", { children: run.score }), _jsxs("td", { children: [run.durationMs, "ms"] }), _jsx("td", { children: run.totalSteps }), _jsxs("td", { children: [new Date(run.startedAt).toLocaleString(), index > 0 && runs[index - 1].scenarioId === run.scenarioId ? (_jsx("div", { className: "muted", children: _jsx("a", { href: `/compare?baseline=${runs[index - 1].id}&candidate=${run.id}`, children: "compare previous" }) })) : null] })] }, run.id))) })] })) : null] }));
+    return (_jsxs("section", { children: [_jsxs("div", { className: "hero", children: [_jsx("h1", { children: "Runs" }), _jsx("p", { children: "Inspect local alpha runs, filter failures, and compare behavior changes." })] }), _jsxs("div", { className: "filters", children: [_jsx("input", { value: suite, onChange: (event) => setSuite(event.target.value), placeholder: "Suite" }), _jsxs("select", { value: status, onChange: (event) => setStatus(event.target.value), children: [_jsx("option", { value: "", children: "All statuses" }), _jsx("option", { value: "pass", children: "Pass" }), _jsx("option", { value: "fail", children: "Fail" }), _jsx("option", { value: "error", children: "Error" })] }), _jsxs("select", { value: provider, onChange: (event) => setProvider(event.target.value), children: [_jsx("option", { value: "", children: "All providers" }), _jsx("option", { value: "mock", children: "Mock" }), _jsx("option", { value: "openai", children: "OpenAI" }), _jsx("option", { value: "external_process", children: "External process" })] })] }), runs.length === 0 ? _jsx(EmptyState, { title: "No runs yet", description: "Run a scenario from the CLI to populate the lab." }) : null, runs.length > 0 ? (_jsxs("table", { className: "table", children: [_jsx("thead", { children: _jsxs("tr", { children: [_jsx("th", { children: "Run" }), _jsx("th", { children: "Scenario" }), _jsx("th", { children: "Provider" }), _jsx("th", { children: "Status" }), _jsx("th", { children: "Score" }), _jsx("th", { children: "Runtime" }), _jsx("th", { children: "Steps" }), _jsx("th", { children: "Started" })] }) }), _jsx("tbody", { children: runs.map((run, index) => (_jsxs("tr", { children: [_jsx("td", { children: _jsx("a", { href: `/runs/${run.id}`, children: run.id }) }), _jsx("td", { children: run.scenarioId }), _jsxs("td", { children: [run.provider ?? "-", _jsx("div", { className: "muted", children: run.modelId ?? run.agentLabel ?? "" })] }), _jsx("td", { children: _jsx("span", { className: `pill ${run.status}`, children: run.status }) }), _jsx("td", { children: run.score }), _jsxs("td", { children: [run.durationMs, "ms"] }), _jsx("td", { children: run.totalSteps }), _jsxs("td", { children: [new Date(run.startedAt).toLocaleString(), index > 0 && runs[index - 1].scenarioId === run.scenarioId ? (_jsx("div", { className: "muted", children: _jsx("a", { href: `/compare?baseline=${runs[index - 1].id}&candidate=${run.id}`, children: "compare previous" }) })) : null, index > 0 &&
+                                            runs[index - 1].suite === run.suite &&
+                                            runs[index - 1].suiteBatchId &&
+                                            run.suiteBatchId &&
+                                            runs[index - 1].suiteBatchId !== run.suiteBatchId ? (_jsx("div", { className: "muted", children: _jsx("a", { href: `/compare-suite?baselineBatch=${runs[index - 1].suiteBatchId}&candidateBatch=${run.suiteBatchId}`, children: "compare suite batch" }) })) : null] })] }, run.id))) })] })) : null] }));
 }
 function RunDetailPage(props) {
     const [detail, setDetail] = useState(null);
@@ -33,7 +37,18 @@ function RunDetailPage(props) {
     if (!detail) {
         return _jsx(EmptyState, { title: "Loading run", description: "Fetching run detail from the local lab." });
     }
-    return (_jsxs("section", { children: [_jsxs("div", { className: "hero", children: [_jsx("h1", { children: detail.run.id }), _jsx("p", { children: detail.run.scenarioId })] }), _jsxs("div", { className: "stats", children: [_jsx(Stat, { label: "Status", value: _jsx("span", { className: `pill ${detail.run.status}`, children: detail.run.status }) }), _jsx(Stat, { label: "Score", value: detail.run.score }), _jsx(Stat, { label: "Runtime", value: `${detail.run.durationMs}ms` }), _jsx(Stat, { label: "Steps", value: detail.run.totalSteps })] }), _jsxs("div", { className: "panel-grid", children: [_jsxs("section", { className: "panel", children: [_jsx("h2", { children: "Summary" }), _jsxs("p", { children: [_jsx("strong", { children: "Provider:" }), " ", detail.agentVersion?.provider ?? "-"] }), _jsxs("p", { children: [_jsx("strong", { children: "Model:" }), " ", detail.agentVersion?.modelId ?? "-"] }), detail.agentVersion?.command ? (_jsxs("p", { children: [_jsx("strong", { children: "Command:" }), " ", detail.agentVersion.command, " ", (detail.agentVersion.args ?? []).join(" ")] })) : null, _jsxs("p", { children: [_jsx("strong", { children: "Termination:" }), " ", detail.run.terminationReason] }), detail.errorDetail ? _jsxs("p", { children: [_jsx("strong", { children: "Error:" }), " ", detail.errorDetail] }) : null, _jsx("p", { children: _jsx("strong", { children: "Final output:" }) }), _jsx("pre", { children: detail.run.finalOutput || "(none)" })] }), _jsxs("section", { className: "panel", children: [_jsx("h2", { children: "Evaluators" }), _jsx("ul", { className: "stack", children: detail.evaluatorResults.map((result) => (_jsxs("li", { children: [_jsx("span", { className: `pill ${result.status}`, children: result.status }), " ", result.evaluatorId, _jsx("div", { className: "muted", children: result.message })] }, result.evaluatorId))) })] })] }), _jsxs("section", { className: "panel", children: [_jsx("h2", { children: "Tool Calls" }), detail.toolCalls.length === 0 ? _jsx("p", { className: "muted", children: "No tool calls recorded." }) : null, _jsx("ul", { className: "stack", children: detail.toolCalls.map((call) => (_jsxs("li", { children: [_jsx("strong", { children: call.toolName }), " ", _jsx("span", { className: `pill ${call.status}`, children: call.status }), _jsx("pre", { children: JSON.stringify({ input: call.input, output: call.output }, null, 2) })] }, call.id))) })] }), _jsxs("section", { className: "panel", children: [_jsx("h2", { children: "Trace" }), _jsx("ol", { className: "timeline", children: detail.traceEvents.map((event) => (_jsxs("li", { children: [_jsxs("div", { children: [_jsxs("strong", { children: [event.stepIndex, ". ", event.type] }), " ", _jsx("span", { className: "muted", children: event.source })] }), _jsx("pre", { children: JSON.stringify(event.payload, null, 2) })] }, event.eventId))) })] })] }));
+    return (_jsxs("section", { children: [_jsxs("div", { className: "hero", children: [_jsx("h1", { children: detail.run.id }), _jsx("p", { children: detail.run.scenarioId })] }), _jsx(FailureSummaryPanel, { detail: detail }), _jsxs("div", { className: "stats", children: [_jsx(Stat, { label: "Status", value: _jsx("span", { className: `pill ${detail.run.status}`, children: detail.run.status }) }), _jsx(Stat, { label: "Score", value: detail.run.score }), _jsx(Stat, { label: "Runtime", value: `${detail.run.durationMs}ms` }), _jsx(Stat, { label: "Steps", value: detail.run.totalSteps })] }), _jsxs("div", { className: "panel-grid", children: [_jsxs("section", { className: "panel", children: [_jsx("h2", { children: "Summary" }), _jsxs("p", { children: [_jsx("strong", { children: "Provider:" }), " ", detail.agentVersion?.provider ?? "-"] }), _jsxs("p", { children: [_jsx("strong", { children: "Model:" }), " ", detail.agentVersion?.modelId ?? "-"] }), _jsx(RunIdentitySummary, { detail: detail }), detail.agentVersion?.command ? (_jsxs("p", { children: [_jsx("strong", { children: "Command:" }), " ", detail.agentVersion.command, " ", (detail.agentVersion.args ?? []).join(" ")] })) : null, _jsxs("p", { children: [_jsx("strong", { children: "Termination:" }), " ", detail.run.terminationReason] }), detail.errorDetail ? _jsxs("p", { children: [_jsx("strong", { children: "Error:" }), " ", detail.errorDetail] }) : null, _jsx("p", { children: _jsx("strong", { children: "Final output:" }) }), _jsx("pre", { children: detail.run.finalOutput || "(none)" })] }), _jsxs("section", { className: "panel", children: [_jsx("h2", { children: "Evaluators" }), _jsx("ul", { className: "stack", children: detail.evaluatorResults.map((result) => (_jsxs("li", { children: [_jsx("span", { className: `pill ${result.status}`, children: result.status }), " ", result.evaluatorId, _jsx("div", { className: "muted", children: result.message })] }, result.evaluatorId))) })] })] }), _jsxs("section", { className: "panel", children: [_jsx("h2", { children: "Tool Calls" }), detail.toolCalls.length === 0 ? _jsx("p", { className: "muted", children: "No tool calls recorded." }) : null, _jsx("ul", { className: "stack", children: detail.toolCalls.map((call) => (_jsxs("li", { children: [_jsx("strong", { children: call.toolName }), " ", _jsx("span", { className: `pill ${call.status}`, children: call.status }), _jsx("pre", { children: JSON.stringify({ input: call.input, output: call.output }, null, 2) })] }, call.id))) })] }), _jsxs("section", { className: "panel", children: [_jsx("h2", { children: "Trace" }), _jsx("ol", { className: "timeline", children: detail.traceEvents.map((event) => (_jsxs("li", { children: [_jsxs("div", { children: [_jsxs("strong", { children: [event.stepIndex, ". ", event.type] }), " ", _jsx("span", { className: "muted", children: event.source })] }), _jsx("pre", { children: JSON.stringify(event.payload, null, 2) })] }, event.eventId))) })] })] }));
+}
+export function FailureSummaryPanel(props) {
+    const failureItems = getFailureSummaryItems(props.detail);
+    if (failureItems.length === 0) {
+        return null;
+    }
+    return (_jsxs("section", { className: "panel", children: [_jsx("h2", { children: "Failures First" }), _jsxs("p", { children: [_jsx("strong", { children: "Status:" }), " ", _jsx("span", { className: `pill ${props.detail.run.status}`, children: props.detail.run.status })] }), _jsxs("p", { children: [_jsx("strong", { children: "Termination:" }), " ", props.detail.run.terminationReason] }), _jsx("ul", { className: "stack", children: failureItems.map((item) => (_jsx("li", { children: item }, item))) })] }));
+}
+export function RunIdentitySummary(props) {
+    const run = props.detail.run;
+    return (_jsxs(_Fragment, { children: [_jsxs("p", { children: [_jsx("strong", { children: "Variant set:" }), " ", run.variantSetName ?? "-"] }), _jsxs("p", { children: [_jsx("strong", { children: "Variant:" }), " ", run.variantLabel ?? "-"] }), _jsxs("p", { children: [_jsx("strong", { children: "Prompt version:" }), " ", run.promptVersion ?? "-"] }), _jsxs("p", { children: [_jsx("strong", { children: "Model version:" }), " ", run.modelVersion ?? "-"] }), _jsxs("p", { children: [_jsx("strong", { children: "Tool schema version:" }), " ", run.toolSchemaVersion ?? "-"] }), _jsxs("p", { children: [_jsx("strong", { children: "Config label:" }), " ", run.configLabel ?? "-"] }), _jsxs("p", { children: [_jsx("strong", { children: "Runtime profile:" }), " ", run.runtimeProfileName ?? "-"] }), _jsxs("p", { children: [_jsx("strong", { children: "Suite definition:" }), " ", run.suiteDefinitionName ?? "-"] })] }));
 }
 function ComparePage(props) {
     const [data, setData] = useState(null);
@@ -55,17 +70,57 @@ function ComparePage(props) {
     if (!data) {
         return _jsx(EmptyState, { title: "Loading comparison", description: "Fetching both runs and computing deltas." });
     }
-    return (_jsxs("section", { children: [_jsxs("div", { className: "hero", children: [_jsx("h1", { children: "Compare" }), _jsx("p", { children: data.baseline.run.scenarioId })] }), _jsxs("div", { className: "stats", children: [_jsx(Stat, { label: "Score delta", value: signed(data.deltas.score) }), _jsx(Stat, { label: "Runtime delta", value: `${signed(data.deltas.runtimeMs)}ms` }), _jsx(Stat, { label: "Step delta", value: signed(data.deltas.steps) })] }), _jsxs("section", { className: "panel", children: [_jsx("h2", { children: "Notes" }), data.notes.length === 0 ? _jsx("p", { className: "muted", children: "No material differences recorded." }) : null, _jsx("ul", { className: "stack", children: data.notes.map((note) => (_jsx("li", { children: note }, note))) })] }), _jsxs("div", { className: "panel-grid", children: [_jsxs("section", { className: "panel", children: [_jsx("h2", { children: "Evaluator diffs" }), data.evaluatorDiffs.length === 0 ? _jsx("p", { className: "muted", children: "No evaluator changes." }) : null, _jsx("ul", { className: "stack", children: data.evaluatorDiffs.map((diff) => (_jsx("li", { children: diff.note }, diff.evaluatorId))) })] }), _jsxs("section", { className: "panel", children: [_jsx("h2", { children: "Tool diffs" }), data.toolDiffs.length === 0 ? _jsx("p", { className: "muted", children: "No tool usage changes." }) : null, _jsx("ul", { className: "stack", children: data.toolDiffs.map((diff) => (_jsx("li", { children: diff.note }, diff.toolName))) })] })] }), _jsxs("div", { className: "compare-grid", children: [_jsx(RunSide, { title: "Baseline", detail: data.baseline }), _jsx(RunSide, { title: "Candidate", detail: data.candidate })] })] }));
+    return (_jsxs("section", { children: [_jsxs("div", { className: "hero", children: [_jsx("h1", { children: "Compare" }), _jsx("p", { children: data.baseline.run.scenarioId })] }), _jsxs("div", { className: "stats", children: [_jsx(Stat, { label: "Classification", value: data.classification }), _jsx(Stat, { label: "Score delta", value: signed(data.deltas.score) }), _jsx(Stat, { label: "Runtime delta", value: `${signed(data.deltas.runtimeMs)}ms` }), _jsx(Stat, { label: "Step delta", value: signed(data.deltas.steps) })] }), _jsxs("section", { className: "panel", children: [_jsx("h2", { children: "Notes" }), data.notes.length === 0 ? _jsx("p", { className: "muted", children: "No material differences recorded." }) : null, _jsx("ul", { className: "stack", children: data.notes.map((note) => (_jsx("li", { children: note }, note))) })] }), _jsxs("div", { className: "panel-grid", children: [_jsxs("section", { className: "panel", children: [_jsx("h2", { children: "Evaluator diffs" }), data.evaluatorDiffs.length === 0 ? _jsx("p", { className: "muted", children: "No evaluator changes." }) : null, _jsx("ul", { className: "stack", children: data.evaluatorDiffs.map((diff) => (_jsxs("li", { children: [diff.note, diff.hardGate ? " (hard gate)" : ""] }, diff.evaluatorId))) })] }), _jsxs("section", { className: "panel", children: [_jsx("h2", { children: "Tool diffs" }), data.toolDiffs.length === 0 ? _jsx("p", { className: "muted", children: "No tool usage changes." }) : null, _jsx("ul", { className: "stack", children: data.toolDiffs.map((diff) => (_jsx("li", { children: diff.note }, diff.toolName))) })] })] }), _jsxs("div", { className: "compare-grid", children: [_jsx(RunSide, { title: "Baseline", detail: data.baseline }), _jsx(RunSide, { title: "Candidate", detail: data.candidate })] })] }));
 }
 function RunSide(props) {
     return (_jsxs("section", { className: "panel", children: [_jsx("h2", { children: props.title }), _jsxs("p", { children: [_jsx("strong", { children: "Run:" }), " ", _jsx("a", { href: `/runs/${props.detail.run.id}`, children: props.detail.run.id })] }), _jsxs("p", { children: [_jsx("strong", { children: "Status:" }), " ", _jsx("span", { className: `pill ${props.detail.run.status}`, children: props.detail.run.status })] }), _jsxs("p", { children: [_jsx("strong", { children: "Score:" }), " ", props.detail.run.score] }), _jsxs("p", { children: [_jsx("strong", { children: "Runtime:" }), " ", props.detail.run.durationMs, "ms"] }), _jsxs("p", { children: [_jsx("strong", { children: "Termination:" }), " ", props.detail.run.terminationReason] }), _jsxs("p", { children: [_jsx("strong", { children: "Agent:" }), " ", props.detail.agentVersion?.label ?? "-"] }), _jsxs("p", { children: [_jsx("strong", { children: "Provider:" }), " ", props.detail.agentVersion?.provider ?? "-"] }), props.detail.agentVersion?.modelId ? _jsxs("p", { children: [_jsx("strong", { children: "Model:" }), " ", props.detail.agentVersion.modelId] }) : null, props.detail.agentVersion?.command ? (_jsxs("p", { children: [_jsx("strong", { children: "Command:" }), " ", props.detail.agentVersion.command, " ", (props.detail.agentVersion.args ?? []).join(" ")] })) : null, props.detail.errorDetail ? _jsxs("p", { children: [_jsx("strong", { children: "Error:" }), " ", props.detail.errorDetail] }) : null, _jsx("p", { children: _jsx("strong", { children: "Final output:" }) }), _jsx("pre", { children: props.detail.run.finalOutput || "(none)" }), _jsx("h3", { children: "Trace" }), _jsx("ol", { className: "timeline compact", children: props.detail.traceEvents.map((event) => (_jsx("li", { children: _jsxs("strong", { children: [event.stepIndex, ". ", event.type] }) }, event.eventId))) })] }));
 }
+function SuiteComparePage(props) {
+    const [data, setData] = useState(null);
+    useEffect(() => {
+        if (!props.baselineBatch || !props.candidateBatch) {
+            setData(null);
+            return;
+        }
+        const url = new URL("/api/compare-suite", window.location.origin);
+        url.searchParams.set("baselineBatch", props.baselineBatch);
+        url.searchParams.set("candidateBatch", props.candidateBatch);
+        void fetch(url)
+            .then((response) => response.json())
+            .then((payload) => setData(payload));
+    }, [props.baselineBatch, props.candidateBatch]);
+    if (!props.baselineBatch || !props.candidateBatch) {
+        return _jsx(EmptyState, { title: "No suite comparison selected", description: "Open the suite compare page with baseline and candidate batch ids." });
+    }
+    if (!data) {
+        return _jsx(EmptyState, { title: "Loading suite comparison", description: "Fetching suite batches and computing regressions." });
+    }
+    return (_jsxs("section", { children: [_jsxs("div", { className: "hero", children: [_jsx("h1", { children: "Suite Compare" }), _jsx("p", { children: data.suite })] }), _jsxs("div", { className: "stats", children: [_jsx(Stat, { label: "Classification", value: data.classification }), _jsx(Stat, { label: "Pass delta", value: signed(data.deltas.pass) }), _jsx(Stat, { label: "Fail delta", value: signed(data.deltas.fail) }), _jsx(Stat, { label: "Score delta", value: signed(data.deltas.averageScore) }), _jsx(Stat, { label: "Runtime delta", value: `${signed(data.deltas.averageRuntimeMs)}ms` }), _jsx(Stat, { label: "Step delta", value: signed(data.deltas.averageSteps) })] }), _jsxs("section", { className: "panel", children: [_jsx("h2", { children: "Notes" }), data.notes.length === 0 ? _jsx("p", { className: "muted", children: "No suite-level notes recorded." }) : null, _jsx("ul", { className: "stack", children: data.notes.map((note) => (_jsx("li", { children: note }, note))) })] }), _jsxs("div", { className: "panel-grid", children: [_jsx(ScenarioList, { title: "Regressions", items: data.regressions }), _jsx(ScenarioList, { title: "Improvements", items: data.improvements })] }), _jsxs("section", { className: "panel", children: [_jsx("h2", { children: "Missing scenarios" }), _jsxs("p", { children: [_jsx("strong", { children: "Missing from candidate:" }), " ", data.missingFromCandidate.join(", ") || "None"] }), _jsxs("p", { children: [_jsx("strong", { children: "Missing from baseline:" }), " ", data.missingFromBaseline.join(", ") || "None"] })] })] }));
+}
+function ScenarioList(props) {
+    return (_jsxs("section", { className: "panel", children: [_jsx("h2", { children: props.title }), props.items.length === 0 ? _jsx("p", { className: "muted", children: "None." }) : null, _jsx("ul", { className: "stack", children: props.items.map((item) => (_jsxs("li", { children: [_jsx("strong", { children: item.scenarioId }), " ", _jsx("span", { className: "muted", children: item.comparison.classification }), _jsx("div", { children: _jsx("a", { href: `/compare?baseline=${item.comparison.baseline.run.id}&candidate=${item.comparison.candidate.run.id}`, children: "open run compare" }) })] }, item.scenarioId))) })] }));
+}
 function Stat(props) {
     return (_jsxs("div", { className: "stat", children: [_jsx("div", { className: "muted", children: props.label }), _jsx("div", { className: "stat-value", children: props.value })] }));
 }
 function EmptyState(props) {
     return (_jsxs("section", { className: "empty", children: [_jsx("h1", { children: props.title }), _jsx("p", { children: props.description })] }));
 }
+export function getFailureSummaryItems(detail) {
+    const items = [];
+    if (detail.errorDetail) {
+        items.push(`Error: ${detail.errorDetail}`);
+    }
+    for (const result of detail.evaluatorResults) {
+        if (result.status === "fail") {
+            items.push(`Evaluator ${result.evaluatorId}: ${result.message}`);
+        }
+    }
+    if (detail.run.status !== "pass" && items.length === 0) {
+        items.push("Run did not pass. Inspect evaluator results and trace for the first divergence.");
+    }
+    return items;
+}
 function signed(value) {
     return value > 0 ? `+${value}` : `${value}`;
 }
@@ -74,6 +129,13 @@ function getRoute() {
     if (url.pathname.startsWith("/runs/")) {
         return { type: "detail", runId: decodeURIComponent(url.pathname.slice("/runs/".length)) };
     }
+    if (url.pathname === "/compare-suite") {
+        return {
+            type: "compare-suite",
+            baselineBatch: url.searchParams.get("baselineBatch") ?? undefined,
+            candidateBatch: url.searchParams.get("candidateBatch") ?? undefined,
+        };
+    }
     if (url.pathname === "/compare") {
         return {
             type: "compare",

package/dist/ui/server.js

@@ -80,6 +80,10 @@ function handleApi(url, response) {
                     ...comparison.candidate,
                     errorDetail: getRunErrorDetail(comparison.candidate),
                 },
+                classification: comparison.classification,
+                verdictDelta: comparison.verdictDelta,
+                terminationDelta: comparison.terminationDelta,
+                outputChanged: comparison.outputChanged,
                 notes: comparison.notes,
                 deltas: comparison.deltas,
                 evaluatorDiffs: comparison.evaluatorDiffs,
@@ -87,11 +91,25 @@ function handleApi(url, response) {
             });
             return;
         }
+        if (url.pathname === "/api/compare-suite") {
+            const baselineBatch = url.searchParams.get("baselineBatch");
+            const candidateBatch = url.searchParams.get("candidateBatch");
+            if (!baselineBatch || !candidateBatch) {
+                sendJson(response, 400, { error: "Both 'baselineBatch' and 'candidateBatch' query params are required." });
+                return;
+            }
+            const comparison = storage.compareSuites(baselineBatch, candidateBatch);
+            sendJson(response, 200, comparison);
+            return;
+        }
         sendJson(response, 404, { error: "Not found." });
     }
     catch (error) {
         sendJson(response, 500, { error: error instanceof Error ? error.message : String(error) });
     }
+    finally {
+        storage.close();
+    }
 }
 async function buildUiAssets() {
     if (existsSync(PACKAGED_ASSETS_ROOT)) {