agent-regression-lab 0.1.1 → 0.2.0

package/dist/ui/server.js

@@ -80,6 +80,10 @@ function handleApi(url, response) {
                     ...comparison.candidate,
                     errorDetail: getRunErrorDetail(comparison.candidate),
                 },
+                classification: comparison.classification,
+                verdictDelta: comparison.verdictDelta,
+                terminationDelta: comparison.terminationDelta,
+                outputChanged: comparison.outputChanged,
                 notes: comparison.notes,
                 deltas: comparison.deltas,
                 evaluatorDiffs: comparison.evaluatorDiffs,
@@ -87,11 +91,25 @@ function handleApi(url, response) {
             });
             return;
         }
+        if (url.pathname === "/api/compare-suite") {
+            const baselineBatch = url.searchParams.get("baselineBatch");
+            const candidateBatch = url.searchParams.get("candidateBatch");
+            if (!baselineBatch || !candidateBatch) {
+                sendJson(response, 400, { error: "Both 'baselineBatch' and 'candidateBatch' query params are required." });
+                return;
+            }
+            const comparison = storage.compareSuites(baselineBatch, candidateBatch);
+            sendJson(response, 200, comparison);
+            return;
+        }
         sendJson(response, 404, { error: "Not found." });
     }
     catch (error) {
         sendJson(response, 500, { error: error instanceof Error ? error.message : String(error) });
     }
+    finally {
+        storage.close();
+    }
 }
 async function buildUiAssets() {
     if (existsSync(PACKAGED_ASSETS_ROOT)) {

package/dist/ui-assets/client.js

@@ -21731,7 +21731,8 @@ function App() {
     /* @__PURE__ */ (0, import_jsx_runtime.jsxs)("main", { className: "page", children: [
       route.type === "list" ? /* @__PURE__ */ (0, import_jsx_runtime.jsx)(RunListPage, {}) : null,
       route.type === "detail" ? /* @__PURE__ */ (0, import_jsx_runtime.jsx)(RunDetailPage, { runId: route.runId }) : null,
-      route.type === "compare" ? /* @__PURE__ */ (0, import_jsx_runtime.jsx)(ComparePage, { baseline: route.baseline, candidate: route.candidate }) : null
+      route.type === "compare" ? /* @__PURE__ */ (0, import_jsx_runtime.jsx)(ComparePage, { baseline: route.baseline, candidate: route.candidate }) : null,
+      route.type === "compare-suite" ? /* @__PURE__ */ (0, import_jsx_runtime.jsx)(SuiteComparePage, { baselineBatch: route.baselineBatch, candidateBatch: route.candidateBatch }) : null
     ] })
   ] });
 }
@@ -21795,7 +21796,8 @@ function RunListPage() {
         /* @__PURE__ */ (0, import_jsx_runtime.jsx)("td", { children: run.totalSteps }),
         /* @__PURE__ */ (0, import_jsx_runtime.jsxs)("td", { children: [
           new Date(run.startedAt).toLocaleString(),
-          index > 0 && runs[index - 1].scenarioId === run.scenarioId ? /* @__PURE__ */ (0, import_jsx_runtime.jsx)("div", { className: "muted", children: /* @__PURE__ */ (0, import_jsx_runtime.jsx)("a", { href: `/compare?baseline=${runs[index - 1].id}&candidate=${run.id}`, children: "compare previous" }) }) : null
+          index > 0 && runs[index - 1].scenarioId === run.scenarioId ? /* @__PURE__ */ (0, import_jsx_runtime.jsx)("div", { className: "muted", children: /* @__PURE__ */ (0, import_jsx_runtime.jsx)("a", { href: `/compare?baseline=${runs[index - 1].id}&candidate=${run.id}`, children: "compare previous" }) }) : null,
+          index > 0 && runs[index - 1].suite === run.suite && runs[index - 1].suiteBatchId && run.suiteBatchId && runs[index - 1].suiteBatchId !== run.suiteBatchId ? /* @__PURE__ */ (0, import_jsx_runtime.jsx)("div", { className: "muted", children: /* @__PURE__ */ (0, import_jsx_runtime.jsx)("a", { href: `/compare-suite?baselineBatch=${runs[index - 1].suiteBatchId}&candidateBatch=${run.suiteBatchId}`, children: "compare suite batch" }) }) : null
         ] })
       ] }, run.id)) })
     ] }) : null
@@ -21914,6 +21916,7 @@ function ComparePage(props) {
       /* @__PURE__ */ (0, import_jsx_runtime.jsx)("p", { children: data.baseline.run.scenarioId })
     ] }),
     /* @__PURE__ */ (0, import_jsx_runtime.jsxs)("div", { className: "stats", children: [
+      /* @__PURE__ */ (0, import_jsx_runtime.jsx)(Stat, { label: "Classification", value: data.classification }),
       /* @__PURE__ */ (0, import_jsx_runtime.jsx)(Stat, { label: "Score delta", value: signed(data.deltas.score) }),
       /* @__PURE__ */ (0, import_jsx_runtime.jsx)(Stat, { label: "Runtime delta", value: `${signed(data.deltas.runtimeMs)}ms` }),
       /* @__PURE__ */ (0, import_jsx_runtime.jsx)(Stat, { label: "Step delta", value: signed(data.deltas.steps) })
@@ -21927,7 +21930,10 @@ function ComparePage(props) {
       /* @__PURE__ */ (0, import_jsx_runtime.jsxs)("section", { className: "panel", children: [
         /* @__PURE__ */ (0, import_jsx_runtime.jsx)("h2", { children: "Evaluator diffs" }),
         data.evaluatorDiffs.length === 0 ? /* @__PURE__ */ (0, import_jsx_runtime.jsx)("p", { className: "muted", children: "No evaluator changes." }) : null,
-        /* @__PURE__ */ (0, import_jsx_runtime.jsx)("ul", { className: "stack", children: data.evaluatorDiffs.map((diff) => /* @__PURE__ */ (0, import_jsx_runtime.jsx)("li", { children: diff.note }, diff.evaluatorId)) })
+        /* @__PURE__ */ (0, import_jsx_runtime.jsx)("ul", { className: "stack", children: data.evaluatorDiffs.map((diff) => /* @__PURE__ */ (0, import_jsx_runtime.jsxs)("li", { children: [
+          diff.note,
+          diff.hardGate ? " (hard gate)" : ""
+        ] }, diff.evaluatorId)) })
       ] }),
       /* @__PURE__ */ (0, import_jsx_runtime.jsxs)("section", { className: "panel", children: [
         /* @__PURE__ */ (0, import_jsx_runtime.jsx)("h2", { children: "Tool diffs" }),
@@ -22007,6 +22013,73 @@ function RunSide(props) {
     ] }) }, event.eventId)) })
   ] });
 }
+function SuiteComparePage(props) {
+  const [data, setData] = (0, import_react.useState)(null);
+  (0, import_react.useEffect)(() => {
+    if (!props.baselineBatch || !props.candidateBatch) {
+      setData(null);
+      return;
+    }
+    const url = new URL("/api/compare-suite", window.location.origin);
+    url.searchParams.set("baselineBatch", props.baselineBatch);
+    url.searchParams.set("candidateBatch", props.candidateBatch);
+    void fetch(url).then((response) => response.json()).then((payload) => setData(payload));
+  }, [props.baselineBatch, props.candidateBatch]);
+  if (!props.baselineBatch || !props.candidateBatch) {
+    return /* @__PURE__ */ (0, import_jsx_runtime.jsx)(EmptyState, { title: "No suite comparison selected", description: "Open the suite compare page with baseline and candidate batch ids." });
+  }
+  if (!data) {
+    return /* @__PURE__ */ (0, import_jsx_runtime.jsx)(EmptyState, { title: "Loading suite comparison", description: "Fetching suite batches and computing regressions." });
+  }
+  return /* @__PURE__ */ (0, import_jsx_runtime.jsxs)("section", { children: [
+    /* @__PURE__ */ (0, import_jsx_runtime.jsxs)("div", { className: "hero", children: [
+      /* @__PURE__ */ (0, import_jsx_runtime.jsx)("h1", { children: "Suite Compare" }),
+      /* @__PURE__ */ (0, import_jsx_runtime.jsx)("p", { children: data.suite })
+    ] }),
+    /* @__PURE__ */ (0, import_jsx_runtime.jsxs)("div", { className: "stats", children: [
+      /* @__PURE__ */ (0, import_jsx_runtime.jsx)(Stat, { label: "Classification", value: data.classification }),
+      /* @__PURE__ */ (0, import_jsx_runtime.jsx)(Stat, { label: "Pass delta", value: signed(data.deltas.pass) }),
+      /* @__PURE__ */ (0, import_jsx_runtime.jsx)(Stat, { label: "Fail delta", value: signed(data.deltas.fail) }),
+      /* @__PURE__ */ (0, import_jsx_runtime.jsx)(Stat, { label: "Score delta", value: signed(data.deltas.averageScore) }),
+      /* @__PURE__ */ (0, import_jsx_runtime.jsx)(Stat, { label: "Runtime delta", value: `${signed(data.deltas.averageRuntimeMs)}ms` }),
+      /* @__PURE__ */ (0, import_jsx_runtime.jsx)(Stat, { label: "Step delta", value: signed(data.deltas.averageSteps) })
+    ] }),
+    /* @__PURE__ */ (0, import_jsx_runtime.jsxs)("section", { className: "panel", children: [
+      /* @__PURE__ */ (0, import_jsx_runtime.jsx)("h2", { children: "Notes" }),
+      data.notes.length === 0 ? /* @__PURE__ */ (0, import_jsx_runtime.jsx)("p", { className: "muted", children: "No suite-level notes recorded." }) : null,
+      /* @__PURE__ */ (0, import_jsx_runtime.jsx)("ul", { className: "stack", children: data.notes.map((note) => /* @__PURE__ */ (0, import_jsx_runtime.jsx)("li", { children: note }, note)) })
+    ] }),
+    /* @__PURE__ */ (0, import_jsx_runtime.jsxs)("div", { className: "panel-grid", children: [
+      /* @__PURE__ */ (0, import_jsx_runtime.jsx)(ScenarioList, { title: "Regressions", items: data.regressions }),
+      /* @__PURE__ */ (0, import_jsx_runtime.jsx)(ScenarioList, { title: "Improvements", items: data.improvements })
+    ] }),
+    /* @__PURE__ */ (0, import_jsx_runtime.jsxs)("section", { className: "panel", children: [
+      /* @__PURE__ */ (0, import_jsx_runtime.jsx)("h2", { children: "Missing scenarios" }),
+      /* @__PURE__ */ (0, import_jsx_runtime.jsxs)("p", { children: [
+        /* @__PURE__ */ (0, import_jsx_runtime.jsx)("strong", { children: "Missing from candidate:" }),
+        " ",
+        data.missingFromCandidate.join(", ") || "None"
+      ] }),
+      /* @__PURE__ */ (0, import_jsx_runtime.jsxs)("p", { children: [
+        /* @__PURE__ */ (0, import_jsx_runtime.jsx)("strong", { children: "Missing from baseline:" }),
+        " ",
+        data.missingFromBaseline.join(", ") || "None"
+      ] })
+    ] })
+  ] });
+}
+function ScenarioList(props) {
+  return /* @__PURE__ */ (0, import_jsx_runtime.jsxs)("section", { className: "panel", children: [
+    /* @__PURE__ */ (0, import_jsx_runtime.jsx)("h2", { children: props.title }),
+    props.items.length === 0 ? /* @__PURE__ */ (0, import_jsx_runtime.jsx)("p", { className: "muted", children: "None." }) : null,
+    /* @__PURE__ */ (0, import_jsx_runtime.jsx)("ul", { className: "stack", children: props.items.map((item) => /* @__PURE__ */ (0, import_jsx_runtime.jsxs)("li", { children: [
+      /* @__PURE__ */ (0, import_jsx_runtime.jsx)("strong", { children: item.scenarioId }),
+      " ",
+      /* @__PURE__ */ (0, import_jsx_runtime.jsx)("span", { className: "muted", children: item.comparison.classification }),
+      /* @__PURE__ */ (0, import_jsx_runtime.jsx)("div", { children: /* @__PURE__ */ (0, import_jsx_runtime.jsx)("a", { href: `/compare?baseline=${item.comparison.baseline.run.id}&candidate=${item.comparison.candidate.run.id}`, children: "open run compare" }) })
+    ] }, item.scenarioId)) })
+  ] });
+}
 function Stat(props) {
   return /* @__PURE__ */ (0, import_jsx_runtime.jsxs)("div", { className: "stat", children: [
     /* @__PURE__ */ (0, import_jsx_runtime.jsx)("div", { className: "muted", children: props.label }),
@@ -22027,6 +22100,13 @@ function getRoute() {
   if (url.pathname.startsWith("/runs/")) {
     return { type: "detail", runId: decodeURIComponent(url.pathname.slice("/runs/".length)) };
   }
+  if (url.pathname === "/compare-suite") {
+    return {
+      type: "compare-suite",
+      baselineBatch: url.searchParams.get("baselineBatch") ?? void 0,
+      candidateBatch: url.searchParams.get("candidateBatch") ?? void 0
+    };
+  }
   if (url.pathname === "/compare") {
     return {
       type: "compare",

package/docs/agents.md

@@ -0,0 +1,152 @@
+# Agents
+
+Named agents are configured in `agentlab.config.yaml`.
+
+This repo currently supports three provider modes:
+
+- `mock`
+- `openai`
+- `external_process`
+
+## Named Agent Config
+
+Example:
+
+```yaml
+agents:
+  - name: mock-default
+    provider: mock
+    label: mock-default
+
+  - name: openai-cheap
+    provider: openai
+    model: gpt-4o-mini
+    label: openai-cheap
+
+  - name: custom-node-agent
+    provider: external_process
+    command: node
+    args:
+      - custom_agents/node_agent.mjs
+    label: custom-node-agent
+```
+
+Run a named agent with:
+
+```bash
+agentlab run support.refund-correct-order --agent mock-default
+```
+
+## Mock
+
+The built-in mock adapter is the best path for deterministic smoke tests and baseline examples.
+
+Use it when you want:
+
+- fast local verification
+- stable docs examples
+- predictable benchmark behavior
+
+## OpenAI
+
+The OpenAI path uses your API key and a configured model.
+
+Requirements:
+
+- `OPENAI_API_KEY` in the environment
+- a named `openai` agent in `agentlab.config.yaml`, or equivalent CLI runtime settings
+
+Example:
+
+```bash
+export OPENAI_API_KEY=...
+agentlab run support.refund-correct-order --agent openai-cheap
+```
+
+The OpenAI path is useful, but less deterministic than the mock path.
+
+## External Process
+
+External-process agents communicate with the runner over line-delimited JSON on stdin/stdout.
+
+The runner stays in control of:
+
+- tool execution
+- stopping conditions
+- runtime limits
+- persisted run state
+
+The external agent decides what tool to call next or when to return a final answer.
+
+### Protocol
+
+Runner events:
+
+- `run_started`
+- `tool_result`
+- `runner_error`
+
+Agent responses:
+
+- `tool_call`
+- `final`
+- `error`
+
+Minimal flow:
+
+1. the runner sends `run_started`
+2. the agent returns `tool_call` or `final`
+3. the runner executes the tool and sends `tool_result`
+4. the agent continues until it returns `final` or `error`
+
+Working examples:
+
+- `custom_agents/node_agent.mjs`
+- `custom_agents/python_agent.py`
+
+Run one of them with:
+
+```bash
+agentlab run support.refund-via-config-tool --agent custom-node-agent
+```
+
+## Environment Allowlist
+
+External-process agents can optionally define `envAllowlist`.
+
+Use it when a child process needs specific environment variables passed through.
+
+Example shape:
+
+```yaml
+agents:
+  - name: custom-agent
+    provider: external_process
+    command: node
+    args:
+      - custom_agents/node_agent.mjs
+    envAllowlist:
+      - OPENAI_API_KEY
+```
+
+Only allow through what the child actually needs.
+
+## Best Practices
+
+- use named agents instead of ad hoc local command strings
+- keep labels stable so compare output stays readable
+- prefer the mock path for smoke tests and docs
+- use external-process agents when you want to wrap a local Node or Python agent implementation
+- keep the runner authoritative for tools and termination
+
+## Common Errors
+
+Typical failures:
+
+- missing `OPENAI_API_KEY`
+- unsupported provider name
+- missing external-process `command`
+- invalid `args` or `envAllowlist`
+- child process returning invalid JSON
+
+See [troubleshooting.md](troubleshooting.md) for fixes.

package/docs/release-checklist.md

@@ -0,0 +1,64 @@
+# Release Checklist
+
+Use this before publishing a new npm version or telling users to upgrade.
+
+## Verification
+
+Run the full release gate:
+
+```bash
+npm run check
+npm test
+npm run build
+npm run smoke:cli
+npm pack --dry-run
+```
+
+## Manual CLI Flow
+
+Verify the canonical workflow:
+
+```bash
+agentlab list scenarios
+agentlab run support.refund-correct-order --agent mock-default
+agentlab show <run-id>
+agentlab run support.refund-correct-order --agent mock-default
+agentlab compare <baseline-run-id> <candidate-run-id>
+agentlab run --suite support --agent mock-default
+agentlab run --suite support --agent mock-default
+agentlab compare --suite <baseline-batch-id> <candidate-batch-id>
+agentlab ui
+```
+
+## Extension Smoke
+
+Verify at least one extension path:
+
+- run `support.refund-via-config-tool` with `custom-node-agent`, or
+- verify a repo-local custom tool still loads from `agentlab.config.yaml`
+
+## Docs Verification
+
+Confirm these files match current behavior:
+
+- `README.md`
+- `docs/scenarios.md`
+- `docs/tools.md`
+- `docs/agents.md`
+- `docs/troubleshooting.md`
+
+Requirements:
+
+- every command works as written
+- every referenced path exists
+- limitations are stated honestly
+- `compare --suite` is documented using suite batch ids, not run ids
+
+## Publish Hygiene
+
+Before `npm publish`:
+
+- confirm the package version is correct
+- confirm the git tree contains the intended release changes
+- confirm packaged UI assets are included in the tarball
+- confirm the npm metadata still points at the correct repo, homepage, and issues URL

package/docs/scenarios.md

@@ -0,0 +1,172 @@
+# Scenarios
+
+Scenarios are YAML files under `scenarios/`. They are the core authoring interface for the product.
+
+Each scenario should describe one narrow job for the agent, not a vague capability test.
+
+## Required Shape
+
+Each scenario should define:
+
+- `id`
+- `name`
+- `suite`
+- `task`
+- `tools`
+- `runtime`
+- `evaluators`
+
+Common optional fields already used in this repo:
+
+- `description`
+- `difficulty`
+- `tags`
+- task `context`
+
+## Example
+
+```yaml
+id: support.refund-correct-order
+name: Refund The Correct Order
+suite: support
+difficulty: easy
+description: Refund only the duplicated charge.
+tags:
+  - refund
+  - support
+task:
+  instructions: |
+    The customer says they were charged twice.
+    Find the duplicated charge and refund only that order.
+  context:
+    customer_email: alice@example.com
+tools:
+  allowed:
+    - crm.search_customer
+    - orders.list
+    - orders.refund
+runtime:
+  max_steps: 8
+  timeout_seconds: 60
+evaluators:
+  - id: refund-created
+    type: tool_call_assertion
+    mode: hard_gate
+    config:
+      tool: orders.refund
+      match:
+        order_id: ord_1024
+  - id: mentions-order
+    type: final_answer_contains
+    mode: weighted
+    weight: 1
+    config:
+      required_substrings:
+        - ord_1024
+```
+
+## Suites In This Repo
+
+Current benchmark domains:
+
+- `support`
+- `coding`
+- `research`
+- `ops`
+
+Use a suite when scenarios belong to one behavior family and should be runnable together with:
+
+```bash
+agentlab run --suite support --agent mock-default
+```
+
+`run --suite` creates a suite batch id. That id is later used for:
+
+```bash
+agentlab compare --suite <baseline-batch-id> <candidate-batch-id>
+```
+
+Suite comparison is strict. Only compare batches from the same suite.
+
+## Tools
+
+Each scenario declares its allowed tools:
+
+```yaml
+tools:
+  allowed:
+    - crm.search_customer
+    - orders.list
+    - orders.refund
+```
+
+Keep the tool allowlist as narrow as possible. A broad allowlist weakens the benchmark and makes regressions harder to interpret.
+
+This repo supports both:
+
+- built-in deterministic tools
+- repo-local custom tools registered in `agentlab.config.yaml`
+
+The launch benchmark now includes built-in tools for:
+
+- support
+- coding
+- research
+- ops
+
+See [tools.md](tools.md) for custom tool registration.
+
+## Runtime Limits
+
+Scenarios can enforce:
+
+- `max_steps`
+- `timeout_seconds`
+
+Example:
+
+```yaml
+runtime:
+  max_steps: 8
+  timeout_seconds: 60
+```
+
+These limits are enforced by the runner. Use them to keep runs bounded and comparisons meaningful.
+
+## Evaluators
+
+Use deterministic evaluators only.
+
+The current evaluator set includes:
+
+- `tool_call_assertion`
+- `forbidden_tool`
+- `final_answer_contains`
+- `exact_final_answer`
+- `step_count_max`
+
+Guidance:
+
+- use hard gates for non-negotiable behavior
+- use weighted evaluators for softer quality checks
+- prefer tool assertions or exact output checks over vague answer checks when possible
+
+## Authoring Conventions
+
+Use these defaults:
+
+- `id` format: `<suite>.<short-name>`
+- keep scenario jobs narrow and concrete
+- keep fixture-backed context in `task.context`
+- prefer deterministic fixture references over open-ended prompts
+- include `difficulty`, `description`, and `tags` for every launch scenario
+
+## Current Examples
+
+Useful scenario references in this repo:
+
+- support: `scenarios/support/refund-correct-order.yaml`
+- support with config tool: `scenarios/support/refund-via-config-tool.yaml`
+- coding: `scenarios/coding/fix-add-function.yaml`
+- research: `scenarios/research/remote-work-policy.yaml`
+- ops: `scenarios/ops/payments-api-alert.yaml`

package/docs/tools.md

@@ -0,0 +1,102 @@
+# Custom Tools
+
+Custom tools are registered in `agentlab.config.yaml` and loaded from repo-local JS or TS modules.
+
+This is the main extension point when built-in tools are not enough.
+
+## What A Tool Registration Needs
+
+Each tool entry must define:
+
+- `name`
+- `modulePath`
+- `exportName`
+- `description`
+- `inputSchema`
+
+Example:
+
+```yaml
+tools:
+  - name: support.find_duplicate_charge
+    modulePath: user_tools/findDuplicateCharge.ts
+    exportName: findDuplicateCharge
+    description: Find the duplicated charge order id for a given customer.
+    inputSchema:
+      type: object
+      additionalProperties: false
+      properties:
+        customer_id:
+          type: string
+          description: Customer id to inspect for duplicated charges.
+      required:
+        - customer_id
+```
+
+## Tool Module Shape
+
+The exported function should be async and should return JSON-serializable output.
+
+Minimal example:
+
+```ts
+export async function myTool(input: unknown): Promise<{ ok: boolean }> {
+  return { ok: true };
+}
+```
+
+The existing working example is:
+
+- `user_tools/findDuplicateCharge.ts`
+
+## Important Constraints
+
+- `modulePath` must stay within the repo
+- the module must exist at load time
+- the named export must exist
+- tool input should be validated defensively inside the tool
+- tool output should be deterministic and JSON-serializable
+
+For launch usage, treat tools as fixture-backed local functions, not live integrations.
+
+## Recommended Pattern
+
+Use this approach:
+
+1. read fixture data from `fixtures/`
+2. validate the input shape
+3. return a small structured result
+4. throw a clear error for missing fixture state or invalid input
+
+The current `findDuplicateCharge` tool shows that pattern.
+
+## Wiring A Tool Into A Scenario
+
+1. register the tool in `agentlab.config.yaml`
+2. add the tool name to the scenario allowlist
+3. add an evaluator that confirms the tool was used correctly if the behavior is important
+
+Example scenario:
+
+- `scenarios/support/refund-via-config-tool.yaml`
+
+## Best Practices
+
+- keep tool names stable and descriptive
+- keep tools scenario-agnostic where possible
+- prefer read-only or sandboxed behavior
+- do not mutate global machine state
+- do not call live external systems in benchmark paths
+- keep schemas narrow so agent tool calls are easy to validate and compare
+
+## Common Errors
+
+Typical config failures:
+
+- duplicate tool names
+- repo-external module paths
+- missing module files
+- missing exports
+- invalid `inputSchema` shape
+
+See [troubleshooting.md](troubleshooting.md) for failure examples and fixes.