@post-print/agent-test 0.3.4 → 0.3.5

package/dist/run-suite.js

@@ -1,17 +1,21 @@
 import { createRequire } from "node:module";
 import { basename, dirname, join, resolve } from "node:path";
-import { cancelActiveClaudeRun, cancelActiveCursorRun, captureWorkingTreeStatus, createScenarioWorktree, enrichTrace, filterWorkingTreeLeaks, findWorkingTreeLeak, formatWorkingTreeLeak, judgeTrace, loadContext, loadUnifiedDiffPaths, mergeMcpServers, partitionSeedCollateralLeaks, porcelainPathsFromLines, resolveHarnessArtifactIgnoreRoots, restoreWorkingTreePaths, runAgent, traceEditsOutsideWorktree, traceHasUserInputTool, } from "@post-print/agent-harness";
+import { cancelActiveClaudeRun, cancelActiveCursorRun, cancelActiveOpenaiRun, captureWorkingTreeStatus, createSealedWorkspace, defaultSealedOverlayPaths, enrichTrace, filterWorkingTreeLeaks, findWorkingTreeLeak, formatWorkingTreeLeak, judgeCompareTraces, judgeTrace, loadContext, loadUnifiedDiffPaths, mergeMcpServers, parseScenarioWorkspace, partitionSeedCollateralLeaks, porcelainPathsFromLines, resolveAllowUserSkills, resolveHarnessArtifactIgnoreRoots, restoreWorkingTreePaths, runAgent, skillInvokeJudgeCriteria, skillPathsFromSetting, sumUsageParts, toolPathsOutsideWorkspace, traceEditsOutsideWorktree, traceHasUserInputTool, } from "@post-print/agent-harness";
+import { applyCompareArm, applySidecarCompareDurations, assertCompareMetrics, compareArmLabel, compareStoryFields, prefixCompareFailures, } from "./compare-scenario.js";
 import { collectDebugEnvironment, getDebugBundleDir, writeDebugBundle } from "./debug-bundle.js";
 import { discoverSuites } from "./discover-suites.js";
 import { assertRubric } from "./expect.js";
 import { assertionFailure } from "./failures.js";
+import { resolveSuiteHosts, scenarioRunsOnHost, uniqueHosts } from "./hosts.js";
 import { failuresForLiveSubprocessExit, killActiveLiveChildren, liveScenarioIsolationEnabled, parentScenarioCounters, spawnLiveScenario, } from "./live-isolation.js";
 import { resolveLiveTimeoutMs } from "./live-timeout.js";
 import { loadSuiteFile } from "./load-suite.js";
-import { formatDuration, logPhase, logProgress, logVerdict, withHeartbeat } from "./progress.js";
+import { bindMcpServersToCaller, mcpStdioScriptPaths } from "./mcp-config.js";
+import { formatDuration, logLive, logPhase, logProgress, logVerdict, refreshHeartbeat, withHeartbeat, } from "./progress.js";
 import { cleanupStagingSession, createLiveStagingSessionId, getLiveStagingRootOverride, getLiveStagingSessionRoot, getStagingAgentStartPath, getStagingResultPath, getStagingTracePath, loadStagingResult, loadStagingTrace, recordTrace, resolveRecordingPath, setLiveStagingRootOverride, writeAgentStartMarker, writeStagingResult, } from "./record-trace.js";
 import { resolveScenarioRetryMaxAttempts, shouldRetryAnnounceStopFlake } from "./scenario-retry.js";
 import { captureCallerHead, restoreCallerHeadIfSeedCommit, seedScenarioWorktree, } from "./scenario-seed.js";
+import { buildScenarioStory, pathFromArgs, quoteExcerpt } from "./scenario-story.js";
 import { buildScenarioResultUsage, totalTokensFromScenarioUsage } from "./scenario-usage.js";
 import { summarizeReportResults } from "./suite-summary.js";
 import { theme } from "./theme.js";
@@ -38,6 +42,14 @@ async function restoreActiveCallerHead() {
 function isChildProcess() {
     return process.env.AGENT_TEST_CHILD === "1";
 }
+function resolveMaxConversationTurns() {
+    const raw = process.env.AGENT_TEST_MAX_TURNS?.trim();
+    if (!raw) {
+        return undefined;
+    }
+    const parsed = Number(raw);
+    return Number.isInteger(parsed) && parsed > 0 ? parsed : undefined;
+}
 /** Parent process prints suite headers and final verdicts; children only print phases. */
 export function shouldPrintSuiteChrome() {
     return !isChildProcess();
@@ -52,6 +64,7 @@ export function registerLiveRunHandlers() {
         killActiveLiveChildren();
         cancelActiveCursorRun();
         cancelActiveClaudeRun();
+        cancelActiveOpenaiRun();
         const cleanup = activeWorktreeCleanup;
         const headRestore = activeCallerHeadRestore;
         if (!cleanup && !headRestore) {
@@ -99,6 +112,82 @@ function normalizeJudgeCriteria(judge) {
         return { id: item.id ?? `judge-${index}`, question: item.question };
     });
 }
+/** Explicit judge questions plus skill-invoke questions for mustInvokeSkill. */
+export function collectJudgeCriteria(rubric) {
+    return [
+        ...normalizeJudgeCriteria(rubric.judge),
+        ...skillInvokeJudgeCriteria(rubric.mustInvokeSkill ?? []),
+    ];
+}
+/** Pairwise judge questions only. Skill-follow checks stay per-arm and deterministic. */
+export function collectCompareJudgeCriteria(rubric) {
+    return normalizeJudgeCriteria(rubric.judge);
+}
+/** Judge auth and the judge call apply only when a rubric has judge work. */
+export function judgeAuthRequired(judge, rubrics) {
+    return judge !== false && rubrics.some((rubric) => collectJudgeCriteria(rubric).length > 0);
+}
+/** Pairwise judge on compare. Skill-follow judge questions stay on non-compare scenarios. */
+export function scenarioNeedsJudge(judge, scenario) {
+    if (judge === false) {
+        return false;
+    }
+    if (scenario.compare) {
+        return collectCompareJudgeCriteria(scenario.rubric).length > 0;
+    }
+    return collectJudgeCriteria(scenario.rubric).length > 0;
+}
+/** True when the selected live run will call the judge. */
+export async function selectedRunNeedsJudge(options) {
+    if (options.judge === false) {
+        return false;
+    }
+    const suitePaths = await discoverSuites(resolve(options.cwd, options.suitesDir));
+    const filtered = options.filter
+        ? suitePaths.filter((suitePath) => {
+            const suiteName = suiteNameFromPath(suitePath);
+            return suiteName === options.filter || suitePath.includes(`/${options.filter}/`);
+        })
+        : suitePaths;
+    for (const suitePath of filtered) {
+        const suite = await loadSuiteFile(suitePath, { rubricsDir: options.rubricsDir });
+        for (const scenario of suite.scenarios) {
+            if (scenario.skip) {
+                continue;
+            }
+            if (options.scenarioFilter && scenario.name !== options.scenarioFilter) {
+                continue;
+            }
+            if (scenarioNeedsJudge(true, scenario)) {
+                return true;
+            }
+        }
+    }
+    return false;
+}
+export async function loadSelectedRubrics(options) {
+    const suitePaths = await discoverSuites(resolve(options.cwd, options.suitesDir));
+    const filtered = options.filter
+        ? suitePaths.filter((suitePath) => {
+            const suiteName = suiteNameFromPath(suitePath);
+            return suiteName === options.filter || suitePath.includes(`/${options.filter}/`);
+        })
+        : suitePaths;
+    const rubrics = [];
+    for (const suitePath of filtered) {
+        const suite = await loadSuiteFile(suitePath, { rubricsDir: options.rubricsDir });
+        for (const scenario of suite.scenarios) {
+            if (scenario.skip) {
+                continue;
+            }
+            if (options.scenarioFilter && scenario.name !== options.scenarioFilter) {
+                continue;
+            }
+            rubrics.push(scenario.rubric);
+        }
+    }
+    return rubrics;
+}
 function questionForCriterion(criteria, id) {
     return criteria.find((c) => c.id === id)?.question ?? id;
 }
@@ -145,6 +234,7 @@ function emitScenarioVerdict(options) {
         judgeVerdicts: options.judgeVerdicts,
         rubricFailures: rubricFailuresOnly(options.failures),
         failureCategory: options.failures[0]?.category,
+        story: options.story,
         debug: options.debug,
         debugBundleDir: options.debugBundleDir,
     }));
@@ -156,7 +246,7 @@ async function maybeWriteDebugBundle(options) {
     if (!options.stagingSessionId) {
         return undefined;
     }
-    const dir = getDebugBundleDir(options.stagingSessionId, options.suiteName, options.scenario.name, getLiveStagingSessionRoot);
+    const dir = getDebugBundleDir(options.stagingSessionId, options.suiteName, options.scenario.name, getLiveStagingSessionRoot, options.host);
     const cliPath = process.argv[1] ?? resolve(options.cwd, "node_modules/@post-print/agent-test/dist/cli.js");
     try {
         await writeDebugBundle({
@@ -235,12 +325,20 @@ async function runSuiteBody(options) {
     }
     const defaultHost = options.host ?? suite.defaults?.host ?? "cursor";
     const results = [];
-    const scenarios = options.scenarioFilter
+    const selected = options.scenarioFilter
         ? suite.scenarios.filter((scenario) => scenario.name === options.scenarioFilter)
         : suite.scenarios;
-    if (options.scenarioFilter && scenarios.length === 0) {
+    if (options.scenarioFilter && selected.length === 0) {
         throw new Error(`Scenario not found: ${options.scenarioFilter}`);
     }
+    const hostLocked = options.hostLocked === true;
+    const scenarios = hostLocked
+        ? selected.filter((scenario) => scenarioRunsOnHost(scenario, defaultHost, true))
+        : selected;
+    if (hostLocked && options.scenarioFilter && selected.length > 0 && scenarios.length === 0) {
+        const pinned = selected[0]?.host;
+        throw new Error(`Scenario ${options.scenarioFilter} is pinned to ${pinned} (this run is ${defaultHost})`);
+    }
     const filteredTotal = scenarios.length;
     const parentCounters = parentScenarioCounters();
     const displayTotal = parentCounters?.total ?? filteredTotal;
@@ -249,7 +347,7 @@ async function runSuiteBody(options) {
     if (shouldPrintSuiteChrome()) {
         logProgress(`\n${theme.suiteHeader(suite.name, defaultHost, displayTotal)}`);
         if (isolateLive) {
-            logProgress(`  ${theme.isolationNote()}`);
+            logProgress(`    ${theme.isolationNote()}`);
         }
     }
     for (let index = 0; index < scenarios.length; index++) {
@@ -266,10 +364,17 @@ async function runSuiteBody(options) {
                 results.push({
                     suite: suite.name,
                     scenario: scenario.name,
+                    prompt: scenario.prompt,
                     passed: true,
                     failures: [],
                     skipped: true,
                     durationMs: 0,
+                    story: buildScenarioStory({
+                        rubric: scenario.rubric,
+                        passed: true,
+                        skipped: true,
+                        failures: [],
+                    }),
                 });
                 continue;
             }
@@ -282,9 +387,10 @@ async function runSuiteBody(options) {
             let failures = [];
             let scenarioTrace;
             let previousAttemptExitCode = previousIsolatedExitCode;
+            let childSidecar;
             while (true) {
                 attempts++;
-                const exitCode = await spawnLiveScenario({
+                const spawned = await spawnLiveScenario({
                     cwd: options.cwd,
                     suiteName: suite.name,
                     scenarioName: scenario.name,
@@ -296,6 +402,7 @@ async function runSuiteBody(options) {
                     worktree: options.worktree,
                     judge: options.judge,
                     host: defaultHost,
+                    adapterModules: options.adapterModules,
                     scenarioIndex: index + 1,
                     scenarioTotal: filteredTotal,
                     timeoutMs: resolveLiveTimeoutMs(options.timeoutMs),
@@ -305,23 +412,34 @@ async function runSuiteBody(options) {
                     debugDir: options.debugDir,
                     previousExitCode: previousAttemptExitCode,
                 });
-                previousAttemptExitCode = exitCode;
-                previousIsolatedExitCode = exitCode;
+                previousAttemptExitCode = spawned.exitCode;
+                previousIsolatedExitCode = spawned.exitCode;
                 failures = [];
                 scenarioTrace = undefined;
-                if (exitCode !== 0) {
-                    const childResult = options.stagingSessionId !== undefined
+                childSidecar =
+                    options.stagingSessionId !== undefined
                         ? await loadStagingResult(getStagingResultPath(options.stagingSessionId, suite.name, scenario.name))
                         : undefined;
-                    failures.push(...failuresForLiveSubprocessExit(exitCode, childResult));
+                if (spawned.exitCode !== 0) {
+                    failures.push(...failuresForLiveSubprocessExit(spawned.exitCode, childSidecar, spawned.stderr));
                 }
                 if (options.stagingSessionId) {
-                    const tracePath = getStagingTracePath(options.stagingSessionId, suite.name, scenario.name);
-                    try {
-                        scenarioTrace = await loadStagingTrace(tracePath);
+                    if (scenario.compare) {
+                        try {
+                            scenarioTrace = await loadStagingTrace(getStagingTracePath(options.stagingSessionId, suite.name, scenario.name, "a"));
+                        }
+                        catch {
+                            // Trace may be missing when the child crashed before recording.
+                        }
                     }
-                    catch {
-                        // Trace may be missing when the child crashed before recording.
+                    else {
+                        const tracePath = getStagingTracePath(options.stagingSessionId, suite.name, scenario.name);
+                        try {
+                            scenarioTrace = await loadStagingTrace(tracePath);
+                        }
+                        catch {
+                            // Trace may be missing when the child crashed before recording.
+                        }
                     }
                 }
                 const canRetry = failures.length > 0 &&
@@ -334,38 +452,79 @@ async function runSuiteBody(options) {
                 break;
             }
             let judgeVerdicts;
-            if (failures.length === 0 && options.judge !== false && scenarioTrace) {
-                const criteria = normalizeJudgeCriteria(scenario.rubric.judge);
-                if (criteria.length > 0) {
-                    releaseLiveMemory();
-                    logPhase(theme.judgePhase(criteria.length), { last: true });
-                    try {
-                        const judged = await runJudgeRubric(scenarioTrace, scenario.rubric, options.cwd);
-                        failures.push(...judged.failures);
-                        scenarioTrace = judged.trace;
-                        judgeVerdicts = toJudgeVerdictResults(judged.trace, criteria, judged.verdicts);
+            let compareResult;
+            const judgeHost = options.host ?? scenario.host ?? suite.defaults?.host ?? "cursor";
+            if (scenario.compare && options.stagingSessionId) {
+                compareResult = applySidecarCompareDurations(await loadCompareResultFromStaging(options.stagingSessionId, suite.name, scenario), childSidecar);
+                scenarioTrace = compareResult.a.trace ?? scenarioTrace;
+            }
+            if (failures.length === 0 && options.judge !== false) {
+                if (scenario.compare && compareResult?.a.trace && compareResult.b.trace) {
+                    const criteria = collectCompareJudgeCriteria(scenario.rubric);
+                    if (criteria.length > 0) {
+                        releaseLiveMemory();
+                        logPhase(theme.judgePhase(criteria.length), { last: true });
+                        try {
+                            const judged = await runCompareJudgeRubric(compareResult, scenario.rubric, options.cwd, judgeHost);
+                            failures.push(...judged.failures);
+                            judgeVerdicts = toJudgeVerdictResults({ ...compareResult.a.trace, judgeVerdicts: judged.verdicts }, criteria, judged.verdicts);
+                        }
+                        catch (error) {
+                            failures.push(assertionFailure("judge", error instanceof Error ? error.message : "failed to judge compare traces", "judge_infra"));
+                        }
                     }
-                    catch (error) {
-                        failures.push(assertionFailure("judge", error instanceof Error ? error.message : "failed to load staging trace for judge", "judge_infra"));
+                }
+                else if (scenarioTrace && !scenario.compare) {
+                    const criteria = collectJudgeCriteria(scenario.rubric);
+                    if (criteria.length > 0) {
+                        releaseLiveMemory();
+                        logPhase(theme.judgePhase(criteria.length), { last: true });
+                        try {
+                            const judged = await runJudgeRubric(scenarioTrace, scenario.rubric, options.cwd, judgeHost);
+                            failures.push(...judged.failures);
+                            scenarioTrace = judged.trace;
+                            judgeVerdicts = toJudgeVerdictResults(judged.trace, criteria, judged.verdicts);
+                        }
+                        catch (error) {
+                            failures.push(assertionFailure("judge", error instanceof Error ? error.message : "failed to load staging trace for judge", "judge_infra"));
+                        }
                     }
                 }
             }
             const durationMs = Math.round(performance.now() - started);
             const passed = failures.length === 0;
             const usageFields = buildScenarioResultUsage({
-                agentUsage: scenarioTrace?.usage,
+                agentUsage: compareResult
+                    ? sumUsageParts([compareResult.a.trace?.usage, compareResult.b.trace?.usage])
+                    : scenarioTrace?.usage,
                 judgeVerdicts,
             });
+            const story = buildScenarioStory({
+                rubric: scenario.rubric,
+                trace: scenarioTrace,
+                passed,
+                failures,
+                judgeVerdicts,
+                compare: compareResult
+                    ? {
+                        ...compareStoryFields(scenario),
+                        aTrace: compareResult.a.trace,
+                        bTrace: compareResult.b.trace,
+                    }
+                    : undefined,
+            });
             const scenarioResult = {
                 suite: suite.name,
                 scenario: scenario.name,
-                compareId: scenario.compareId,
+                prompt: scenario.prompt,
                 passed,
                 failures,
                 durationMs,
                 attempts,
                 judgeVerdicts,
                 trace: scenarioTrace,
+                compare: compareResult,
+                story,
                 ...usageFields,
             };
             const debugBundleDir = await maybeWriteDebugBundle({
@@ -396,6 +555,7 @@ async function runSuiteBody(options) {
                 totalTokens: totalTokensFromScenarioUsage(scenarioResult.usage, scenarioTrace?.usage),
                 judgeVerdicts,
                 failures,
+                story,
                 debug,
                 debugBundleDir,
             });
@@ -435,10 +595,58 @@ async function runSuiteBody(options) {
         summary: summarizeReportResults(results),
     };
 }
+/** Suite trees and MCP scripts from the caller, so uncommitted fixtures reach the host. */
+function liveOverlayExtras(cwd, suitesDir, contextSources, mcpServers) {
+    const extras = [...(contextSources ?? [])];
+    const suitesRel = overlayRelPath(cwd, suitesDir);
+    if (suitesRel && !extras.includes(suitesRel)) {
+        extras.push(suitesRel);
+    }
+    for (const script of mcpStdioScriptPaths(mcpServers)) {
+        if (!extras.includes(script)) {
+            extras.push(script);
+        }
+    }
+    return extras;
+}
+function overlayRelPath(cwd, path) {
+    const trimmed = path.replace(/^\.\//, "").trim();
+    if (trimmed.length === 0) {
+        return undefined;
+    }
+    if (!trimmed.startsWith("/")) {
+        return trimmed;
+    }
+    const root = resolve(cwd);
+    if (trimmed === root) {
+        return undefined;
+    }
+    if (trimmed.startsWith(`${root}/`)) {
+        return trimmed.slice(root.length + 1);
+    }
+    return undefined;
+}
+function resolveRunWorkspace(defaultWorkspace, scenarioWorkspace) {
+    const raw = scenarioWorkspace !== undefined ? scenarioWorkspace : defaultWorkspace;
+    const parsed = parseScenarioWorkspace(raw);
+    if (!parsed.ok) {
+        throw new Error(parsed.message);
+    }
+    return parsed.rel;
+}
 function mergeContextSources(defaults, scenarioSources) {
     const merged = [...(defaults ?? []), ...(scenarioSources ?? [])].filter((value) => typeof value === "string" && value.trim().length > 0);
     return merged.length > 0 ? merged : undefined;
 }
+function defaultProfileForHost(host) {
+    if (host === "cursor") {
+        return "cursor";
+    }
+    if (host === "claude") {
+        return "claude";
+    }
+    return "shared";
+}
 /** Run one real agent scenario. JSON suites delegate to this same execution boundary. */
 export async function runAgentTest(options) {
     const previousStagingRoot = getLiveStagingRootOverride();
@@ -462,7 +670,7 @@ async function runAgentTestBody(options) {
     if (legacyScenario.host === "replay" ||
         legacyDefaults?.host === "replay" ||
         "replayTrace" in legacyScenario) {
-        throw new Error("Replay-based testing is deprecated and no longer supported; use Cursor or Claude.");
+        throw new Error("Replay-based testing is deprecated and no longer supported; use Cursor, Claude, or OpenAI.");
     }
     const suiteName = options.suiteName ?? "direct";
     const defaultHost = options.host ?? options.defaults?.host ?? "cursor";
@@ -471,7 +679,7 @@ async function runAgentTestBody(options) {
     let result;
     while (true) {
         attempts++;
-        result = await runAgentTestOnce(options.cwd, suiteName, options.scenario, defaultHost, options.defaults?.profile, options.defaults?.skills, options.defaults?.contextSources, options.defaults?.mcpServers, options.judge ?? true, options.worktree ?? true, options.stagingSessionId, options.scenarioIndex, options.scenarioTotal, options.timeoutMs, options.allowUserInput, options.debug, options.debugDir, options.suitesDir ?? "agent-suites", options.keepRecordings, options.rubricsDir, { suppressEmit: maxAttempts > 1 });
+        result = await runAgentTestOnce(options.cwd, suiteName, options.scenario, defaultHost, options.defaults?.profile, options.defaults?.skills, options.defaults?.contextSources, options.defaults?.mcpServers, options.defaults?.workspace, options.defaults?.allowUserSkills, options.judge ?? true, options.worktree ?? true, options.stagingSessionId, options.scenarioIndex, options.scenarioTotal, options.timeoutMs, options.allowUserInput, options.debug, options.debugDir, options.suitesDir ?? "agent-suites", options.keepRecordings, options.rubricsDir, { suppressEmit: maxAttempts > 1 });
         const canRetry = !result.skipped &&
             !result.passed &&
             attempts < maxAttempts &&
@@ -512,13 +720,14 @@ async function runAgentTestBody(options) {
             totalTokens: totalTokensFromScenarioUsage(result.usage, result.trace?.usage),
             judgeVerdicts: result.judgeVerdicts,
             failures: result.failures,
+            story: result.story,
             debug,
             debugBundleDir,
         });
     }
     return result;
 }
-async function runAgentTestOnce(cwd, suiteName, scenario, defaultHost, defaultProfile, defaultSkills, defaultContextSources, defaultMcpServers, judge, worktree, stagingSessionId, scenarioIndex, scenarioTotal, timeoutMs, allowUserInput, debugFlag, debugDir, suitesDir = "agent-suites", keepRecordings, rubricsDir, runOptions) {
+async function runAgentTestOnce(cwd, suiteName, scenario, defaultHost, defaultProfile, defaultSkills, defaultContextSources, defaultMcpServers, defaultWorkspace, defaultAllowUserSkills, judge, worktree, stagingSessionId, scenarioIndex, scenarioTotal, timeoutMs, allowUserInput, debugFlag, debugDir, suitesDir = "agent-suites", keepRecordings, rubricsDir, runOptions) {
     const started = performance.now();
     const debug = isDebugEnabled({ debug: debugFlag });
     const suppressEmit = runOptions?.suppressEmit === true;
@@ -530,20 +739,38 @@ async function runAgentTestOnce(cwd, suiteName, scenario, defaultHost, defaultPr
         return {
             suite: suiteName,
             scenario: scenario.name,
+            prompt: scenario.prompt,
             passed: true,
             failures: [],
             skipped: true,
             durationMs: 0,
+            story: buildScenarioStory({
+                rubric: scenario.rubric,
+                passed: true,
+                skipped: true,
+                failures: [],
+            }),
         };
     }
+    if (scenario.compare && runOptions?.compareArm === undefined) {
+        return runCompareAgentTestOnce(cwd, suiteName, scenario, defaultHost, defaultProfile, defaultSkills, defaultContextSources, defaultMcpServers, defaultWorkspace, defaultAllowUserSkills, judge, worktree, stagingSessionId, scenarioIndex, scenarioTotal, timeoutMs, allowUserInput, debugFlag, debugDir, suitesDir, keepRecordings, rubricsDir, { suppressEmit });
+    }
     const host = scenario.host ?? defaultHost;
-    const profile = scenario.profile ?? defaultProfile ?? (host === "cursor" ? "cursor" : "shared");
+    const profile = scenario.profile ?? defaultProfile ?? defaultProfileForHost(host);
     const skills = scenario.skills ?? defaultSkills;
     const contextSources = mergeContextSources(defaultContextSources, scenario.contextSources);
-    const mcpServers = mergeMcpServers(defaultMcpServers, scenario.mcpServers);
+    const workspaceRel = resolveRunWorkspace(defaultWorkspace, scenario.workspace);
+    const fixtureWorkspace = workspaceRel !== undefined;
+    const allowUserSkills = resolveAllowUserSkills(scenario.allowUserSkills, defaultAllowUserSkills);
+    const mcpServers = fixtureWorkspace
+        ? bindMcpServersToCaller(mergeMcpServers(defaultMcpServers, scenario.mcpServers), cwd)
+        : mergeMcpServers(defaultMcpServers, scenario.mcpServers);
     const liveTimeoutMs = resolveLiveTimeoutMs(timeoutMs);
     const failOnUserInput = !allowUserInput;
-    if (scenarioIndex !== undefined && scenarioTotal !== undefined) {
+    if (runOptions?.compareArm) {
+        logPhase(theme.phase("arm", `${runOptions.compareArm.toUpperCase()} ${host}`));
+    }
+    else if (scenarioIndex !== undefined && scenarioTotal !== undefined) {
         logProgress(theme.scenarioTitle(scenarioIndex, scenarioTotal, scenario.name, host));
     }
     else {
@@ -558,9 +785,14 @@ async function runAgentTestOnce(cwd, suiteName, scenario, defaultHost, defaultPr
             callerHeadBefore = await captureCallerHead(cwd);
             setCallerHeadRestore(cwd, callerHeadBefore);
         }
-        worktreeHandle = await createScenarioWorktree(cwd, `${suiteName}-${scenario.name}`);
+        worktreeHandle = await createSealedWorkspace({
+            callerCwd: cwd,
+            workspace: workspaceRel,
+            overlayPaths: fixtureWorkspace
+                ? undefined
+                : defaultSealedOverlayPaths(liveOverlayExtras(cwd, suitesDir, contextSources, mcpServers), skillPathsFromSetting(skills)),
+        });
         activeWorktreeCleanup = worktreeHandle.cleanup;
-        logPhase(theme.phase("worktree", theme.path(worktreeHandle.path)));
         if (scenario.seedPatch) {
             logPhase(theme.phase("seed", theme.basename(scenario.seedPatch)));
             await seedScenarioWorktree(cwd, worktreeHandle.path, scenario.seedPatch, {
@@ -571,24 +803,21 @@ async function runAgentTestOnce(cwd, suiteName, scenario, defaultHost, defaultPr
     else {
         logPhase(theme.phase("worktree", theme.phaseDim("disabled (AGENT_TEST_ALLOW_IN_PLACE=1)")));
     }
-    const runCwd = worktreeHandle?.path ?? cwd;
+    const runCwd = worktreeHandle?.path ?? (workspaceRel ? resolve(cwd, workspaceRel) : cwd);
     try {
-        logPhase(theme.phase("context"));
-        // Worktree runs code in an isolated checkout; load rules/AGENTS from caller cwd
-        // so uncommitted .cursor/rules and AGENTS.md edits apply during dogfood.
-        const contextRoot = useWorktree ? cwd : runCwd;
         const context = await loadContext({
-            cwd: contextRoot,
+            cwd: runCwd,
             profile,
             skills,
             contextSources,
         });
-        logPhase(theme.phase("agent"));
         const outputContract = outputContractForRubric(scenario.rubric);
         const agentStartMarkerPath = isChildProcess() && stagingSessionId
             ? getStagingAgentStartPath(stagingSessionId, suiteName, scenario.name)
             : undefined;
+        logPhase(theme.phase("agent", theme.phaseDim("started")));
         const agentStarted = performance.now();
+        let livePreview;
         const session = await withHeartbeat(runAgent({
             host,
             cwd: runCwd,
@@ -597,12 +826,25 @@ async function runAgentTestOnce(cwd, suiteName, scenario, defaultHost, defaultPr
             prompt: scenario.prompt,
             outputContract,
             mcpServers,
+            allowUserSkills,
             timeoutMs: liveTimeoutMs,
             failOnUserInput,
+            maxConversationTurns: resolveMaxConversationTurns(),
             onDeadlineStart: agentStartMarkerPath
                 ? () => writeAgentStartMarker(agentStartMarkerPath)
                 : undefined,
-        }), { started: agentStarted });
+            onAgentEvent: (event) => {
+                if (event.type === "tool") {
+                    logLive(theme.liveTool(event.name, pathFromArgs(event.args)));
+                    livePreview = undefined;
+                    refreshHeartbeat();
+                    return;
+                }
+                // Clock tick paints the preview. Do not rewrite here — a long
+                // line wraps and `\r` cannot clear the leftover row.
+                livePreview = quoteExcerpt(event.text);
+            },
+        }), { started: agentStarted, preview: () => livePreview });
         logPhase(theme.phase("agent", `${theme.statusCompleted(session.status)} ${theme.duration(formatDuration(performance.now() - agentStarted))}`));
         let trace = enrichTrace(session.trace);
         const failures = [];
@@ -629,10 +871,15 @@ async function runAgentTestOnce(cwd, suiteName, scenario, defaultHost, defaultPr
         else if (failOnUserInput && traceHasUserInputTool(trace.toolCalls)) {
             failures.push(assertionFailure("runAgent", "agent trace contains AskQuestion-style user-input tool in headless mode", "agent_runtime", `toolCalls=${trace.toolCalls.map((call) => call.name).join(", ")}`));
         }
-        logPhase(theme.phase("rubric"));
         failures.push(...assertRubric(trace, scenario.rubric, {
             skillsMode: context.skillsMode,
         }));
+        if (worktreeHandle) {
+            const escaped = toolPathsOutsideWorkspace(trace, worktreeHandle.path);
+            if (escaped.length > 0) {
+                failures.push(assertionFailure("workingTreeLeak", `agent used paths outside the sealed workspace: ${escaped.join(", ")}`, "worktree_leak", `escaped=${escaped.join(", ")}`));
+            }
+        }
         if (useWorktree && callerTreeBefore !== undefined) {
             const callerTreeAfter = await captureWorkingTreeStatus(cwd);
             const ignoreRoots = resolveHarnessArtifactIgnoreRoots(cwd, getLiveStagingRootOverride());
@@ -663,11 +910,13 @@ async function runAgentTestOnce(cwd, suiteName, scenario, defaultHost, defaultPr
                 ].join("\n")));
             }
         }
-        const stagingTracePath = resolveRecordingPath(suiteName, scenario.name, stagingSessionId);
+        const stagingTracePath = resolveRecordingPath(suiteName, scenario.name, stagingSessionId, runOptions?.compareArm);
         if (stagingTracePath) {
             try {
                 const path = await recordTrace(stagingTracePath, trace);
-                logPhase(theme.phase("trace", theme.path(path)));
+                if (keepRecordings || debug) {
+                    logPhase(theme.phase("trace", theme.path(path)));
+                }
             }
             catch (error) {
                 failures.push(assertionFailure("recordTrace", error instanceof Error ? error.message : "failed to record trace", "recording_error"));
@@ -676,17 +925,17 @@ async function runAgentTestOnce(cwd, suiteName, scenario, defaultHost, defaultPr
         const deferJudgeToParent = isChildProcess();
         let judgeVerdicts;
         if (judge && !deferJudgeToParent) {
-            const criteria = normalizeJudgeCriteria(scenario.rubric.judge);
+            const criteria = collectJudgeCriteria(scenario.rubric);
             if (criteria.length > 0) {
                 logPhase(theme.judgePhase(criteria.length), { last: true });
             }
-            const judged = await runJudgeRubric(trace, scenario.rubric, runCwd);
+            const judged = await runJudgeRubric(trace, scenario.rubric, runCwd, host);
             failures.push(...judged.failures);
             trace = judged.trace;
             judgeVerdicts = toJudgeVerdictResults(judged.trace, criteria, judged.verdicts);
         }
         const durationMs = Math.round(performance.now() - started);
-        if (isChildProcess() && stagingSessionId) {
+        if (isChildProcess() && stagingSessionId && runOptions?.writeSidecar !== false) {
             await writeStagingResult(getStagingResultPath(stagingSessionId, suiteName, scenario.name), {
                 passed: failures.length === 0,
                 failures,
@@ -694,10 +943,6 @@ async function runAgentTestOnce(cwd, suiteName, scenario, defaultHost, defaultPr
             });
         }
         if (worktreeHandle) {
-            const willJudge = Boolean(judge) &&
-                !isChildProcess() &&
-                normalizeJudgeCriteria(scenario.rubric.judge).length > 0;
-            logPhase(theme.phase("cleanup"), { last: !willJudge });
             await worktreeHandle.cleanup();
             if (activeWorktreeCleanup === worktreeHandle.cleanup) {
                 activeWorktreeCleanup = undefined;
@@ -707,15 +952,24 @@ async function runAgentTestOnce(cwd, suiteName, scenario, defaultHost, defaultPr
             }
             worktreeHandle = undefined;
         }
+        const passed = failures.length === 0;
+        const story = buildScenarioStory({
+            rubric: scenario.rubric,
+            trace,
+            passed,
+            failures,
+            judgeVerdicts,
+        });
         const scenarioResult = {
             suite: suiteName,
             scenario: scenario.name,
-            compareId: scenario.compareId,
-            passed: failures.length === 0,
+            prompt: scenario.prompt,
+            passed,
             failures,
             durationMs,
             judgeVerdicts,
             trace,
+            story,
             ...buildScenarioResultUsage({
                 agentUsage: session.usage ?? trace?.usage,
                 judgeVerdicts,
@@ -742,7 +996,7 @@ async function runAgentTestOnce(cwd, suiteName, scenario, defaultHost, defaultPr
             });
             scenarioResult.debugBundleDir = debugBundleDir;
             emitScenarioVerdict({
-                passed: failures.length === 0,
+                passed,
                 index: scenarioIndex,
                 total: scenarioTotal,
                 name: scenario.name,
@@ -750,6 +1004,7 @@ async function runAgentTestOnce(cwd, suiteName, scenario, defaultHost, defaultPr
                 totalTokens: totalTokensFromScenarioUsage(scenarioResult.usage, trace?.usage),
                 judgeVerdicts,
                 failures,
+                story,
                 debug,
                 debugBundleDir,
             });
@@ -766,7 +1021,6 @@ async function runAgentTestOnce(cwd, suiteName, scenario, defaultHost, defaultPr
             }
         }
         if (worktreeHandle) {
-            logPhase(theme.phase("cleanup"), { last: true });
             await worktreeHandle.cleanup();
             if (activeWorktreeCleanup === worktreeHandle.cleanup) {
                 activeWorktreeCleanup = undefined;
@@ -777,12 +1031,215 @@ async function runAgentTestOnce(cwd, suiteName, scenario, defaultHost, defaultPr
         }
     }
 }
-async function runJudgeRubric(trace, rubric, runCwd) {
-    const criteria = normalizeJudgeCriteria(rubric.judge);
+async function runCompareAgentTestOnce(cwd, suiteName, scenario, defaultHost, defaultProfile, defaultSkills, defaultContextSources, defaultMcpServers, defaultWorkspace, defaultAllowUserSkills, judge, worktree, stagingSessionId, scenarioIndex, scenarioTotal, timeoutMs, allowUserInput, debugFlag, debugDir, suitesDir = "agent-suites", keepRecordings, rubricsDir, runOptions) {
+    const started = performance.now();
+    const debug = isDebugEnabled({ debug: debugFlag });
+    const suppressEmit = runOptions?.suppressEmit === true;
+    const host = scenario.host ?? defaultHost;
+    if (scenarioIndex !== undefined && scenarioTotal !== undefined) {
+        logProgress(theme.scenarioTitle(scenarioIndex, scenarioTotal, scenario.name, host));
+    }
+    else {
+        logProgress(theme.scenarioLabel(scenario.name, host));
+    }
+    const aScenario = applyCompareArm(scenario, "a");
+    const bScenario = applyCompareArm(scenario, "b");
+    const aLabel = compareArmLabel(scenario.compare?.a, "a");
+    const bLabel = compareArmLabel(scenario.compare?.b, "b");
+    const shared = {
+        cwd,
+        suiteName,
+        defaultHost,
+        defaultProfile,
+        defaultSkills,
+        defaultContextSources,
+        defaultMcpServers,
+        defaultWorkspace,
+        defaultAllowUserSkills,
+        worktree,
+        stagingSessionId,
+        timeoutMs,
+        allowUserInput,
+        debugFlag,
+        debugDir,
+        suitesDir,
+        keepRecordings,
+        rubricsDir,
+    };
+    const aResult = await runAgentTestOnce(shared.cwd, shared.suiteName, aScenario, shared.defaultHost, shared.defaultProfile, shared.defaultSkills, shared.defaultContextSources, shared.defaultMcpServers, shared.defaultWorkspace, shared.defaultAllowUserSkills, false, shared.worktree, shared.stagingSessionId, undefined, undefined, shared.timeoutMs, shared.allowUserInput, shared.debugFlag, shared.debugDir, shared.suitesDir, shared.keepRecordings, shared.rubricsDir, { suppressEmit: true, compareArm: "a", writeSidecar: false });
+    const bResult = await runAgentTestOnce(shared.cwd, shared.suiteName, bScenario, shared.defaultHost, shared.defaultProfile, shared.defaultSkills, shared.defaultContextSources, shared.defaultMcpServers, shared.defaultWorkspace, shared.defaultAllowUserSkills, false, shared.worktree, shared.stagingSessionId, undefined, undefined, shared.timeoutMs, shared.allowUserInput, shared.debugFlag, shared.debugDir, shared.suitesDir, shared.keepRecordings, shared.rubricsDir, { suppressEmit: true, compareArm: "b", writeSidecar: false });
+    const failures = [
+        ...prefixCompareFailures(aLabel, aResult.failures),
+        ...prefixCompareFailures(bLabel, bResult.failures),
+    ];
+    const compareResult = {
+        a: {
+            id: "a",
+            label: aLabel,
+            prompt: aScenario.prompt,
+            trace: aResult.trace,
+            durationMs: aResult.durationMs,
+        },
+        b: {
+            id: "b",
+            label: bLabel,
+            prompt: bScenario.prompt,
+            trace: bResult.trace,
+            durationMs: bResult.durationMs,
+        },
+    };
+    if (scenario.compare) {
+        failures.push(...assertCompareMetrics(scenario.compare, compareResult));
+    }
+    const deferJudgeToParent = isChildProcess();
+    let judgeVerdicts;
+    if (judge &&
+        !deferJudgeToParent &&
+        failures.length === 0 &&
+        compareResult.a.trace &&
+        compareResult.b.trace) {
+        const criteria = collectCompareJudgeCriteria(scenario.rubric);
+        if (criteria.length > 0) {
+            logPhase(theme.judgePhase(criteria.length), { last: true });
+        }
+        const judged = await runCompareJudgeRubric(compareResult, scenario.rubric, cwd, host);
+        failures.push(...judged.failures);
+        judgeVerdicts = toJudgeVerdictResults({ ...compareResult.a.trace, judgeVerdicts: judged.verdicts }, criteria, judged.verdicts);
+    }
+    const durationMs = Math.round(performance.now() - started);
+    if (isChildProcess() && stagingSessionId) {
+        await writeStagingResult(getStagingResultPath(stagingSessionId, suiteName, scenario.name), {
+            passed: failures.length === 0,
+            failures,
+            durationMs,
+            compare: {
+                a: { durationMs: aResult.durationMs },
+                b: { durationMs: bResult.durationMs },
+            },
+        });
+    }
+    const passed = failures.length === 0;
+    const story = buildScenarioStory({
+        rubric: scenario.rubric,
+        trace: compareResult.a.trace,
+        passed,
+        failures,
+        judgeVerdicts,
+        compare: {
+            ...compareStoryFields(scenario),
+            aTrace: compareResult.a.trace,
+            bTrace: compareResult.b.trace,
+        },
+    });
+    const scenarioResult = {
+        suite: suiteName,
+        scenario: scenario.name,
+        prompt: scenario.prompt,
+        passed,
+        failures,
+        durationMs,
+        judgeVerdicts,
+        trace: compareResult.a.trace,
+        compare: compareResult,
+        story,
+        ...buildScenarioResultUsage({
+            agentUsage: sumUsageParts([aResult.agentUsage, bResult.agentUsage]),
+            judgeVerdicts,
+        }),
+    };
+    if (!suppressEmit) {
+        const debugBundleDir = await maybeWriteDebugBundle({
+            debug,
+            cwd,
+            suitesDir,
+            rubricsDir,
+            stagingSessionId,
+            debugDir,
+            suiteName,
+            scenario,
+            host,
+            result: scenarioResult,
+            trace: compareResult.a.trace,
+            timeoutMs,
+            worktree,
+            judge,
+            allowUserInput,
+            keepRecordings,
+        });
+        scenarioResult.debugBundleDir = debugBundleDir;
+        emitScenarioVerdict({
+            passed,
+            index: scenarioIndex,
+            total: scenarioTotal,
+            name: scenario.name,
+            durationMs,
+            totalTokens: totalTokensFromScenarioUsage(scenarioResult.usage, compareResult.a.trace?.usage),
+            judgeVerdicts,
+            failures,
+            story,
+            debug,
+            debugBundleDir,
+        });
+    }
+    return scenarioResult;
+}
+async function loadCompareResultFromStaging(stagingSessionId, suiteName, scenario) {
+    const aLabel = compareArmLabel(scenario.compare?.a, "a");
+    const bLabel = compareArmLabel(scenario.compare?.b, "b");
+    const aScenario = applyCompareArm(scenario, "a");
+    const bScenario = applyCompareArm(scenario, "b");
+    const loadArm = async (side) => {
+        try {
+            return await loadStagingTrace(getStagingTracePath(stagingSessionId, suiteName, scenario.name, side));
+        }
+        catch {
+            return undefined;
+        }
+    };
+    return {
+        a: { id: "a", label: aLabel, prompt: aScenario.prompt, trace: await loadArm("a") },
+        b: { id: "b", label: bLabel, prompt: bScenario.prompt, trace: await loadArm("b") },
+    };
+}
+async function runCompareJudgeRubric(compare, rubric, runCwd, host) {
+    const criteria = collectCompareJudgeCriteria(rubric);
+    if (criteria.length === 0 || !compare.a.trace || !compare.b.trace) {
+        return { failures: [], verdicts: [] };
+    }
+    const result = await judgeCompareTraces({
+        aLabel: compare.a.label,
+        a: compare.a.trace,
+        bLabel: compare.b.label,
+        b: compare.b.trace,
+    }, criteria, { cwd: runCwd, host });
+    if (result.skipped) {
+        return {
+            failures: [assertionFailure("judge", result.error ?? "judge skipped", "judge_infra")],
+            verdicts: [],
+        };
+    }
+    const failures = [];
+    for (const verdict of result.verdicts) {
+        if (!verdict.pass) {
+            const category = verdict.infraError
+                ? "judge_infra"
+                : verdict.parseError
+                    ? "judge_parse"
+                    : "rubric_miss";
+            failures.push(assertionFailure(`judge:${verdict.id}`, verdict.rationale, category));
+        }
+    }
+    if (result.error && !result.verdicts.some((verdict) => !verdict.pass)) {
+        failures.push(assertionFailure("judge", result.error, "judge_infra"));
+    }
+    return { failures, verdicts: result.verdicts };
+}
+async function runJudgeRubric(trace, rubric, runCwd, host) {
+    const criteria = collectJudgeCriteria(rubric);
     if (criteria.length === 0) {
         return { trace, failures: [], verdicts: [] };
     }
-    const result = await judgeTrace(trace, criteria, { cwd: runCwd });
+    const result = await judgeTrace(trace, criteria, { cwd: runCwd, host });
     if (result.skipped) {
         return {
             trace,
@@ -820,26 +1277,38 @@ export async function runAllSuites(options) {
             return suiteName === options.filter || suitePath.includes(`/${options.filter}/`);
         })
         : suitePaths;
+    const cliHosts = uniqueHosts(options.hosts ?? (options.host ? [options.host] : undefined));
     const reports = [];
     for (const suitePath of filtered) {
-        reports.push(await runSuite({
-            cwd: options.cwd,
-            suitePath,
-            host: options.host,
-            scenarioFilter: options.scenarioFilter,
-            judge: options.judge,
-            worktree: options.worktree,
-            stagingSessionId: options.stagingSessionId,
-            keepRecordings: options.keepRecordings,
-            suitesDir: options.suitesDir,
-            suiteFilter: options.filter,
-            timeoutMs: options.timeoutMs,
-            allowUserInput: options.allowUserInput,
-            debug: options.debug,
-            debugDir: options.debugDir,
-            scenarioRetries: options.scenarioRetries,
-            rubricsDir: options.rubricsDir,
-        }));
+        const suite = await loadSuiteFile(suitePath, { rubricsDir: options.rubricsDir });
+        const hosts = resolveSuiteHosts({
+            cliHosts,
+            suiteHosts: suite.hosts,
+            defaultHost: suite.defaults?.host,
+        });
+        const hostLocked = hosts.length > 1;
+        for (const host of hosts) {
+            reports.push(await runSuite({
+                cwd: options.cwd,
+                suitePath,
+                host,
+                hostLocked,
+                scenarioFilter: options.scenarioFilter,
+                judge: options.judge,
+                worktree: options.worktree,
+                stagingSessionId: options.stagingSessionId,
+                keepRecordings: options.keepRecordings,
+                suitesDir: options.suitesDir,
+                suiteFilter: options.filter,
+                timeoutMs: options.timeoutMs,
+                allowUserInput: options.allowUserInput,
+                debug: options.debug,
+                debugDir: options.debugDir,
+                scenarioRetries: options.scenarioRetries,
+                rubricsDir: options.rubricsDir,
+                adapterModules: options.adapterModules,
+            }));
+        }
     }
     return reports;
 }