@post-print/agent-test 0.3.12 → 1.0.0

package/dist/run-suite.js

@@ -1,7 +1,7 @@
 import { createRequire } from "node:module";
 import { basename, dirname, join, resolve } from "node:path";
 import { cancelActiveClaudeRun, cancelActiveCursorRun, cancelActiveOpenaiRun, captureWorkingTreeStatus, createSealedWorkspace, defaultSealedOverlayPaths, enrichTrace, filterWorkingTreeLeaks, findWorkingTreeLeak, formatWorkingTreeLeak, getProcessAuthMode, judgeCompareTraces, judgeTrace, loadContext, loadUnifiedDiffPaths, mergeMcpServers, parseScenarioWorkspace, partitionSeedCollateralLeaks, porcelainPathsFromLines, resolveAllowUserSkills, resolveHarnessArtifactIgnoreRoots, restoreWorkingTreePaths, runAgent, setProcessAuthMode, skillInvokeJudgeCriteria, skillPathsFromSetting, sumUsageParts, toolPathsOutsideWorkspace, traceEditsOutsideWorktree, traceHasUserInputTool, } from "@post-print/agent-harness";
-import { applyCompareArm, applySidecarCompareDurations, assertCompareMetrics, attachCompareStoryResults, buildCompareResult, compareArmDescription, compareArmLabel, compareResultArms, compareStoryFields, plainDescription, prefixCompareFailures, resolveCompareArms, resolveCompareMetricPairs, } from "./compare-scenario.js";
+import { applyCompareArm, applySidecarCompareDurations, assertCompareGates, attachCompareStoryResults, buildCompareResult, compareArmDescription, compareArmLabel, compareArmTokens, compareArmTurns, compareResultArms, compareStoryFields, evaluateCompareGates, plainDescription, prefixCompareFailures, requireCompareArm, resolveCompareArms, } from "./compare-scenario.js";
 import { collectDebugEnvironment, getDebugBundleDir, writeDebugBundle } from "./debug-bundle.js";
 import { discoverSuites } from "./discover-suites.js";
 import { assertRubric } from "./expect.js";
@@ -20,6 +20,9 @@ import { buildScenarioResultUsage, totalTokensFromScenarioUsage } from "./scenar
 import { summarizeReportResults } from "./suite-summary.js";
 import { theme } from "./theme.js";
 import { validateSuiteFile } from "./validate-suite.js";
+import { viewerContextFiles } from "./viewer/context-files.js";
+import { emitViewerEvent } from "./viewer/emit.js";
+import { DEFAULT_CLI_WORKERS, runWorkerPool } from "./worker-pool.js";
 const require = createRequire(import.meta.url);
 const packageVersion = require("../package.json").version;
 let activeWorktreeCleanup;
@@ -142,7 +145,8 @@ export function scenarioNeedsJudge(judge, scenario) {
         return false;
     }
     if (scenario.compare) {
-        return collectCompareJudgeCriteria(scenario.rubric).length > 0;
+        return (collectCompareJudgeCriteria(scenario.rubric).length > 0 ||
+            (scenario.compare.judgeMetrics?.length ?? 0) > 0);
     }
     return collectJudgeCriteria(scenario.rubric).length > 0;
 }
@@ -330,26 +334,26 @@ async function runSuiteBody(options) {
     const parentCounters = parentScenarioCounters();
     const displayTotal = parentCounters?.total ?? filteredTotal;
     const isolateLive = liveScenarioIsolationEnabled() && !options.scenarioFilter && filteredTotal > 1;
-    let previousIsolatedExitCode;
+    const workers = Math.max(1, options.workers ?? DEFAULT_CLI_WORKERS);
     if (shouldPrintSuiteChrome()) {
         logProgress(`\n${theme.suiteHeader(suite.name, defaultHost, displayTotal)}`);
         if (isolateLive) {
             logProgress(`    ${theme.isolationNote()}`);
         }
     }
-    for (let index = 0; index < scenarios.length; index++) {
-        const scenario = scenarios[index];
-        if (!scenario) {
-            continue;
-        }
-        const scenarioIndex = parentCounters?.index ?? index + 1;
-        const scenarioTotal = displayTotal;
-        if (isolateLive) {
+    if (isolateLive) {
+        const isolateSlots = new Array(scenarios.length);
+        await runWorkerPool(scenarios.map((scenario, index) => ({ scenario, index })), workers, async ({ scenario, index }) => {
+            if (!scenario) {
+                return;
+            }
+            const scenarioIndex = parentCounters?.index ?? index + 1;
+            const scenarioTotal = displayTotal;
             if (scenario.skip) {
                 const skipLabel = `[${scenarioIndex}/${scenarioTotal}] ${scenario.name}`;
                 logProgress(theme.skipped(skipLabel));
                 logScenarioDescription(scenario.description);
-                results.push({
+                isolateSlots[index] = {
                     suite: suite.name,
                     scenario: scenario.name,
                     description: plainDescription(scenario.description),
@@ -364,8 +368,8 @@ async function runSuiteBody(options) {
                         skipped: true,
                         failures: [],
                     }),
-                });
-                continue;
+                };
+                return;
             }
             const started = performance.now();
             const debug = isDebugEnabled(options);
@@ -375,7 +379,7 @@ async function runSuiteBody(options) {
             let attempts = 0;
             let failures = [];
             let scenarioTrace;
-            let previousAttemptExitCode = previousIsolatedExitCode;
+            let previousAttemptExitCode;
             let childSidecar;
             while (true) {
                 attempts++;
@@ -403,7 +407,6 @@ async function runSuiteBody(options) {
                     authMode: options.authMode ?? getProcessAuthMode(),
                 });
                 previousAttemptExitCode = spawned.exitCode;
-                previousIsolatedExitCode = spawned.exitCode;
                 failures = [];
                 scenarioTrace = undefined;
                 childSidecar =
@@ -451,7 +454,32 @@ async function runSuiteBody(options) {
                 compareResult = applySidecarCompareDurations(await loadCompareResultFromStaging(options.stagingSessionId, suite.name, scenario), childSidecar);
                 scenarioTrace = compareResultArms(compareResult)[0]?.trace ?? scenarioTrace;
             }
-            if (failures.length === 0 && options.judge !== false) {
+            if (compareResult && options.judge !== false && scenario.compare?.judgeMetrics?.length) {
+                for (const arm of compareResult.arms) {
+                    if (!arm.trace)
+                        continue;
+                    try {
+                        const judged = await runJudgeRubric(arm.trace, { judge: scenario.compare.judgeMetrics }, options.cwd, judgeHost);
+                        arm.trace = judged.trace;
+                        arm.judgeVerdicts = toJudgeVerdictResults(judged.trace, scenario.compare.judgeMetrics, judged.verdicts);
+                        arm.failures = [...(arm.failures ?? []), ...judged.failures];
+                        failures.push(...prefixCompareFailures(arm.label, judged.failures.filter((failure) => failure.category !== "rubric_miss")));
+                    }
+                    catch (error) {
+                        failures.push(assertionFailure("judge", error instanceof Error ? error.message : "failed to judge compare arm", "judge_infra"));
+                    }
+                }
+            }
+            if (compareResult) {
+                compareResult.gateResults = evaluateCompareGates(compareResult.gates, compareResult);
+                if (failures.every((failure) => failure.category === "rubric_miss")) {
+                    failures.push(...assertCompareGates(compareResult.gates, compareResult));
+                }
+            }
+            if ((options.judge !== false && failures.length === 0) ||
+                (options.judge !== false &&
+                    scenario.compare &&
+                    failures.every((failure) => failure.category === "rubric_miss"))) {
                 const compareTraces = compareResult ? compareResultArms(compareResult) : [];
                 if (scenario.compare &&
                     compareResult &&
@@ -492,7 +520,9 @@ async function runSuiteBody(options) {
                             judgeVerdicts = toJudgeVerdictResults(judged.trace, criteria, judged.verdicts);
                         }
                         catch (error) {
-                            failures.push(assertionFailure("judge", error instanceof Error ? error.message : "failed to load staging trace for judge", "judge_infra"));
+                            failures.push(assertionFailure("judge", error instanceof Error
+                                ? error.message
+                                : "failed to load staging trace for judge", "judge_infra"));
                         }
                     }
                 }
@@ -563,31 +593,47 @@ async function runSuiteBody(options) {
                 debug,
                 debugBundleDir,
             });
-            results.push(scenarioResult);
+            isolateSlots[index] = scenarioResult;
+            releaseLiveMemory();
+            return;
+        });
+        for (const result of isolateSlots) {
+            if (result) {
+                results.push(result);
+            }
+        }
+    }
+    else {
+        for (let index = 0; index < scenarios.length; index++) {
+            const scenario = scenarios[index];
+            if (!scenario) {
+                continue;
+            }
+            const scenarioIndex = parentCounters?.index ?? index + 1;
+            const scenarioTotal = displayTotal;
+            results.push(await runAgentTest({
+                cwd: options.cwd,
+                suiteName: suite.name,
+                scenario,
+                defaults: suite.defaults,
+                host: options.host,
+                judge: options.judge,
+                worktree: options.worktree,
+                stagingSessionId: options.stagingSessionId,
+                scenarioIndex,
+                scenarioTotal,
+                timeoutMs: options.timeoutMs,
+                allowUserInput: options.allowUserInput,
+                debug: options.debug,
+                debugDir: options.debugDir,
+                suitesDir: options.suitesDir,
+                keepRecordings: options.keepRecordings,
+                rubricsDir: options.rubricsDir,
+                scenarioRetries: isChildProcess() ? 0 : options.scenarioRetries,
+                compareArm: options.compareArm,
+            }));
             releaseLiveMemory();
-            continue;
         }
-        results.push(await runAgentTest({
-            cwd: options.cwd,
-            suiteName: suite.name,
-            scenario,
-            defaults: suite.defaults,
-            host: options.host,
-            judge: options.judge,
-            worktree: options.worktree,
-            stagingSessionId: options.stagingSessionId,
-            scenarioIndex,
-            scenarioTotal,
-            timeoutMs: options.timeoutMs,
-            allowUserInput: options.allowUserInput,
-            debug: options.debug,
-            debugDir: options.debugDir,
-            suitesDir: options.suitesDir,
-            keepRecordings: options.keepRecordings,
-            rubricsDir: options.rubricsDir,
-            scenarioRetries: isChildProcess() ? 0 : options.scenarioRetries,
-        }));
-        releaseLiveMemory();
     }
     return {
         suite: suite.name,
@@ -690,7 +736,7 @@ async function runAgentTestBody(options) {
     let result;
     while (true) {
         attempts++;
-        result = await runAgentTestOnce(options.cwd, suiteName, options.scenario, defaultHost, options.defaults?.profile, options.defaults?.skills, options.defaults?.contextSources, options.defaults?.mcpServers, options.defaults?.workspace, options.defaults?.allowUserSkills, options.judge ?? true, options.worktree ?? true, options.stagingSessionId, options.scenarioIndex, options.scenarioTotal, options.timeoutMs, options.allowUserInput, options.debug, options.debugDir, options.suitesDir ?? "agent-suites", options.keepRecordings, options.rubricsDir, { suppressEmit: maxAttempts > 1 });
+        result = await runAgentTestOnce(options.cwd, suiteName, options.scenario, defaultHost, options.defaults?.profile, options.defaults?.skills, options.defaults?.contextSources, options.defaults?.mcpServers, options.defaults?.workspace, options.defaults?.allowUserSkills, options.judge ?? true, options.worktree ?? true, options.stagingSessionId, options.scenarioIndex, options.scenarioTotal, options.timeoutMs, options.allowUserInput, options.debug, options.debugDir, options.suitesDir ?? "agent-suites", options.keepRecordings, options.rubricsDir, { suppressEmit: maxAttempts > 1, compareArm: options.compareArm });
         const canRetry = !result.skipped &&
             !result.passed &&
             attempts < maxAttempts &&
@@ -769,6 +815,9 @@ async function runAgentTestOnce(cwd, suiteName, scenario, defaultHost, defaultPr
     if (scenario.compare && runOptions?.compareArm === undefined) {
         return runCompareAgentTestOnce(cwd, suiteName, scenario, defaultHost, defaultProfile, defaultSkills, defaultContextSources, defaultMcpServers, defaultWorkspace, defaultAllowUserSkills, judge, worktree, stagingSessionId, scenarioIndex, scenarioTotal, timeoutMs, allowUserInput, debugFlag, debugDir, suitesDir, keepRecordings, rubricsDir, { suppressEmit });
     }
+    if (runOptions?.compareArm && scenario.compare) {
+        scenario = requireCompareArm(scenario, runOptions.compareArm);
+    }
     const host = scenario.host ?? defaultHost;
     const profile = scenario.profile ?? defaultProfile ?? defaultProfileForHost(host);
     const skills = scenario.skills ?? defaultSkills;
@@ -781,6 +830,12 @@ async function runAgentTestOnce(cwd, suiteName, scenario, defaultHost, defaultPr
         : mergeMcpServers(defaultMcpServers, scenario.mcpServers);
     const liveTimeoutMs = resolveLiveTimeoutMs(timeoutMs);
     const failOnUserInput = !allowUserInput;
+    const viewerEnvelope = {
+        suite: suiteName,
+        scenario: scenario.name,
+        host,
+        ...(runOptions?.compareArm ? { arm: runOptions.compareArm } : {}),
+    };
     if (runOptions?.compareArm) {
         logPhase(theme.phase("arm", `${runOptions.compareArm.toUpperCase()} ${host}`));
     }
@@ -797,6 +852,7 @@ async function runAgentTestOnce(cwd, suiteName, scenario, defaultHost, defaultPr
     let callerHeadBefore;
     const callerTreeBefore = useWorktree ? await captureWorkingTreeStatus(cwd) : undefined;
     if (useWorktree) {
+        emitViewerEvent({ type: "status", text: "Creating sealed workspace.", ...viewerEnvelope });
         if (scenario.seedPatch) {
             callerHeadBefore = await captureCallerHead(cwd);
             setCallerHeadRestore(cwd, callerHeadBefore);
@@ -811,9 +867,15 @@ async function runAgentTestOnce(cwd, suiteName, scenario, defaultHost, defaultPr
         activeWorktreeCleanup = worktreeHandle.cleanup;
         if (scenario.seedPatch) {
             logPhase(theme.phase("seed", theme.basename(scenario.seedPatch)));
+            emitViewerEvent({
+                type: "status",
+                text: `Applying seed ${basename(scenario.seedPatch)}.`,
+                ...viewerEnvelope,
+            });
             await seedScenarioWorktree(cwd, worktreeHandle.path, scenario.seedPatch, {
                 stageOnly: scenario.seedStageOnly === true,
             });
+            emitViewerEvent({ type: "status", text: "Seed applied.", ...viewerEnvelope });
         }
     }
     else {
@@ -829,8 +891,20 @@ async function runAgentTestOnce(cwd, suiteName, scenario, defaultHost, defaultPr
         });
         const outputContract = outputContractForRubric(scenario.rubric);
         const agentStartMarkerPath = isChildProcess() && stagingSessionId
-            ? getStagingAgentStartPath(stagingSessionId, suiteName, scenario.name)
+            ? getStagingAgentStartPath(stagingSessionId, suiteName, scenario.name, runOptions?.compareArm)
             : undefined;
+        emitViewerEvent({ type: "cell_started", ...viewerEnvelope });
+        emitViewerEvent({ type: "status", text: "Starting host agent.", ...viewerEnvelope });
+        const contextFiles = viewerContextFiles(context.preamble, context.sources, contextSources ?? []);
+        if (contextFiles.length > 0) {
+            emitViewerEvent({
+                type: "status",
+                text: contextFiles.map((file) => file.why).join(" "),
+                ...viewerEnvelope,
+            });
+            emitViewerEvent({ type: "context", files: contextFiles, ...viewerEnvelope });
+        }
+        emitViewerEvent({ type: "prompt", text: scenario.prompt, ...viewerEnvelope });
         logPhase(theme.phase("agent", theme.phaseDim("started")));
         const agentStarted = performance.now();
         let livePreview;
@@ -852,11 +926,18 @@ async function runAgentTestOnce(cwd, suiteName, scenario, defaultHost, defaultPr
                 : undefined,
             onAgentEvent: (event) => {
                 if (event.type === "tool") {
+                    emitViewerEvent({
+                        type: "tool",
+                        name: event.name,
+                        args: event.args,
+                        ...viewerEnvelope,
+                    });
                     logLive(theme.liveTool(event.name, pathFromArgs(event.args)));
                     livePreview = undefined;
                     refreshHeartbeat();
                     return;
                 }
+                emitViewerEvent({ type: "text", text: event.text, ...viewerEnvelope });
                 // Clock tick paints the preview. Do not rewrite here — a long
                 // line wraps and `\r` cannot clear the leftover row.
                 livePreview = quoteExcerpt(event.text);
@@ -953,7 +1034,7 @@ async function runAgentTestOnce(cwd, suiteName, scenario, defaultHost, defaultPr
         }
         const durationMs = Math.round(performance.now() - started);
         if (isChildProcess() && stagingSessionId && runOptions?.writeSidecar !== false) {
-            await writeStagingResult(getStagingResultPath(stagingSessionId, suiteName, scenario.name), {
+            await writeStagingResult(getStagingResultPath(stagingSessionId, suiteName, scenario.name, runOptions?.compareArm), {
                 passed: failures.length === 0,
                 failures,
                 durationMs,
@@ -993,6 +1074,42 @@ async function runAgentTestOnce(cwd, suiteName, scenario, defaultHost, defaultPr
                 judgeVerdicts,
             }),
         };
+        const armMetrics = {
+            id: runOptions?.compareArm ?? "cell",
+            label: runOptions?.compareArm ?? "cell",
+            prompt: scenario.prompt,
+            trace,
+        };
+        const turns = compareArmTurns(armMetrics);
+        const tokens = compareArmTokens(armMetrics);
+        const tools = trace.toolCalls.length;
+        emitViewerEvent({
+            type: "cell_finished",
+            ...viewerEnvelope,
+            passed,
+            durationMs,
+            failures: failures.map((failure) => ({
+                matcher: failure.matcher,
+                message: failure.message,
+            })),
+            metrics: {
+                ...(turns !== undefined ? { turns } : {}),
+                ...(tokens !== undefined ? { tokens } : {}),
+                tools,
+            },
+        });
+        if (judgeVerdicts && judgeVerdicts.length > 0) {
+            emitViewerEvent({
+                type: "judge",
+                ...viewerEnvelope,
+                verdicts: judgeVerdicts.map((verdict) => ({
+                    id: verdict.id,
+                    question: verdict.question,
+                    pass: verdict.pass,
+                    rationale: verdict.rationale,
+                })),
+            });
+        }
         if (!suppressEmit) {
             const debugBundleDir = await maybeWriteDebugBundle({
                 debug,
@@ -1090,8 +1207,10 @@ async function runCompareAgentTestOnce(cwd, suiteName, scenario, defaultHost, de
         const result = await runAgentTestOnce(shared.cwd, shared.suiteName, armScenario, shared.defaultHost, shared.defaultProfile, shared.defaultSkills, shared.defaultContextSources, shared.defaultMcpServers, shared.defaultWorkspace, shared.defaultAllowUserSkills, false, shared.worktree, shared.stagingSessionId, undefined, undefined, shared.timeoutMs, shared.allowUserInput, shared.debugFlag, shared.debugDir, shared.suitesDir, shared.keepRecordings, shared.rubricsDir, { suppressEmit: true, compareArm: entry.id, writeSidecar: false });
         armRuns.push({ id: entry.id, label, scenario: armScenario, result });
     }
-    const failures = armRuns.flatMap((arm) => prefixCompareFailures(arm.label, arm.result.failures));
-    const armIds = resolvedArms.map((entry) => entry.id);
+    const experimentMode = (scenario.compare?.gates?.length ?? 0) > 0;
+    const failures = armRuns.flatMap((arm) => prefixCompareFailures(arm.label, experimentMode
+        ? arm.result.failures.filter((failure) => failure.category !== "rubric_miss")
+        : arm.result.failures));
     const compareResult = buildCompareResult(armRuns.map((arm) => ({
         id: arm.id,
         label: arm.label,
@@ -1099,12 +1218,25 @@ async function runCompareAgentTestOnce(cwd, suiteName, scenario, defaultHost, de
         prompt: arm.scenario.prompt,
         trace: arm.result.trace,
         durationMs: arm.result.durationMs,
-    })), {
-        faster: resolveCompareMetricPairs(scenario.compare?.faster, armIds),
-        cheaper: resolveCompareMetricPairs(scenario.compare?.cheaper, armIds),
-    });
-    if (scenario.compare) {
-        failures.push(...assertCompareMetrics(scenario.compare, compareResult));
+        passed: !arm.result.failures.some((failure) => failure.category === "rubric_miss"),
+        failures: arm.result.failures,
+    })), scenario.compare?.gates);
+    if (judge && !isChildProcess() && scenario.compare?.judgeMetrics?.length) {
+        for (const arm of compareResult.arms) {
+            if (!arm.trace)
+                continue;
+            const judged = await runJudgeRubric(arm.trace, { judge: scenario.compare.judgeMetrics }, cwd, host);
+            arm.trace = judged.trace;
+            arm.judgeVerdicts = toJudgeVerdictResults(judged.trace, scenario.compare.judgeMetrics, judged.verdicts);
+            arm.failures = [...(arm.failures ?? []), ...judged.failures];
+            failures.push(...prefixCompareFailures(arm.label, judged.failures.filter((failure) => failure.category !== "rubric_miss")));
+        }
+    }
+    if (!isChildProcess()) {
+        compareResult.gateResults = evaluateCompareGates(compareResult.gates, compareResult);
+        if (failures.every((failure) => failure.category === "rubric_miss")) {
+            failures.push(...assertCompareGates(compareResult.gates, compareResult));
+        }
     }
     const deferJudgeToParent = isChildProcess();
     const compareArms = compareResultArms(compareResult);
@@ -1112,7 +1244,7 @@ async function runCompareAgentTestOnce(cwd, suiteName, scenario, defaultHost, de
     let judgeVerdicts;
     if (judge &&
         !deferJudgeToParent &&
-        failures.length === 0 &&
+        failures.every((failure) => failure.category === "rubric_miss") &&
         compareArms.length > 0 &&
         compareArms.every((arm) => arm.trace)) {
         const criteria = collectCompareJudgeCriteria(scenario.rubric);
@@ -1207,7 +1339,6 @@ async function runCompareAgentTestOnce(cwd, suiteName, scenario, defaultHost, de
 }
 async function loadCompareResultFromStaging(stagingSessionId, suiteName, scenario) {
     const resolved = resolveCompareArms(scenario.compare);
-    const armIds = resolved.map((entry) => entry.id);
     const arms = [];
     for (const entry of resolved) {
         const armScenario = applyCompareArm(scenario, entry.id);
@@ -1224,12 +1355,11 @@ async function loadCompareResultFromStaging(stagingSessionId, suiteName, scenari
             description: compareArmDescription(entry.arm),
             prompt: armScenario.prompt,
             trace,
+            passed: trace ? assertRubric(trace, armScenario.rubric).length === 0 : undefined,
+            failures: trace ? assertRubric(trace, armScenario.rubric) : undefined,
         });
     }
-    return buildCompareResult(arms, {
-        faster: resolveCompareMetricPairs(scenario.compare?.faster, armIds),
-        cheaper: resolveCompareMetricPairs(scenario.compare?.cheaper, armIds),
-    });
+    return buildCompareResult(arms, scenario.compare?.gates);
 }
 async function runCompareJudgeRubric(compare, rubric, runCwd, host) {
     const criteria = collectCompareJudgeCriteria(rubric);
@@ -1358,6 +1488,8 @@ async function runAllSuitesBody(options) {
                 rubricsDir: options.rubricsDir,
                 adapterModules: options.adapterModules,
                 authMode: options.authMode ?? getProcessAuthMode(),
+                compareArm: options.compareArm,
+                workers: options.workers,
             }));
         }
     }