@post-print/agent-test 1.1.1 → 1.1.3

package/dist/run-suite.js

@@ -1,7 +1,7 @@
 import { createRequire } from "node:module";
 import { basename, dirname, join, resolve } from "node:path";
 import { buildHostPrompt, cancelActiveClaudeRun, cancelActiveCursorRun, cancelActiveOpenaiRun, captureWorkingTreeStatus, createSealedWorkspace, defaultSealedOverlayPaths, enrichTrace, filterWorkingTreeLeaks, findWorkingTreeLeak, formatWorkingTreeLeak, getProcessAuthMode, isBuiltinAgentHost, judgeCompareTraces, judgeTrace, loadContext, loadUnifiedDiffPaths, mergeMcpServers, parseScenarioWorkspace, partitionSeedCollateralLeaks, porcelainPathsFromLines, resolveAllowUserSkills, resolveHarnessArtifactIgnoreRoots, restoreWorkingTreePaths, runAgent, setProcessAuthMode, skillInvokeJudgeCriteria, skillPathsFromSetting, sumUsageParts, toolPathsOutsideWorkspace, traceEditsOutsideWorktree, traceHasUserInputTool, } from "@post-print/agent-harness";
-import { applyCompareArm, applySidecarCompareDurations, assertCompareGates, attachCompareStoryResults, buildCompareResult, compareArmDescription, compareArmLabel, compareArmTokens, compareArmTurns, compareResultArms, compareStoryFields, evaluateCompareGates, plainDescription, prefixCompareFailures, requireCompareArm, resolveCompareArms, } from "./compare-scenario.js";
+import { applyCompareArm, applySidecarCompareDurations, assertCompareGates, attachCompareStoryResults, buildCompareResult, compareArmDescription, compareArmLabel, compareArmTokens, compareArmTurns, compareResultArms, compareStoryFields, evaluateCompareGates, finalizeCompareGates, finalizeCompareOutcome, plainDescription, prefixCompareFailures, requireCompareArm, resolveCompareArms, } from "./compare-scenario.js";
 import { collectDebugEnvironment, getDebugBundleDir, writeDebugBundle } from "./debug-bundle.js";
 import { discoverSuites } from "./discover-suites.js";
 import { assertRubric } from "./expect.js";
@@ -13,10 +13,11 @@ import { loadSuiteFile } from "./load-suite.js";
 import { bindMcpServersToCaller, mcpStdioScriptPaths } from "./mcp-config.js";
 import { formatDuration, logLive, logPhase, logProgress, logVerdict, refreshHeartbeat, withHeartbeat, } from "./progress.js";
 import { cleanupStagingSession, createLiveStagingSessionId, getLiveStagingRootOverride, getLiveStagingSessionRoot, getStagingAgentStartPath, getStagingResultPath, getStagingTracePath, loadStagingResult, loadStagingTrace, recordTrace, resolveRecordingPath, setLiveStagingRootOverride, writeAgentStartMarker, writeStagingResult, } from "./record-trace.js";
+import { finalizeScenarioResult } from "./scenario-finalizer.js";
 import { resolveScenarioRetryMaxAttempts, shouldRetryAnnounceStopFlake } from "./scenario-retry.js";
 import { captureCallerHead, restoreCallerHeadIfSeedCommit, seedScenarioWorktree, } from "./scenario-seed.js";
 import { buildScenarioStory, pathFromArgs, quoteExcerpt } from "./scenario-story.js";
-import { buildScenarioResultUsage, totalTokensFromScenarioUsage } from "./scenario-usage.js";
+import { totalTokensFromScenarioUsage } from "./scenario-usage.js";
 import { summarizeReportResults } from "./suite-summary.js";
 import { theme } from "./theme.js";
 import { validateSuiteFile } from "./validate-suite.js";
@@ -190,6 +191,9 @@ function toJudgeVerdictResults(trace, criteria, verdictsFromJudge) {
             question: questionForCriterion(criteria, extended.id),
             pass: extended.pass,
             rationale: extended.rationale,
+            evidence: extended.evidence,
+            prompt: extended.prompt,
+            response: extended.response,
             infraError: extended.infraError,
             parseError: extended.parseError,
             rawSdkStatus: extended.rawSdkStatus,
@@ -529,44 +533,29 @@ async function runSuiteBody(options) {
                 }
             }
             const durationMs = Math.round(performance.now() - started);
-            const passed = failures.length === 0;
-            const usageFields = buildScenarioResultUsage({
-                agentUsage: compareResult
-                    ? sumUsageParts(compareResultArms(compareResult).map((arm) => arm.trace?.usage))
-                    : scenarioTrace?.usage,
-                judgeVerdicts,
-            });
-            const story = buildScenarioStory({
-                rubric: scenario.rubric,
-                trace: scenarioTrace,
-                passed,
-                failures,
-                judgeVerdicts,
-                compare: compareResult
-                    ? attachCompareStoryResults(compareStoryFields(scenario), compareResult)
-                    : undefined,
-            });
-            const scenarioResult = {
+            const scenarioResult = finalizeScenarioResult({
                 suite: suite.name,
-                scenario: scenario.name,
-                description: plainDescription(scenario.description),
-                prompt: scenario.prompt,
+                scenario,
                 contextMode: childSidecar?.contextMode ??
                     scenario.contextMode ??
                     suite.defaults?.contextMode ??
                     "harness-preamble",
                 contextFiles: childSidecar?.contextFiles,
                 hostInput: childSidecar?.hostInput,
-                passed,
                 failures,
                 durationMs,
                 attempts,
                 judgeVerdicts,
                 trace: scenarioTrace,
                 compare: compareResult,
-                story,
-                ...usageFields,
-            };
+                storyCompare: compareResult
+                    ? attachCompareStoryResults(compareStoryFields(scenario), compareResult)
+                    : undefined,
+                agentUsage: compareResult
+                    ? sumUsageParts(compareResultArms(compareResult).map((arm) => arm.trace?.usage))
+                    : scenarioTrace?.usage,
+            });
+            const { passed, story } = scenarioResult;
             const debugBundleDir = await maybeWriteDebugBundle({
                 debug,
                 cwd: options.cwd,
@@ -937,6 +926,7 @@ async function runAgentTestOnce(cwd, suiteName, scenario, defaultHost, defaultCo
             outputContract,
             mcpServers,
             allowUserSkills,
+            networkAccess: scenario.networkAccess === true,
             authMode: getProcessAuthMode(),
             timeoutMs: liveTimeoutMs,
             failOnUserInput,
@@ -1073,33 +1063,19 @@ async function runAgentTestOnce(cwd, suiteName, scenario, defaultHost, defaultCo
             }
             worktreeHandle = undefined;
         }
-        const passed = failures.length === 0;
-        const story = buildScenarioStory({
-            rubric: scenario.rubric,
-            trace,
-            passed,
-            failures,
-            judgeVerdicts,
-        });
-        const scenarioResult = {
+        const scenarioResult = finalizeScenarioResult({
             suite: suiteName,
-            scenario: scenario.name,
-            description: plainDescription(scenario.description),
-            prompt: scenario.prompt,
+            scenario,
             contextMode,
             contextFiles,
             hostInput,
-            passed,
             failures,
             durationMs,
             judgeVerdicts,
             trace,
-            story,
-            ...buildScenarioResultUsage({
-                agentUsage: session.usage ?? trace?.usage,
-                judgeVerdicts,
-            }),
-        };
+            agentUsage: session.usage ?? trace?.usage,
+        });
+        const { passed, story } = scenarioResult;
         const armMetrics = {
             id: runOptions?.compareArm ?? "cell",
             label: runOptions?.compareArm ?? "cell",
@@ -1157,6 +1133,11 @@ async function runAgentTestOnce(cwd, suiteName, scenario, defaultHost, defaultCo
                 authMode: getProcessAuthMode(),
             });
             scenarioResult.debugBundleDir = debugBundleDir;
+            emitViewerEvent({
+                type: "scenario_result",
+                ...viewerEnvelope,
+                result: scenarioResult,
+            });
             emitScenarioVerdict({
                 passed,
                 index: scenarioIndex,
@@ -1234,60 +1215,18 @@ async function runCompareAgentTestOnce(cwd, suiteName, scenario, defaultHost, de
         const result = await runAgentTestOnce(shared.cwd, shared.suiteName, armScenario, shared.defaultHost, shared.defaultContextMode, shared.defaultProfile, shared.defaultSkills, shared.defaultContextSources, shared.defaultMcpServers, shared.defaultWorkspace, shared.defaultAllowUserSkills, false, shared.worktree, shared.stagingSessionId, undefined, undefined, shared.timeoutMs, shared.allowUserInput, shared.debugFlag, shared.debugDir, shared.suitesDir, shared.keepRecordings, shared.rubricsDir, { suppressEmit: true, compareArm: entry.id, writeSidecar: false });
         armRuns.push({ id: entry.id, label, scenario: armScenario, result });
     }
-    const experimentMode = (scenario.compare?.gates?.length ?? 0) > 0;
-    const failures = armRuns.flatMap((arm) => prefixCompareFailures(arm.label, experimentMode
-        ? arm.result.failures.filter((failure) => failure.category !== "rubric_miss")
-        : arm.result.failures));
-    const compareResult = buildCompareResult(armRuns.map((arm) => ({
-        id: arm.id,
-        label: arm.label,
-        description: compareArmDescription(resolvedArms.find((entry) => entry.id === arm.id)?.arm),
-        prompt: arm.scenario.prompt,
-        trace: arm.result.trace,
-        contextMode: arm.result.contextMode,
-        contextFiles: arm.result.contextFiles,
-        hostInput: arm.result.hostInput,
-        durationMs: arm.result.durationMs,
-        passed: !arm.result.failures.some((failure) => failure.category === "rubric_miss"),
-        failures: arm.result.failures,
-    })), scenario.compare?.gates);
-    if (judge && !isChildProcess() && scenario.compare?.judgeMetrics?.length) {
-        for (const arm of compareResult.arms) {
-            if (!arm.trace)
-                continue;
-            const judged = await runJudgeRubric(arm.trace, { judge: scenario.compare.judgeMetrics }, cwd, host);
-            arm.trace = judged.trace;
-            arm.judgeVerdicts = toJudgeVerdictResults(judged.trace, scenario.compare.judgeMetrics, judged.verdicts);
-            arm.failures = [...(arm.failures ?? []), ...judged.failures];
-            failures.push(...prefixCompareFailures(arm.label, judged.failures.filter((failure) => failure.category !== "rubric_miss")));
-        }
-    }
-    if (!isChildProcess()) {
-        compareResult.gateResults = evaluateCompareGates(compareResult.gates, compareResult);
-        if (failures.every((failure) => failure.category === "rubric_miss")) {
-            failures.push(...assertCompareGates(compareResult.gates, compareResult));
-        }
-    }
-    const deferJudgeToParent = isChildProcess();
-    const compareArms = compareResultArms(compareResult);
-    const firstTrace = compareArms[0]?.trace;
-    let judgeVerdicts;
-    if (judge &&
-        !deferJudgeToParent &&
-        failures.every((failure) => failure.category === "rubric_miss") &&
-        compareArms.length > 0 &&
-        compareArms.every((arm) => arm.trace)) {
-        const criteria = collectCompareJudgeCriteria(scenario.rubric);
-        if (criteria.length > 0) {
-            logPhase(theme.judgePhase(criteria.length), { last: true });
-        }
-        const judged = await runCompareJudgeRubric(compareResult, scenario.rubric, cwd, host);
-        failures.push(...judged.failures);
-        if (firstTrace) {
-            judgeVerdicts = toJudgeVerdictResults({ ...firstTrace, judgeVerdicts: judged.verdicts }, criteria, judged.verdicts);
-        }
-    }
     const durationMs = Math.round(performance.now() - started);
+    const scenarioResult = await finalizeCompareArmRuns({
+        cwd,
+        suite: suiteName,
+        scenario,
+        host,
+        armRuns,
+        durationMs,
+        judge: Boolean(judge && !isChildProcess()),
+        evaluateGates: !isChildProcess(),
+    });
+    const { passed, story, failures, trace: firstTrace } = scenarioResult;
     if (isChildProcess() && stagingSessionId) {
         const sidecarArms = {};
         for (const arm of armRuns) {
@@ -1309,32 +1248,6 @@ async function runCompareAgentTestOnce(cwd, suiteName, scenario, defaultHost, de
             },
         });
     }
-    const passed = failures.length === 0;
-    const story = buildScenarioStory({
-        rubric: scenario.rubric,
-        trace: firstTrace,
-        passed,
-        failures,
-        judgeVerdicts,
-        compare: attachCompareStoryResults(compareStoryFields(scenario), compareResult),
-    });
-    const scenarioResult = {
-        suite: suiteName,
-        scenario: scenario.name,
-        description: plainDescription(scenario.description),
-        prompt: scenario.prompt,
-        passed,
-        failures,
-        durationMs,
-        judgeVerdicts,
-        trace: firstTrace,
-        compare: compareResult,
-        story,
-        ...buildScenarioResultUsage({
-            agentUsage: sumUsageParts(armRuns.map((arm) => arm.result.agentUsage)),
-            judgeVerdicts,
-        }),
-    };
     if (!suppressEmit) {
         const debugBundleDir = await maybeWriteDebugBundle({
             debug,
@@ -1356,6 +1269,13 @@ async function runCompareAgentTestOnce(cwd, suiteName, scenario, defaultHost, de
             authMode: getProcessAuthMode(),
         });
         scenarioResult.debugBundleDir = debugBundleDir;
+        emitViewerEvent({
+            type: "scenario_result",
+            suite: suiteName,
+            scenario: scenario.name,
+            host,
+            result: scenarioResult,
+        });
         emitScenarioVerdict({
             passed,
             index: scenarioIndex,
@@ -1363,7 +1283,7 @@ async function runCompareAgentTestOnce(cwd, suiteName, scenario, defaultHost, de
             name: scenario.name,
             durationMs,
             totalTokens: totalTokensFromScenarioUsage(scenarioResult.usage, firstTrace?.usage),
-            judgeVerdicts,
+            judgeVerdicts: scenarioResult.judgeVerdicts,
             failures,
             story,
             debug,
@@ -1372,6 +1292,107 @@ async function runCompareAgentTestOnce(cwd, suiteName, scenario, defaultHost, de
     }
     return scenarioResult;
 }
+async function finalizeCompareArmRuns(options) {
+    const resolvedArms = resolveCompareArms(options.scenario.compare);
+    const finalized = finalizeCompareOutcome(options.armRuns.map((arm) => ({
+        id: arm.id,
+        label: arm.label,
+        description: compareArmDescription(resolvedArms.find((entry) => entry.id === arm.id)?.arm),
+        prompt: arm.scenario.prompt,
+        trace: arm.result.trace,
+        contextMode: arm.result.contextMode,
+        contextFiles: arm.result.contextFiles,
+        hostInput: arm.result.hostInput,
+        durationMs: arm.result.durationMs,
+        passed: !arm.result.failures.some((failure) => failure.category === "rubric_miss"),
+        failures: arm.result.failures,
+    })), options.scenario.compare?.gates, { evaluateGates: false });
+    const compareResult = finalized.compare;
+    const failures = finalized.failures;
+    if (options.judge && options.scenario.compare?.judgeMetrics?.length) {
+        for (const arm of compareResult.arms) {
+            if (!arm.trace)
+                continue;
+            const judged = await runJudgeRubric(arm.trace, { judge: options.scenario.compare.judgeMetrics }, options.cwd, options.host);
+            arm.trace = judged.trace;
+            arm.judgeVerdicts = toJudgeVerdictResults(judged.trace, options.scenario.compare.judgeMetrics, judged.verdicts);
+            arm.failures = [...(arm.failures ?? []), ...judged.failures];
+            failures.push(...prefixCompareFailures(arm.label, judged.failures.filter((failure) => failure.category !== "rubric_miss")));
+        }
+    }
+    if (options.evaluateGates)
+        finalizeCompareGates(compareResult, failures);
+    const compareArms = compareResultArms(compareResult);
+    const firstTrace = compareArms[0]?.trace;
+    let judgeVerdicts;
+    if (options.judge &&
+        failures.every((failure) => failure.category === "rubric_miss") &&
+        compareArms.length > 0 &&
+        compareArms.every((arm) => arm.trace)) {
+        const criteria = collectCompareJudgeCriteria(options.scenario.rubric);
+        if (criteria.length > 0)
+            logPhase(theme.judgePhase(criteria.length), { last: true });
+        const judged = await runCompareJudgeRubric(compareResult, options.scenario.rubric, options.cwd, options.host);
+        failures.push(...judged.failures);
+        if (firstTrace) {
+            judgeVerdicts = toJudgeVerdictResults({ ...firstTrace, judgeVerdicts: judged.verdicts }, criteria, judged.verdicts);
+        }
+    }
+    return finalizeScenarioResult({
+        suite: options.suite,
+        scenario: options.scenario,
+        failures,
+        durationMs: options.durationMs,
+        judgeVerdicts,
+        trace: firstTrace,
+        compare: compareResult,
+        storyCompare: attachCompareStoryResults(compareStoryFields(options.scenario), compareResult),
+        agentUsage: sumUsageParts(options.armRuns.map((arm) => arm.result.agentUsage)),
+    });
+}
+/** Finalize viewer-scheduled compare arms through the same domain and judge path as the CLI. */
+export async function finalizeScheduledCompareScenario(options) {
+    const compareArms = options.scenario.compare ?? [];
+    const scenario = {
+        name: options.scenario.name,
+        description: options.scenario.description,
+        prompt: options.scenario.prompt,
+        rubric: options.scenario.rubric,
+        compare: {
+            arms: compareArms.map((arm) => ({
+                id: arm.id,
+                label: arm.label,
+                description: arm.description,
+                prompt: arm.prompt,
+            })),
+            gates: options.scenario.gates,
+            judgeMetrics: options.scenario.judgeMetrics,
+        },
+    };
+    const armRuns = compareArms.flatMap((arm) => {
+        const result = options.armResults.get(arm.id);
+        return result
+            ? [
+                {
+                    id: arm.id,
+                    label: arm.label,
+                    scenario: applyCompareArm(scenario, arm.id),
+                    result,
+                },
+            ]
+            : [];
+    });
+    return finalizeCompareArmRuns({
+        cwd: options.cwd,
+        suite: options.suite,
+        scenario,
+        host: options.host,
+        armRuns,
+        durationMs: Math.max(0, ...armRuns.map((arm) => arm.result.durationMs)),
+        judge: options.judge !== false,
+        evaluateGates: true,
+    });
+}
 async function loadCompareResultFromStaging(stagingSessionId, suiteName, scenario) {
     const resolved = resolveCompareArms(scenario.compare);
     const arms = [];