@post-print/agent-test 0.3.3 → 0.3.5

package/dist/cli.js

@@ -1,42 +1,107 @@
 #!/usr/bin/env -S node --disable-warning=ExperimentalWarning
-import { access, mkdir, writeFile } from "node:fs/promises";
 import { join, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
-import { CLAUDE_AUTH_MODE_ENV, cleanupStaleScenarioWorktrees, isPathUnderRoot, parseClaudeAuthMode, } from "@post-print/agent-harness";
+import { cleanupStaleScenarioWorktrees, isPathUnderRoot, knownAgentHosts, missingClassifierAuth, } from "@post-print/agent-harness";
+import { formatCheckReport, formatCheckSummary, missingHostsAuth, runCheck } from "./check.js";
 import { isCliMain } from "./cli-entry.js";
-import { compareSuiteReports, labelForCompareSide, loadSuiteRunReport, parseComparePairToken, writeCompareReport, } from "./compare.js";
-import { discoverSuites } from "./discover-suites.js";
-import { runDoctor } from "./doctor.js";
+import { missingAgentAuth } from "./doctor.js";
+import { installHostLogFilter } from "./host-log.js";
+import { parseHostList, uniqueHosts } from "./hosts.js";
 import { writeHtmlReport } from "./html-report.js";
-import { assertLiveDogfoodPreflight } from "./preflight.js";
+import { loadHostAdapters } from "./load-adapters.js";
+import { assertDirectAgentPreflight } from "./preflight.js";
 import { logProgress } from "./progress.js";
-import { cleanupLegacyRepoRecordings, cleanupStagingSession, createLiveStagingSessionId, getLiveStagingSessionRoot, setLiveStagingRootOverride, } from "./record-trace.js";
-import { registerLiveRunHandlers, runAllSuites, runSuite } from "./run-suite.js";
+import { cleanupStagingSession, createLiveStagingSessionId, getLiveStagingSessionRoot, setLiveStagingRootOverride, } from "./record-trace.js";
+import { shouldStartReportPreview, startDetachedReportPreview } from "./report-preview.js";
+import { registerLiveRunHandlers, runAllSuites, selectedRunNeedsJudge } from "./run-suite.js";
 import { formatRunSummary, shouldFailScenario, summarizeReports, } from "./suite-summary.js";
 import { configureCliColor, theme } from "./theme.js";
-import { formatSeedValidationReport, validateSeedPatches } from "./validate-seeds.js";
-import { formatValidationReport, validateSuitePaths } from "./validate-suite.js";
 import { suppressNoisyRuntimeWarnings } from "./warnings.js";
 suppressNoisyRuntimeWarnings();
 configureCliColor();
+function splitArgvFlag(token) {
+    if (!token.startsWith("-") || token === "-") {
+        return undefined;
+    }
+    if (token.startsWith("--")) {
+        const eq = token.indexOf("=");
+        if (eq === -1) {
+            return { flag: token };
+        }
+        return { flag: token.slice(0, eq), inline: token.slice(eq + 1) };
+    }
+    return { flag: token };
+}
+function readFlagValue(flag, inline, argv, index) {
+    if (inline !== undefined) {
+        if (inline.length === 0) {
+            throw new Error(`${flag} requires a value`);
+        }
+        return { value: inline, nextIndex: index };
+    }
+    const next = argv[index + 1];
+    if (!next || next.startsWith("--") || /^-[A-Za-z]/.test(next)) {
+        throw new Error(`${flag} requires a value`);
+    }
+    return { value: next, nextIndex: index + 1 };
+}
+function enableCheckMode(target) {
+    target.check = true;
+    target.validateOnly = true;
+    target.validatePaths = true;
+    target.validateSeeds = true;
+}
+/** Usage text for `--help`. */
+export function formatHelp() {
+    return [
+        "agent-test [options] [suite]",
+        "",
+        "Launch a host agent and score the transcript.",
+        "A run checks the suite and host first.",
+        "",
+        "  --check                         Check the suite and host. Do not launch an agent.",
+        "  --help, -h                      Print this help.",
+        "",
+        "  --host cursor|claude|openai|all Host adapter (default: suite or cursor).",
+        "                                  Repeat or comma-separate for a matrix.",
+        "  --adapter <module>              Load a consumer host adapter module.",
+        "  --suites-dir <path>             Suite root (default: agent-suites)",
+        "  --suite <name>                  Run one suite",
+        "  --scenario <name>               Run one scenario",
+        "  --no-judge                      Skip the judge",
+        "  --fail-on all|behavior|infra-only",
+        "  --allow-user-input              Let a user agent answer AskQuestion tools",
+        "  --timeout-ms <n>                Agent deadline",
+        "  --no-timeout                    Disable the deadline",
+        "  --scenario-retries <n>          Announce-stop retries",
+        "  --debug                         Keep recordings and write a debug bundle",
+        "  --debug-dir <path>              Debug session parent directory",
+        "  --report-out <path>             HTML report file or directory",
+        "  --no-html-report                Skip the HTML report",
+        "  --no-worktree                   Run in the caller checkout (needs AGENT_TEST_ALLOW_IN_PLACE=1)",
+        "",
+        "--doctor, --validate-only, --validate-paths, and --validate-seeds run --check.",
+    ].join("\n");
+}
 /** Parse agent-test CLI argv (exported for unit tests). */
 export function parseCliArgs(argv) {
     const cwd = process.cwd();
     let suitesDir = "agent-suites";
     let rubricsDir;
-    let host;
+    const parsedHosts = [];
+    let hostAll = false;
+    const adapterModules = [];
     let filter;
     let scenarioFilter;
     let stagingSessionId;
-    let record = false;
-    let recordFixtures = false;
-    let live = false;
     let judge;
     let worktree;
     let keepRecordings = false;
     let timeoutMs;
     let noTimeout = false;
     let allowUserInput = false;
+    let check = false;
+    let help = false;
     let doctor = false;
     let htmlReport = true;
     let reportOut;
@@ -47,141 +112,164 @@ export function parseCliArgs(argv) {
     let validatePaths = false;
     let failOn = "all";
     let scenarioRetries;
-    let compareMode = false;
-    let compareA;
-    let compareB;
-    let comparePairs;
-    let compareOutDir;
-    const startIndex = argv[2] === "compare" ? 3 : 2;
     if (argv[2] === "compare") {
-        compareMode = true;
+        throw new Error("The compare subcommand is removed. Add compare.a and compare.b on one scenario. Set compare.faster or compare.cheaper when one arm must win on time or tokens. Add rubric.judge only when you want a pairwise judge.");
     }
+    const startIndex = 2;
+    const checkMode = {
+        check: false,
+        validateOnly: false,
+        validatePaths: false,
+        validateSeeds: false,
+    };
     for (let i = startIndex; i < argv.length; i++) {
         const token = argv[i];
-        if (token === "--host" && argv[i + 1]) {
-            host = argv[++i];
-        }
-        else if (token === "--suites-dir" && argv[i + 1]) {
-            suitesDir = argv[++i];
-        }
-        else if (token === "--rubrics-dir" && argv[i + 1]) {
-            rubricsDir = argv[++i];
-        }
-        else if (token === "--suite" && argv[i + 1]) {
-            filter = argv[++i];
-        }
-        else if (token === "--scenario" && argv[i + 1]) {
-            scenarioFilter = argv[++i];
-        }
-        else if (token === "--staging-session-id" && argv[i + 1]) {
-            stagingSessionId = argv[++i];
-        }
-        else if (token === "--record") {
-            record = true;
-        }
-        else if (token === "--record-fixtures") {
-            record = true;
-            recordFixtures = true;
+        if (!token) {
+            continue;
         }
-        else if (token === "--live") {
-            live = true;
-        }
-        else if (token === "--keep-recordings") {
-            keepRecordings = true;
-        }
-        else if (token === "--judge") {
-            judge = true;
-        }
-        else if (token === "--no-judge") {
-            judge = false;
-        }
-        else if (token === "--no-worktree") {
-            worktree = false;
-        }
-        else if (token === "--timeout-ms" && argv[i + 1]) {
-            const parsed = Number(argv[++i]);
-            if (Number.isFinite(parsed) && parsed > 0) {
-                timeoutMs = parsed;
+        const parsed = splitArgvFlag(token);
+        if (!parsed) {
+            filter = token;
+            continue;
+        }
+        const { flag, inline } = parsed;
+        const read = () => {
+            const result = readFlagValue(flag, inline, argv, i);
+            i = result.nextIndex;
+            return result.value;
+        };
+        switch (flag) {
+            case "--help":
+            case "-h":
+                help = true;
+                break;
+            case "--check":
+                enableCheckMode(checkMode);
+                break;
+            case "--doctor":
+                doctor = true;
+                enableCheckMode(checkMode);
+                break;
+            case "--validate-only":
+                enableCheckMode(checkMode);
+                break;
+            case "--validate-seeds":
+                enableCheckMode(checkMode);
+                break;
+            case "--validate-paths":
+                enableCheckMode(checkMode);
+                break;
+            case "--host": {
+                const value = read();
+                if (value
+                    .split(",")
+                    .map((token) => token.trim().toLowerCase())
+                    .includes("all")) {
+                    hostAll = true;
+                }
+                parsedHosts.push(...parseHostList(value));
+                break;
             }
-        }
-        else if (token === "--no-timeout") {
-            noTimeout = true;
-        }
-        else if (token === "--allow-user-input") {
-            allowUserInput = true;
-        }
-        else if (token === "--doctor") {
-            doctor = true;
-        }
-        else if (token === "--validate-only") {
-            validateOnly = true;
-        }
-        else if (token === "--validate-seeds") {
-            validateSeeds = true;
-        }
-        else if (token === "--validate-paths") {
-            validatePaths = true;
-        }
-        else if (token === "--fail-on" && argv[i + 1]) {
-            const mode = argv[++i];
-            if (mode !== "all" && mode !== "behavior" && mode !== "infra-only") {
-                throw new Error("--fail-on must be all|behavior|infra-only");
+            case "--adapter":
+                adapterModules.push(resolve(cwd, read()));
+                break;
+            case "--suites-dir":
+                suitesDir = read();
+                break;
+            case "--rubrics-dir":
+                rubricsDir = read();
+                break;
+            case "--suite":
+                filter = read();
+                break;
+            case "--scenario":
+                scenarioFilter = read();
+                break;
+            case "--staging-session-id":
+                stagingSessionId = read();
+                break;
+            case "--live":
+                throw new Error("--live was removed because agent-test now always runs a real agent");
+            case "--record":
+                throw new Error("--record was removed; direct runs capture transient traces automatically (use --keep-recordings to retain them)");
+            case "--record-fixtures":
+                throw new Error("--record-fixtures was removed because replay-based testing is deprecated and no longer supported");
+            case "--keep-recordings":
+                keepRecordings = true;
+                break;
+            case "--judge":
+                judge = true;
+                break;
+            case "--no-judge":
+                judge = false;
+                break;
+            case "--no-worktree":
+                worktree = false;
+                break;
+            case "--timeout-ms": {
+                const parsedTimeout = Number(read());
+                if (Number.isFinite(parsedTimeout) && parsedTimeout > 0) {
+                    timeoutMs = parsedTimeout;
+                }
+                break;
             }
-            failOn = mode;
-        }
-        else if (token === "--scenario-retries" && argv[i + 1]) {
-            const parsed = Number(argv[++i]);
-            if (!Number.isInteger(parsed) || parsed < 0) {
-                throw new Error("--scenario-retries must be an integer >= 0");
+            case "--no-timeout":
+                noTimeout = true;
+                break;
+            case "--allow-user-input":
+                allowUserInput = true;
+                break;
+            case "--fail-on": {
+                const mode = read();
+                if (mode !== "all" && mode !== "behavior" && mode !== "infra-only") {
+                    throw new Error("--fail-on must be all|behavior|infra-only");
+                }
+                failOn = mode;
+                break;
             }
-            scenarioRetries = parsed;
-        }
-        else if (token === "--compare-pairs" && argv[i + 1]) {
-            comparePairs = argv[++i];
-        }
-        else if ((token === "--a" || token === "--compare-a") && argv[i + 1]) {
-            compareA = argv[++i];
-            compareMode = true;
-        }
-        else if ((token === "--b" || token === "--compare-b") && argv[i + 1]) {
-            compareB = argv[++i];
-            compareMode = true;
-        }
-        else if ((token === "--out-dir" || token === "--compare-out") && argv[i + 1]) {
-            compareOutDir = argv[++i];
-        }
-        else if (token === "--report-out" && argv[i + 1]) {
-            reportOut = argv[++i];
-        }
-        else if (token === "--no-html-report") {
-            htmlReport = false;
-        }
-        else if (token === "--debug") {
-            debug = true;
-        }
-        else if (token === "--debug-dir") {
-            const value = argv[++i];
-            if (!value || value.startsWith("-")) {
-                throw new Error("--debug-dir requires a non-empty path argument");
+            case "--scenario-retries": {
+                const parsedRetries = Number(read());
+                if (!Number.isInteger(parsedRetries) || parsedRetries < 0) {
+                    throw new Error("--scenario-retries must be an integer >= 0");
+                }
+                scenarioRetries = parsedRetries;
+                break;
             }
-            debugDir = value;
-            debug = true;
-        }
-        else if (token && !token.startsWith("-")) {
-            filter = token;
-        }
-    }
-    if (compareMode && !comparePairs) {
-        if (!compareA || !compareB) {
-            throw new Error("compare requires --a <report.json> and --b <report.json>");
+            case "--compare-pairs":
+            case "--a":
+            case "--b":
+            case "--compare-a":
+            case "--compare-b":
+            case "--out-dir":
+            case "--compare-out":
+                throw new Error(`${flag} is removed. Add compare.a and compare.b on one scenario. Set compare.faster or compare.cheaper when one arm must win on time or tokens. Add rubric.judge only when you want a pairwise judge.`);
+            case "--report-out":
+                reportOut = read();
+                break;
+            case "--no-html-report":
+                htmlReport = false;
+                break;
+            case "--debug":
+                debug = true;
+                break;
+            case "--debug-dir": {
+                const value = read();
+                debugDir = value;
+                debug = true;
+                break;
+            }
+            default:
+                throw new Error(`Unknown flag: ${flag}`);
         }
     }
-    if (live) {
-        host = host ?? "cursor";
-        record = true;
-        judge = judge ?? true;
-        worktree = worktree ?? true;
-    }
+    check = checkMode.check;
+    validateOnly = checkMode.validateOnly;
+    validatePaths = checkMode.validatePaths;
+    validateSeeds = checkMode.validateSeeds;
+    const hosts = uniqueHosts(parsedHosts);
+    const host = hosts?.[0];
+    judge = judge ?? true;
+    worktree = worktree ?? true;
     if (debug) {
         keepRecordings = true;
         process.env.AGENT_TEST_DEBUG = "1";
@@ -193,18 +281,20 @@ export function parseCliArgs(argv) {
         suitesDir,
         rubricsDir,
         host,
+        hosts,
+        hostAll,
+        adapterModules: adapterModules.length > 0 ? adapterModules : undefined,
         filter,
         scenarioFilter,
         stagingSessionId,
-        record,
-        recordFixtures,
-        live,
         judge,
         worktree,
         keepRecordings,
         timeoutMs: noTimeout ? 0 : timeoutMs,
         noTimeout,
         allowUserInput,
+        check,
+        help,
         doctor,
         htmlReport,
         reportOut: reportOut ? resolve(cwd, reportOut) : undefined,
@@ -215,70 +305,9 @@ export function parseCliArgs(argv) {
         validatePaths,
         failOn,
         scenarioRetries,
-        compareMode,
-        compareA,
-        compareB,
-        comparePairs,
-        compareOutDir: compareOutDir ? resolve(cwd, compareOutDir) : undefined,
     };
 }
-async function pathExists(path) {
-    try {
-        await access(path);
-        return true;
-    }
-    catch {
-        return false;
-    }
-}
-async function resolveSuiteScenariosPath(cwd, suitesDir, side) {
-    const candidates = [
-        resolve(cwd, side, "scenarios.json"),
-        resolve(cwd, suitesDir, side, "scenarios.json"),
-        resolve(side, "scenarios.json"),
-    ];
-    for (const candidate of candidates) {
-        if (await pathExists(candidate)) {
-            return candidate;
-        }
-    }
-    throw new Error(`compare side "${side}" is not a suite report JSON or suite dir (looked under ${suitesDir}/ and cwd)`);
-}
-async function loadOrRunCompareSide(args, side, stagingSessionId) {
-    const resolved = resolve(args.cwd, side);
-    if (side.endsWith(".json") || resolved.endsWith(".json")) {
-        const jsonPath = side.endsWith(".json") ? resolved : resolve(args.cwd, `${side}.json`);
-        const path = (await pathExists(resolved)) ? resolved : jsonPath;
-        if (!(await pathExists(path))) {
-            throw new Error(`compare report not found: ${side}`);
-        }
-        return loadSuiteRunReport(path);
-    }
-    const suitePath = await resolveSuiteScenariosPath(args.cwd, args.suitesDir, side);
-    return runSuite({
-        cwd: args.cwd,
-        suitePath,
-        host: args.host,
-        scenarioFilter: args.scenarioFilter,
-        record: args.record,
-        recordFixtures: args.recordFixtures,
-        judge: args.judge,
-        worktree: args.worktree,
-        stagingSessionId,
-        keepRecordings: args.keepRecordings,
-        suitesDir: args.suitesDir,
-        rubricsDir: args.rubricsDir,
-        timeoutMs: args.timeoutMs,
-        allowUserInput: args.allowUserInput,
-        debug: args.debug,
-        debugDir: args.debugDir,
-        scenarioRetries: args.scenarioRetries,
-    });
-}
-/**
- * Split `--report-out` into an HTML file path and, when a directory was given,
- * the directory every other report artifact is written to.
- */
+/** Resolve an explicit report target into the HTML path and optional artifact directory. */
 export function resolveReportOutput(reportOut) {
     if (!reportOut) {
         return {};
@@ -288,13 +317,16 @@ export function resolveReportOutput(reportOut) {
     }
     return { htmlPath: join(reportOut, "report.html"), outDir: reportOut };
 }
-async function writeSuiteReportDump(outDir, label, report) {
-    await mkdir(outDir, { recursive: true });
-    const path = join(outDir, `${label}.suite-report.json`);
-    await writeFile(path, `${JSON.stringify(report, null, 2)}\n`, "utf8");
-    return path;
+async function htmlReportTip(label, reportPath) {
+    if (shouldStartReportPreview()) {
+        const url = await startDetachedReportPreview(reportPath);
+        if (url) {
+            return theme.fileTip(label, url);
+        }
+    }
+    return theme.fileTip(label, reportPath);
 }
-async function cleanupLiveRunArtifacts(cwd, stagingSessionRoot, keepRecordings) {
+async function cleanupRunArtifacts(stagingSessionRoot, keepRecordings) {
     if (keepRecordings) {
         return;
     }
@@ -306,123 +338,116 @@ async function cleanupLiveRunArtifacts(cwd, stagingSessionRoot, keepRecordings)
             // best-effort
         }
     }
-    const legacyRemoved = await cleanupLegacyRepoRecordings(cwd);
-    if (legacyRemoved.length > 0) {
-        console.log(`Removed legacy in-repo recording dir(s):\n  ${legacyRemoved.join("\n  ")}`);
-    }
 }
 async function main() {
     let args;
     try {
         args = parseCliArgs(process.argv);
+        installHostLogFilter();
     }
     catch (error) {
         console.error(error instanceof Error ? error.message : error);
         return 1;
     }
-    if (args.doctor) {
-        const report = runDoctor();
-        for (const message of report.messages) {
-            console.log(message);
+    if (args.help) {
+        console.log(formatHelp());
+        return 0;
+    }
+    try {
+        await loadHostAdapters({ cwd: args.cwd, adapterModules: args.adapterModules });
+        if (args.hostAll) {
+            args.hosts = uniqueHosts(knownAgentHosts());
+            args.host = args.hosts?.[0];
         }
-        return report.ok ? 0 : 1;
     }
-    if (args.compareMode && !args.comparePairs) {
-        const aPath = resolve(args.cwd, args.compareA);
-        const bPath = resolve(args.cwd, args.compareB);
-        const aReport = await loadSuiteRunReport(aPath);
-        const bReport = await loadSuiteRunReport(bPath);
-        const outDir = args.compareOutDir ??
-            resolveReportOutput(args.reportOut).outDir ??
-            resolve(args.cwd, "compare-out");
-        const compare = compareSuiteReports({
-            aLabel: labelForCompareSide(aPath),
-            bLabel: labelForCompareSide(bPath),
-            a: aReport,
-            b: bReport,
-        });
-        const written = await writeCompareReport({ outDir, report: compare });
-        console.log(theme.tip(`compare JSON: ${written.jsonPath}`));
-        console.log(theme.tip(`compare markdown: ${written.markdownPath}`));
-        console.log(theme.fileTip("compare HTML", written.htmlPath));
-        return compare.summary.passRegressions > 0 ? 1 : 0;
+    catch (error) {
+        console.error(error instanceof Error ? error.message : error);
+        return 1;
     }
-    if (args.validateOnly || args.validateSeeds) {
-        if (args.validateOnly) {
-            const suitePaths = await discoverSuites(resolve(args.cwd, args.suitesDir));
-            const filtered = args.filter
-                ? suitePaths.filter((path) => path.includes(`/${args.filter}/`) || path.endsWith(`/${args.filter}/scenarios.json`))
-                : suitePaths;
-            const report = await validateSuitePaths(filtered, {
-                validatePaths: args.validatePaths,
-                repoRoot: args.cwd,
-                rubricsDir: args.rubricsDir,
-            });
-            console.log(formatValidationReport(report));
-            if (!report.ok) {
-                return 1;
-            }
-        }
-        if (args.validateSeeds) {
-            const report = await validateSeedPatches({
-                cwd: args.cwd,
-                suitesDir: args.suitesDir,
-                rubricsDir: args.rubricsDir,
-                filter: args.filter,
-            });
-            console.log(formatSeedValidationReport(report));
-            if (!report.ok) {
-                return 1;
-            }
-        }
-        return 0;
+    if (args.check || args.doctor || args.validateOnly || args.validateSeeds) {
+        const report = await runCheck({
+            cwd: args.cwd,
+            suitesDir: args.suitesDir,
+            filter: args.filter,
+            rubricsDir: args.rubricsDir,
+            host: args.host,
+            hosts: args.hosts,
+        });
+        console.log(formatCheckReport(report));
+        return report.ok ? 0 : 1;
     }
     if (args.debugDir) {
         setLiveStagingRootOverride(args.debugDir);
     }
     const isChild = process.env.AGENT_TEST_CHILD === "1";
     if (args.debugDir && isPathUnderRoot(args.debugDir, args.cwd) && !isChild) {
-        console.warn(theme.warn(`--debug-dir is inside the repo (${args.debugDir}). Default is $TMPDIR/agent-spec — prefer that for live runs so debug output stays out of git status.`));
+        console.warn(theme.warn(`--debug-dir is inside the repo (${args.debugDir}). Default is $TMPDIR/agent-spec — prefer that for direct runs so debug output stays out of git status.`));
     }
     const verbose = args.debug || process.env.AGENT_TEST_VERBOSE === "1" || process.env.AGENT_TEST_DEBUG === "1";
     const stagingSessionId = args.stagingSessionId?.trim() ||
         process.env.AGENT_TEST_STAGING_SESSION_ID?.trim() ||
-        (args.live || args.debug || (args.record && !args.recordFixtures)
-            ? createLiveStagingSessionId()
-            : undefined);
+        createLiveStagingSessionId();
     const stagingSessionRoot = stagingSessionId
         ? getLiveStagingSessionRoot(stagingSessionId)
         : undefined;
-    const reportOutput = resolveReportOutput(args.reportOut);
     try {
-        if (args.live) {
-            const host = args.host ?? "cursor";
-            if (host === "claude") {
-                try {
-                    const authMode = parseClaudeAuthMode(process.env[CLAUDE_AUTH_MODE_ENV]);
-                    if (authMode === "api-key" && !process.env.ANTHROPIC_API_KEY?.trim()) {
-                        console.error(`${CLAUDE_AUTH_MODE_ENV}=api-key requires ANTHROPIC_API_KEY for --live --host claude`);
-                        return 1;
-                    }
-                }
-                catch (error) {
-                    console.error(error instanceof Error ? error.message : error);
-                    return 1;
-                }
+        let checkHosts = args.hosts ?? (args.host ? [args.host] : ["cursor"]);
+        if (!isChild) {
+            const report = await runCheck({
+                cwd: args.cwd,
+                suitesDir: args.suitesDir,
+                filter: args.filter,
+                rubricsDir: args.rubricsDir,
+                host: args.host,
+                hosts: args.hosts,
+            });
+            console.log(theme.tip(formatCheckSummary(report)));
+            if (!report.ok) {
+                console.error(formatCheckReport(report));
+                return 1;
             }
-            if (host === "cursor" && !process.env.CURSOR_API_KEY?.trim()) {
-                console.error("CURSOR_API_KEY required for --live (Cursor SDK runs)");
+            checkHosts = report.hosts;
+            const missingHost = missingHostsAuth(report.hosts);
+            if (missingHost) {
+                console.error(missingHost);
                 return 1;
             }
-            // Judge classifiers still use the Cursor SDK.
-            if (args.judge !== false && !process.env.CURSOR_API_KEY?.trim()) {
-                console.error("CURSOR_API_KEY required for live judge classifiers (use --no-judge to skip)");
+        }
+        else {
+            const missingHost = missingAgentAuth(args.host ?? "cursor");
+            if (missingHost) {
+                console.error(missingHost);
                 return 1;
             }
         }
-        if (args.live) {
+        if (args.judge !== false) {
+            let needsJudge = false;
             try {
-                await assertLiveDogfoodPreflight(args.cwd, args.suitesDir);
+                needsJudge = await selectedRunNeedsJudge({
+                    cwd: args.cwd,
+                    suitesDir: args.suitesDir,
+                    filter: args.filter,
+                    scenarioFilter: args.scenarioFilter,
+                    rubricsDir: args.rubricsDir,
+                    judge: args.judge,
+                });
+            }
+            catch {
+                needsJudge = false;
+            }
+            if (needsJudge) {
+                for (const host of checkHosts) {
+                    const missingJudge = missingClassifierAuth(host);
+                    if (missingJudge) {
+                        console.error(`${missingJudge} (use --no-judge to skip)`);
+                        return 1;
+                    }
+                }
+            }
+        }
+        {
+            try {
+                await assertDirectAgentPreflight(args.cwd, args.suitesDir);
             }
             catch (error) {
                 console.error(error instanceof Error ? error.message : error);
@@ -433,7 +458,7 @@ async function main() {
                 process.env.AGENT_TEST_NO_WORKTREE === "1" ||
                 process.env.AGENT_TEST_NO_WORKTREE === "true";
             if (worktreeDisabled && !inPlaceAllowed) {
-                console.error("Live dogfood requires git worktree isolation. Set AGENT_TEST_ALLOW_IN_PLACE=1 to run in repo cwd (--no-worktree leaks agent edits into your working tree).");
+                console.error("Direct agent tests require a sealed temp workspace. Set AGENT_TEST_ALLOW_IN_PLACE=1 to run in repo cwd (--no-worktree leaks agent edits into your working tree).");
                 return 1;
             }
             registerLiveRunHandlers();
@@ -442,70 +467,32 @@ async function main() {
                 if (removed.length > 0) {
                     console.log(theme.warn(`Cleaned ${removed.length} stale agent-test worktree(s) from a prior crash`));
                 }
-                console.log(theme.banner(args.debug ? "live debug" : "live"));
+                console.log(theme.banner(args.debug ? "direct debug" : "direct"));
                 if (stagingSessionRoot) {
                     console.log(theme.bannerSession(stagingSessionRoot));
                 }
-                console.log(`  ${theme.tip("Ctrl+C cancels in-flight scenarios")}`);
-                if (worktreeDisabled) {
-                    console.warn(theme.warn("running in repo cwd — agent file edits will persist in your working tree"));
-                }
+                const hints = ["Ctrl+C cancels"];
                 if (args.debug || process.env.AGENT_TEST_VERBOSE === "1") {
-                    if (!args.keepRecordings) {
-                        console.log(`  ${theme.tip("traces removed on exit unless --keep-recordings")}`);
-                    }
-                    else {
-                        console.log(`  ${theme.tip("debug: recordings kept")}`);
-                    }
-                    console.log(`  ${theme.tip("exit 137 = macOS OOM — isolated subprocesses (AGENT_TEST_NO_ISOLATE=1 to disable)")}`);
+                    hints.push(args.keepRecordings ? "recordings kept" : "traces removed on exit");
+                    hints.push("exit 137 = macOS OOM");
                 }
                 else if (!args.keepRecordings) {
-                    console.log(`  ${theme.tip("traces removed on exit unless --keep-recordings")}`);
+                    hints.push("traces removed on exit");
+                }
+                console.log(theme.bannerHints(hints));
+                if (worktreeDisabled) {
+                    console.warn(theme.warn("running in repo cwd — agent file edits will persist in your working tree"));
                 }
             }
         }
-        else if (args.debug && !isChild && stagingSessionRoot) {
-            console.log(theme.banner("debug"));
-            console.log(theme.bannerSession(stagingSessionRoot));
-        }
-        let reports;
-        let comparePassRegressions = 0;
-        if (args.comparePairs) {
-            const { a, b } = parseComparePairToken(args.comparePairs);
-            const aReport = await loadOrRunCompareSide(args, a, stagingSessionId);
-            const bReport = await loadOrRunCompareSide(args, b, stagingSessionId);
-            reports = [aReport, bReport];
-            if (!isChild) {
-                const outDir = args.compareOutDir ??
-                    reportOutput.outDir ??
-                    (stagingSessionRoot
-                        ? join(stagingSessionRoot, "compare")
-                        : resolve(args.cwd, "compare-out"));
-                await writeSuiteReportDump(outDir, labelForCompareSide(a), aReport);
-                await writeSuiteReportDump(outDir, labelForCompareSide(b), bReport);
-                const compare = compareSuiteReports({
-                    aLabel: labelForCompareSide(a),
-                    bLabel: labelForCompareSide(b),
-                    a: aReport,
-                    b: bReport,
-                });
-                comparePassRegressions = compare.summary.passRegressions;
-                const written = await writeCompareReport({ outDir, report: compare });
-                console.log(`\n${theme.tip(`compare JSON: ${written.jsonPath}`)}`);
-                console.log(theme.tip(`compare markdown: ${written.markdownPath}`));
-                console.log(theme.fileTip("compare HTML", written.htmlPath));
-            }
-        }
-        else {
-            reports = await runAllSuites({
-                ...args,
-                stagingSessionId,
-                timeoutMs: args.timeoutMs,
-                allowUserInput: args.allowUserInput,
-                debug: args.debug,
-                debugDir: args.debugDir,
-            });
-        }
+        const reports = await runAllSuites({
+            ...args,
+            stagingSessionId,
+            timeoutMs: args.timeoutMs,
+            allowUserInput: args.allowUserInput,
+            debug: args.debug,
+            debugDir: args.debugDir,
+        });
         let exitCode = 0;
         if (!isChild) {
             for (const report of reports) {
@@ -531,7 +518,10 @@ async function main() {
                 console.log(theme.summary(report.suite, report.passed, report.failed, report.skipped));
             }
             const runSummary = summarizeReports(reports);
-            console.log(`\n${formatRunSummary(runSummary)}`);
+            const runSummaryText = formatRunSummary(runSummary);
+            if (runSummaryText) {
+                console.log(`\n${theme.runSummary(runSummaryText)}`);
+            }
             if (args.failOn === "behavior" && runSummary.infraFailures > 0 && exitCode === 0) {
                 console.log(theme.tip(`${runSummary.infraFailures} infra failure(s) ignored (--fail-on=behavior)`));
             }
@@ -540,30 +530,16 @@ async function main() {
             }
             if (args.htmlReport && reports.length > 0) {
                 try {
-                    const pair = args.comparePairs ? parseComparePairToken(args.comparePairs) : undefined;
                     const reportPath = await writeHtmlReport(reports, {
                         host: args.host,
                         suitesDir: args.suitesDir,
-                        includeCompare: Boolean(args.comparePairs),
-                        compareALabel: pair ? labelForCompareSide(pair.a) : undefined,
-                        compareBLabel: pair ? labelForCompareSide(pair.b) : undefined,
-                    }, reportOutput.htmlPath);
-                    console.log(`\n${theme.fileTip("HTML report", reportPath)}`);
-                    // A directory collects everything, not just the HTML.
-                    if (reportOutput.outDir && !args.comparePairs) {
-                        for (const report of reports) {
-                            const dumpPath = await writeSuiteReportDump(reportOutput.outDir, report.suite, report);
-                            console.log(theme.tip(`suite JSON: ${dumpPath}`));
-                        }
-                    }
+                    });
+                    console.log(`\n${await htmlReportTip("HTML report", reportPath)}`);
                 }
                 catch (error) {
                     console.warn(theme.warn(`HTML report failed: ${error instanceof Error ? error.message : String(error)}`));
                 }
             }
-            if (comparePassRegressions > 0) {
-                exitCode = 1;
-            }
         }
         else {
             for (const report of reports) {
@@ -581,7 +557,7 @@ async function main() {
     }
     finally {
         if (stagingSessionId && !isChild) {
-            await cleanupLiveRunArtifacts(args.cwd, stagingSessionRoot, args.keepRecordings);
+            await cleanupRunArtifacts(stagingSessionRoot, args.keepRecordings);
         }
     }
 }