@post-print/agent-test 1.1.9 → 2.0.0

package/dist/viewer/run-controller.js

@@ -20,113 +20,135 @@ export function followViewerRun(controller, runId, onEvent) {
     flush();
     return unsubscribe;
 }
-export function createViewerRunController(options) {
-    const maxParallel = options.maxParallelAgents ?? VIEWER_MAX_PARALLEL_AGENTS;
-    const sessions = new Map();
-    let activeId;
-    for (const record of options.initialRuns ?? []) {
-        sessions.set(record.id, {
-            record: structuredClone(record),
-            events: [],
-            listeners: new Set(),
-            done: Promise.resolve(),
-            armResults: new Map(),
-            finalizing: new Set(),
-        });
+class RunController {
+    options;
+    sessions = new Map();
+    activeId;
+    maxParallel;
+    constructor(options) {
+        this.options = options;
+        this.maxParallel = this.options.maxParallelAgents ?? VIEWER_MAX_PARALLEL_AGENTS;
+        for (const record of this.options.initialRuns ?? []) {
+            this.sessions.set(record.id, {
+                record: structuredClone(record),
+                events: [],
+                listeners: new Set(),
+                done: Promise.resolve(),
+                armResults: new Map(),
+                finalizing: new Set(),
+            });
+        }
     }
-    const emit = (target, event) => {
+    emit = (target, event) => {
         target.events.push(event);
         if (event.type === "scenario_result") {
-            addScenarioResult(target, event, options.catalog);
+            addScenarioResult(target, event, this.options.catalog);
         }
         for (const listener of target.listeners)
             listener(event);
     };
-    return {
-        activeRunId: () => activeId,
-        maxParallelAgents: () => maxParallel,
-        runs: () => [...sessions.values()].map(({ record }) => structuredClone(record)),
-        run: (runId) => {
-            const record = sessions.get(runId)?.record;
-            return record ? structuredClone(record) : undefined;
-        },
-        start(request) {
-            if (activeId)
-                throw new Error("A viewer run is already in progress");
-            if (request.workers !== undefined &&
-                (!Number.isInteger(request.workers) || request.workers < 1 || request.workers > maxParallel)) {
-                throw new Error(`workers must be an integer 1-${maxParallel}`);
-            }
-            const jobs = expandViewerJobs(options.catalog, request);
-            if (jobs.length === 0)
-                throw new Error("No matching scenarios to run");
-            const runId = `run-${Date.now()}`;
-            const abort = new AbortController();
-            const current = {
-                record: {
-                    id: runId,
-                    request: structuredClone(request),
-                    status: "running",
-                    startedAt: new Date().toISOString(),
-                    reports: [],
-                },
-                events: [],
-                listeners: new Set(),
-                abort,
-                done: Promise.resolve(),
-                armResults: new Map(),
-                finalizing: new Set(),
-            };
-            sessions.set(runId, current);
-            activeId = runId;
-            emit(current, { type: "run_started", runId });
-            current.done = runJobBatches({
-                jobs,
-                catalog: options.catalog,
-                hosts: request.hosts ?? options.catalog.defaultSelectedHosts,
-                parallelHosts: request.parallelHosts === true,
-                maxParallel: request.workers ?? maxParallel,
-                signal: abort.signal,
-                runner: options.runner,
-                forward: (event) => emit(current, event),
-                afterJob: () => finalizeScheduledCompares(current, options.catalog, options.runner, emit),
-            })
-                .catch((error) => emit(current, {
-                type: "error",
-                message: error instanceof Error ? error.message : String(error),
-            }))
-                .finally(() => {
-                const totals = summarizeRun(current.record.reports, current.events);
-                current.record.status = abort.signal.aborted ? "cancelled" : "completed";
-                current.record.finishedAt = new Date().toISOString();
-                emit(current, { type: "run_finished", runId, status: current.record.status, ...totals });
-                if (activeId === runId)
-                    activeId = undefined;
-            });
-            return { runId };
-        },
-        cancel(runId) {
-            const target = sessions.get(runId);
-            if (!target?.abort || target.record.status !== "running")
-                return false;
-            target.record.status = "cancelling";
-            target.abort.abort();
-            return true;
-        },
-        history(runId) {
-            const target = sessions.get(runId);
-            return target ? [...target.events] : undefined;
-        },
-        subscribe(runId, listener) {
-            const target = sessions.get(runId);
-            if (!target)
-                return () => undefined;
-            target.listeners.add(listener);
-            return () => {
-                target.listeners.delete(listener);
-            };
-        },
+    activeRunId = () => this.activeId;
+    maxParallelAgents = () => this.maxParallel;
+    runs = () => [...this.sessions.values()].map(({ record }) => structuredClone(record));
+    run = (runId) => {
+        const record = this.sessions.get(runId)?.record;
+        return record ? structuredClone(record) : undefined;
     };
+    start(request) {
+        if (this.activeId)
+            throw new Error("A viewer run is already in progress");
+        if (request.workers !== undefined &&
+            (!Number.isInteger(request.workers) ||
+                request.workers < 1 ||
+                request.workers > this.maxParallel)) {
+            throw new Error(`workers must be an integer 1-${this.maxParallel}`);
+        }
+        const jobs = expandViewerJobs(this.options.catalog, request);
+        if (jobs.length === 0)
+            throw new Error("No matching scenarios to run");
+        const runId = `run-${Date.now()}`;
+        const abort = new AbortController();
+        const current = {
+            record: {
+                id: runId,
+                request: structuredClone(request),
+                status: "running",
+                startedAt: new Date().toISOString(),
+                reports: [],
+            },
+            events: [],
+            listeners: new Set(),
+            abort,
+            done: Promise.resolve(),
+            armResults: new Map(),
+            finalizing: new Set(),
+        };
+        this.sessions.set(runId, current);
+        this.activeId = runId;
+        this.emit(current, { type: "run_started", runId });
+        this.schedule(current, jobs, abort);
+        return { runId };
+    }
+    schedule(current, jobs, abort) {
+        const runId = current.record.id;
+        const request = current.record.request;
+        current.done = runJobBatches({
+            jobs,
+            catalog: this.options.catalog,
+            hosts: request.hosts ?? this.options.catalog.defaultSelectedHosts,
+            parallelHosts: request.parallelHosts === true,
+            maxParallel: request.workers ?? this.maxParallel,
+            signal: abort.signal,
+            runner: this.options.runner,
+            forward: (event) => this.emit(current, event),
+            afterJob: () => finalizeScheduledCompares(current, {
+                catalog: this.options.catalog,
+                runner: this.options.runner,
+                emit: this.emit,
+            }),
+        })
+            .catch((error) => this.emit(current, {
+            type: "error",
+            message: error instanceof Error ? error.message : String(error),
+        }))
+            .finally(() => {
+            const totals = summarizeRun(current.record.reports, current.events);
+            current.record.status = abort.signal.aborted ? "cancelled" : "completed";
+            current.record.finishedAt = new Date().toISOString();
+            this.emit(current, {
+                type: "run_finished",
+                runId,
+                status: current.record.status,
+                ...totals,
+            });
+            if (this.activeId === runId)
+                this.activeId = undefined;
+        });
+    }
+    cancel(runId) {
+        const target = this.sessions.get(runId);
+        if (!target?.abort || target.record.status !== "running")
+            return false;
+        target.record.status = "cancelling";
+        target.abort.abort();
+        return true;
+    }
+    history(runId) {
+        const target = this.sessions.get(runId);
+        return target ? [...target.events] : undefined;
+    }
+    subscribe(runId, listener) {
+        const target = this.sessions.get(runId);
+        if (!target)
+            return () => undefined;
+        target.listeners.add(listener);
+        return () => {
+            target.listeners.delete(listener);
+        };
+    }
+}
+export function createViewerRunController(options) {
+    return new RunController(options);
 }
 function addScenarioResult(session, event, catalog) {
     const report = ensureReport(session.record, event.suite, event.host);
@@ -151,8 +173,8 @@ function addScenarioResult(session, event, catalog) {
     group.results.set(event.arm, event.result);
     session.armResults.set(key, group);
 }
-function buildViewerCompareResult(suite, scenarioName, scenario, stored) {
-    const arms = (scenario.compare ?? []).flatMap((arm) => {
+function viewerCompareArms(scenario, stored) {
+    return (scenario.compare ?? []).flatMap((arm) => {
         const result = stored.get(arm.id);
         return result
             ? [
@@ -170,6 +192,9 @@ function buildViewerCompareResult(suite, scenarioName, scenario, stored) {
             ]
             : [];
     });
+}
+function buildViewerCompareResult(suite, { scenarioName, scenario, stored, }) {
+    const arms = viewerCompareArms(scenario, stored);
     const { compare, failures } = finalizeCompareOutcome(arms, scenario.gates);
     const passed = failures.length === 0;
     const result = {
@@ -206,7 +231,7 @@ function buildViewerCompareResult(suite, scenarioName, scenario, stored) {
     };
     return result;
 }
-async function finalizeScheduledCompares(session, catalog, runner, emit) {
+async function finalizeScheduledCompares(session, { catalog, runner, emit, }) {
     for (const [key, group] of session.armResults) {
         if (session.abort?.signal.aborted)
             return;
@@ -218,79 +243,97 @@ async function finalizeScheduledCompares(session, catalog, runner, emit) {
         if (!scenario?.compare?.length || group.results.size < scenario.compare.length)
             continue;
         session.finalizing.add(key);
-        emit(session, {
-            type: "cell_started",
-            suite: group.suite,
-            scenario: group.scenario,
-            host: group.host,
-        });
-        emit(session, {
-            type: "scenario_finalizing",
-            suite: group.suite,
-            scenario: group.scenario,
-            host: group.host,
-        });
-        emit(session, {
-            type: "status",
-            suite: group.suite,
-            scenario: group.scenario,
-            host: group.host,
-            text: "Finalizing comparison and judge.",
-        });
-        let result;
-        try {
-            const armResults = new Map();
-            for (const [armId, armResult] of group.results) {
-                const resultWithWorkspace = structuredClone(armResult);
-                const workspace = group.judgeWorkspaces.get(armId);
-                if (workspace) {
-                    Object.defineProperty(resultWithWorkspace, "judgeWorkspace", {
-                        value: workspace,
-                        enumerable: false,
-                    });
-                }
-                armResults.set(armId, resultWithWorkspace);
-            }
-            result = runner.finalizeCompare
-                ? await runner.finalizeCompare({
-                    suite: group.suite,
-                    scenario,
-                    host: group.host,
-                    armResults,
-                })
-                : buildViewerCompareResult(group.suite, group.scenario, scenario, group.results);
-        }
-        catch (error) {
-            result = buildViewerCompareResult(group.suite, group.scenario, scenario, group.results);
-            result.failures.push({
-                matcher: "compareFinalize",
-                message: error instanceof Error ? error.message : String(error),
-                category: "agent_runtime",
+        await finalizeGroup(session, { group, scenario, runner, emit });
+    }
+}
+function comparisonInputs(group) {
+    const armResults = new Map();
+    for (const [armId, armResult] of group.results) {
+        const resultWithWorkspace = structuredClone(armResult);
+        const workspace = group.judgeWorkspaces.get(armId);
+        if (workspace) {
+            Object.defineProperty(resultWithWorkspace, "judgeWorkspace", {
+                value: workspace,
+                enumerable: false,
             });
-            result.passed = false;
         }
-        if (session.abort?.signal.aborted)
-            return;
-        emit(session, {
-            type: "cell_finished",
-            suite: group.suite,
-            scenario: group.scenario,
-            host: group.host,
-            passed: result.passed,
-            durationMs: result.durationMs,
-            failures: result.failures.map((failure) => ({
-                matcher: failure.matcher,
-                message: failure.message,
-            })),
+        armResults.set(armId, resultWithWorkspace);
+    }
+    return armResults;
+}
+function publishComparison(session, { group, result, emit, }) {
+    emit(session, {
+        type: "cell_finished",
+        suite: group.suite,
+        scenario: group.scenario,
+        host: group.host,
+        passed: result.passed,
+        durationMs: result.durationMs,
+        failures: result.failures.map((failure) => ({
+            matcher: failure.matcher,
+            message: failure.message,
+        })),
+    });
+    emit(session, {
+        type: "scenario_result",
+        suite: group.suite,
+        scenario: group.scenario,
+        host: group.host,
+        result,
+    });
+}
+async function finalizeGroup(session, { group, scenario, runner, emit }) {
+    emit(session, {
+        type: "cell_started",
+        suite: group.suite,
+        scenario: group.scenario,
+        host: group.host,
+    });
+    emit(session, {
+        type: "scenario_finalizing",
+        suite: group.suite,
+        scenario: group.scenario,
+        host: group.host,
+    });
+    emit(session, {
+        type: "status",
+        suite: group.suite,
+        scenario: group.scenario,
+        host: group.host,
+        text: "Finalizing comparison and judge.",
+    });
+    let result;
+    try {
+        const armResults = comparisonInputs(group);
+        result = runner.finalizeCompare
+            ? await runner.finalizeCompare({
+                suite: group.suite,
+                scenario,
+                host: group.host,
+                armResults,
+            })
+            : buildViewerCompareResult(group.suite, {
+                scenarioName: group.scenario,
+                scenario: scenario,
+                stored: group.results,
+            });
+    }
+    catch (error) {
+        result = buildViewerCompareResult(group.suite, {
+            scenarioName: group.scenario,
+            scenario: scenario,
+            stored: group.results,
         });
-        emit(session, {
-            type: "scenario_result",
-            suite: group.suite,
-            scenario: group.scenario,
-            host: group.host,
-            result,
+        result.failures.push({
+            matcher: "compareFinalize",
+            message: error instanceof Error ? error.message : String(error),
+            category: "agent_runtime",
         });
+        result.passed = false;
     }
+    if (session.abort?.signal.aborted)
+        return;
+    publishComparison(session, { group, result, emit });
 }
 function ensureReport(record, suite, host) {
     let report = record.reports.find((item) => item.suite === suite && item.host === host);
@@ -341,86 +384,92 @@ async function runJobBatches(options) {
     for (const batch of batches) {
         if (options.signal.aborted)
             return;
-        await runWorkerPool(batch, options.maxParallel, async (job) => {
-            try {
-                let completedResult;
-                await options.runner.runJob(job, (event) => {
-                    if (event.type === "scenario_result" && !job.arm) {
-                        completedResult = event.result;
-                        return;
-                    }
-                    options.forward(event);
-                }, options.signal);
-                if (!completedResult ||
-                    job.arm ||
-                    !options.runner.finalizeScenario ||
-                    options.signal.aborted) {
-                    if (completedResult && !options.signal.aborted)
-                        options.forward({ type: "scenario_result", ...job, result: completedResult });
-                    return;
-                }
-                const scenario = options.catalog.suites
-                    .find((suite) => suite.name === job.suite)
-                    ?.scenarios.find((item) => item.name === job.scenario);
-                if (!scenario || scenario.compare?.length) {
-                    options.forward({ type: "scenario_result", ...job, result: completedResult });
-                    return;
-                }
-                options.forward({ type: "scenario_finalizing", ...job });
-                const finalized = await options.runner.finalizeScenario({ suite: job.suite, scenario, host: job.host, result: completedResult }, options.forward);
-                // The child emits an agent-only completion before parent-process judging.
-                // Replace that provisional status so the cell cannot remain green when
-                // judging later fails or becomes unavailable.
-                options.forward({
-                    type: "cell_finished",
-                    ...job,
-                    passed: finalized.passed,
-                    durationMs: finalized.durationMs,
-                    failures: finalized.failures.map((failure) => ({
-                        matcher: failure.matcher,
-                        message: failure.message,
-                    })),
-                });
-                options.forward({ type: "scenario_result", ...job, result: finalized });
-            }
-            catch (error) {
-                if (options.signal.aborted)
-                    return;
-                const message = error instanceof Error ? error.message : String(error);
-                const envelope = {
-                    suite: job.suite,
-                    scenario: job.scenario,
-                    host: job.host,
-                    ...(job.arm ? { arm: job.arm } : {}),
-                };
-                const failures = [
-                    { matcher: "liveScenario", message, category: "agent_runtime" },
-                ];
-                options.forward({ type: "error", ...envelope, message });
-                options.forward({
-                    type: "cell_finished",
-                    ...envelope,
-                    passed: false,
-                    failures,
-                    durationMs: 0,
-                });
-                options.forward({
-                    type: "scenario_result",
-                    ...envelope,
-                    result: {
-                        suite: job.suite,
-                        scenario: job.scenario,
-                        prompt: job.prompt,
-                        passed: false,
-                        failures,
-                        durationMs: 0,
-                    },
-                });
-            }
-            finally {
-                await options.afterJob();
+        await runWorkerPool(batch, {
+            workers: options.maxParallel,
+            worker: (job) => runViewerJob(job, options),
+            signal: options.signal,
+        });
+    }
+}
+async function runViewerJob(job, options) {
+    try {
+        let completedResult;
+        await options.runner.runJob(job, (event) => {
+            if (event.type === "scenario_result" && !job.arm) {
+                completedResult = event.result;
+                return;
             }
+            options.forward(event);
         }, options.signal);
+        await finalizeViewerJob(job, options, completedResult);
+    }
+    catch (error) {
+        publishJobFailure(job, options, error);
     }
+    finally {
+        await options.afterJob();
+    }
+}
+async function finalizeViewerJob(job, options, completedResult) {
+    if (!completedResult || job.arm || !options.runner.finalizeScenario || options.signal.aborted) {
+        if (completedResult && !options.signal.aborted)
+            options.forward({ type: "scenario_result", ...job, result: completedResult });
+        return;
+    }
+    const scenario = options.catalog.suites
+        .find((suite) => suite.name === job.suite)
+        ?.scenarios.find((item) => item.name === job.scenario);
+    if (!scenario || scenario.compare?.length) {
+        options.forward({ type: "scenario_result", ...job, result: completedResult });
+        return;
+    }
+    options.forward({ type: "scenario_finalizing", ...job });
+    const finalized = await options.runner.finalizeScenario({ suite: job.suite, scenario, host: job.host, result: completedResult }, options.forward);
+    // The child emits an agent-only completion before parent-process judging.
+    // Replace that provisional status so the cell cannot remain green when
+    // judging later fails or becomes unavailable.
+    options.forward({
+        type: "cell_finished",
+        ...job,
+        passed: finalized.passed,
+        durationMs: finalized.durationMs,
+        failures: finalized.failures.map((failure) => ({
+            matcher: failure.matcher,
+            message: failure.message,
+        })),
+    });
+    options.forward({ type: "scenario_result", ...job, result: finalized });
+}
+function publishJobFailure(job, options, error) {
+    if (options.signal.aborted)
+        return;
+    const message = error instanceof Error ? error.message : String(error);
+    const envelope = {
+        suite: job.suite,
+        scenario: job.scenario,
+        host: job.host,
+        ...(job.arm ? { arm: job.arm } : {}),
+    };
+    const failures = [{ matcher: "liveScenario", message, category: "agent_runtime" }];
+    options.forward({ type: "error", ...envelope, message });
+    options.forward({
+        type: "cell_finished",
+        ...envelope,
+        passed: false,
+        failures,
+        durationMs: 0,
+    });
+    options.forward({
+        type: "scenario_result",
+        ...envelope,
+        result: {
+            suite: job.suite,
+            scenario: job.scenario,
+            prompt: job.prompt,
+            passed: false,
+            failures,
+            durationMs: 0,
+        },
+    });
 }
 //# sourceMappingURL=run-controller.js.map