@hue-run/sdk 0.19.0 → 0.20.0

package/dist/cli/eval.js

@@ -7,8 +7,8 @@ import { pathToFileURL } from "node:url";
 import { parseArgs } from "node:util";
 import { randomUUID } from "node:crypto";
 import { createHue } from "../client.js";
-import { createEnvironmentClient } from "../environment/client.js";
-import { agentEnvironment, isHueControlPlaneCredential, stripHueControlPlaneCredentials, withoutWorldVariables, writeMcpConfig, } from "../environment/world.js";
+import { createEnvironmentClient } from "../world/client.js";
+import { agentEnvironment, isHueControlPlaneCredential, stripHueControlPlaneCredentials, withoutWorldVariables, writeMcpConfig, } from "../world/world.js";
 import { EvaluationClient, HueApiError } from "../evals/client.js";
 import { TargetResult } from "../evals/types.js";
 import { CheckpointIdentityError, CheckpointStore, checkpointPath } from "../evals/checkpoint.js";
@@ -96,6 +96,8 @@ Output and limits:
                                   arrive, which Hue attaches once its export is complete
   --timeout <seconds>             Per-case --command timeout (default: 600)
   --wait <seconds>                Verdict wait after the run finishes (default: 300)
+  --judges                        Request pinned hosted model judges after outputs upload
+                                  (off by default; uses the project's judge budget)
   -h, --help                      Show this help
 
 HUE_API_KEY must be a "Read and write" project key; it is never printed.
@@ -149,6 +151,7 @@ function parse(argv) {
                 content: { type: "boolean" },
                 "no-content": { type: "boolean", default: false },
                 "no-output": { type: "boolean", default: false },
+                judges: { type: "boolean", default: false },
                 "trace-not-accepted": { type: "string" },
                 "save-version": { type: "boolean", default: false },
                 check: { type: "boolean", default: false },
@@ -1054,8 +1057,133 @@ function withTelemetryFailures(verdicts, telemetry = []) {
         totals[item.state]++;
     return { ...verdicts, summary: { cases, totals } };
 }
-/** Waits for Hue's verdicts, prints them (table or JSON, with the optional baseline) and returns the exit code. */
-async function reportVerdicts(client, values, run, baselineId, output, signal) {
+/** Requests only the run's pinned model judges. Stable batches let an interrupted CLI resume
+ * dispatch without creating a second paid job for the same scoring item and evaluator. */
+export async function dispatchPinnedJudges(client, experimentId, subjectIds, signal) {
+    signal?.throwIfAborted();
+    const experiment = await client.getExperiment(experimentId);
+    const run = experiment.evaluation;
+    const scorerVersionIds = [
+        ...new Set(run.scorerVersions
+            .filter((version) => version.definition.kind === "llm_judge")
+            .map((version) => version.id)),
+    ].sort();
+    if (!scorerVersionIds.length)
+        return 0;
+    const subjects = new Set(subjectIds);
+    const items = new Set();
+    let after;
+    do {
+        signal?.throwIfAborted();
+        const page = await client.listEvaluationItems(run.id, { after });
+        for (const item of page.items)
+            if (item.hasOutput && subjects.has(item.subjectId))
+                items.add(item.id);
+        after = page.nextCursor ?? undefined;
+    } while (after);
+    if (!items.size)
+        return 0;
+    let count = 0;
+    let jobs = [];
+    const dispatch = async () => {
+        signal?.throwIfAborted();
+        try {
+            // Admission belongs to the server: an accepted keyed request must remain replayable
+            // even if the project's budget is disabled before its acknowledgement reaches us.
+            await client.createJudgeJobs(run.id, {
+                idempotencyKey: `cli-judges-${digest({ runId: run.id, jobs })}`,
+                jobs,
+            });
+        }
+        catch (error) {
+            if (error instanceof HueApiError && error.status === 402)
+                throw new Error("Hosted judges are unavailable for this project; check its judge budget in Hue", { cause: error });
+            throw error;
+        }
+        count += jobs.length;
+        jobs = [];
+    };
+    for (const evaluationItemId of [...items].sort()) {
+        for (const scorerVersionId of scorerVersionIds) {
+            jobs.push({ evaluationItemId, scorerVersionId });
+            if (jobs.length === 100)
+                await dispatch();
+        }
+    }
+    if (jobs.length)
+        await dispatch();
+    return count;
+}
+/** Durable handoff before the original attempt marker or queue acknowledgement. */
+async function checkpointJudges(judging, run, completion) {
+    await judging.store.write("pending", {
+        selectionDigest: judging.selectionDigest,
+        run: {
+            experimentId: run.experimentId,
+            subjectIds: run.subjectIds,
+            runUrl: run.runUrl,
+            ...(run.extra ? { extra: run.extra } : {}),
+            ...(run.telemetryNotAccepted ? { telemetryNotAccepted: run.telemetryNotAccepted } : {}),
+        },
+        completion,
+    });
+}
+/** Finish the original handoff if the process stopped between the two durable writes. */
+async function completeJudgeHandoff(client, judging, pending) {
+    const store = await CheckpointStore.acquire(judging.source.directory, judging.source.identity);
+    try {
+        if (pending.completion.kind === "worker") {
+            try {
+                await client.completeLocalAgentRun({
+                    runId: pending.completion.runId,
+                    workerId: pending.completion.workerId,
+                    state: "completed",
+                });
+            }
+            catch (error) {
+                // Released or cancelled work cannot be completed by this worker. Temporary failures
+                // still throw, leaving the durable handoff available for the unchanged command.
+                if (error instanceof HueApiError && error.status === 409)
+                    return false;
+                throw error;
+            }
+        }
+        else {
+            const attempt = await store.read("active-attempt");
+            if (!attempt || attempt.experimentId !== pending.run.experimentId)
+                throw new Error("Hosted judge checkpoint does not match the uploaded attempt");
+            if (attempt.stage !== "completed")
+                await store.write("active-attempt", { ...attempt, stage: "completed" });
+        }
+        return true;
+    }
+    finally {
+        await store.release();
+    }
+}
+/** Discards only the matching terminal worker claim; a retry retains its handoff. */
+async function discardWorkerJudgeHandoff(judging, runId) {
+    const pending = await judging.store.read("pending");
+    if (pending?.completion.kind === "worker" && pending.completion.runId === runId)
+        await judging.store.write("pending", null);
+}
+async function requestRunJudges(client, run, output, signal, judging) {
+    const count = await dispatchPinnedJudges(client, run.experimentId, run.subjectIds, signal);
+    await judging.store.write("pending", null);
+    if (count)
+        output.log(`Requested ${count} hosted judge check${count === 1 ? "" : "s"}.`);
+}
+/** Dispatch failures retain the handoff; reading verdicts does not dispatch again. */
+async function reportVerdicts(client, values, run, baselineId, output, signal, judging) {
+    if (values.judges) {
+        if (!judging)
+            throw new Error("Hosted judge recovery checkpoint is unavailable");
+        await requestRunJudges(client, run, output, signal, judging);
+    }
+    return readAndReportVerdicts(client, values, run, baselineId, output, signal);
+}
+/** Waits for Hue's verdicts, prints them and returns the exit code. */
+async function readAndReportVerdicts(client, values, run, baselineId, output, signal) {
     const wait = integer("wait", values.wait, 300, 0, 86_400);
     output.log("Waiting for Hue checks...");
     const verdicts = withTelemetryFailures(await collectExperimentVerdicts(client, {
@@ -1127,88 +1255,125 @@ async function runOnce(values, connection, agents, agent, hue, output, signal) {
     }
     const runName = values.name ?? defaultRunName(agent);
     const project = await client.checkConnection();
-    if (selection.direct)
-        return runDirect({
+    const checkpointDirectory = await prepareCheckpointDirectory(values["checkpoint-dir"], agent.key, project.id, selection.direct ? "direct" : "simulation");
+    const judging = values.judges
+        ? {
+            store: await CheckpointStore.acquire(checkpointPath(checkpointDirectory, ["judge-dispatch", "judge checkpoint"]), { kind: "cli-judge-dispatch", projectId: project.id, baseUrl: connection.baseUrl }),
+            selectionDigest: digest({
+                datasetVersionId: pins.datasetVersionId,
+                scorerVersionIds: [...pins.scorerVersionIds].sort(),
+                agent,
+            }),
+            source: {
+                directory: checkpointDirectory,
+                identity: {
+                    kind: selection.direct ? "direct" : "simulation",
+                    projectId: project.id,
+                    baseUrl: connection.baseUrl,
+                },
+            },
+        }
+        : undefined;
+    try {
+        const pending = await judging?.store.read("pending");
+        if (pending) {
+            if (pending.selectionDigest !== judging.selectionDigest)
+                throw new Error("Recover the pending judge dispatch with the unchanged command or use another --checkpoint-dir");
+            output.log(`Resuming hosted checks: ${pending.run.runUrl}`);
+            await completeJudgeHandoff(client, judging, pending);
+            return await reportVerdicts(client, values, pending.run, baselineId, output, signal, judging);
+        }
+        if (selection.direct)
+            return await runDirect({
+                client,
+                values,
+                pins,
+                runName,
+                projectId: project.id,
+                agent,
+                agents,
+                hue,
+                output,
+                signal,
+                concurrency,
+                baselineId,
+                judging,
+            }, connection);
+        const environmentClient = createEnvironmentClient(connection);
+        const caseKeys = new Map();
+        let runUrl = "";
+        const telemetry = telemetryReporter(output);
+        const report = await runSimulation({
             client,
-            values,
-            pins,
-            runName,
-            projectId: project.id,
-            agent,
-            agents,
+            environmentClient,
             hue,
-            output,
-            signal,
+            checkpointDirectory,
+            definition: {
+                kind: "pins",
+                datasetVersionId: pins.datasetVersionId,
+                scorerVersionIds: pins.scorerVersionIds,
+            },
+            runName,
+            // Outputs, error messages and explanations are stored unless opted out; --content governs
+            // only the telemetry.
+            persistResultContent: !values["no-output"],
+            traceEvidence: { mode: "required" },
+            // A case whose telemetry Hue did not accept fails instead of staying started.
+            traceNotAccepted: traceNotAcceptedPolicy(values["trace-not-accepted"]),
+            onTelemetryNotAccepted: telemetry.report,
             concurrency,
-            baselineId,
-        }, connection);
-    const environmentClient = createEnvironmentClient(connection);
-    const checkpointDirectory = await prepareCheckpointDirectory(values["checkpoint-dir"], agent.key, project.id, "simulation");
-    const caseKeys = new Map();
-    let runUrl = "";
-    const telemetry = telemetryReporter(output);
-    const report = await runSimulation({
-        client,
-        environmentClient,
-        hue,
-        checkpointDirectory,
-        definition: {
-            kind: "pins",
-            datasetVersionId: pins.datasetVersionId,
-            scorerVersionIds: pins.scorerVersionIds,
-        },
-        runName,
-        // Outputs, error messages and explanations are stored unless opted out; --content governs
-        // only the telemetry.
-        persistResultContent: !values["no-output"],
-        traceEvidence: { mode: "required" },
-        // A case whose telemetry Hue did not accept fails instead of staying started.
-        traceNotAccepted: traceNotAcceptedPolicy(values["trace-not-accepted"]),
-        onTelemetryNotAccepted: telemetry.report,
-        concurrency,
-        agentRevision: agent.revision,
-        // The CLI adapts to whatever the deployment serves; the library warning is for code that
-        // still reads the legacy capability itself.
-        deprecationWarnings: false,
-        signal,
-        target: agents.simulation,
-        async onProgress(event) {
-            if (event.type === "run_created") {
-                runUrl = event.runUrl;
-                output.log(`Run: ${event.runUrl}`);
-                output.log(`Experiment: ${event.experimentId}`);
-                try {
-                    let after;
-                    do {
-                        const page = await client.listExperimentItems(event.experimentId, { after });
-                        for (const item of page.items)
-                            caseKeys.set(item.id, item.externalKey);
-                        after = page.nextCursor ?? undefined;
-                    } while (after);
+            agentRevision: agent.revision,
+            // The CLI adapts to whatever the deployment serves; the library warning is for code that
+            // still reads the legacy capability itself.
+            deprecationWarnings: false,
+            signal,
+            target: agents.simulation,
+            ...(judging
+                ? {
+                    onUploaded: (report) => checkpointJudges(judging, report, { kind: "attempt" }),
                 }
-                catch {
-                    // Progress labels fall back to case IDs; the run itself is unaffected.
+                : {}),
+            async onProgress(event) {
+                if (event.type === "run_created") {
+                    runUrl = event.runUrl;
+                    output.log(`Run: ${event.runUrl}`);
+                    output.log(`Experiment: ${event.experimentId}`);
+                    try {
+                        let after;
+                        do {
+                            const page = await client.listExperimentItems(event.experimentId, { after });
+                            for (const item of page.items)
+                                caseKeys.set(item.id, item.externalKey);
+                            after = page.nextCursor ?? undefined;
+                        } while (after);
+                    }
+                    catch {
+                        // Progress labels fall back to case IDs; the run itself is unaffected.
+                    }
+                    return;
                 }
-                return;
-            }
-            const label = caseKeys.get(event.caseId) ?? event.caseId;
-            if (event.type === "world_created")
-                output.log(`[${label}] world created`);
-            else if (event.type === "target_started")
-                output.log(`[${label}] agent started`);
-            else if (event.type === "world_sealed")
-                output.log(`[${label}] world sealed`);
-            else if (event.type === "attempt_prepared")
-                output.log(`[${label}] attempt prepared: ${event.status}`);
-        },
-    });
-    telemetry.rest(report.telemetryNotAccepted);
-    return reportVerdicts(client, values, {
-        experimentId: report.experimentId,
-        subjectIds: report.subjectIds,
-        runUrl: runUrl || report.runUrl,
-        telemetryNotAccepted: report.telemetryNotAccepted,
-    }, baselineId, output, signal);
+                const label = caseKeys.get(event.caseId) ?? event.caseId;
+                if (event.type === "world_created")
+                    output.log(`[${label}] world created`);
+                else if (event.type === "target_started")
+                    output.log(`[${label}] agent started`);
+                else if (event.type === "world_sealed")
+                    output.log(`[${label}] world sealed`);
+                else if (event.type === "attempt_prepared")
+                    output.log(`[${label}] attempt prepared: ${event.status}`);
+            },
+        });
+        telemetry.rest(report.telemetryNotAccepted);
+        return await reportVerdicts(client, values, {
+            experimentId: report.experimentId,
+            subjectIds: report.subjectIds,
+            runUrl: runUrl || report.runUrl,
+            telemetryNotAccepted: report.telemetryNotAccepted,
+        }, baselineId, output, signal, judging);
+    }
+    finally {
+        await judging?.store.release();
+    }
 }
 /**
  * `--check`: everything a one-shot run resolves before it creates anything (the key's project,
@@ -1352,6 +1517,14 @@ async function runDirect(run, connection) {
                 return result;
             },
         });
+        if (run.judging)
+            await checkpointJudges(run.judging, {
+                experimentId,
+                subjectIds: report.subjectIds,
+                runUrl,
+                extra: { mode: "direct", deferredScorerVersionIds: report.deferredScorerVersionIds },
+                telemetryNotAccepted: report.telemetryNotAccepted,
+            }, { kind: "attempt" });
         attempt.stage = "completed";
         await store.write("active-attempt", attempt);
     }
@@ -1367,126 +1540,199 @@ async function runDirect(run, connection) {
         runUrl,
         extra: { mode: "direct", deferredScorerVersionIds: report.deferredScorerVersionIds },
         telemetryNotAccepted: report.telemetryNotAccepted,
-    }, run.baselineId, output, signal);
+    }, run.baselineId, output, signal, run.judging);
 }
 async function runWorker(values, connection, agents, agent, hue, output, signal) {
     const client = new ObservedClient(connection);
     const environmentClient = createEnvironmentClient(connection);
     const wait = integer("wait", values.wait, 300, 0, 86_400);
     const concurrency = integer("concurrency", values.concurrency, 1, 1, 100);
-    const maxRuns = values["max-runs"] === undefined
+    let maxRuns = values["max-runs"] === undefined
         ? undefined
         : integer("max-runs", values["max-runs"], 1, 1, 1_000_000);
     const project = await client.checkConnection();
     const checkpointDirectory = await prepareCheckpointDirectory(values["checkpoint-dir"], agent.key, project.id, "worker", slug(agent.revision) || "dev");
-    let current;
-    client.onRegistered = (registered) => {
-        output.log(`Registered agent ${registered.key} (revision ${registered.revision}) with ${connection.baseUrl}; waiting for runs launched from Hue${maxRuns ? ` (stops after ${maxRuns})` : " (Ctrl+C to stop)"}`);
-    };
-    client.onClaimed = (claim) => {
-        current = claim;
-        output.log(`Claimed run ${claim.runId}: ${new URL(`/runs/${claim.experimentId}`, connection.baseUrl).toString()}`);
+    const judging = values.judges
+        ? {
+            store: await CheckpointStore.acquire(checkpointPath(checkpointDirectory, ["judge-dispatch", "judge checkpoint"]), {
+                kind: "cli-judge-dispatch",
+                projectId: project.id,
+                baseUrl: connection.baseUrl,
+            }),
+            selectionDigest: digest(agent),
+            source: {
+                directory: checkpointDirectory,
+                identity: {
+                    kind: "local-agent-worker",
+                    projectId: project.id,
+                    baseUrl: connection.baseUrl,
+                    agent: { key: agent.key, revision: agent.revision },
+                },
+            },
+        }
+        : undefined;
+    const reportJudgedRun = async (run) => {
+        if (!judging)
+            throw new Error("Hosted judge recovery checkpoint is unavailable");
+        await requestRunJudges(client, run, output, signal, judging);
+        try {
+            await readAndReportVerdicts(client, values, run, undefined, output, signal);
+        }
+        catch (error) {
+            output.error(`Unable to read verdicts: ${explain(error)}`);
+        }
     };
-    const telemetry = telemetryReporter(output);
-    // A command can use the same stdin contract for both case kinds. Existing adapter files
-    // may require a world, so their direct context remains an explicit capability opt-in.
-    const direct = values.command !== undefined || values.capability?.includes("direct:v1");
-    await runLocalAgent({
-        client,
-        environmentClient,
-        hue,
-        checkpointDirectory,
-        agent: {
-            key: agent.key,
-            name: agent.name,
-            revision: agent.revision,
-            capabilities: ["environment:v1", ...(values.capability ?? [])],
-        },
-        scorers: [],
-        concurrency,
-        traceNotAccepted: traceNotAcceptedPolicy(values["trace-not-accepted"]),
-        onTelemetryNotAccepted: telemetry.report,
-        deprecationWarnings: false,
-        signal,
-        ...(maxRuns === undefined ? {} : { maxRuns }),
-        onPollError(error) {
-            output.error(`Hue did not answer the poll (${explain(error)}); polling again`);
-        },
-        onRunFailed(failure) {
-            const run = `Run ${failure.runId}`;
-            if (failure.outcome === "retry") {
-                const waitMillis = failure.waitMillis ?? 0;
-                output.error(`${run} attempt ${failure.attempt} failed (${explain(failure.error)}); resuming in ${Math.round(waitMillis / 1000)} s`);
+    try {
+        const pending = await judging?.store.read("pending");
+        if (pending) {
+            if (pending.selectionDigest !== judging.selectionDigest)
+                throw new Error("Recover the pending judge dispatch with the unchanged command or use another --checkpoint-dir");
+            output.log(`Resuming hosted checks: ${pending.run.runUrl}`);
+            if (await completeJudgeHandoff(client, judging, pending)) {
+                await reportJudgedRun(pending.run);
             }
-            else if (failure.outcome === "attention")
-                output.error(`${run} needs attention (${explain(failure.error)}); requeue or cancel it from its run page`);
-            else
-                output.error(`${run} was released or cancelled in Hue; its outcome here is not recorded`);
-        },
-        target(inputs, tools, context) {
-            output.log(`[${context.item.externalKey}] agent started`);
-            return agents.simulation(inputs, {
-                config: context.config,
-                item: context.item,
-                executionId: context.executionId,
-                environmentRunId: context.environmentRunId,
-                tools,
-                ...(context.world ? { world: context.world } : {}),
-                ...(context.mcp ? { mcp: context.mcp } : {}),
-                ...(context.connectionBundle ? { connectionBundle: context.connectionBundle } : {}),
-                files: context.files,
-                outputDirectory: context.outputDirectory,
-                // The case's own signal: the worker stopping, or the world's deadline passing, both
-                // end a command that would otherwise keep running past its timed-out case.
-                signal: context.signal ?? signal,
-            });
-        },
-        ...(direct
-            ? {
-                directTarget(inputs, context) {
-                    output.log(`[${context.item.externalKey}] agent started`);
-                    return agents.direct(inputs, {
-                        mode: "direct",
-                        config: context.config,
-                        item: context.item,
-                        executionId: context.executionId,
-                        files: context.files,
-                        outputDirectory: context.outputDirectory,
-                        signal: context.signal ?? signal,
-                    });
-                },
+            else if (pending.completion.kind === "worker") {
+                await discardWorkerJudgeHandoff(judging, pending.completion.runId);
+                output.error(`Run ${pending.completion.runId} was released or cancelled in Hue`);
             }
-            : {}),
-        async onCompleted(report) {
-            output.log(`Run ${report.runId} completed: ${report.subjectIds.length} case${report.subjectIds.length === 1 ? "" : "s"}`);
-            telemetry.rest(report.telemetryNotAccepted);
-            if (!current)
-                return;
-            output.log("Waiting for Hue checks...");
-            try {
-                const verdicts = withTelemetryFailures(await collectExperimentVerdicts(client, {
-                    experimentId: current.experimentId,
-                    subjectIds: report.subjectIds,
-                    timeoutMillis: wait * 1000,
-                    signal,
-                }), report.telemetryNotAccepted);
-                if (signal.aborted)
-                    return;
-                if (values.json)
-                    process.stdout.write(`${JSON.stringify(toJson(verdicts, new URL(`/runs/${current.experimentId}`, connection.baseUrl).toString(), undefined, {}, report.telemetryNotAccepted))}\n`);
+            if (signal.aborted)
+                return 130;
+            if (maxRuns !== undefined && --maxRuns === 0)
+                return 0;
+        }
+        let current;
+        client.onRegistered = (registered) => {
+            output.log(`Registered agent ${registered.key} (revision ${registered.revision}) with ${connection.baseUrl}; waiting for runs launched from Hue${maxRuns ? ` (stops after ${maxRuns})` : " (Ctrl+C to stop)"}`);
+        };
+        client.onClaimed = (claim) => {
+            current = claim;
+            output.log(`Claimed run ${claim.runId}: ${new URL(`/runs/${claim.experimentId}`, connection.baseUrl).toString()}`);
+        };
+        const telemetry = telemetryReporter(output);
+        // A command can use the same stdin contract for both case kinds. Existing adapter files
+        // may require a world, so their direct context remains an explicit capability opt-in.
+        const direct = values.command !== undefined || values.capability?.includes("direct:v1");
+        await runLocalAgent({
+            client,
+            environmentClient,
+            hue,
+            checkpointDirectory,
+            agent: {
+                key: agent.key,
+                name: agent.name,
+                revision: agent.revision,
+                capabilities: ["environment:v1", ...(values.capability ?? [])],
+            },
+            scorers: [],
+            concurrency,
+            traceNotAccepted: traceNotAcceptedPolicy(values["trace-not-accepted"]),
+            onTelemetryNotAccepted: telemetry.report,
+            deprecationWarnings: false,
+            signal,
+            ...(maxRuns === undefined ? {} : { maxRuns }),
+            onPollError(error) {
+                output.error(`Hue did not answer the poll (${explain(error)}); polling again`);
+            },
+            async onRunFailed(failure) {
+                if (judging && failure.outcome !== "retry")
+                    await discardWorkerJudgeHandoff(judging, failure.runId);
+                const run = `Run ${failure.runId}`;
+                if (failure.outcome === "retry") {
+                    const waitMillis = failure.waitMillis ?? 0;
+                    output.error(`${run} attempt ${failure.attempt} failed (${explain(failure.error)}); resuming in ${Math.round(waitMillis / 1000)} s`);
+                }
+                else if (failure.outcome === "attention")
+                    output.error(`${run} needs attention (${explain(failure.error)}); requeue or cancel it from its run page`);
                 else
-                    renderTable(verdicts, output);
-            }
-            catch (error) {
-                output.error(`Unable to read verdicts: ${explain(error)}`);
-            }
-        },
-    });
-    if (signal.aborted) {
-        process.stderr.write("Interrupted.\n");
-        return 130;
+                    output.error(`${run} was released or cancelled in Hue; its outcome here is not recorded`);
+            },
+            target(inputs, tools, context) {
+                output.log(`[${context.item.externalKey}] agent started`);
+                return agents.simulation(inputs, {
+                    config: context.config,
+                    item: context.item,
+                    executionId: context.executionId,
+                    environmentRunId: context.environmentRunId,
+                    tools,
+                    ...(context.world ? { world: context.world } : {}),
+                    ...(context.mcp ? { mcp: context.mcp } : {}),
+                    ...(context.connectionBundle ? { connectionBundle: context.connectionBundle } : {}),
+                    files: context.files,
+                    outputDirectory: context.outputDirectory,
+                    // The case's own signal: the worker stopping, or the world's deadline passing, both
+                    // end a command that would otherwise keep running past its timed-out case.
+                    signal: context.signal ?? signal,
+                });
+            },
+            ...(direct
+                ? {
+                    directTarget(inputs, context) {
+                        output.log(`[${context.item.externalKey}] agent started`);
+                        return agents.direct(inputs, {
+                            mode: "direct",
+                            config: context.config,
+                            item: context.item,
+                            executionId: context.executionId,
+                            files: context.files,
+                            outputDirectory: context.outputDirectory,
+                            signal: context.signal ?? signal,
+                        });
+                    },
+                }
+                : {}),
+            ...(judging
+                ? {
+                    onUploaded: (report, claim) => checkpointJudges(judging, {
+                        experimentId: claim.experimentId,
+                        subjectIds: report.subjectIds,
+                        runUrl: new URL(`/runs/${claim.experimentId}`, connection.baseUrl).toString(),
+                        telemetryNotAccepted: report.telemetryNotAccepted,
+                    }, { kind: "worker", runId: claim.runId, workerId: claim.workerId }),
+                }
+                : {}),
+            async onCompleted(report) {
+                output.log(`Run ${report.runId} completed: ${report.subjectIds.length} case${report.subjectIds.length === 1 ? "" : "s"}`);
+                telemetry.rest(report.telemetryNotAccepted);
+                if (!current)
+                    return;
+                if (values.judges) {
+                    await reportJudgedRun({
+                        experimentId: current.experimentId,
+                        subjectIds: report.subjectIds,
+                        runUrl: new URL(`/runs/${current.experimentId}`, connection.baseUrl).toString(),
+                        telemetryNotAccepted: report.telemetryNotAccepted,
+                    });
+                    return;
+                }
+                try {
+                    output.log("Waiting for Hue checks...");
+                    const verdicts = withTelemetryFailures(await collectExperimentVerdicts(client, {
+                        experimentId: current.experimentId,
+                        subjectIds: report.subjectIds,
+                        timeoutMillis: wait * 1000,
+                        signal,
+                    }), report.telemetryNotAccepted);
+                    if (signal.aborted)
+                        return;
+                    if (values.json)
+                        process.stdout.write(`${JSON.stringify(toJson(verdicts, new URL(`/runs/${current.experimentId}`, connection.baseUrl).toString(), undefined, {}, report.telemetryNotAccepted))}\n`);
+                    else
+                        renderTable(verdicts, output);
+                }
+                catch (error) {
+                    output.error(`Unable to read verdicts: ${explain(error)}`);
+                }
+            },
+        });
+        if (signal.aborted) {
+            process.stderr.write("Interrupted.\n");
+            return 130;
+        }
+        return 0;
+    }
+    finally {
+        await judging?.store.release();
     }
-    return 0;
 }
 /**
  * `hue eval`: runs a local adapter or command against a Scenario or eval set through
@@ -1529,6 +1775,8 @@ export async function runEvalCommand(argv) {
         worker = values.worker;
         if (values.content && values["no-content"])
             throw new UsageError("Choose either --content or --no-content");
+        if (values.judges && values["no-output"])
+            throw new UsageError("--judges requires stored case outputs; remove --no-output");
         // A usage error before anything is prepared or created: no run is left to recover from it.
         traceNotAcceptedPolicy(values["trace-not-accepted"]);
         json = values.json;