@hue-run/sdk 0.4.2 → 0.5.1

package/dist/cli/eval.js

@@ -0,0 +1,985 @@
+import { execFileSync, spawn } from "node:child_process";
+import { access, mkdir, writeFile } from "node:fs/promises";
+import { basename, extname, join, resolve } from "node:path";
+import { pathToFileURL } from "node:url";
+import { parseArgs } from "node:util";
+import { randomUUID } from "node:crypto";
+import { createHue } from "../client.js";
+import { createEnvironmentClient } from "../environment/client.js";
+import { EvaluationClient, HueApiError } from "../evals/client.js";
+import { TargetResult } from "../evals/types.js";
+import { CheckpointStore } from "../evals/checkpoint.js";
+import { digest } from "../evals/json.js";
+import { runLocalAgent } from "../evals/local-worker.js";
+import { runExperiment, TargetCancelledError } from "../evals/runner.js";
+import { matchByName, parseScenarioSelector, resolveEvalSetPins, resolveScenarioPins, } from "../evals/scenarios.js";
+import { collectDirectOutputs, stageDirectCase } from "./eval-direct.js";
+import { runSimulation, } from "../evals/simulation.js";
+import { collectExperimentVerdicts, compareVerdicts, metricPassed, } from "../evals/verdicts.js";
+const USAGE = `Usage: hue eval [adapter-file] [options]
+
+Run a local agent against a Hue Scenario or eval set, then print Hue's verdicts.
+
+Selection (exactly one, not used with --worker):
+  --scenario <name|id|url>        Published Scenario to run
+  --set <name|id|url>             Saved eval set; requires --scorer or --scorer-version
+  --set-version <n>               Saved version number of the eval set (default: latest saved)
+  --dataset-version <id>          Frozen dataset version; requires --scorer or --scorer-version
+  --scorer <slug|name|id>         Evaluator to pin at its latest published version (repeatable)
+  --scorer-version <id>           Scorer version to pin (repeatable)
+  --mode <auto|direct|simulation> Case kind; auto picks direct for sets whose cases pin no world
+
+Agent (exactly one):
+  <adapter-file>                  Module exporting default or runMyAgent(inputs, context)
+  --command "<shell command>"     Simulation: spawned per case with HUE_MCP_URL, HUE_MCP_TOKEN,
+                                  HUE_MCP_EXPIRES_AT, HUE_EXECUTION_ID, HUE_ENVIRONMENT_RUN_ID,
+                                  HUE_CASE_ID and HUE_CASE_KEY set; {"inputs","config"} on stdin.
+                                  Direct: spawned in a private case directory with HUE_CASE_DIR,
+                                  HUE_CASE_INPUTS, HUE_CASE_OUTPUT_DIR, HUE_CASE_ID, HUE_CASE_KEY
+                                  and HUE_EXECUTION_ID set; files/<role>/ hold the pinned inputs
+                                  and every file written to output/ is uploaded to Hue
+
+Modes:
+  --worker                        Register the agent and poll for runs launched from Hue
+  --max-runs <n>                  Stop the worker after n completed runs
+  --agent-key <key>               Agent key (default: slug of the adapter filename)
+  --agent-name <name>             Agent display name (default: the key)
+  --revision <id>                 Agent revision (default: AGENT_REVISION, git HEAD or "dev")
+
+Connection:
+  --env-file <path>               Load a dotenv file (HUE_API_KEY, HUE_BASE_URL) first
+  --origin <url>                  Hue origin (default: HUE_BASE_URL or https://app.hue.run)
+
+Output and limits:
+  --name <run name>               Experiment name (default: <scenario> · <agent key> · <revision>)
+  --baseline <experiment id|url>  Compare verdicts with a previous experiment
+  --json                          Print one JSON document on stdout; progress goes to stderr
+  --content                       Capture telemetry content; one-shot also persists
+                                  outputs/explanations (--worker always persists them)
+  --save-version                  Freeze an unsaved eval-set version before running
+  --checkpoint-dir <path>         Private checkpoint directory (default: .hue/eval/<agent-key>)
+  --concurrency <n>               Cases in flight, 1-16 (default: 1)
+  --timeout <seconds>             Per-case --command timeout (default: 600)
+  --wait <seconds>                Verdict wait after the run finishes (default: 300)
+  -h, --help                      Show this help
+
+HUE_API_KEY must be a "Read and write" project key; it is never printed.
+Code evaluators pinned to a direct run are graded by Hue's executor after the upload; the wait
+covers them. Exit codes: 0 every case passed, 1 a case failed, errored or is incomplete,
+2 usage error, 130 interrupted.
+`;
+/** Thrown for invalid arguments or configuration; exits with status 2. */
+class UsageError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = "UsageError";
+    }
+}
+function parse(argv) {
+    try {
+        return parseArgs({
+            args: argv,
+            allowPositionals: true,
+            strict: true,
+            options: {
+                scenario: { type: "string" },
+                set: { type: "string" },
+                "set-version": { type: "string" },
+                "dataset-version": { type: "string" },
+                scorer: { type: "string", multiple: true },
+                "scorer-version": { type: "string", multiple: true },
+                mode: { type: "string" },
+                command: { type: "string" },
+                worker: { type: "boolean", default: false },
+                "max-runs": { type: "string" },
+                "agent-key": { type: "string" },
+                "agent-name": { type: "string" },
+                revision: { type: "string" },
+                "env-file": { type: "string" },
+                origin: { type: "string" },
+                name: { type: "string" },
+                baseline: { type: "string" },
+                json: { type: "boolean", default: false },
+                content: { type: "boolean", default: false },
+                "save-version": { type: "boolean", default: false },
+                "checkpoint-dir": { type: "string" },
+                concurrency: { type: "string" },
+                timeout: { type: "string" },
+                wait: { type: "string" },
+                help: { type: "boolean", short: "h", default: false },
+            },
+        });
+    }
+    catch (error) {
+        throw new UsageError(error instanceof Error ? error.message : "Invalid arguments");
+    }
+}
+function integer(name, value, fallback, min, max) {
+    if (value === undefined)
+        return fallback;
+    const parsed = Number(value);
+    if (!Number.isInteger(parsed) || parsed < min || parsed > max)
+        throw new UsageError(`--${name} must be an integer between ${min} and ${max}`);
+    return parsed;
+}
+function slug(value) {
+    return value
+        .toLowerCase()
+        .replace(/[^a-z0-9]+/g, "-")
+        .replace(/^-+|-+$/g, "")
+        .slice(0, 64);
+}
+const INTERPRETERS = new Set([
+    "node",
+    "nodejs",
+    "bun",
+    "bunx",
+    "deno",
+    "npx",
+    "pnpx",
+    "yarn",
+    "pnpm",
+    "npm",
+    "tsx",
+    "ts-node",
+    "python",
+    "python3",
+    "uv",
+    "uvx",
+    "sh",
+    "bash",
+    "zsh",
+    "env",
+]);
+/** Agent key from the adapter filename, or the first script-like token of a command. */
+function derivedAgentKey(adapterFile, command) {
+    if (adapterFile)
+        return slug(basename(adapterFile, extname(adapterFile)));
+    const tokens = (command ?? "").trim().split(/\s+/);
+    const names = tokens.map((token) => token.split(/[\\/]/).pop() ?? "");
+    const script = names.find((name) => name && !name.startsWith("-") && !INTERPRETERS.has(name.toLowerCase())) ??
+        names[0] ??
+        "";
+    return slug(basename(script, extname(script)));
+}
+function gitRevision() {
+    try {
+        const value = execFileSync("git", ["rev-parse", "--short=12", "HEAD"], {
+            encoding: "utf8",
+            stdio: ["ignore", "pipe", "ignore"],
+            timeout: 5000,
+        }).trim();
+        return /^[0-9a-f]{7,40}$/.test(value) ? value : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+async function loadAdapter(file) {
+    const path = resolve(file);
+    try {
+        await access(path);
+    }
+    catch {
+        throw new UsageError(`Adapter file not found: ${file}`);
+    }
+    let loaded;
+    try {
+        loaded = (await import(pathToFileURL(path).href));
+    }
+    catch (error) {
+        const code = error.code;
+        const hint = code === "ERR_UNKNOWN_FILE_EXTENSION" ||
+            code === "ERR_UNSUPPORTED_NODE_MODULES_TYPE_STRIPPING"
+            ? " (this Node.js version needs --experimental-strip-types or --import tsx for TypeScript adapters; non-erasable syntax such as enums or parameter properties always needs a loader)"
+            : "";
+        throw new Error(`Unable to load the adapter ${file}: ${error instanceof Error ? error.message : String(error)}${hint}`);
+    }
+    const candidate = loaded.default ?? loaded.runMyAgent;
+    if (typeof candidate !== "function")
+        throw new UsageError(`${file} must export a default function or runMyAgent(inputs, context)`);
+    return candidate;
+}
+/** Grace between the stop signal and SIGKILL for an agent command that ignores SIGTERM. */
+const COMMAND_KILL_GRACE_MS = 5_000;
+/**
+ * Spawns the agent command once in its own process group and returns its trimmed stdout. A
+ * timeout or Ctrl+C stops the agent the shell started, not only the shell: a survivor would still
+ * hold a world token and could write after Hue recorded the case as failed. Windows has no
+ * process group to signal, so the child alone is stopped there.
+ */
+function spawnAgentCommand(command, options) {
+    return new Promise((resolvePromise, reject) => {
+        const group = process.platform !== "win32";
+        const child = spawn(command, {
+            shell: true,
+            detached: group,
+            ...(options.cwd ? { cwd: options.cwd } : {}),
+            env: options.env,
+            stdio: ["pipe", "pipe", "inherit"],
+        });
+        const chunks = [];
+        let size = 0;
+        let timedOut = false;
+        let oversized = false;
+        let escalation;
+        const signalTree = (signal) => {
+            if (child.exitCode !== null || child.signalCode !== null)
+                return;
+            try {
+                if (group && child.pid !== undefined)
+                    process.kill(-child.pid, signal);
+                else
+                    child.kill(signal);
+            }
+            catch {
+                // The group is already gone; nothing is left to stop.
+            }
+        };
+        const stop = (signal) => {
+            signalTree(signal);
+            escalation ??= setTimeout(() => signalTree("SIGKILL"), COMMAND_KILL_GRACE_MS).unref();
+        };
+        const timer = setTimeout(() => {
+            timedOut = true;
+            stop("SIGTERM");
+        }, options.timeoutSeconds * 1000);
+        const cancel = () => stop("SIGTERM");
+        options.signal?.addEventListener("abort", cancel, { once: true });
+        child.stdout.on("data", (chunk) => {
+            size += chunk.byteLength;
+            if (size > 4 * 1024 * 1024) {
+                oversized = true;
+                stop("SIGTERM");
+                return;
+            }
+            chunks.push(chunk);
+        });
+        child.stdin.on("error", () => undefined);
+        if (options.stdin !== undefined)
+            child.stdin.end(options.stdin);
+        else
+            child.stdin.end();
+        child.on("error", (error) => {
+            clearTimeout(timer);
+            clearTimeout(escalation);
+            options.signal?.removeEventListener("abort", cancel);
+            reject(new Error(`Unable to start the agent command: ${error.message}`));
+        });
+        child.on("close", (code, signal) => {
+            clearTimeout(timer);
+            clearTimeout(escalation);
+            options.signal?.removeEventListener("abort", cancel);
+            if (options.signal?.aborted)
+                return reject(new TargetCancelledError());
+            if (timedOut)
+                return reject(new Error(`The agent command timed out after ${options.timeoutSeconds} seconds`));
+            if (oversized)
+                return reject(new Error("The agent command printed more than 4 MiB"));
+            if (code !== 0)
+                return reject(new Error(signal
+                    ? `The agent command was stopped by ${signal}`
+                    : `The agent command exited with code ${code}`));
+            resolvePromise(Buffer.concat(chunks).toString("utf8").trim());
+        });
+    });
+}
+function parseAnswer(text) {
+    if (!text)
+        return undefined;
+    try {
+        return JSON.parse(text);
+    }
+    catch {
+        return text;
+    }
+}
+/** Runs the shell command once per case; the MCP token travels only through the child's environment. */
+function commandAdapter(command, timeoutSeconds) {
+    return async (inputs, context) => parseAnswer(await spawnAgentCommand(command, {
+        env: {
+            ...process.env,
+            HUE_MCP_URL: context.mcp.url,
+            HUE_MCP_TOKEN: context.mcp.token,
+            HUE_MCP_EXPIRES_AT: context.mcp.expiresAt,
+            HUE_EXECUTION_ID: context.executionId,
+            HUE_ENVIRONMENT_RUN_ID: context.environmentRunId,
+            HUE_CASE_ID: context.item.id,
+            HUE_CASE_KEY: context.item.externalKey,
+        },
+        stdin: JSON.stringify({ inputs, config: context.config }),
+        timeoutSeconds,
+        ...(context.signal ? { signal: context.signal } : {}),
+    }));
+}
+/**
+ * Direct cases: the command works in a private case directory and writes its documents to
+ * `output/`. Its stdout is only used as the JSON output when it wrote no result or summary file.
+ */
+function directCommandAdapter(command, timeoutSeconds) {
+    return async (inputs, context) => {
+        const layout = await stageDirectCase(context.outputDirectory, {
+            inputs,
+            config: context.config,
+            item: context.item,
+            executionId: context.executionId,
+            files: context.files,
+        });
+        const stdout = await spawnAgentCommand(command, {
+            cwd: layout.caseDirectory,
+            env: {
+                ...process.env,
+                HUE_CASE_DIR: layout.caseDirectory,
+                HUE_CASE_INPUTS: layout.inputsPath,
+                HUE_CASE_OUTPUT_DIR: layout.outputDirectory,
+                HUE_CASE_ID: context.item.id,
+                HUE_CASE_KEY: context.item.externalKey,
+                HUE_EXECUTION_ID: context.executionId,
+            },
+            stdin: JSON.stringify({ inputs, config: context.config }),
+            timeoutSeconds,
+            ...(context.signal ? { signal: context.signal } : {}),
+        });
+        return collectDirectOutputs(layout.outputDirectory, parseAnswer(stdout));
+    };
+}
+function metricText(metric) {
+    if (typeof metric.value === "boolean")
+        return metricPassed(metric) ? "PASS" : "FAIL";
+    if (typeof metric.value === "number")
+        return Number.isInteger(metric.value) ? String(metric.value) : metric.value.toFixed(3);
+    const text = metric.value.replace(/\s+/g, " ");
+    return text.length > 40 ? `${text.slice(0, 37)}...` : text;
+}
+const STATE_LABEL = {
+    passed: "PASSED",
+    failed: "FAILED",
+    error: "ERROR",
+    skipped: "SKIPPED",
+    pending: "PENDING",
+};
+function renderTable(verdicts, output) {
+    const names = [];
+    for (const item of verdicts.summary.cases)
+        for (const metric of item.metrics)
+            if (!names.includes(metric.name))
+                names.push(metric.name);
+    const header = ["Case", ...names, "Result"];
+    const rows = verdicts.summary.cases.map((item) => [
+        item.externalKey,
+        ...names.map((name) => {
+            const metric = item.metrics.find((candidate) => candidate.name === name);
+            return metric ? metricText(metric) : "-";
+        }),
+        STATE_LABEL[item.state],
+    ]);
+    const widths = header.map((cell, index) => Math.max(cell.length, ...rows.map((row) => row[index].length)));
+    const line = (cells) => cells
+        .map((cell, index) => cell.padEnd(widths[index]))
+        .join("  ")
+        .trimEnd();
+    output.log(line(header));
+    output.log(widths.map((width) => "-".repeat(width)).join("  "));
+    for (const row of rows)
+        output.log(line(row));
+    for (const item of verdicts.summary.cases) {
+        if (item.state === "passed")
+            continue;
+        const details = [...item.errors.map((type) => `scorer error ${type}`), ...item.explanations];
+        for (const detail of details.length ? details : [STATE_LABEL[item.state].toLowerCase()])
+            output.log(`  ${item.externalKey}: ${detail}`);
+    }
+    const totals = verdicts.summary.totals;
+    const extra = [
+        totals.error ? `${totals.error} error` : "",
+        totals.skipped ? `${totals.skipped} skipped` : "",
+        totals.pending ? `${totals.pending} pending` : "",
+    ].filter(Boolean);
+    output.log(`${totals.passed} of ${totals.cases} case${totals.cases === 1 ? "" : "s"} passed${extra.length ? ` (${extra.join(", ")})` : ""}`);
+    if (!verdicts.results.complete)
+        output.log("Hue checks are still running; rerun with a longer --wait or open the run URL.");
+}
+function renderComparison(baselineId, comparison, output) {
+    output.log(`Baseline ${baselineId}: ${comparison.improvements} improved, ${comparison.regressions} regressed, ${comparison.unchanged} unchanged`);
+    for (const item of comparison.cases)
+        if (item.change !== "unchanged")
+            output.log(`  ${item.externalKey}: ${item.before} -> ${item.after} (${item.change})`);
+}
+function redact(message, secrets) {
+    let text = message;
+    for (const secret of secrets)
+        if (secret)
+            text = text.replaceAll(secret, "[redacted]");
+    return text;
+}
+function explain(error) {
+    if (error instanceof HueApiError && (error.status === 401 || error.status === 403))
+        return `${error.message}. Check that HUE_API_KEY is a "Read and write" project key for this origin.`;
+    if (error instanceof Error) {
+        const causes = [];
+        let cause = error.cause;
+        while (cause instanceof Error && causes.length < 3) {
+            causes.push(cause.message);
+            cause = cause.cause;
+        }
+        return causes.length ? `${error.message} (${causes.join("; ")})` : error.message;
+    }
+    return String(error);
+}
+/** `.hue/eval/<agent-key>/<project>/<leaf>` unless overridden; the SDK binds each store to one
+ * project and origin, so switching keys or deployments must not collide. */
+async function prepareCheckpointDirectory(explicit, agentKey, projectId, leaf) {
+    if (explicit)
+        return join(resolve(explicit), projectId, leaf);
+    const root = resolve(".hue", "eval");
+    await mkdir(root, { recursive: true, mode: 0o700 });
+    const ignore = join(root, ".gitignore");
+    try {
+        await access(ignore);
+    }
+    catch {
+        await writeFile(ignore, "*\n", { flag: "wx", mode: 0o600 }).catch(() => undefined);
+    }
+    return join(root, agentKey, projectId, leaf);
+}
+/** Worker-side client that reports registration and claims without changing the worker. */
+class ObservedClient extends EvaluationClient {
+    registered = false;
+    onRegistered;
+    onClaimed;
+    async registerLocalAgent(input) {
+        const agent = await super.registerLocalAgent(input);
+        if (!this.registered) {
+            this.registered = true;
+            this.onRegistered?.(agent);
+        }
+        return agent;
+    }
+    async claimLocalAgentRun(input) {
+        const claim = await super.claimLocalAgentRun(input);
+        if (claim)
+            this.onClaimed?.(claim);
+        return claim;
+    }
+}
+const MAX_LISTED_SCORERS = 1000;
+/** Newest published version of an evaluator named by ID, slug or display name. */
+async function resolveScorerVersion(client, selector) {
+    const parsed = parseScenarioSelector(selector, ["scorers", "evaluators"]);
+    let scorer;
+    if (parsed.kind === "id")
+        scorer = await client.getScorer(parsed.id);
+    else {
+        const candidates = [];
+        let after;
+        for (;;) {
+            const page = await client.listScorers({ after, limit: 100 });
+            candidates.push(...page.items.filter((item) => !item.archivedAt));
+            if (!page.nextCursor || candidates.length >= MAX_LISTED_SCORERS)
+                break;
+            after = page.nextCursor;
+        }
+        const wanted = parsed.name.trim().toLowerCase();
+        const bySlug = candidates.filter((item) => item.slug.toLowerCase() === wanted);
+        const { matches } = bySlug.length ? { matches: bySlug } : matchByName(candidates, parsed.name);
+        if (matches.length > 1)
+            throw new UsageError(`Several evaluators match "${parsed.name}"; pass a slug or ID: ${matches.map((item) => item.slug).join(", ")}`);
+        if (!matches.length)
+            throw new UsageError(candidates.length
+                ? `No evaluator matches "${parsed.name}". Evaluators: ${candidates.map((item) => item.slug).join(", ")}`
+                : `No evaluator matches "${parsed.name}"`);
+        scorer = await client.getScorer(matches[0].id);
+    }
+    const versions = scorer.versions ?? [];
+    if (!versions.length)
+        throw new UsageError(`Evaluator "${scorer.slug}" has no published version to pin`);
+    // Servers list versions newest first; prefer the version number when the response carries it.
+    const latest = versions.reduce((best, item) => {
+        const candidate = item;
+        return best.version !== undefined && candidate.version !== undefined
+            ? candidate.version > best.version
+                ? candidate
+                : best
+            : best;
+    }, versions[0]);
+    return latest.id;
+}
+async function resolveSelection(client, values) {
+    const extra = [...(values["scorer-version"] ?? [])];
+    for (const selector of values.scorer ?? [])
+        extra.push(await resolveScorerVersion(client, selector));
+    if (values.scenario) {
+        if (values["set-version"])
+            throw new UsageError("--set-version applies to --set only");
+        const pins = await resolveScenarioPins(client, values.scenario);
+        pins.scorerVersionIds = [...new Set([...pins.scorerVersionIds, ...extra])];
+        return pins;
+    }
+    if (!extra.length)
+        throw new UsageError(`${values.set ? "--set" : "--dataset-version"} needs at least one --scorer or --scorer-version; use --scenario for published pins`);
+    if (values.set) {
+        const pins = await resolveEvalSetPins(client, values.set, { scorerVersionIds: extra });
+        if (values["set-version"] === undefined)
+            return pins;
+        const wanted = integer("set-version", values["set-version"], 1, 1, 1_000_000);
+        const dataset = await client.getDataset(pins.datasetId);
+        const version = dataset.versions.find((item) => item.version === wanted);
+        if (!version)
+            throw new UsageError(`Eval set "${dataset.name}" has no version ${wanted}; versions: ${dataset.versions.map((item) => item.version).join(", ")}`);
+        return {
+            ...pins,
+            datasetVersionId: version.id,
+            saved: version.frozenAt !== null,
+            revision: version.revision,
+        };
+    }
+    if (values["set-version"])
+        throw new UsageError("--set-version applies to --set only");
+    const version = await client.getDatasetVersion(values["dataset-version"]);
+    const dataset = await client.getDataset(version.datasetId);
+    return {
+        scenarioId: null,
+        name: dataset.name,
+        datasetId: dataset.id,
+        datasetVersionId: version.id,
+        scorerVersionIds: [...new Set(extra)],
+        environmentVersionId: null,
+        saved: version.frozenAt !== null,
+        revision: version.revision,
+    };
+}
+function parseMode(value) {
+    if (value === undefined)
+        return "auto";
+    if (value === "auto" || value === "direct" || value === "simulation")
+        return value;
+    throw new UsageError("--mode must be auto, direct or simulation");
+}
+/** Direct when nothing pins a simulated world: not a Scenario, and no case of the version does. */
+async function detectDirect(client, pins, mode) {
+    if (mode !== "auto")
+        return mode === "direct";
+    if (pins.scenarioId || pins.environmentVersionId)
+        return false;
+    let after;
+    do {
+        const page = await client.listCases(pins.datasetVersionId, { after });
+        if (page.items.some((item) => item.environmentVersionId))
+            return false;
+        after = page.nextCursor ?? undefined;
+    } while (after);
+    return true;
+}
+function toJson(verdicts, runUrl, baseline, extra = {}) {
+    return {
+        experimentId: verdicts.experimentId,
+        runId: verdicts.runId,
+        runUrl,
+        complete: verdicts.results.complete,
+        cases: verdicts.summary.cases,
+        totals: verdicts.summary.totals,
+        ...extra,
+        ...(baseline
+            ? { baseline: { experimentId: baseline.experimentId, ...baseline.comparison } }
+            : {}),
+    };
+}
+/** Waits for Hue's verdicts, prints them (table or JSON, with the optional baseline) and returns the exit code. */
+async function reportVerdicts(client, values, run, baselineId, output, signal) {
+    const wait = integer("wait", values.wait, 300, 0, 86_400);
+    output.log("Waiting for Hue checks...");
+    const verdicts = await collectExperimentVerdicts(client, {
+        experimentId: run.experimentId,
+        subjectIds: run.subjectIds,
+        timeoutMillis: wait * 1000,
+        signal,
+    });
+    // The wait returns its partial state on abort rather than throwing, so Ctrl+C here must not
+    // fall through to a baseline read and a verdict table that nobody asked to finish.
+    if (signal.aborted) {
+        process.stderr.write("Interrupted.\n");
+        return 130;
+    }
+    let baseline;
+    if (baselineId) {
+        const previous = await collectExperimentVerdicts(client, {
+            experimentId: baselineId,
+            timeoutMillis: 0,
+        });
+        baseline = {
+            experimentId: baselineId,
+            comparison: compareVerdicts(verdicts.summary, previous.summary),
+        };
+    }
+    if (values.json) {
+        process.stdout.write(`${JSON.stringify(toJson(verdicts, run.runUrl, baseline, run.extra))}\n`);
+    }
+    else {
+        renderTable(verdicts, output);
+        if (baseline)
+            renderComparison(baseline.experimentId, baseline.comparison, output);
+        output.log(`Run: ${run.runUrl}`);
+    }
+    const totals = verdicts.summary.totals;
+    return verdicts.results.complete && totals.cases > 0 && totals.passed === totals.cases ? 0 : 1;
+}
+function parseBaseline(value) {
+    if (!value)
+        return undefined;
+    const parsed = parseScenarioSelector(value, ["experiments"]);
+    if (parsed.kind !== "id")
+        throw new UsageError("--baseline must be an experiment ID or its Hue URL");
+    return parsed.id;
+}
+async function runOnce(values, connection, agents, agent, hue, output, signal) {
+    const client = new EvaluationClient(connection);
+    const concurrency = integer("concurrency", values.concurrency, 1, 1, 16);
+    const baselineId = parseBaseline(values.baseline);
+    const mode = parseMode(values.mode);
+    const pins = await resolveSelection(client, values);
+    if (!pins.saved) {
+        if (!values["save-version"]) {
+            output.error(`The ${pins.scenarioId ? "Scenario" : "eval set"} "${pins.name}" points at an unsaved version. Choose "Save eval-set version" in Hue, or pass --save-version to freeze it now, then rerun.`);
+            return 1;
+        }
+        const frozen = await client.freezeDatasetVersion(pins.datasetVersionId, pins.revision);
+        output.log(`Saved "${pins.name}" version ${frozen.version}.`);
+    }
+    const runName = values.name ?? `${pins.name} · ${agent.key} · ${agent.revision}`;
+    const project = await client.checkConnection();
+    if (await detectDirect(client, pins, mode))
+        return runDirect({
+            client,
+            values,
+            pins,
+            runName,
+            projectId: project.id,
+            agent,
+            agents,
+            hue,
+            output,
+            signal,
+            concurrency,
+            baselineId,
+        }, connection);
+    const environmentClient = createEnvironmentClient(connection);
+    const checkpointDirectory = await prepareCheckpointDirectory(values["checkpoint-dir"], agent.key, project.id, "simulation");
+    const caseKeys = new Map();
+    let runUrl = "";
+    const report = await runSimulation({
+        client,
+        environmentClient,
+        hue,
+        checkpointDirectory,
+        definition: {
+            kind: "pins",
+            datasetVersionId: pins.datasetVersionId,
+            scorerVersionIds: pins.scorerVersionIds,
+        },
+        runName,
+        persistResultContent: values.content,
+        traceEvidence: { mode: "required" },
+        concurrency,
+        signal,
+        target: agents.simulation,
+        async onProgress(event) {
+            if (event.type === "run_created") {
+                runUrl = event.runUrl;
+                output.log(`Run: ${event.runUrl}`);
+                output.log(`Experiment: ${event.experimentId}`);
+                try {
+                    let after;
+                    do {
+                        const page = await client.listExperimentItems(event.experimentId, { after });
+                        for (const item of page.items)
+                            caseKeys.set(item.id, item.externalKey);
+                        after = page.nextCursor ?? undefined;
+                    } while (after);
+                }
+                catch {
+                    // Progress labels fall back to case IDs; the run itself is unaffected.
+                }
+                return;
+            }
+            const label = caseKeys.get(event.caseId) ?? event.caseId;
+            if (event.type === "world_created")
+                output.log(`[${label}] world created`);
+            else if (event.type === "target_started")
+                output.log(`[${label}] agent started`);
+            else if (event.type === "world_sealed")
+                output.log(`[${label}] world sealed`);
+            else if (event.type === "attempt_prepared")
+                output.log(`[${label}] attempt prepared: ${event.status}`);
+        },
+    });
+    return reportVerdicts(client, values, {
+        experimentId: report.experimentId,
+        subjectIds: report.subjectIds,
+        runUrl: runUrl || report.runUrl,
+    }, baselineId, output, signal);
+}
+/**
+ * Direct cases: one ordinary experiment through `runExperiment`. The agent receives the case's
+ * pinned files and returns generated documents; code evaluators pinned to the run stay deferred
+ * for Hue's executor (`deferUnboundLocalScorers`), so no evaluator source runs on this machine.
+ * The attempt is checkpointed as in simulation mode: rerunning the same selection resumes the
+ * saved experiment without invoking the agent again for its finished cases.
+ */
+async function runDirect(run, connection) {
+    const { client, values, pins, output, signal } = run;
+    const config = { agentKey: run.agent.key, agentRevision: run.agent.revision };
+    const checkpointDirectory = await prepareCheckpointDirectory(values["checkpoint-dir"], run.agent.key, run.projectId, "direct");
+    const store = await CheckpointStore.acquire(checkpointDirectory, {
+        kind: "direct",
+        projectId: run.projectId,
+        baseUrl: connection.baseUrl,
+    });
+    let experimentId = "";
+    let runUrl = "";
+    let report;
+    try {
+        const selectionDigest = digest({
+            datasetVersionId: pins.datasetVersionId,
+            scorerVersionIds: [...pins.scorerVersionIds].sort(),
+            config,
+        });
+        let attempt = await store.read("active-attempt");
+        if (attempt && attempt.stage !== "completed" && attempt.selectionDigest !== selectionDigest)
+            throw new Error("Recover the unfinished direct run before running a changed selection");
+        if (!attempt || attempt.stage === "completed") {
+            attempt = { selectionDigest, idempotencyKey: randomUUID(), stage: "preparing" };
+            await store.write("active-attempt", attempt);
+        }
+        if (!attempt.experimentId) {
+            const experiment = await client.createExperiment({
+                idempotencyKey: attempt.idempotencyKey,
+                name: run.runName.slice(0, 100),
+                datasetVersionId: pins.datasetVersionId,
+                scorerVersionIds: pins.scorerVersionIds,
+                config,
+            });
+            attempt.experimentId = experiment.id;
+            attempt.stage = "running";
+            await store.write("active-attempt", attempt);
+        }
+        experimentId = attempt.experimentId;
+        runUrl = new URL(`/experiments/${experimentId}`, connection.baseUrl).toString();
+        output.log(`Run: ${runUrl}`);
+        output.log(`Experiment: ${experimentId}`);
+        report = await runExperiment({
+            client,
+            hue: run.hue,
+            experimentId,
+            checkpointDirectory: join(store.directory, experimentId),
+            persistResultContent: values.content,
+            traceEvidence: { mode: "required" },
+            concurrency: run.concurrency,
+            scorers: [],
+            deferUnboundLocalScorers: true,
+            environmentEvidence: "when_pinned",
+            async target(inputs, context) {
+                if (signal.aborted)
+                    throw new TargetCancelledError();
+                output.log(`[${context.item.externalKey}] agent started`);
+                const result = await run.agents.direct(inputs, {
+                    mode: "direct",
+                    config: structuredClone(context.config),
+                    item: {
+                        id: context.item.id,
+                        externalKey: context.item.externalKey,
+                    },
+                    executionId: context.executionId,
+                    files: structuredClone(context.files),
+                    outputDirectory: context.outputDirectory,
+                    signal,
+                });
+                const count = result instanceof TargetResult ? result.files.length : result === undefined ? 0 : null;
+                output.log(`[${context.item.externalKey}] ${count === null ? "answer returned" : `${count} file${count === 1 ? "" : "s"} produced`}`);
+                return result;
+            },
+        });
+        attempt.stage = "completed";
+        await store.write("active-attempt", attempt);
+    }
+    finally {
+        await store.release();
+    }
+    if (report.deferredScorerVersionIds.length)
+        output.log(`${report.deferredScorerVersionIds.length} evaluator version${report.deferredScorerVersionIds.length === 1 ? "" : "s"} left to Hue's executor`);
+    return reportVerdicts(client, values, {
+        experimentId,
+        subjectIds: report.subjectIds,
+        runUrl,
+        extra: { mode: "direct", deferredScorerVersionIds: report.deferredScorerVersionIds },
+    }, run.baselineId, output, signal);
+}
+async function runWorker(values, connection, adapter, agent, hue, output, signal) {
+    const client = new ObservedClient(connection);
+    const environmentClient = createEnvironmentClient(connection);
+    const wait = integer("wait", values.wait, 300, 0, 86_400);
+    const concurrency = integer("concurrency", values.concurrency, 1, 1, 16);
+    const maxRuns = values["max-runs"] === undefined
+        ? undefined
+        : integer("max-runs", values["max-runs"], 1, 1, 1_000_000);
+    const project = await client.checkConnection();
+    const checkpointDirectory = await prepareCheckpointDirectory(values["checkpoint-dir"], agent.key, project.id, join("worker", slug(agent.revision) || "dev"));
+    let current;
+    client.onRegistered = (registered) => {
+        output.log(`Registered agent ${registered.key} (revision ${registered.revision}) with ${connection.baseUrl}; waiting for runs launched from Hue${maxRuns ? ` (stops after ${maxRuns})` : " (Ctrl+C to stop)"}`);
+    };
+    client.onClaimed = (claim) => {
+        current = claim;
+        output.log(`Claimed run ${claim.runId}: ${new URL(`/experiments/${claim.experimentId}`, connection.baseUrl).toString()}`);
+    };
+    await runLocalAgent({
+        client,
+        environmentClient,
+        hue,
+        checkpointDirectory,
+        agent: {
+            key: agent.key,
+            name: agent.name,
+            revision: agent.revision,
+            capabilities: ["environment:v1"],
+        },
+        scorers: [],
+        concurrency,
+        signal,
+        ...(maxRuns === undefined ? {} : { maxRuns }),
+        target(inputs, tools, context) {
+            output.log(`[${context.item.externalKey}] agent started`);
+            return adapter(inputs, {
+                config: context.config,
+                item: context.item,
+                executionId: context.executionId,
+                environmentRunId: context.environmentRunId,
+                tools,
+                mcp: context.mcp,
+                ...(context.connectionBundle ? { connectionBundle: context.connectionBundle } : {}),
+                signal,
+            });
+        },
+        async onCompleted(report) {
+            output.log(`Run ${report.runId} completed: ${report.subjectIds.length} case${report.subjectIds.length === 1 ? "" : "s"}`);
+            if (!current)
+                return;
+            output.log("Waiting for Hue checks...");
+            try {
+                const verdicts = await collectExperimentVerdicts(client, {
+                    experimentId: current.experimentId,
+                    subjectIds: report.subjectIds,
+                    timeoutMillis: wait * 1000,
+                    signal,
+                });
+                if (signal.aborted)
+                    return;
+                if (values.json)
+                    process.stdout.write(`${JSON.stringify(toJson(verdicts, new URL(`/experiments/${current.experimentId}`, connection.baseUrl).toString()))}\n`);
+                else
+                    renderTable(verdicts, output);
+            }
+            catch (error) {
+                output.error(`Unable to read verdicts: ${explain(error)}`);
+            }
+        },
+    });
+    if (signal.aborted) {
+        process.stderr.write("Interrupted.\n");
+        return 130;
+    }
+    return 0;
+}
+/**
+ * `hue eval`: runs a local adapter or command against a Scenario or eval set through
+ * `runSimulation`, or registers it as an outbound worker through `runLocalAgent`.
+ * Returns the process exit code.
+ */
+export async function runEvalCommand(argv) {
+    const secrets = [];
+    const controller = new AbortController();
+    const interrupt = () => controller.abort(new Error("Interrupted"));
+    process.once("SIGINT", interrupt);
+    process.once("SIGTERM", interrupt);
+    let hue;
+    let json = false;
+    try {
+        const { values, positionals } = parse(argv);
+        json = values.json;
+        const output = {
+            log: (line) => (json ? process.stderr : process.stdout).write(`${line}\n`),
+            error: (line) => process.stderr.write(`${line}\n`),
+        };
+        if (values.help) {
+            process.stdout.write(USAGE);
+            return 0;
+        }
+        if (positionals.length > 1)
+            throw new UsageError("Pass at most one adapter file");
+        const adapterFile = positionals[0];
+        if ((adapterFile ? 1 : 0) + (values.command ? 1 : 0) !== 1)
+            throw new UsageError("Pass exactly one agent: an adapter file or --command");
+        const selections = [values.scenario, values.set, values["dataset-version"]].filter((value) => value !== undefined).length;
+        if (values.worker && selections)
+            throw new UsageError("--worker takes no selection; Hue chooses the run to execute");
+        if (!values.worker && selections !== 1)
+            throw new UsageError("Pass exactly one of --scenario, --set or --dataset-version");
+        if (values["env-file"]) {
+            try {
+                process.loadEnvFile(resolve(values["env-file"]));
+            }
+            catch (error) {
+                throw new UsageError(`Unable to load ${values["env-file"]}: ${error instanceof Error ? error.message : String(error)}`);
+            }
+        }
+        const apiKey = process.env.HUE_API_KEY?.trim();
+        if (!apiKey)
+            throw new UsageError('HUE_API_KEY is required: a "Read and write" project key, set in the environment or an ignored --env-file');
+        secrets.push(apiKey);
+        const baseUrl = values.origin ?? process.env.HUE_BASE_URL?.trim() ?? "https://app.hue.run";
+        const connection = { apiKey, baseUrl };
+        const timeout = integer("timeout", values.timeout, 600, 1, 86_400);
+        const key = values["agent-key"] ?? (derivedAgentKey(adapterFile, values.command) || "agent");
+        const agent = {
+            key,
+            name: values["agent-name"] ?? key,
+            revision: values.revision ?? process.env.AGENT_REVISION?.trim() ?? gitRevision() ?? "dev",
+        };
+        if (values.worker && (values.mode || values["set-version"] || values.scorer?.length))
+            throw new UsageError("--worker takes no selection; Hue chooses the run to execute");
+        const loaded = adapterFile ? await loadAdapter(adapterFile) : undefined;
+        // One adapter module serves both case kinds; the direct context announces itself with `mode`.
+        const agents = {
+            simulation: loaded
+                ? async (inputs, context) => {
+                    const answer = await loaded(inputs, context);
+                    if (answer instanceof TargetResult)
+                        throw new Error("The adapter returned generated files for a simulated-world case");
+                    return answer;
+                }
+                : commandAdapter(values.command, timeout),
+            direct: loaded ?? directCommandAdapter(values.command, timeout),
+        };
+        hue = createHue({ apiKey, baseUrl, serviceName: key, captureContent: values.content });
+        return values.worker
+            ? await runWorker(values, connection, agents.simulation, agent, hue, output, controller.signal)
+            : await runOnce(values, connection, agents, agent, hue, output, controller.signal);
+    }
+    catch (error) {
+        if (controller.signal.aborted || error instanceof TargetCancelledError) {
+            process.stderr.write("Interrupted.\n");
+            return 130;
+        }
+        if (error instanceof UsageError) {
+            process.stderr.write(`${redact(error.message, secrets)}\n${json ? "" : USAGE}`);
+            return 2;
+        }
+        process.stderr.write(`Error: ${redact(explain(error), secrets)}\n`);
+        return 1;
+    }
+    finally {
+        process.removeListener("SIGINT", interrupt);
+        process.removeListener("SIGTERM", interrupt);
+        if (hue)
+            await hue.shutdownSafe({ timeoutMillis: 5000 });
+    }
+}