@hue-run/sdk 0.9.0 → 0.11.0

package/dist/cli/eval.js

@@ -1,18 +1,23 @@
 import { execFileSync, spawn } from "node:child_process";
-import { access, mkdir, writeFile } from "node:fs/promises";
+import { mkdtempSync, rmSync } from "node:fs";
+import { access, mkdir, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
 import { basename, extname, join, resolve } from "node:path";
 import { pathToFileURL } from "node:url";
 import { parseArgs } from "node:util";
 import { randomUUID } from "node:crypto";
 import { createHue } from "../client.js";
 import { createEnvironmentClient } from "../environment/client.js";
-import { agentEnvironment, stripHueControlPlaneCredentials, writeMcpConfig, } from "../environment/world.js";
+import { agentEnvironment, isHueControlPlaneCredential, stripHueControlPlaneCredentials, writeMcpConfig, } from "../environment/world.js";
 import { EvaluationClient, HueApiError } from "../evals/client.js";
 import { TargetResult } from "../evals/types.js";
-import { CheckpointStore } from "../evals/checkpoint.js";
+import { CheckpointIdentityError, CheckpointStore } from "../evals/checkpoint.js";
+import { onForcedExit, runForcedExitCleanups } from "../evals/exit-cleanup.js";
+import { safeFilename } from "../evals/files.js";
 import { digest } from "../evals/json.js";
 import { runLocalAgent } from "../evals/local-worker.js";
-import { runExperiment, TargetCancelledError } from "../evals/runner.js";
+import { describeTelemetryIssues, runExperiment, TargetCancelledError, TELEMETRY_NOT_ACCEPTED, telemetryIssueCounts, } from "../evals/runner.js";
+import { HueExportError } from "../transport.js";
 import { matchByName, parseScenarioSelector, resolveEvalSetPins, resolveScenarioPins, } from "../evals/scenarios.js";
 import { collectDirectOutputs, stageDirectCase } from "./eval-direct.js";
 import { runSimulation, } from "../evals/simulation.js";
@@ -41,12 +46,17 @@ Agent (exactly one):
                                   HUE_ENVIRONMENT_RUN_ID, HUE_CASE_ID and HUE_CASE_KEY set;
                                   {"inputs","config"} on stdin. HUE_API_KEY and other Hue
                                   control-plane credentials are removed from the child.
+                                  HUE_CASE_DIR, HUE_CASE_INPUTS and HUE_CASE_OUTPUT_DIR name the
+                                  case directory as for direct cases: the case's agent-visible
+                                  files and output/, uploaded after the case
   --allow-hue-credentials         Keep HUE_API_KEY and other Hue control-plane credentials in
                                   the --command child (off by default)
                                   Direct: spawned in a private case directory with HUE_CASE_DIR,
                                   HUE_CASE_INPUTS, HUE_CASE_OUTPUT_DIR, HUE_CASE_ID, HUE_CASE_KEY
                                   and HUE_EXECUTION_ID set; files/<role>/ hold the pinned inputs
                                   and every file written to output/ is uploaded to Hue
+                                  Stdout is the case's answer and is stored: print no secrets or
+                                  debug logs there (credentials hue eval knows are redacted)
 
 Modes:
   --worker                        Register the agent and poll for runs launched from Hue
@@ -54,6 +64,8 @@ Modes:
   --agent-key <key>               Agent key (default: slug of the adapter filename)
   --agent-name <name>             Agent display name (default: the key)
   --revision <id>                 Agent revision (default: AGENT_REVISION, git HEAD or "dev")
+  --capability <value>            Extra capability to register (repeatable), for example
+                                  environment-files:v1 and input:pdf for world cases with files
 
 Connection:
   --env-file <path>               Load a dotenv file (HUE_API_KEY, HUE_BASE_URL) first
@@ -64,8 +76,10 @@ Output and limits:
   --name <run name>               Run name (default: <agent key> @ <revision>, hashes shortened)
   --baseline <experiment id|url>  Compare verdicts with a previous experiment
   --json                          Print one JSON document on stdout; progress goes to stderr
-  --content                       Capture telemetry content; one-shot also persists
-                                  outputs/explanations (--worker always persists them)
+  --content                       Capture telemetry content (span inputs, outputs and messages)
+  --no-output                     One-shot: do not store case outputs, error messages and
+                                  explanations (stored by default; --worker always stores); with
+                                  --content the case span still carries the output
   --save-version                  Freeze an unsaved eval-set version before running
   --checkpoint-dir <path>         Private checkpoint directory (default: .hue/eval/<agent-key>)
   --concurrency <n>               Cases in flight, 1-64 (default: 1)
@@ -106,6 +120,7 @@ function parse(argv) {
                 "max-runs": { type: "string" },
                 "agent-key": { type: "string" },
                 "agent-name": { type: "string" },
+                capability: { type: "string", multiple: true },
                 revision: { type: "string" },
                 ...envFileOptions,
                 origin: { type: "string" },
@@ -113,6 +128,7 @@ function parse(argv) {
                 baseline: { type: "string" },
                 json: { type: "boolean", default: false },
                 content: { type: "boolean", default: false },
+                "no-output": { type: "boolean", default: false },
                 "save-version": { type: "boolean", default: false },
                 "checkpoint-dir": { type: "string" },
                 concurrency: { type: "string" },
@@ -220,13 +236,32 @@ async function loadAdapter(file) {
 }
 /** Grace between the stop signal and SIGKILL for an agent command that ignores SIGTERM. */
 const COMMAND_KILL_GRACE_MS = 5_000;
+/** How long after SIGKILL a stopped command waits for its group to be gone before settling. A
+ * killed process that nobody reaps stays in the group as a zombie, which can no longer run. */
+const COMMAND_REAP_MS = 2_000;
+const runningCommands = new Set();
+/** Stops every agent command still running and resolves once each has settled. */
+async function settleCommands() {
+    for (const command of runningCommands)
+        command.stop();
+    while (runningCommands.size)
+        await Promise.all([...runningCommands].map((c) => c.settled));
+}
+function warn(message) {
+    process.stderr.write(`Warning: ${message}\n`);
+}
 /**
  * Spawns the agent command once in its own process group and returns its trimmed stdout. A
  * timeout or Ctrl+C stops the agent the shell started, not only the shell: a survivor would still
- * hold a world token and could write after Hue recorded the case as failed. Windows has no
- * process group to signal, so the child alone is stopped there.
+ * hold a world token and could write after Hue recorded the case as failed. The group is signalled
+ * even once the shell has exited, since a compound command's agent can outlive it, and a stopped
+ * command settles only when nothing in the group is left. A command that exits normally but
+ * leaves processes behind has them stopped the same way before it settles, so none can touch its
+ * outputs while they are collected. Once the group is found empty it is never signalled again,
+ * since its ID could be reused. Windows has no process group to signal, so the child alone is
+ * stopped there.
  */
-function spawnAgentCommand(command, options) {
+export function spawnAgentCommand(command, options) {
     return new Promise((resolvePromise, reject) => {
         const group = process.platform !== "win32";
         const child = spawn(command, {
@@ -240,35 +275,108 @@ function spawnAgentCommand(command, options) {
         let size = 0;
         let timedOut = false;
         let oversized = false;
-        let escalation;
-        const signalTree = (signal) => {
-            if (child.exitCode !== null || child.signalCode !== null)
-                return;
+        let stopping = false;
+        let stopped = false;
+        let settled = false;
+        let groupGone = false;
+        let poll;
+        let watch;
+        let closed;
+        let warned = false;
+        /** Records a vanished group; any other failure to signal it is reported, once. */
+        const gone = (error, signal) => {
+            const code = error.code;
+            if (code === "ESRCH")
+                groupGone = true;
+            else if (!warned) {
+                warned = true;
+                warn(`could not signal the agent command's process group (${signal}, ${code})`);
+            }
+        };
+        /** Whether anything the command started is left: its whole group, or the child alone. */
+        const running = () => {
+            if (!group || child.pid === undefined)
+                return child.exitCode === null && child.signalCode === null;
+            if (groupGone)
+                return false;
             try {
-                if (group && child.pid !== undefined)
-                    process.kill(-child.pid, signal);
-                else
-                    child.kill(signal);
+                process.kill(-child.pid, 0);
+                return true;
             }
-            catch {
-                // The group is already gone; nothing is left to stop.
+            catch (error) {
+                gone(error, 0);
+                return !groupGone;
             }
         };
-        const stop = (signal) => {
-            signalTree(signal);
-            escalation ??= setTimeout(() => signalTree("SIGKILL"), COMMAND_KILL_GRACE_MS).unref();
+        const signalTree = (signal) => {
+            if (group && child.pid !== undefined) {
+                if (groupGone)
+                    return;
+                try {
+                    process.kill(-child.pid, signal);
+                }
+                catch (error) {
+                    gone(error, signal);
+                }
+            }
+            else if (running())
+                child.kill(signal);
+        };
+        /** SIGTERM, then SIGKILL for whatever is still running when the grace ends; the command
+         * settles once the group is gone. */
+        const stop = () => {
+            if (stopping || settled)
+                return;
+            stopping = true;
+            clearTimeout(timer);
+            signalTree("SIGTERM");
+            const killAt = Date.now() + COMMAND_KILL_GRACE_MS;
+            let reapBy;
+            poll = setInterval(() => {
+                const left = running();
+                if (left && reapBy === undefined && Date.now() >= killAt) {
+                    signalTree("SIGKILL");
+                    reapBy = Date.now() + COMMAND_REAP_MS;
+                }
+                if (left && (reapBy === undefined || Date.now() < reapBy))
+                    return;
+                clearInterval(poll);
+                if (left)
+                    warn("the agent command's process group still has members after SIGKILL");
+                stopped = true;
+                settle();
+            }, 50);
+        };
+        let markSettled = () => { };
+        const handle = {
+            stop,
+            kill: () => signalTree("SIGKILL"),
+            settled: new Promise((done) => (markSettled = done)),
+        };
+        runningCommands.add(handle);
+        const release = () => {
+            settled = true;
+            clearTimeout(timer);
+            clearInterval(poll);
+            clearInterval(watch);
+            options.signal?.removeEventListener("abort", cancel);
+            runningCommands.delete(handle);
+            markSettled();
         };
         const timer = setTimeout(() => {
             timedOut = true;
-            stop("SIGTERM");
+            stop();
         }, options.timeoutSeconds * 1000);
-        const cancel = () => stop("SIGTERM");
+        const cancel = () => stop();
         options.signal?.addEventListener("abort", cancel, { once: true });
+        // An abort that landed before the listener existed still stops the command.
+        if (options.signal?.aborted)
+            stop();
         child.stdout.on("data", (chunk) => {
             size += chunk.byteLength;
             if (size > 4 * 1024 * 1024) {
                 oversized = true;
-                stop("SIGTERM");
+                stop();
                 return;
             }
             chunks.push(chunk);
@@ -279,26 +387,51 @@ function spawnAgentCommand(command, options) {
         else
             child.stdin.end();
         child.on("error", (error) => {
-            clearTimeout(timer);
-            clearTimeout(escalation);
-            options.signal?.removeEventListener("abort", cancel);
+            if (settled)
+                return;
+            release();
             reject(new Error(`Unable to start the agent command: ${error.message}`));
         });
-        child.on("close", (code, signal) => {
-            clearTimeout(timer);
-            clearTimeout(escalation);
-            options.signal?.removeEventListener("abort", cancel);
+        // From the shell's exit, notice the moment its group empties, so a later stop never signals
+        // an ID another process may have taken.
+        child.on("exit", () => {
+            if (!group || !running())
+                return;
+            watch = setInterval(() => {
+                if (!running())
+                    clearInterval(watch);
+            }, 50).unref();
+        });
+        /** Settles once the shell's output closed or, after a stop, once its group is gone. A stopped
+         * command's output no longer matters, and a process that left the group may hold the pipe. */
+        const settle = () => {
+            if (settled || (stopping ? !stopped : !closed))
+                return;
+            release();
+            if (stopping) {
+                child.stdout.destroy();
+                child.stdin.destroy();
+            }
             if (options.signal?.aborted)
                 return reject(new TargetCancelledError());
             if (timedOut)
                 return reject(new Error(`The agent command timed out after ${options.timeoutSeconds} seconds`));
             if (oversized)
                 return reject(new Error("The agent command printed more than 4 MiB"));
+            const { code, signal } = closed;
             if (code !== 0)
                 return reject(new Error(signal
                     ? `The agent command was stopped by ${signal}`
                     : `The agent command exited with code ${code}`));
             resolvePromise(Buffer.concat(chunks).toString("utf8").trim());
+        };
+        child.on("close", (code, signal) => {
+            closed = { code, signal };
+            // Whatever the command left running is stopped before its answer and files are read, so
+            // nothing it started can still write, swap or link files while they are collected.
+            if (group && !stopping && running())
+                return stop();
+            settle();
         });
     });
 }
@@ -312,13 +445,142 @@ function parseAnswer(text) {
         return text;
     }
 }
-/**
- * Runs the shell command once per case. The world token and mirror URLs travel only through the
- * child's environment and an owner-only MCP configuration file that is removed after the run;
- * Hue control-plane credentials (`HUE_API_KEY`, `HUE_MCP_KEY`, any `hue_sk_` value) stay with
- * the CLI unless `--allow-hue-credentials` is passed. A world created while the gateway is off
- * still gets the `hue_sim_` capability under the same `HUE_MCP_*` names.
- */
+const MIN_SECRET_LENGTH = 16;
+/** Each export of the CLI's own telemetry, retries included, may take this long. A case's trace
+ * is required evidence, and Hue can acknowledge it slowly or ask for a retry after a few seconds,
+ * which the 10-second default leaves no room for. A refusal still fails at once. */
+const CASE_TRACE_EXPORT_MILLIS = 30_000;
+function caseSecrets(context) {
+    const values = new Set();
+    // Hue issues long credentials; a short value would redact ordinary text around it.
+    const add = (value) => {
+        if (typeof value === "string" && value.trim().length >= MIN_SECRET_LENGTH)
+            values.add(value).add(value.trim());
+    };
+    for (const [name, value] of Object.entries(process.env))
+        if (isHueControlPlaneCredential(name, value))
+            add(value);
+    add(context.world?.token);
+    for (const server of Object.values(context.world?.mcpConfig?.mcpServers ?? {}))
+        if (server.headers && typeof server.headers === "object")
+            for (const header of Object.values(server.headers))
+                add(header);
+    add(context.mcp?.token);
+    const bearers = (value, depth) => {
+        if (depth > 16 || !value || typeof value !== "object")
+            return;
+        for (const [key, item] of Object.entries(value))
+            if (key === "bearer" || key === "token")
+                add(item);
+            else
+                bearers(item, depth + 1);
+    };
+    bearers(context.connectionBundle, 0);
+    return [...values].sort((a, b) => b.length - a.length);
+}
+/** Each secret's exact value, as written and as JSON escapes it, replaced with `[redacted]`. */
+function redactSecrets(text, secrets) {
+    for (const secret of secrets) {
+        text = text.replaceAll(secret, "[redacted]");
+        const escaped = JSON.stringify(secret).slice(1, -1);
+        if (escaped !== secret)
+            text = text.replaceAll(escaped, "[redacted]");
+    }
+    return text;
+}
+const TEXT_CONTENT_TYPES = new Set(["text/plain", "text/csv", "application/json"]);
+/** A generated file with its name, and a text document held in memory with its text, cleared of
+ * the case's credentials. Other documents, a file declared by path and text that is not UTF-8
+ * are uploaded as written. */
+function redactFile(file, secrets) {
+    // A name that is not a string is left for staging to refuse.
+    const filename = typeof file.filename === "string" ? redactSecrets(file.filename, secrets) : file.filename;
+    if (filename !== file.filename)
+        file = { ...file, filename };
+    if (!file.bytes || !TEXT_CONTENT_TYPES.has(file.contentType))
+        return file;
+    let text;
+    try {
+        text = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(file.bytes);
+    }
+    catch {
+        return file;
+    }
+    const redacted = redactSecrets(text, secrets);
+    return redacted === text ? file : { ...file, bytes: new TextEncoder().encode(redacted) };
+}
+/** Generated files cleared of the case's credentials. A name redaction made equal to another
+ * file's, as staging compares them, gains `-2`, `-3`, … before its extension, so every file is
+ * still taken. */
+function redactFiles(files, secrets) {
+    const redacted = files.map((file) => redactFile(file, secrets));
+    const renamed = redacted.map((file, index) => file.filename !== files[index].filename);
+    const taken = new Set(redacted.flatMap((file, index) => !renamed[index] && typeof file.filename === "string" ? [safeFilename(file.filename)] : []));
+    return redacted.map((file, index) => {
+        if (!renamed[index])
+            return file;
+        const dot = file.filename.lastIndexOf(".");
+        const stem = dot > 0 ? file.filename.slice(0, dot) : file.filename;
+        const extension = dot > 0 ? file.filename.slice(dot) : "";
+        let filename = file.filename;
+        for (let copy = 2; taken.has(safeFilename(filename)); copy++)
+            filename = `${stem}-${copy}${extension}`;
+        taken.add(safeFilename(filename));
+        return filename === file.filename ? file : { ...file, filename };
+    });
+}
+/** An answer with its strings redacted, keys included. Deeper than any output the runner
+ * accepts, a value is left for the runner to refuse. */
+function redactAnswer(value, secrets, depth = 0) {
+    if (typeof value === "string")
+        return redactSecrets(value, secrets);
+    if (depth > 64 || !value || typeof value !== "object")
+        return value;
+    if (value instanceof TargetResult)
+        return new TargetResult(redactAnswer(value.output, secrets, depth + 1), redactFiles(value.files, secrets));
+    if (Array.isArray(value))
+        return value.map((item) => redactAnswer(item, secrets, depth + 1));
+    if (Object.getPrototypeOf(value) !== Object.prototype && Object.getPrototypeOf(value) !== null)
+        return value;
+    return Object.fromEntries(Object.entries(value).map(([key, item]) => [
+        redactSecrets(key, secrets),
+        redactAnswer(item, secrets, depth + 1),
+    ]));
+}
+/** Runs an adapter with its answer and any error it throws cleared of the case's credentials,
+ * before either reaches Hue, the case's error message or a checkpoint. */
+function redacting(adapter) {
+    return async (inputs, context) => {
+        // A direct case's context carries no world; its answer is still cleared of the CLI's own.
+        const secrets = caseSecrets(context);
+        try {
+            return redactAnswer(await adapter(inputs, context), secrets);
+        }
+        catch (error) {
+            if (!(error instanceof Error))
+                throw error;
+            // The name is exported too, as the span's error type, so it is cleared like the message.
+            const message = redactSecrets(error.message, secrets);
+            const name = redactSecrets(String(error.name), secrets);
+            if (message === error.message && name === String(error.name))
+                throw error;
+            try {
+                if (message !== error.message)
+                    error.message = message;
+                if (name !== String(error.name))
+                    error.name = name;
+            }
+            catch {
+                // A frozen error, or one whose message or name is a getter, keeps its text; replace it.
+            }
+            if (error.message === message && String(error.name) === name)
+                throw error;
+            const replacement = new Error(message);
+            replacement.name = name;
+            throw replacement;
+        }
+    };
+}
 /** The parent environment an agent child starts from: without Hue control-plane credentials
  * unless `--allow-hue-credentials` was passed. */
 function parentEnvironment(allowHueCredentials) {
@@ -329,14 +591,36 @@ function parentEnvironment(allowHueCredentials) {
 function commandAdapter(command, timeoutSeconds, options) {
     return async (inputs, context) => {
         const parent = parentEnvironment(options.allowHueCredentials);
+        // The case directory direct cases get: inputs, the verified agent-visible files and
+        // output/. It holds no world credential and the runner removes it after the case.
+        const layout = await stageDirectCase(context.outputDirectory, {
+            inputs,
+            config: context.config,
+            item: context.item,
+            executionId: context.executionId,
+            files: context.files,
+        });
         const identity = {
             HUE_EXECUTION_ID: context.executionId,
             HUE_ENVIRONMENT_RUN_ID: context.environmentRunId,
             HUE_CASE_ID: context.item.id,
             HUE_CASE_KEY: context.item.externalKey,
+            HUE_CASE_DIR: layout.caseDirectory,
+            HUE_CASE_INPUTS: layout.inputsPath,
+            HUE_CASE_OUTPUT_DIR: layout.outputDirectory,
         };
-        const configFile = context.world ? await writeMcpConfig(context.world) : undefined;
+        // The token-bearing file is written inside a private directory registered before any of it
+        // exists, so a forced exit at any point removes it.
+        const configDirectory = context.world
+            ? mkdtempSync(join(tmpdir(), "hue-mcp-config-"))
+            : undefined;
+        const untrack = configDirectory
+            ? onForcedExit(() => rmSync(configDirectory, { recursive: true, force: true }))
+            : undefined;
         try {
+            const configFile = context.world
+                ? await writeMcpConfig(context.world, { directory: configDirectory })
+                : undefined;
             const env = context.world
                 ? {
                     ...agentEnvironment(context.world, { parent, includeHueCredentials: true }),
@@ -354,15 +638,18 @@ function commandAdapter(command, timeoutSeconds, options) {
                         : {}),
                     ...identity,
                 };
-            return parseAnswer(await spawnAgentCommand(command, {
+            const stdout = await spawnAgentCommand(command, {
                 env,
                 stdin: JSON.stringify({ inputs, config: context.config }),
                 timeoutSeconds,
                 ...(context.signal ? { signal: context.signal } : {}),
-            }));
+            });
+            return collectDirectOutputs(layout.outputDirectory, parseAnswer(stdout));
         }
         finally {
-            await configFile?.dispose();
+            if (configDirectory)
+                await rm(configDirectory, { recursive: true, force: true });
+            untrack?.();
         }
     };
 }
@@ -412,18 +699,39 @@ const STATE_LABEL = {
     skipped: "SKIPPED",
     pending: "PENDING",
 };
-function renderTable(verdicts, output) {
-    const names = [];
-    for (const item of verdicts.summary.cases)
-        for (const metric of item.metrics)
-            if (!names.includes(metric.name))
-                names.push(metric.name);
-    const header = ["Case", ...names, "Result"];
-    const rows = verdicts.summary.cases.map((item) => [
+export function renderTable(verdicts, output) {
+    const cases = verdicts.summary.cases;
+    // A column per metric name; a name several evaluators report, such as two judges' `verdict`,
+    // gets a column per evaluator so each one's value shows.
+    const reporters = new Map();
+    for (const item of cases)
+        for (const metric of item.metrics) {
+            const versions = reporters.get(metric.name) ?? [];
+            if (!versions.includes(metric.scorerVersionId))
+                versions.push(metric.scorerVersionId);
+            reporters.set(metric.name, versions);
+        }
+    const columns = [...reporters].flatMap(([name, versions]) => versions.map((version) => ({
+        name,
+        version,
+        label: versions.length === 1 ? name : `${name} ${version.slice(0, 8)}`,
+    })));
+    // An advisory evaluator's values are shown, marked, and never decide a case.
+    const advisory = new Set(cases.flatMap((item) => item.advisory ?? []));
+    const header = [
+        "Case",
+        ...columns.map((column) => advisory.has(column.version) ? `${column.label} (advisory)` : column.label),
+        "Result",
+    ];
+    const rows = cases.map((item) => [
         item.externalKey,
-        ...names.map((name) => {
-            const metric = item.metrics.find((candidate) => candidate.name === name);
-            return metric ? metricText(metric) : "-";
+        ...columns.map((column) => {
+            const metric = item.metrics.find((candidate) => candidate.name === column.name && candidate.scorerVersionId === column.version);
+            if (metric)
+                return metricText(metric);
+            // A case the column's evaluator does not apply to shows n/a; one it applies to without
+            // reporting the metric shows "-".
+            return item.notApplicable?.includes(column.version) ? "n/a" : "-";
         }),
         STATE_LABEL[item.state],
     ]);
@@ -444,10 +752,18 @@ function renderTable(verdicts, output) {
             output.log(`  ${item.externalKey}: ${detail}`);
     }
     const totals = verdicts.summary.totals;
+    const advisoryFailures = cases.reduce((count, item) => count +
+        item.metrics.filter((metric) => item.advisory?.includes(metric.scorerVersionId) && !metricPassed(metric)).length, 0);
     const extra = [
         totals.error ? `${totals.error} error` : "",
         totals.skipped ? `${totals.skipped} skipped` : "",
         totals.pending ? `${totals.pending} pending` : "",
+        totals.notApplicable
+            ? `${totals.notApplicable} evaluator result${totals.notApplicable === 1 ? "" : "s"} not applicable`
+            : "",
+        advisoryFailures
+            ? `${advisoryFailures} advisory failure${advisoryFailures === 1 ? "" : "s"} not counted`
+            : "",
     ].filter(Boolean);
     output.log(`${totals.passed} of ${totals.cases} case${totals.cases === 1 ? "" : "s"} passed${extra.length ? ` (${extra.join(", ")})` : ""}`);
     if (!verdicts.results.complete)
@@ -466,9 +782,20 @@ function redact(message, secrets) {
             text = text.replaceAll(secret, "[redacted]");
     return text;
 }
-function explain(error) {
+export function explain(error) {
     if (error instanceof HueApiError && (error.status === 401 || error.status === 403))
         return `${error.message}. Check that HUE_API_KEY is a "Read and write" project key for this origin.`;
+    // An unfinished run keeps the content policy it started with; name the flags that resume it.
+    if (error instanceof CheckpointIdentityError && error.startedWith) {
+        const flags = [
+            error.startedWith.persistResultContent === false ? "--no-output" : "",
+            error.startedWith.captureContent === true ? "--content" : "",
+        ].filter(Boolean);
+        return `This unfinished run was started ${flags.length ? `with ${flags.join(" and ")}` : "without --no-output or --content"}; rerun with the same flags to resume it, or remove its checkpoint directory to start over`;
+    }
+    // The error's own message points at an in-memory report the user cannot reach.
+    if (error instanceof HueExportError)
+        return `Hue could not accept all telemetry (${describeTelemetryIssues(telemetryIssueCounts(error))})`;
     if (error instanceof Error) {
         const causes = [];
         let cause = error.cause;
@@ -625,13 +952,19 @@ async function detectDirect(client, pins, mode) {
     } while (after);
     return true;
 }
-function toJson(verdicts, runUrl, baseline, extra = {}) {
+function toJson(verdicts, runUrl, baseline, extra = {}, telemetry = []) {
+    const notAccepted = new Map(telemetry.map((entry) => [entry.caseId, entry.issues]));
     return {
         experimentId: verdicts.experimentId,
         runId: verdicts.runId,
         runUrl,
         complete: verdicts.results.complete,
-        cases: verdicts.summary.cases,
+        cases: verdicts.summary.cases.map((item) => notAccepted.has(item.caseId)
+            ? {
+                ...item,
+                telemetry: { code: TELEMETRY_NOT_ACCEPTED, issues: notAccepted.get(item.caseId) },
+            }
+            : item),
         totals: verdicts.summary.totals,
         ...extra,
         ...(baseline
@@ -639,16 +972,52 @@ function toJson(verdicts, runUrl, baseline, extra = {}) {
             : {}),
     };
 }
+/** Says, once per execution, that a case failed because Hue did not accept its telemetry, with
+ * sanitized counts: as the case completes, or from the report for a case an earlier, interrupted
+ * run completed. */
+function telemetryReporter(output) {
+    const reported = new Set();
+    const report = (entry) => {
+        if (reported.has(entry.executionId))
+            return;
+        reported.add(entry.executionId);
+        output.error(`[${entry.caseKey}] telemetry not accepted, case failed: ${describeTelemetryIssues(entry.issues)}`);
+    };
+    return {
+        report,
+        rest: (entries) => entries?.forEach(report),
+    };
+}
+/** A case without trace evidence fails whatever its scores say: a scorer that grades only the
+ * world or ignores the execution state could still pass it. */
+function withTelemetryFailures(verdicts, telemetry = []) {
+    if (!telemetry.length)
+        return verdicts;
+    const failed = new Set(telemetry.map((entry) => entry.caseId));
+    const cases = verdicts.summary.cases.map((item) => failed.has(item.caseId) ? { ...item, state: "error", passed: false } : item);
+    const totals = {
+        cases: cases.length,
+        passed: 0,
+        failed: 0,
+        error: 0,
+        skipped: 0,
+        pending: 0,
+        notApplicable: verdicts.summary.totals.notApplicable,
+    };
+    for (const item of cases)
+        totals[item.state]++;
+    return { ...verdicts, summary: { cases, totals } };
+}
 /** Waits for Hue's verdicts, prints them (table or JSON, with the optional baseline) and returns the exit code. */
 async function reportVerdicts(client, values, run, baselineId, output, signal) {
     const wait = integer("wait", values.wait, 300, 0, 86_400);
     output.log("Waiting for Hue checks...");
-    const verdicts = await collectExperimentVerdicts(client, {
+    const verdicts = withTelemetryFailures(await collectExperimentVerdicts(client, {
         experimentId: run.experimentId,
         subjectIds: run.subjectIds,
         timeoutMillis: wait * 1000,
         signal,
-    });
+    }), run.telemetryNotAccepted);
     // The wait returns its partial state on abort rather than throwing, so Ctrl+C here must not
     // fall through to a baseline read and a verdict table that nobody asked to finish.
     if (signal.aborted) {
@@ -667,7 +1036,7 @@ async function reportVerdicts(client, values, run, baselineId, output, signal) {
         };
     }
     if (values.json) {
-        process.stdout.write(`${JSON.stringify(toJson(verdicts, run.runUrl, baseline, run.extra))}\n`);
+        process.stdout.write(`${JSON.stringify(toJson(verdicts, run.runUrl, baseline, run.extra, run.telemetryNotAccepted))}\n`);
     }
     else {
         renderTable(verdicts, output);
@@ -676,6 +1045,8 @@ async function reportVerdicts(client, values, run, baselineId, output, signal) {
         output.log(`Run: ${run.runUrl}`);
     }
     const totals = verdicts.summary.totals;
+    if (run.telemetryNotAccepted?.length)
+        return 1;
     return verdicts.results.complete && totals.cases > 0 && totals.passed === totals.cases ? 0 : 1;
 }
 function parseBaseline(value) {
@@ -721,6 +1092,7 @@ async function runOnce(values, connection, agents, agent, hue, output, signal) {
     const checkpointDirectory = await prepareCheckpointDirectory(values["checkpoint-dir"], agent.key, project.id, "simulation");
     const caseKeys = new Map();
     let runUrl = "";
+    const telemetry = telemetryReporter(output);
     const report = await runSimulation({
         client,
         environmentClient,
@@ -732,8 +1104,13 @@ async function runOnce(values, connection, agents, agent, hue, output, signal) {
             scorerVersionIds: pins.scorerVersionIds,
         },
         runName,
-        persistResultContent: values.content,
+        // Outputs, error messages and explanations are stored unless opted out; --content governs
+        // only the telemetry.
+        persistResultContent: !values["no-output"],
         traceEvidence: { mode: "required" },
+        // A case whose telemetry Hue did not accept fails instead of staying started.
+        traceNotAccepted: "fail_case",
+        onTelemetryNotAccepted: telemetry.report,
         concurrency,
         agentRevision: agent.revision,
         // The CLI adapts to whatever the deployment serves; the library warning is for code that
@@ -771,10 +1148,12 @@ async function runOnce(values, connection, agents, agent, hue, output, signal) {
                 output.log(`[${label}] attempt prepared: ${event.status}`);
         },
     });
+    telemetry.rest(report.telemetryNotAccepted);
     return reportVerdicts(client, values, {
         experimentId: report.experimentId,
         subjectIds: report.subjectIds,
         runUrl: runUrl || report.runUrl,
+        telemetryNotAccepted: report.telemetryNotAccepted,
     }, baselineId, output, signal);
 }
 /**
@@ -796,6 +1175,7 @@ async function runDirect(run, connection) {
     let experimentId = "";
     let runUrl = "";
     let report;
+    const telemetry = telemetryReporter(output);
     try {
         const selectionDigest = digest({
             datasetVersionId: pins.datasetVersionId,
@@ -830,8 +1210,12 @@ async function runDirect(run, connection) {
             hue: run.hue,
             experimentId,
             checkpointDirectory: join(store.directory, experimentId),
-            persistResultContent: values.content,
+            // Outputs, error messages and explanations are stored unless opted out; --content governs
+            // only the telemetry.
+            persistResultContent: !values["no-output"],
             traceEvidence: { mode: "required" },
+            traceNotAccepted: "fail_case",
+            onTelemetryNotAccepted: telemetry.report,
             concurrency: run.concurrency,
             scorers: [],
             deferUnboundLocalScorers: true,
@@ -865,11 +1249,13 @@ async function runDirect(run, connection) {
     }
     if (report.deferredScorerVersionIds.length)
         output.log(`${report.deferredScorerVersionIds.length} evaluator version${report.deferredScorerVersionIds.length === 1 ? "" : "s"} left to Hue's executor`);
+    telemetry.rest(report.telemetryNotAccepted);
     return reportVerdicts(client, values, {
         experimentId,
         subjectIds: report.subjectIds,
         runUrl,
         extra: { mode: "direct", deferredScorerVersionIds: report.deferredScorerVersionIds },
+        telemetryNotAccepted: report.telemetryNotAccepted,
     }, run.baselineId, output, signal);
 }
 async function runWorker(values, connection, adapter, agent, hue, output, signal) {
@@ -890,6 +1276,7 @@ async function runWorker(values, connection, adapter, agent, hue, output, signal
         current = claim;
         output.log(`Claimed run ${claim.runId}: ${new URL(`/experiments/${claim.experimentId}`, connection.baseUrl).toString()}`);
     };
+    const telemetry = telemetryReporter(output);
     await runLocalAgent({
         client,
         environmentClient,
@@ -899,10 +1286,12 @@ async function runWorker(values, connection, adapter, agent, hue, output, signal
             key: agent.key,
             name: agent.name,
             revision: agent.revision,
-            capabilities: ["environment:v1"],
+            capabilities: ["environment:v1", ...(values.capability ?? [])],
         },
         scorers: [],
         concurrency,
+        traceNotAccepted: "fail_case",
+        onTelemetryNotAccepted: telemetry.report,
         deprecationWarnings: false,
         signal,
         ...(maxRuns === undefined ? {} : { maxRuns }),
@@ -917,25 +1306,28 @@ async function runWorker(values, connection, adapter, agent, hue, output, signal
                 ...(context.world ? { world: context.world } : {}),
                 ...(context.mcp ? { mcp: context.mcp } : {}),
                 ...(context.connectionBundle ? { connectionBundle: context.connectionBundle } : {}),
+                files: context.files,
+                outputDirectory: context.outputDirectory,
                 signal,
             });
         },
         async onCompleted(report) {
             output.log(`Run ${report.runId} completed: ${report.subjectIds.length} case${report.subjectIds.length === 1 ? "" : "s"}`);
+            telemetry.rest(report.telemetryNotAccepted);
             if (!current)
                 return;
             output.log("Waiting for Hue checks...");
             try {
-                const verdicts = await collectExperimentVerdicts(client, {
+                const verdicts = withTelemetryFailures(await collectExperimentVerdicts(client, {
                     experimentId: current.experimentId,
                     subjectIds: report.subjectIds,
                     timeoutMillis: wait * 1000,
                     signal,
-                });
+                }), report.telemetryNotAccepted);
                 if (signal.aborted)
                     return;
                 if (values.json)
-                    process.stdout.write(`${JSON.stringify(toJson(verdicts, new URL(`/experiments/${current.experimentId}`, connection.baseUrl).toString()))}\n`);
+                    process.stdout.write(`${JSON.stringify(toJson(verdicts, new URL(`/experiments/${current.experimentId}`, connection.baseUrl).toString(), undefined, {}, report.telemetryNotAccepted))}\n`);
                 else
                     renderTable(verdicts, output);
             }
@@ -958,9 +1350,31 @@ async function runWorker(values, connection, adapter, agent, hue, output, signal
 export async function runEvalCommand(argv) {
     const secrets = [];
     const controller = new AbortController();
-    const interrupt = () => controller.abort(new Error("Interrupted"));
-    process.once("SIGINT", interrupt);
-    process.once("SIGTERM", interrupt);
+    let firstInterrupt = 0;
+    // The first interrupt stops the agents within their grace; a repeated one kills their process
+    // groups at once and exits, so no agent is left running with its world token. `npm run` and
+    // `npx` forward the terminal's SIGINT a moment after the terminal delivers it, so a repeat
+    // within 50 ms is the same Ctrl+C.
+    const interrupt = () => {
+        if (!controller.signal.aborted) {
+            firstInterrupt = performance.now();
+            if (runningCommands.size)
+                process.stderr.write("Stopping the agent; press Ctrl+C again to force.\n");
+            controller.abort(new Error("Interrupted"));
+            return;
+        }
+        if (performance.now() - firstInterrupt < 50)
+            return;
+        for (const command of runningCommands)
+            command.kill();
+        // The exit cannot wait for the normal cleanup: remove the world cases' files, the MCP
+        // configurations holding world tokens and the checkpoint locks now.
+        runForcedExitCleanups();
+        process.stderr.write("Interrupted.\n");
+        process.exit(130);
+    };
+    process.on("SIGINT", interrupt);
+    process.on("SIGTERM", interrupt);
     let hue;
     let json = false;
     try {
@@ -1014,25 +1428,31 @@ export async function runEvalCommand(argv) {
         };
         if (values.worker && (values.mode || values["set-version"] || values.scorer?.length))
             throw new UsageError("--worker takes no selection; Hue chooses the run to execute");
+        if (values.worker && values["no-output"])
+            throw new UsageError("--no-output applies to one-shot runs; a run launched from Hue always stores its outputs");
+        if (!values.worker && values.capability?.length)
+            throw new UsageError("--capability applies to --worker only");
         const loaded = adapterFile ? await loadAdapter(adapterFile) : undefined;
         // One adapter module serves both case kinds; the direct context announces itself with `mode`.
+        // Outputs are stored by default, so every answer and error message is cleared of the
+        // credentials the case handed the agent before it leaves the CLI.
         const agents = {
-            simulation: loaded
-                ? async (inputs, context) => {
-                    const answer = await loaded(inputs, context);
-                    if (answer instanceof TargetResult)
-                        throw new Error("The adapter returned generated files for a simulated-world case");
-                    return answer;
-                }
-                : commandAdapter(values.command, timeout, {
+            simulation: redacting(loaded ??
+                commandAdapter(values.command, timeout, {
                     allowHueCredentials: values["allow-hue-credentials"],
-                }),
-            direct: loaded ??
+                })),
+            direct: redacting(loaded ??
                 directCommandAdapter(values.command, timeout, {
                     allowHueCredentials: values["allow-hue-credentials"],
-                }),
+                })),
         };
-        hue = createHue({ apiKey, baseUrl, serviceName: key, captureContent: values.content });
+        hue = createHue({
+            apiKey,
+            baseUrl,
+            serviceName: key,
+            captureContent: values.content,
+            timeoutMillis: CASE_TRACE_EXPORT_MILLIS,
+        });
         return values.worker
             ? await runWorker(values, connection, agents.simulation, agent, hue, output, controller.signal)
             : await runOnce(values, connection, agents, agent, hue, output, controller.signal);
@@ -1050,6 +1470,9 @@ export async function runEvalCommand(argv) {
         return 1;
     }
     finally {
+        // The handlers stay until every agent command has settled, so an interrupt during a stop's
+        // grace still reaches it.
+        await settleCommands();
         process.removeListener("SIGINT", interrupt);
         process.removeListener("SIGTERM", interrupt);
         if (hue)