@hue-run/sdk 0.15.2 → 0.17.0

package/dist/cli/eval.js

@@ -1,5 +1,5 @@
 import { execFileSync, spawn } from "node:child_process";
-import { mkdtempSync, rmSync } from "node:fs";
+import { existsSync, mkdtempSync, rmSync } from "node:fs";
 import { access, mkdir, rm, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { basename, extname, join, resolve } from "node:path";
@@ -22,7 +22,7 @@ import { matchByName, parseScenarioSelector, resolveEvalSetPins, resolveScenario
 import { collectDirectOutputs, stageDirectCase } from "./eval-direct.js";
 import { runSimulation, } from "../evals/simulation.js";
 import { collectExperimentVerdicts, compareVerdicts, metricPassed, } from "../evals/verdicts.js";
-import { envFileArgument, envFileOptions } from "./env-file.js";
+import { DEFAULT_ENV_FILE, envFileArgument, envFileOptions } from "./env-file.js";
 const USAGE = `Usage: hue eval [adapter-file] [options]
 
 Run a local agent against a published case or eval set, then print Hue's verdicts.
@@ -70,13 +70,15 @@ Modes:
                                   environment-files:v1 and input:pdf for world cases with files
 
 Connection:
-  --env-file <path>               Load a dotenv file (HUE_API_KEY, HUE_BASE_URL) first
+  --env-file <path>               Load a dotenv file (HUE_API_KEY, HUE_BASE_URL) first;
+                                  ./.env.hue loads by default when HUE_API_KEY is not
+                                  set. Variables already set in the environment win
   --env-path <path>               Same as --env-file
   --origin <url>                  Hue origin (default: HUE_BASE_URL or https://app.hue.run)
 
 Output and limits:
   --name <run name>               Run name (default: <agent key> @ <revision>, hashes shortened)
-  --baseline <experiment id|url>  Compare verdicts with a previous experiment
+  --baseline <run id|url>         Compare verdicts with a previous run
   --json                          Print one JSON document on stdout; progress goes to stderr
   --content                       Capture telemetry content (the default)
   --no-content                    Omit telemetry inputs, outputs and messages
@@ -84,6 +86,8 @@ Output and limits:
                                   explanations (stored by default; --worker always stores); with
                                   --content the case span still carries the output
   --save-version                  Freeze an unsaved eval-set version before running
+  --check                         One-shot: check the key, selection and agent, print what the
+                                  run would do and exit without creating a run or a world
   --checkpoint-dir <path>         Private checkpoint directory (default: .hue/eval/<agent-key>)
   --concurrency <n>               Cases in flight, 1-100 (default: 1)
   --trace-not-accepted <policy>   A case whose telemetry Hue did not accept in time: fail_case
@@ -147,6 +151,7 @@ function parse(argv) {
                 "no-output": { type: "boolean", default: false },
                 "trace-not-accepted": { type: "string" },
                 "save-version": { type: "boolean", default: false },
+                check: { type: "boolean", default: false },
                 "checkpoint-dir": { type: "string" },
                 concurrency: { type: "string" },
                 timeout: { type: "string" },
@@ -971,20 +976,27 @@ function parseMode(value) {
         return value;
     throw new UsageError("--mode must be auto, direct or simulation");
 }
-/** Direct when nothing pins a simulated world: not a Scenario, and no case of the version does. */
-async function detectDirect(client, pins, mode) {
-    if (mode !== "auto")
-        return mode === "direct";
-    if (pins.scenarioId || pins.environmentVersionId)
-        return false;
+/** The selection's cases, and whether the run is direct: when `--mode` leaves it open, direct
+ * means nothing pins a simulated world, not a Scenario and no case of the version. A simulation
+ * run cannot execute a case that pins no world, so a selection mixing both kinds is refused. */
+async function inspectSelection(client, pins, mode) {
+    let cases = 0;
+    let worldCases = 0;
     let after;
     do {
         const page = await client.listCases(pins.datasetVersionId, { after });
-        if (page.items.some((item) => item.environmentVersionId))
-            return false;
+        cases += page.items.length;
+        worldCases += page.items.filter((item) => item.environmentVersionId).length;
         after = page.nextCursor ?? undefined;
     } while (after);
-    return true;
+    const direct = mode === "auto"
+        ? !pins.scenarioId && !pins.environmentVersionId && worldCases === 0
+        : mode === "direct";
+    const answerOnly = cases - worldCases;
+    const refusal = !direct && answerOnly > 0
+        ? `${answerOnly} of the ${cases} ${cases === 1 ? "case pins" : "cases pin"} no world, so a simulation run would fail ${answerOnly === 1 ? "it" : "them"} before the agent starts. Put the answer-only cases in their own eval set${mode === "simulation" ? ", or drop --mode simulation" : ""}.`
+        : undefined;
+    return { direct, cases, worldCases, ...(refusal ? { refusal } : {}) };
 }
 function toJson(verdicts, runUrl, baseline, extra = {}, telemetry = []) {
     const notAccepted = new Map(telemetry.map((entry) => [entry.caseId, entry.issues]));
@@ -1055,7 +1067,7 @@ async function reportVerdicts(client, values, run, baselineId, output, signal) {
     // The wait returns its partial state on abort rather than throwing, so Ctrl+C here must not
     // fall through to a baseline read and a verdict table that nobody asked to finish.
     if (signal.aborted) {
-        process.stderr.write("Interrupted.\n");
+        process.stderr.write(`Interrupted. The cases already finished and Hue is still checking them; open ${run.runUrl} instead of rerunning.\n`);
         return 130;
     }
     let baseline;
@@ -1086,7 +1098,7 @@ async function reportVerdicts(client, values, run, baselineId, output, signal) {
 function parseBaseline(value) {
     if (!value)
         return undefined;
-    const parsed = parseScenarioSelector(value, ["experiments"]);
+    const parsed = parseScenarioSelector(value, ["runs", "experiments"]);
     if (parsed.kind !== "id")
         throw new UsageError("--baseline must be an experiment ID or its Hue URL");
     return parsed.id;
@@ -1096,7 +1108,15 @@ async function runOnce(values, connection, agents, agent, hue, output, signal) {
     const concurrency = integer("concurrency", values.concurrency, 1, 1, 100);
     const baselineId = parseBaseline(values.baseline);
     const mode = parseMode(values.mode);
+    integer("wait", values.wait, 300, 0, 86_400);
     const pins = await resolveSelection(client, values);
+    const selection = await inspectSelection(client, pins, mode);
+    if (values.check)
+        return checkRun(client, values, pins, selection, agent, output);
+    if (selection.refusal) {
+        output.error(selection.refusal);
+        return 1;
+    }
     if (!pins.saved) {
         if (!values["save-version"]) {
             output.error(`The ${pins.scenarioId ? "Scenario" : "eval set"} "${pins.name}" points at an unsaved version. Choose "Save eval-set version" in Hue, or pass --save-version to freeze it now, then rerun.`);
@@ -1107,7 +1127,7 @@ async function runOnce(values, connection, agents, agent, hue, output, signal) {
     }
     const runName = values.name ?? defaultRunName(agent);
     const project = await client.checkConnection();
-    if (await detectDirect(client, pins, mode))
+    if (selection.direct)
         return runDirect({
             client,
             values,
@@ -1190,6 +1210,62 @@ async function runOnce(values, connection, agents, agent, hue, output, signal) {
         telemetryNotAccepted: report.telemetryNotAccepted,
     }, baselineId, output, signal);
 }
+/**
+ * `--check`: everything a one-shot run resolves before it creates anything (the key's project,
+ * the selection, the case kind and the agent), printed with the command that would start it.
+ * Nothing is created, frozen or checkpointed.
+ */
+async function checkRun(client, values, pins, selection, agent, output) {
+    const project = await client.checkConnection();
+    for (const id of pins.scorerVersionIds) {
+        const found = await client.getScorerVersion(id).then(() => true, (error) => {
+            if (error instanceof HueApiError && (error.status === 404 || error.status === 403))
+                return false;
+            throw error;
+        });
+        if (!found) {
+            output.error(`Evaluator version ${id} was not found in project "${project.name}".`);
+            return 1;
+        }
+    }
+    const { direct, cases, worldCases, refusal } = selection;
+    const unsaved = !pins.saved && !values["save-version"];
+    const check = {
+        ok: !unsaved && !refusal,
+        project: { id: project.id, name: project.name },
+        selection: {
+            kind: pins.scenarioId ? "case" : "eval_set",
+            name: pins.name,
+            datasetVersionId: pins.datasetVersionId,
+            saved: pins.saved,
+            cases,
+            evaluators: pins.scorerVersionIds.length,
+        },
+        mode: direct ? "direct" : "simulation",
+        worldCases: direct ? 0 : worldCases,
+        agent: { key: agent.key, revision: agent.revision, concurrency: values.concurrency ?? "1" },
+    };
+    if (values.json)
+        process.stdout.write(`${JSON.stringify({ check })}\n`);
+    else {
+        output.log(`Project: ${project.name} (${project.id})`);
+        output.log(`Selection: ${check.selection.kind === "case" ? "case" : "eval set"} "${pins.name}", ${cases} ${cases === 1 ? "case" : "cases"}, ${pins.scorerVersionIds.length} ${pins.scorerVersionIds.length === 1 ? "evaluator" : "evaluators"}`);
+        output.log(direct
+            ? "Mode: direct; the agent answers without a simulated world or app mirrors"
+            : `Mode: simulation; ${worldCases} ${worldCases === 1 ? "case gets" : "cases get"} a fresh world, reached through the HUE_SIM_*_URL mirrors and HUE_WORLD_TOKEN`);
+        output.log(`Agent: ${agent.key} @ ${agent.revision}`);
+    }
+    if (refusal) {
+        output.error(refusal);
+        return 1;
+    }
+    if (unsaved) {
+        output.error(`The ${pins.scenarioId ? "Scenario" : "eval set"} "${pins.name}" points at an unsaved version; add --save-version to freeze it when the run starts.`);
+        return 1;
+    }
+    output.log("Check passed. Nothing was created; run the same command without --check to start one run.");
+    return 0;
+}
 /**
  * Direct cases: one ordinary experiment through `runExperiment`. The agent receives the case's
  * pinned files and returns generated documents; code evaluators pinned to the run stay deferred
@@ -1218,7 +1294,7 @@ async function runDirect(run, connection) {
         });
         let attempt = await store.read("active-attempt");
         if (attempt && attempt.stage !== "completed" && attempt.selectionDigest !== selectionDigest)
-            throw new Error("Recover the unfinished direct run before running a changed selection");
+            throw new Error("Recover the unfinished direct run before running a changed selection: run the unchanged command again to finish it, or pass another --checkpoint-dir");
         if (!attempt || attempt.stage === "completed") {
             attempt = { selectionDigest, idempotencyKey: randomUUID(), stage: "preparing" };
             await store.write("active-attempt", attempt);
@@ -1236,7 +1312,7 @@ async function runDirect(run, connection) {
             await store.write("active-attempt", attempt);
         }
         experimentId = attempt.experimentId;
-        runUrl = new URL(`/experiments/${experimentId}`, connection.baseUrl).toString();
+        runUrl = new URL(`/runs/${experimentId}`, connection.baseUrl).toString();
         output.log(`Run: ${runUrl}`);
         output.log(`Experiment: ${experimentId}`);
         report = await runExperiment({
@@ -1251,6 +1327,7 @@ async function runDirect(run, connection) {
             traceNotAccepted: traceNotAcceptedPolicy(values["trace-not-accepted"]),
             onTelemetryNotAccepted: telemetry.report,
             concurrency: run.concurrency,
+            signal,
             scorers: [],
             deferUnboundLocalScorers: true,
             environmentEvidence: "when_pinned",
@@ -1308,7 +1385,7 @@ async function runWorker(values, connection, agents, agent, hue, output, signal)
     };
     client.onClaimed = (claim) => {
         current = claim;
-        output.log(`Claimed run ${claim.runId}: ${new URL(`/experiments/${claim.experimentId}`, connection.baseUrl).toString()}`);
+        output.log(`Claimed run ${claim.runId}: ${new URL(`/runs/${claim.experimentId}`, connection.baseUrl).toString()}`);
     };
     const telemetry = telemetryReporter(output);
     // A command can use the same stdin contract for both case kinds. Existing adapter files
@@ -1396,7 +1473,7 @@ async function runWorker(values, connection, agents, agent, hue, output, signal)
                 if (signal.aborted)
                     return;
                 if (values.json)
-                    process.stdout.write(`${JSON.stringify(toJson(verdicts, new URL(`/experiments/${current.experimentId}`, connection.baseUrl).toString(), undefined, {}, report.telemetryNotAccepted))}\n`);
+                    process.stdout.write(`${JSON.stringify(toJson(verdicts, new URL(`/runs/${current.experimentId}`, connection.baseUrl).toString(), undefined, {}, report.telemetryNotAccepted))}\n`);
                 else
                     renderTable(verdicts, output);
             }
@@ -1446,8 +1523,10 @@ export async function runEvalCommand(argv) {
     process.on("SIGTERM", interrupt);
     let hue;
     let json = false;
+    let worker = false;
     try {
         const { values, positionals } = parse(argv);
+        worker = values.worker;
         if (values.content && values["no-content"])
             throw new UsageError("Choose either --content or --no-content");
         // A usage error before anything is prepared or created: no run is left to recover from it.
@@ -1478,6 +1557,10 @@ export async function runEvalCommand(argv) {
         catch (error) {
             throw new UsageError(error.message);
         }
+        // A key already in the environment keeps its own origin: the checkout's .env.hue is
+        // consulted only when it has to supply the key, so it cannot point an inherited key elsewhere.
+        if (envFile === undefined && !process.env.HUE_API_KEY && existsSync(DEFAULT_ENV_FILE))
+            envFile = DEFAULT_ENV_FILE;
         if (envFile) {
             try {
                 process.loadEnvFile(resolve(envFile));
@@ -1488,7 +1571,7 @@ export async function runEvalCommand(argv) {
         }
         const apiKey = process.env.HUE_API_KEY?.trim();
         if (!apiKey)
-            throw new UsageError('HUE_API_KEY is required: a "Read and write" project key, set in the environment or an ignored --env-file');
+            throw new UsageError('HUE_API_KEY is required: a "Read and write" project key, set in the environment, ./.env.hue or an ignored --env-file');
         secrets.push(apiKey);
         const baseUrl = values.origin ?? process.env.HUE_BASE_URL?.trim() ?? "https://app.hue.run";
         const connection = { apiKey, baseUrl };
@@ -1503,6 +1586,8 @@ export async function runEvalCommand(argv) {
             throw new UsageError("--worker takes no selection; Hue chooses the run to execute");
         if (values.worker && values["no-output"])
             throw new UsageError("--no-output applies to one-shot runs; a run launched from Hue always stores its outputs");
+        if (values.worker && values.check)
+            throw new UsageError("--check applies to one-shot runs");
         if (!values.worker && values.capability?.length)
             throw new UsageError("--capability applies to --worker only");
         const loaded = adapterFile ? await loadAdapter(adapterFile) : undefined;
@@ -1535,7 +1620,9 @@ export async function runEvalCommand(argv) {
     }
     catch (error) {
         if (controller.signal.aborted || error instanceof TargetCancelledError) {
-            process.stderr.write("Interrupted.\n");
+            process.stderr.write(worker
+                ? "Interrupted.\n"
+                : "Interrupted. The run stays open; run the same command again to resume it.\n");
             return 130;
         }
         if (error instanceof UsageError) {

package/dist/cli/login.js

@@ -7,9 +7,8 @@ import { createInterface } from "node:readline";
 import { Writable } from "node:stream";
 import { parseArgs } from "node:util";
 import { isLoopbackHost } from "../config.js";
-import { envFileArgument, envFileOptions } from "./env-file.js";
+import { DEFAULT_ENV_FILE, envFileArgument, envFileOptions } from "./env-file.js";
 const DEFAULT_ORIGIN = "https://app.hue.run";
-const DEFAULT_ENV_FILE = ".env.hue";
 /** Settings section that lists and creates project service keys. */
 const KEY_SETTINGS_PATH = "/settings/integrations";
 const REQUEST_TIMEOUT_MILLIS = 10_000;
@@ -745,6 +744,6 @@ export async function runLoginCommand(argv, io = {}) {
         // Without --auth it configures sign-in unless HUE_MCP_KEY is exported, so name the key stored.
         out(`  hue mcp install --client claude-code --auth key${mcpUrl === mcpUrlForOrigin(DEFAULT_ORIGIN) ? "" : ` --url ${mcpUrl}`}`);
     if (stored.includes("evaluations"))
-        out(`  hue eval --case "<name>" ./hue-agent.ts --env-file ${envDisplay}`);
+        out(`  hue eval --case "<name>" ./hue-agent.ts${envDisplay === DEFAULT_ENV_FILE ? "" : ` --env-file ${envDisplay}`}`);
     return 0;
 }

package/dist/client.d.ts

@@ -149,13 +149,15 @@ export declare class HueClient {
      * Adds a `hue.file` event to the active (or given) span for a file the work read, received or
      * produced: `hue.file.sha256`, `hue.file.role`, `hue.file.media_type`, `hue.file.size` when known
      * and, when `captureContent` is true, `hue.file.name`. `data` is hashed and measured locally and
-     * never exported. The event is metadata, so it is recorded in both capture modes. An invalid
+     * uploaded only with `upload: true`. The event is metadata, so it is recorded in both capture modes. An invalid
      * record, or one without an active span, is omitted and counted, never thrown; an invalid name
      * alone is omitted and counted while the rest is recorded.
      */
     recordFile(file: FileRecord, explicitContext?: Context): void;
     /** The keys of each span's content values that were cut to Hue's value cap, for `hue.truncated`. */
     private truncatedKeys;
+    /** Distinguishes requests whose helper events otherwise have identical metadata. */
+    private fileEventTimes;
     /** A content value's bounds: Hue's value cap as the receiver last advertised it. */
     private contentLimits;
     /**

package/dist/client.js

@@ -1,5 +1,6 @@
 import { AsyncLocalStorage } from "node:async_hooks";
 import { createHash } from "node:crypto";
+import { isMediaName } from "./inline-files.js";
 import { context, isSpanContextValid, ROOT_CONTEXT, SpanKind, SpanStatusCode, trace, } from "@opentelemetry/api";
 import { defaultTextMapGetter, defaultTextMapSetter } from "@opentelemetry/api";
 import { SeverityNumber } from "@opentelemetry/api-logs";
@@ -10,7 +11,7 @@ import { defaultResource, resourceFromAttributes } from "@opentelemetry/resource
 import { HUE_SCOPE } from "./config.js";
 import { hostedServerAddresses, hostedToolActivity, hostedToolProvider, providerErrorDescription, } from "./provider-tools.js";
 import { toolCatalogSummary } from "./tool-definitions.js";
-import { contentLimitsOf, encodeBoundedContent, encodeContent, noopSpan, safeSpan, } from "./safety.js";
+import { contentLimitsOf, encodeBoundedContent, encodeContent, noopSpan, safeSpan, spanSource, } from "./safety.js";
 import { TRUNCATED_KEY, withTruncatedKeys } from "./privacy.js";
 import { MAX_BLOB_BYTES } from "./blobs.js";
 import { createHueTransport, exportBudgetMillis, HueExportError, } from "./transport.js";
@@ -662,7 +663,7 @@ export class HueClient {
      * Adds a `hue.file` event to the active (or given) span for a file the work read, received or
      * produced: `hue.file.sha256`, `hue.file.role`, `hue.file.media_type`, `hue.file.size` when known
      * and, when `captureContent` is true, `hue.file.name`. `data` is hashed and measured locally and
-     * never exported. The event is metadata, so it is recorded in both capture modes. An invalid
+     * uploaded only with `upload: true`. The event is metadata, so it is recorded in both capture modes. An invalid
      * record, or one without an active span, is omitted and counted, never thrown; an invalid name
      * alone is omitted and counted while the rest is recorded.
      */
@@ -680,26 +681,30 @@ export class HueClient {
                 throw new TypeError("Invalid media type");
             let sha256 = typeof file.sha256 === "string" ? file.sha256.toLowerCase() : file.sha256;
             let byteSize = file.byteSize;
+            let bytes;
             if (data !== undefined) {
-                const bytes = typeof data === "string"
-                    ? (() => {
-                        // Buffer.byteLength measures UTF-8 without allocating the copy that hashing would
-                        // otherwise require. Reject before Buffer.from/createHash can retain large input.
-                        if (data.length > MAX_FILE_DATA_BYTES ||
-                            Buffer.byteLength(data, "utf8") > MAX_FILE_DATA_BYTES)
-                            throw new RangeError("File data exceeds Hue's 25 MiB limit");
-                        return Buffer.from(data, "utf8");
-                    })()
-                    : data instanceof Uint8Array
-                        ? data.byteLength <= MAX_FILE_DATA_BYTES
-                            ? data
-                            : (() => {
+                bytes =
+                    typeof data === "string"
+                        ? (() => {
+                            // Buffer.byteLength measures UTF-8 without allocating the copy that hashing would
+                            // otherwise require. Reject before Buffer.from/createHash can retain large input.
+                            if (data.length > MAX_FILE_DATA_BYTES ||
+                                Buffer.byteLength(data, "utf8") > MAX_FILE_DATA_BYTES)
                                 throw new RangeError("File data exceeds Hue's 25 MiB limit");
-                            })()
-                        : undefined;
+                            return Buffer.from(data, "utf8");
+                        })()
+                        : data instanceof Uint8Array
+                            ? data.byteLength <= MAX_FILE_DATA_BYTES
+                                ? data
+                                : (() => {
+                                    throw new RangeError("File data exceeds Hue's 25 MiB limit");
+                                })()
+                            : undefined;
                 if (!bytes)
                     throw new TypeError("File data must be bytes or a string");
-                const digest = createHash("sha256").update(bytes).digest("hex");
+                const digest = bytes.byteLength > 0 && file.upload === true && this.transport.holdsValues()
+                    ? this.transport.digestMedia(bytes, span.spanContext().traceId).sha256
+                    : createHash("sha256").update(bytes).digest("hex");
                 // A caller-supplied digest or size must describe the same bytes.
                 if ((sha256 !== undefined && sha256 !== digest) ||
                     (byteSize !== undefined && byteSize !== bytes.byteLength))
@@ -719,12 +724,55 @@ export class HueClient {
             if (byteSize !== undefined)
                 attributes["hue.file.size"] = byteSize;
             if (this.captureContent && name !== undefined) {
-                if (isLabel(name))
+                if (isMediaName(name))
                     attributes["hue.file.name"] = name;
                 else
                     this.instrumentationFailed();
             }
-            span.addEvent("hue.file", attributes);
+            // Let OpenTelemetry use the span's wall-clock anchor, then keep this event's exact
+            // timestamp for upload admission. A dropped event cannot authorize an upload.
+            const source = spanSource(span);
+            const previous = this.fileEventTimes.get(source);
+            const after = (time) => {
+                if (previous &&
+                    (time[0] < previous[0] || (time[0] === previous[0] && time[1] <= previous[1]))) {
+                    time[0] = previous[0];
+                    time[1] = previous[1] + 1;
+                    if (time[1] === 1_000_000_000) {
+                        time[0]++;
+                        time[1] = 0;
+                    }
+                }
+                return time;
+            };
+            let eventTime;
+            if (Array.isArray(source.events)) {
+                const previousEvent = source.events.at(-1);
+                span.addEvent("hue.file", attributes);
+                const event = source.events.at(-1);
+                eventTime = event && event !== previousEvent ? after(event.time) : undefined;
+            }
+            else {
+                // A span that does not expose its events (an API-only wrapper in attach mode): stamp the
+                // event with the wall clock and keep that exact time for upload admission.
+                const now = Date.now();
+                eventTime = after([Math.floor(now / 1000), (now % 1000) * 1_000_000]);
+                span.addEvent("hue.file", attributes, [eventTime[0], eventTime[1]]);
+            }
+            if (eventTime)
+                this.fileEventTimes.set(source, eventTime);
+            if (file.upload === true && this.transport.offloadsValues()) {
+                if (!this.transport.stageMedia(span, bytes && eventTime
+                    ? {
+                        bytes,
+                        digest: { sha256, size: bytes.byteLength },
+                        contentType: mediaType,
+                        eventTime,
+                        ...(isMediaName(name) ? { name } : {}),
+                    }
+                    : undefined))
+                    this.instrumentationFailed();
+            }
         }
         catch {
             this.instrumentationFailed();
@@ -732,6 +780,8 @@ export class HueClient {
     }
     /** The keys of each span's content values that were cut to Hue's value cap, for `hue.truncated`. */
     truncatedKeys = new WeakMap();
+    /** Distinguishes requests whose helper events otherwise have identical metadata. */
+    fileEventTimes = new WeakMap();
     /** A content value's bounds: Hue's value cap as the receiver last advertised it. */
     contentLimits() {
         return contentLimitsOf(this.transport.receiverLimits().valueBytes);

package/dist/evals/attempt.d.ts

@@ -1,4 +1,4 @@
-import { z } from "zod";
+import { z } from "zod/v4";
 import type { CoverageGap } from "../environment/types.js";
 import type { SimulationMcpCapability } from "./types.js";
 /** Canonical caller-observed agent configuration after missing evidence is made explicit. */

package/dist/evals/attempt.js

@@ -1,5 +1,5 @@
 import { createHash } from "node:crypto";
-import { z } from "zod";
+import { z } from "zod/v4";
 import { json, valueBounds } from "./json.js";
 const canonicalUuid = z.uuid().transform((value) => value.toLowerCase());
 const sha256Digest = z.string().regex(/^sha256:[0-9a-f]{64}$/);

package/dist/evals/checkpoint.d.ts

@@ -15,11 +15,15 @@ export declare class CheckpointIdentityError extends Error {
         captureContent?: unknown;
     } | undefined);
 }
-/** One owner per directory. A crash leaves .lock for explicit operator recovery; a forced exit of
- * `hue eval`, which stops its agents first, releases it. */
+/** One owner per directory. A forced exit of `hue eval`, which stops its agents first, releases
+ * it; a crash leaves .lock, which the next owner reclaims once the process that held it on this
+ * machine has exited. Any other lock is left for explicit operator recovery. */
 export declare class CheckpointStore {
     readonly directory: string;
     private untrack;
+    /** Whether this store took over the lock of a process that died here, so whatever that process
+     * was running never finished. */
+    reclaimed: boolean;
     private constructor();
     static acquire(directory: string, identity: unknown): Promise<CheckpointStore>;
     read<T>(key: string): Promise<T | undefined>;

package/dist/evals/checkpoint.js

@@ -1,7 +1,8 @@
 import { constants, rmSync } from "node:fs";
-import { lstat, mkdir, open, rename, rm } from "node:fs/promises";
+import { lstat, mkdir, open, readFile, rename, rm } from "node:fs/promises";
+import { hostname } from "node:os";
 import { join, resolve, sep } from "node:path";
-import { randomUUID } from "node:crypto";
+import { createHash, randomUUID } from "node:crypto";
 import { onForcedExit } from "./exit-cleanup.js";
 import { digest } from "./json.js";
 const CONTENT_POLICY = ["persistResultContent", "captureContent"];
@@ -46,11 +47,74 @@ function contentPolicyOnly(prior, identity) {
             return undefined;
     return Object.fromEntries(CONTENT_POLICY.map((key) => [key, before[key]]));
 }
-/** One owner per directory. A crash leaves .lock for explicit operator recovery; a forced exit of
- * `hue eval`, which stops its agents first, releases it. */
+async function createLock(lock) {
+    try {
+        await mkdir(lock, { mode: 0o700 });
+        return true;
+    }
+    catch (error) {
+        if (error.code === "EEXIST")
+            return false;
+        throw error;
+    }
+}
+/** Whether a process with this ID is running; one owned by another user still counts. */
+function running(pid) {
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch (error) {
+        return error.code === "EPERM";
+    }
+}
+/** Takes over a lock whose owner ran on this machine and has exited. The lock moves to a name
+ * derived from its unique owner record, which a rename can claim only once, so of the processes
+ * that saw the same dead owner one takes the lock and no live lock is ever moved; that name stays
+ * behind as the record. A lock without a readable owner, from another machine or from an SDK that
+ * did not record the machine is never reclaimed. */
+async function reclaimLock(root, lock) {
+    const owner = await readFile(join(lock, "owner.json"), "utf8").catch(() => undefined);
+    if (owner === undefined)
+        return false;
+    let pid;
+    let host;
+    try {
+        ({ pid, host } =
+            JSON.parse(owner).value ?? {});
+    }
+    catch {
+        return false;
+    }
+    if (typeof pid !== "number" ||
+        !Number.isInteger(pid) ||
+        pid <= 0 ||
+        pid === process.pid ||
+        host !== hostname() ||
+        running(pid))
+        return false;
+    const dead = join(root, `.lock-dead-${createHash("sha256").update(owner).digest("hex")}`);
+    try {
+        await rename(lock, dead);
+    }
+    catch {
+        return false;
+    }
+    if ((await readFile(join(dead, "owner.json"), "utf8").catch(() => undefined)) !== owner) {
+        await rename(dead, lock).catch(() => { });
+        return false;
+    }
+    return createLock(lock);
+}
+/** One owner per directory. A forced exit of `hue eval`, which stops its agents first, releases
+ * it; a crash leaves .lock, which the next owner reclaims once the process that held it on this
+ * machine has exited. Any other lock is left for explicit operator recovery. */
 export class CheckpointStore {
     directory;
     untrack = () => { };
+    /** Whether this store took over the lock of a process that died here, so whatever that process
+     * was running never finished. */
+    reclaimed = false;
     constructor(directory) {
         this.directory = directory;
     }
@@ -61,15 +125,14 @@ export class CheckpointStore {
         if (!info.isDirectory() || info.isSymbolicLink() || (info.mode & 0o077) !== 0)
             throw new Error("Use a private checkpoint directory (mode 0700, no symlink)");
         const store = new CheckpointStore(root);
+        const lock = join(root, ".lock");
+        const created = await createLock(lock);
+        if (!created && !(await reclaimLock(root, lock)))
+            throw new Error("Checkpoint directory is locked by a process that may still be running; confirm it stopped before explicitly removing .lock");
+        store.reclaimed = !created;
+        store.untrack = onForcedExit(() => rmSync(lock, { recursive: true, force: true }));
         try {
-            await mkdir(join(root, ".lock"), { mode: 0o700 });
-            store.untrack = onForcedExit(() => rmSync(join(root, ".lock"), { recursive: true, force: true }));
-        }
-        catch {
-            throw new Error("Checkpoint directory is locked; confirm its owner stopped before explicitly removing .lock");
-        }
-        try {
-            await store.write(".lock/owner", { pid: process.pid });
+            await store.write(".lock/owner", { pid: process.pid, host: hostname(), id: randomUUID() });
             const expected = { format: 1, identity, digest: digest(identity) };
             const prior = await store.read("manifest");
             if (prior && (prior.format !== 1 || prior.digest !== expected.digest))

package/dist/evals/files.js

@@ -30,6 +30,18 @@ export const outputContentTypes = [
     "image/png",
     "image/jpeg",
     "image/webp",
+    "image/gif",
+    "image/avif",
+    "audio/mpeg",
+    "audio/wav",
+    "audio/ogg",
+    "audio/webm",
+    "audio/mp4",
+    "audio/flac",
+    "audio/opus",
+    "audio/aac",
+    "video/mp4",
+    "video/webm",
 ];
 /** Thrown when the target's declared files cannot be used; recorded as the target's error. */
 export class OutputFileError extends Error {

package/dist/evals/runner.d.ts

@@ -105,6 +105,15 @@ export interface RunExperimentTargetContext {
 export interface RunExperimentOptions extends RunnerOptions {
     /** Hue tracing client; each case runs inside a `hue.experiment.case` span. */
     hue: HueClient;
+    /** Caller cancellation. Once it aborts, no further case starts, a case whose target stops with
+     * `TargetCancelledError` is saved as interrupted rather than completed, and the experiment is
+     * left unfinished: the call rejects with `TargetCancelledError`, and calling it again with the
+     * same checkpoint directory keeps the finished cases and runs the interrupted ones anew. */
+    signal?: AbortSignal;
+    /** Rerun, as a new attempt, a case that a process which died on this machine left running.
+     * Only for targets that give every attempt a fresh world: elsewhere the dead run's agent may
+     * still be acting, so such a case stays uncertain. */
+    rerunCrashedCases?: boolean;
     /** Experiment to run; its dataset version must be frozen. */
     experimentId: string;
     /** Whether each case waits for acknowledged trace export or explicitly omits evidence. Required. */