@hue-run/sdk 0.15.2 → 0.16.0

package/CLI.md

@@ -42,7 +42,7 @@ project manifest are refused because managers can update ancestor locks; Python
 The generated `hue.setup.mjs` or `hue_setup.py` always selects `captureContent: false` /
 `capture_content=False`. For a supported application, setup installs the dependency and adds the
 managed import and middleware registration to the existing entrypoint; an unreferenced helper is
-not a completed integration. TypeScript uses `@hue-run/sdk@0.15.2`, `@opentelemetry/api@1.9.1` and
+not a completed integration. TypeScript uses `@hue-run/sdk@0.16.0`, `@opentelemetry/api@1.9.1` and
 `@opentelemetry/context-async-hooks@2.11.0`; Python setup uses its separately tested package pin.
 Content capture requires an ordinary account-managed key and a later explicit application decision.
 
@@ -469,7 +469,7 @@ node packages/sdk-typescript/scripts/verify-package.mjs --artifacts-dir .artifac
 # Set project to an existing supported fixture; use the same directory on resume.
 project=/absolute/path/to/supported-fixture
 node packages/sdk-typescript/scripts/verify-setup-live.mjs \
-  --archive .artifacts/typescript/hue-run-sdk-0.15.2.tgz \
+  --archive .artifacts/typescript/hue-run-sdk-0.16.0.tgz \
   --origin https://STAGING_ORIGIN \
   --project "$project" --command setup \
   --evidence .context/setup-staging-before-claim.json
@@ -480,7 +480,7 @@ the private local handoff and finish the real browser claim, then reconcile the
 
 ```sh
 node packages/sdk-typescript/scripts/verify-setup-live.mjs \
-  --archive .artifacts/typescript/hue-run-sdk-0.15.2.tgz \
+  --archive .artifacts/typescript/hue-run-sdk-0.16.0.tgz \
   --origin https://STAGING_ORIGIN \
   --project "$project" --command claim \
   --evidence .context/setup-staging-after-claim.json
@@ -510,7 +510,7 @@ and prints Hue's verdicts. It is the command-line form of `runSimulation()` (one
 process, Hue creates one isolated simulated world per case, and Hue-owned outcome checks grade
 the sealed world. Hue never executes the agent. `HUE_API_KEY` must be a **Read and
 write** project key (a **Read** or **Tracing only** key cannot read cases or create runs);
-the CLI never prints it. The optional `zod` peer of `@hue-run/sdk/evals` must be installed.
+the CLI never prints it.
 
 Run the agent's own start command with `--command`. `hue eval` spawns it once per case with the
 world's environment (below) and `{"inputs", "config"}` on stdin, and stores its stdout as the
@@ -519,19 +519,20 @@ committed helper described in [Make your agent eval-ready](https://docs.hue.run/
 reads `HUE_SIM_<SURFACE ID>_URL` and `HUE_WORLD_TOKEN` when `HUE_WORLD_TOKEN` is set, fails
 closed when a needed variable is missing, and returns the production URL and credential
 otherwise. Never hand the agent Hue-native tools (`context.tools`, `context.mcp`) or a tool its
-production agent lacks. Pin the SDK and bring its `zod` peer along, so the agent's repository
-gains no dependency:
+production agent lacks. Pin the SDK, so the agent's repository gains no dependency:
 
 ```sh
-npx --yes --package @hue-run/sdk@0.15.2 --package "zod@^4.6.5" hue eval --case "Refund an eligible charge" --command "node agent.js" --env-file .env.hue
-npx --yes --package @hue-run/sdk@0.15.2 --package "zod@^4.6.5" hue eval --case https://app.hue.run/projects/demo/scenarios/<id> --command "python agent.py" --timeout 120 --baseline <experiment id>
-npx --yes --package @hue-run/sdk@0.15.2 --package "zod@^4.6.5" hue eval --set "Billing regressions" --scorer-version <id> --command "node agent.js" --save-version
-npx --yes --package @hue-run/sdk@0.15.2 --package "zod@^4.6.5" hue eval --worker --command "node agent.js" --agent-key support-agent --revision <new-agent-revision> --env-file .env.hue
+npx -y @hue-run/sdk@0.16.0 eval --case "Refund an eligible charge" --command "node agent.js"
+npx -y @hue-run/sdk@0.16.0 eval --case https://app.hue.run/projects/demo/scenarios/<id> --command "python agent.py" --timeout 120 --baseline <run id>
+npx -y @hue-run/sdk@0.16.0 eval --set "Billing regressions" --scorer-version <id> --command "node agent.js" --save-version
+npx -y @hue-run/sdk@0.16.0 eval --worker --command "node agent.js" --agent-key support-agent --revision <new-agent-revision>
 ```
 
-Without the second `--package`, `hue eval` exits with `hue eval needs zod`, even for `--help`;
-installing zod in the project does not help when the SDK itself is not installed there. An
-unpinned `--package @hue-run/sdk` runs a local, possibly older, copy in a project that already
+One invocation creates one run holding every selected case; it prints the run URL
+(`https://app.hue.run/runs/<id>`) once the run exists. Add `--check` to confirm the key, the
+selection, the case kind and the agent first: it prints what the run would do and exits without
+creating a run, a world or a checkpoint. An
+unpinned `npx @hue-run/sdk` runs a local, possibly older, copy in a project that already
 depends on it.
 
 An adapter module is the in-process alternative for a TypeScript agent. It exports `default` or
@@ -562,7 +563,7 @@ export default function runMyAgent(inputs: JsonValue, context: SimulationTargetC
 ```
 
 ```sh
-npx --yes --package @hue-run/sdk@0.15.2 --package "zod@^4.6.5" hue eval --case "Refund an eligible charge" ./hue-agent.ts --content --env-file .env.hue
+npx -y @hue-run/sdk@0.16.0 eval --case "Refund an eligible charge" ./hue-agent.ts --content
 ```
 
 `--scenario` remains an alias for `--case` for existing scripts. Pass one selection flag.
@@ -593,7 +594,7 @@ required judge to decide answer-only cases. Any
 other evaluator's result, an error, or a metric that carries `passed` is never advisory, whatever
 its evidence says. A metric name that
 several evaluators report, such as two judges' `verdict`, gets a column per evaluator, labelled
-with the version's first eight characters. `--baseline <experiment id|url>` adds improvement,
+with the version's first eight characters. `--baseline <run id|url>` adds improvement,
 regression and unchanged counts with per-case deltas. `--json` prints one JSON document
 (`experimentId`, `runId`, `runUrl`, `complete`, `cases`, `totals`, optional `baseline`) on stdout
 and sends progress to stderr; each case lists its not-applicable evaluator versions in
@@ -670,7 +671,8 @@ the sealed world at once and attaches the trace once its export is complete. The
 A Scenario or eval set whose dataset version is not saved cannot back an experiment: the command
 exits 1 and asks for **Save eval-set version** in Hue or `--save-version`, which freezes that
 version at its current revision. Connection settings are `HUE_API_KEY` and `HUE_BASE_URL`
-(default `https://app.hue.run`), loaded from `--env-file <path>` (or its alias `--env-path`) first when given; `--origin`
+(default `https://app.hue.run`), loaded from `--env-file <path>` (or its alias `--env-path`) first when given, otherwise from
+`./.env.hue` when the shell has no `HUE_API_KEY` (a key in the shell keeps its own origin); `--origin`
 overrides the origin. Since TypeScript 0.14.0, telemetry content capture defaults on; `--no-content` opts out.
 Earlier releases require `--content`; the examples include it for compatibility. Model and tool spans inside the agent come
 only from the agent's own instrumentation. Case outputs, error messages and explanations are
@@ -734,8 +736,8 @@ published version, beside or instead of explicit `--scorer-version` IDs.
 ```sh
 hue eval --set gia-d1-citation --scorer gia-d1-citation \
   --command "pnpm --filter @august/frontend run hue:gia-agent" \
-  --revision prompt-v10 --wait 1800 --content --json --env-file .env.hue
-hue eval --set gia-d1-citation --set-version 1 --scorer gia-d1-citation ./hue-agent.ts --content --baseline <experiment id>
+  --revision prompt-v10 --wait 1800 --content --json
+hue eval --set gia-d1-citation --set-version 1 --scorer gia-d1-citation ./hue-agent.ts --content --baseline <run id>
 ```
 
 For each case the command is spawned once **inside a private case directory** with

package/EVALUATIONS.md

@@ -7,7 +7,7 @@ Install the optional runtime-contract peer with the SDK before importing
 npm install @hue-run/sdk zod
 ```
 
-The SDK executes targets and scorers on your machine. Hue stores pinned definitions, experiment progress and results. It does not execute uploaded source code. Follow the [installation guide](https://docs.hue.run/installation) to add `@hue-run/sdk` to your application.
+The SDK executes targets and scorers on your machine. Hue stores pinned definitions, experiment progress and results. It does not execute uploaded source code. Follow [Install the SDK](https://docs.hue.run/quickstart#2-install-the-sdk) to add `@hue-run/sdk` to your application.
 
 Create a **Read and write** project service key under **Settings → Integrations & API keys** and expose it to this server-side process as `HUE_API_KEY`. A **Tracing only** key cannot author datasets or evaluation runs.
 

package/README.md

@@ -677,15 +677,12 @@ conditions only. Use Hue's UI to inspect captured values and redaction.
 
 ## Dependencies
 
-The tracing core depends only on official `@opentelemetry/*` packages. The optional
-`@hue-run/sdk/evals` entry point uses `zod` for its bounded runtime contracts; install that peer
-when you use evaluations or simulations. JSON Schema scoring also uses `ajv`, an optional peer
-loaded inside a worker only when `builtins.jsonSchema` scores a case; without it that scorer
+The tracing core depends only on official `@opentelemetry/*` packages and `zod`, which the
+`@hue-run/sdk/evals` entry point and `hue eval` use for their bounded runtime contracts and which
+installs with the SDK. JSON Schema scoring also uses `ajv`, an optional peer loaded inside a worker only when `builtins.jsonSchema` scores a case; without it that scorer
 reports `SchemaValidatorUnavailable`. Install it when you use that scorer:
 
 ```bash
-npm install zod
-# Add ajv too when using builtins.jsonSchema.
 npm install ajv
 ```
 
@@ -795,6 +792,6 @@ existing-provider flush callbacks and recovery. Local/CI runners remain availabl
 
 Use `createHueSafe(options)` for best-effort startup. Invalid initialization returns a disabled client that keeps your `onExportIssue` hook and records the reason as an instrumentation failure. Pass `enabled: false` to disable Hue without a key or a `captureContent` choice; disabled helpers still execute the application callback, and `inject()` keeps propagating the application's own trace context. `flushSafe({ timeoutMillis: 1000 })` and `shutdownSafe({ timeoutMillis: 1000 })` return `{ ok, timedOut, report }` without rejecting. Strict initialization, connection checks and `flush()` remain available for diagnostics; do not gate application readiness or responses on them.
 
-Capture/serialization/redaction/provider failures omit unsafe telemetry, record failures, and preserve the original business result/error. Async diagnostic rejections are contained; diagnostics are rate-limited. The default `maxQueueBytes` is 8 MiB across traces/logs including in-flight work, alongside the existing record cap. `pendingBytes` is a current queue gauge; `droppedSpans`, `droppedLogs` and `instrumentationFailures` are cumulative failure counters; `uploadedValues` and `uploadFallbacks` count the values over the inline limit uploaded apart from their spans and exported cut instead. This is a telemetry budget, not a total process memory ceiling. A timeout bounds the caller and does not cancel a borrowed provider. Never retry the business operation to recover telemetry. See [production safety](https://docs.hue.run/guides/production-safety).
+Capture/serialization/redaction/provider failures omit unsafe telemetry, record failures, and preserve the original business result/error. Async diagnostic rejections are contained; diagnostics are rate-limited. The default `maxQueueBytes` is 8 MiB across traces/logs including in-flight work, alongside the existing record cap. `pendingBytes` is a current queue gauge; `droppedSpans`, `droppedLogs` and `instrumentationFailures` are cumulative failure counters; `uploadedValues` and `uploadFallbacks` count the values over the inline limit uploaded apart from their spans and exported cut instead. This is a telemetry budget, not a total process memory ceiling. A timeout bounds the caller and does not cancel a borrowed provider. Never retry the business operation to recover telemetry. See [production setup](https://docs.hue.run/sdks/typescript#production-setup).
 
 Queued records snapshot supported telemetry values when a span ends or a log is emitted; later caller mutations cannot change queued data. Resource attributes still awaiting detection are omitted with a sanitized warning. Later records include them after detection finishes; await resource detection before instrumentation when those attributes are required.

package/dist/cli/env-file.d.ts

@@ -1,3 +1,5 @@
+/** The key file `hue login` writes; `hue eval` loads it when no env file is named and the shell has no HUE_API_KEY. */
+export declare const DEFAULT_ENV_FILE = ".env.hue";
 /**
  * Node 22 and 24 scan the whole command line for `--env-file`, including arguments meant for the
  * script, and exit with "node: <path>: not found" before the CLI runs when that file does not

package/dist/cli/env-file.js

@@ -1,4 +1,6 @@
 import { resolve } from "node:path";
+/** The key file `hue login` writes; `hue eval` loads it when no env file is named and the shell has no HUE_API_KEY. */
+export const DEFAULT_ENV_FILE = ".env.hue";
 /**
  * Node 22 and 24 scan the whole command line for `--env-file`, including arguments meant for the
  * script, and exit with "node: <path>: not found" before the CLI runs when that file does not

package/dist/cli/eval.js

@@ -1,5 +1,5 @@
 import { execFileSync, spawn } from "node:child_process";
-import { mkdtempSync, rmSync } from "node:fs";
+import { existsSync, mkdtempSync, rmSync } from "node:fs";
 import { access, mkdir, rm, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { basename, extname, join, resolve } from "node:path";
@@ -22,7 +22,7 @@ import { matchByName, parseScenarioSelector, resolveEvalSetPins, resolveScenario
 import { collectDirectOutputs, stageDirectCase } from "./eval-direct.js";
 import { runSimulation, } from "../evals/simulation.js";
 import { collectExperimentVerdicts, compareVerdicts, metricPassed, } from "../evals/verdicts.js";
-import { envFileArgument, envFileOptions } from "./env-file.js";
+import { DEFAULT_ENV_FILE, envFileArgument, envFileOptions } from "./env-file.js";
 const USAGE = `Usage: hue eval [adapter-file] [options]
 
 Run a local agent against a published case or eval set, then print Hue's verdicts.
@@ -70,13 +70,15 @@ Modes:
                                   environment-files:v1 and input:pdf for world cases with files
 
 Connection:
-  --env-file <path>               Load a dotenv file (HUE_API_KEY, HUE_BASE_URL) first
+  --env-file <path>               Load a dotenv file (HUE_API_KEY, HUE_BASE_URL) first;
+                                  ./.env.hue loads by default when HUE_API_KEY is not
+                                  set. Variables already set in the environment win
   --env-path <path>               Same as --env-file
   --origin <url>                  Hue origin (default: HUE_BASE_URL or https://app.hue.run)
 
 Output and limits:
   --name <run name>               Run name (default: <agent key> @ <revision>, hashes shortened)
-  --baseline <experiment id|url>  Compare verdicts with a previous experiment
+  --baseline <run id|url>         Compare verdicts with a previous run
   --json                          Print one JSON document on stdout; progress goes to stderr
   --content                       Capture telemetry content (the default)
   --no-content                    Omit telemetry inputs, outputs and messages
@@ -84,6 +86,8 @@ Output and limits:
                                   explanations (stored by default; --worker always stores); with
                                   --content the case span still carries the output
   --save-version                  Freeze an unsaved eval-set version before running
+  --check                         One-shot: check the key, selection and agent, print what the
+                                  run would do and exit without creating a run or a world
   --checkpoint-dir <path>         Private checkpoint directory (default: .hue/eval/<agent-key>)
   --concurrency <n>               Cases in flight, 1-100 (default: 1)
   --trace-not-accepted <policy>   A case whose telemetry Hue did not accept in time: fail_case
@@ -147,6 +151,7 @@ function parse(argv) {
                 "no-output": { type: "boolean", default: false },
                 "trace-not-accepted": { type: "string" },
                 "save-version": { type: "boolean", default: false },
+                check: { type: "boolean", default: false },
                 "checkpoint-dir": { type: "string" },
                 concurrency: { type: "string" },
                 timeout: { type: "string" },
@@ -971,20 +976,27 @@ function parseMode(value) {
         return value;
     throw new UsageError("--mode must be auto, direct or simulation");
 }
-/** Direct when nothing pins a simulated world: not a Scenario, and no case of the version does. */
-async function detectDirect(client, pins, mode) {
-    if (mode !== "auto")
-        return mode === "direct";
-    if (pins.scenarioId || pins.environmentVersionId)
-        return false;
+/** The selection's cases, and whether the run is direct: when `--mode` leaves it open, direct
+ * means nothing pins a simulated world, not a Scenario and no case of the version. A simulation
+ * run cannot execute a case that pins no world, so a selection mixing both kinds is refused. */
+async function inspectSelection(client, pins, mode) {
+    let cases = 0;
+    let worldCases = 0;
     let after;
     do {
         const page = await client.listCases(pins.datasetVersionId, { after });
-        if (page.items.some((item) => item.environmentVersionId))
-            return false;
+        cases += page.items.length;
+        worldCases += page.items.filter((item) => item.environmentVersionId).length;
         after = page.nextCursor ?? undefined;
     } while (after);
-    return true;
+    const direct = mode === "auto"
+        ? !pins.scenarioId && !pins.environmentVersionId && worldCases === 0
+        : mode === "direct";
+    const answerOnly = cases - worldCases;
+    const refusal = !direct && answerOnly > 0
+        ? `${answerOnly} of the ${cases} ${cases === 1 ? "case pins" : "cases pin"} no world, so a simulation run would fail ${answerOnly === 1 ? "it" : "them"} before the agent starts. Put the answer-only cases in their own eval set${mode === "simulation" ? ", or drop --mode simulation" : ""}.`
+        : undefined;
+    return { direct, cases, worldCases, ...(refusal ? { refusal } : {}) };
 }
 function toJson(verdicts, runUrl, baseline, extra = {}, telemetry = []) {
     const notAccepted = new Map(telemetry.map((entry) => [entry.caseId, entry.issues]));
@@ -1055,7 +1067,7 @@ async function reportVerdicts(client, values, run, baselineId, output, signal) {
     // The wait returns its partial state on abort rather than throwing, so Ctrl+C here must not
     // fall through to a baseline read and a verdict table that nobody asked to finish.
     if (signal.aborted) {
-        process.stderr.write("Interrupted.\n");
+        process.stderr.write(`Interrupted. The cases already finished and Hue is still checking them; open ${run.runUrl} instead of rerunning.\n`);
         return 130;
     }
     let baseline;
@@ -1086,7 +1098,7 @@ async function reportVerdicts(client, values, run, baselineId, output, signal) {
 function parseBaseline(value) {
     if (!value)
         return undefined;
-    const parsed = parseScenarioSelector(value, ["experiments"]);
+    const parsed = parseScenarioSelector(value, ["runs", "experiments"]);
     if (parsed.kind !== "id")
         throw new UsageError("--baseline must be an experiment ID or its Hue URL");
     return parsed.id;
@@ -1096,7 +1108,15 @@ async function runOnce(values, connection, agents, agent, hue, output, signal) {
     const concurrency = integer("concurrency", values.concurrency, 1, 1, 100);
     const baselineId = parseBaseline(values.baseline);
     const mode = parseMode(values.mode);
+    integer("wait", values.wait, 300, 0, 86_400);
     const pins = await resolveSelection(client, values);
+    const selection = await inspectSelection(client, pins, mode);
+    if (values.check)
+        return checkRun(client, values, pins, selection, agent, output);
+    if (selection.refusal) {
+        output.error(selection.refusal);
+        return 1;
+    }
     if (!pins.saved) {
         if (!values["save-version"]) {
             output.error(`The ${pins.scenarioId ? "Scenario" : "eval set"} "${pins.name}" points at an unsaved version. Choose "Save eval-set version" in Hue, or pass --save-version to freeze it now, then rerun.`);
@@ -1107,7 +1127,7 @@ async function runOnce(values, connection, agents, agent, hue, output, signal) {
     }
     const runName = values.name ?? defaultRunName(agent);
     const project = await client.checkConnection();
-    if (await detectDirect(client, pins, mode))
+    if (selection.direct)
         return runDirect({
             client,
             values,
@@ -1190,6 +1210,62 @@ async function runOnce(values, connection, agents, agent, hue, output, signal) {
         telemetryNotAccepted: report.telemetryNotAccepted,
     }, baselineId, output, signal);
 }
+/**
+ * `--check`: everything a one-shot run resolves before it creates anything (the key's project,
+ * the selection, the case kind and the agent), printed with the command that would start it.
+ * Nothing is created, frozen or checkpointed.
+ */
+async function checkRun(client, values, pins, selection, agent, output) {
+    const project = await client.checkConnection();
+    for (const id of pins.scorerVersionIds) {
+        const found = await client.getScorerVersion(id).then(() => true, (error) => {
+            if (error instanceof HueApiError && (error.status === 404 || error.status === 403))
+                return false;
+            throw error;
+        });
+        if (!found) {
+            output.error(`Evaluator version ${id} was not found in project "${project.name}".`);
+            return 1;
+        }
+    }
+    const { direct, cases, worldCases, refusal } = selection;
+    const unsaved = !pins.saved && !values["save-version"];
+    const check = {
+        ok: !unsaved && !refusal,
+        project: { id: project.id, name: project.name },
+        selection: {
+            kind: pins.scenarioId ? "case" : "eval_set",
+            name: pins.name,
+            datasetVersionId: pins.datasetVersionId,
+            saved: pins.saved,
+            cases,
+            evaluators: pins.scorerVersionIds.length,
+        },
+        mode: direct ? "direct" : "simulation",
+        worldCases: direct ? 0 : worldCases,
+        agent: { key: agent.key, revision: agent.revision, concurrency: values.concurrency ?? "1" },
+    };
+    if (values.json)
+        process.stdout.write(`${JSON.stringify({ check })}\n`);
+    else {
+        output.log(`Project: ${project.name} (${project.id})`);
+        output.log(`Selection: ${check.selection.kind === "case" ? "case" : "eval set"} "${pins.name}", ${cases} ${cases === 1 ? "case" : "cases"}, ${pins.scorerVersionIds.length} ${pins.scorerVersionIds.length === 1 ? "evaluator" : "evaluators"}`);
+        output.log(direct
+            ? "Mode: direct; the agent answers without a simulated world or app mirrors"
+            : `Mode: simulation; ${worldCases} ${worldCases === 1 ? "case gets" : "cases get"} a fresh world, reached through the HUE_SIM_*_URL mirrors and HUE_WORLD_TOKEN`);
+        output.log(`Agent: ${agent.key} @ ${agent.revision}`);
+    }
+    if (refusal) {
+        output.error(refusal);
+        return 1;
+    }
+    if (unsaved) {
+        output.error(`The ${pins.scenarioId ? "Scenario" : "eval set"} "${pins.name}" points at an unsaved version; add --save-version to freeze it when the run starts.`);
+        return 1;
+    }
+    output.log("Check passed. Nothing was created; run the same command without --check to start one run.");
+    return 0;
+}
 /**
  * Direct cases: one ordinary experiment through `runExperiment`. The agent receives the case's
  * pinned files and returns generated documents; code evaluators pinned to the run stay deferred
@@ -1218,7 +1294,7 @@ async function runDirect(run, connection) {
         });
         let attempt = await store.read("active-attempt");
         if (attempt && attempt.stage !== "completed" && attempt.selectionDigest !== selectionDigest)
-            throw new Error("Recover the unfinished direct run before running a changed selection");
+            throw new Error("Recover the unfinished direct run before running a changed selection: run the unchanged command again to finish it, or pass another --checkpoint-dir");
         if (!attempt || attempt.stage === "completed") {
             attempt = { selectionDigest, idempotencyKey: randomUUID(), stage: "preparing" };
             await store.write("active-attempt", attempt);
@@ -1236,7 +1312,7 @@ async function runDirect(run, connection) {
             await store.write("active-attempt", attempt);
         }
         experimentId = attempt.experimentId;
-        runUrl = new URL(`/experiments/${experimentId}`, connection.baseUrl).toString();
+        runUrl = new URL(`/runs/${experimentId}`, connection.baseUrl).toString();
         output.log(`Run: ${runUrl}`);
         output.log(`Experiment: ${experimentId}`);
         report = await runExperiment({
@@ -1251,6 +1327,7 @@ async function runDirect(run, connection) {
             traceNotAccepted: traceNotAcceptedPolicy(values["trace-not-accepted"]),
             onTelemetryNotAccepted: telemetry.report,
             concurrency: run.concurrency,
+            signal,
             scorers: [],
             deferUnboundLocalScorers: true,
             environmentEvidence: "when_pinned",
@@ -1308,7 +1385,7 @@ async function runWorker(values, connection, agents, agent, hue, output, signal)
     };
     client.onClaimed = (claim) => {
         current = claim;
-        output.log(`Claimed run ${claim.runId}: ${new URL(`/experiments/${claim.experimentId}`, connection.baseUrl).toString()}`);
+        output.log(`Claimed run ${claim.runId}: ${new URL(`/runs/${claim.experimentId}`, connection.baseUrl).toString()}`);
     };
     const telemetry = telemetryReporter(output);
     // A command can use the same stdin contract for both case kinds. Existing adapter files
@@ -1396,7 +1473,7 @@ async function runWorker(values, connection, agents, agent, hue, output, signal)
                 if (signal.aborted)
                     return;
                 if (values.json)
-                    process.stdout.write(`${JSON.stringify(toJson(verdicts, new URL(`/experiments/${current.experimentId}`, connection.baseUrl).toString(), undefined, {}, report.telemetryNotAccepted))}\n`);
+                    process.stdout.write(`${JSON.stringify(toJson(verdicts, new URL(`/runs/${current.experimentId}`, connection.baseUrl).toString(), undefined, {}, report.telemetryNotAccepted))}\n`);
                 else
                     renderTable(verdicts, output);
             }
@@ -1446,8 +1523,10 @@ export async function runEvalCommand(argv) {
     process.on("SIGTERM", interrupt);
     let hue;
     let json = false;
+    let worker = false;
     try {
         const { values, positionals } = parse(argv);
+        worker = values.worker;
         if (values.content && values["no-content"])
             throw new UsageError("Choose either --content or --no-content");
         // A usage error before anything is prepared or created: no run is left to recover from it.
@@ -1478,6 +1557,10 @@ export async function runEvalCommand(argv) {
         catch (error) {
             throw new UsageError(error.message);
         }
+        // A key already in the environment keeps its own origin: the checkout's .env.hue is
+        // consulted only when it has to supply the key, so it cannot point an inherited key elsewhere.
+        if (envFile === undefined && !process.env.HUE_API_KEY && existsSync(DEFAULT_ENV_FILE))
+            envFile = DEFAULT_ENV_FILE;
         if (envFile) {
             try {
                 process.loadEnvFile(resolve(envFile));
@@ -1488,7 +1571,7 @@ export async function runEvalCommand(argv) {
         }
         const apiKey = process.env.HUE_API_KEY?.trim();
         if (!apiKey)
-            throw new UsageError('HUE_API_KEY is required: a "Read and write" project key, set in the environment or an ignored --env-file');
+            throw new UsageError('HUE_API_KEY is required: a "Read and write" project key, set in the environment, ./.env.hue or an ignored --env-file');
         secrets.push(apiKey);
         const baseUrl = values.origin ?? process.env.HUE_BASE_URL?.trim() ?? "https://app.hue.run";
         const connection = { apiKey, baseUrl };
@@ -1503,6 +1586,8 @@ export async function runEvalCommand(argv) {
             throw new UsageError("--worker takes no selection; Hue chooses the run to execute");
         if (values.worker && values["no-output"])
             throw new UsageError("--no-output applies to one-shot runs; a run launched from Hue always stores its outputs");
+        if (values.worker && values.check)
+            throw new UsageError("--check applies to one-shot runs");
         if (!values.worker && values.capability?.length)
             throw new UsageError("--capability applies to --worker only");
         const loaded = adapterFile ? await loadAdapter(adapterFile) : undefined;
@@ -1535,7 +1620,9 @@ export async function runEvalCommand(argv) {
     }
     catch (error) {
         if (controller.signal.aborted || error instanceof TargetCancelledError) {
-            process.stderr.write("Interrupted.\n");
+            process.stderr.write(worker
+                ? "Interrupted.\n"
+                : "Interrupted. The run stays open; run the same command again to resume it.\n");
             return 130;
         }
         if (error instanceof UsageError) {

package/dist/cli/login.js

@@ -7,9 +7,8 @@ import { createInterface } from "node:readline";
 import { Writable } from "node:stream";
 import { parseArgs } from "node:util";
 import { isLoopbackHost } from "../config.js";
-import { envFileArgument, envFileOptions } from "./env-file.js";
+import { DEFAULT_ENV_FILE, envFileArgument, envFileOptions } from "./env-file.js";
 const DEFAULT_ORIGIN = "https://app.hue.run";
-const DEFAULT_ENV_FILE = ".env.hue";
 /** Settings section that lists and creates project service keys. */
 const KEY_SETTINGS_PATH = "/settings/integrations";
 const REQUEST_TIMEOUT_MILLIS = 10_000;
@@ -745,6 +744,6 @@ export async function runLoginCommand(argv, io = {}) {
         // Without --auth it configures sign-in unless HUE_MCP_KEY is exported, so name the key stored.
         out(`  hue mcp install --client claude-code --auth key${mcpUrl === mcpUrlForOrigin(DEFAULT_ORIGIN) ? "" : ` --url ${mcpUrl}`}`);
     if (stored.includes("evaluations"))
-        out(`  hue eval --case "<name>" ./hue-agent.ts --env-file ${envDisplay}`);
+        out(`  hue eval --case "<name>" ./hue-agent.ts${envDisplay === DEFAULT_ENV_FILE ? "" : ` --env-file ${envDisplay}`}`);
     return 0;
 }

package/dist/evals/attempt.d.ts

@@ -1,4 +1,4 @@
-import { z } from "zod";
+import { z } from "zod/v4";
 import type { CoverageGap } from "../environment/types.js";
 import type { SimulationMcpCapability } from "./types.js";
 /** Canonical caller-observed agent configuration after missing evidence is made explicit. */

package/dist/evals/attempt.js

@@ -1,5 +1,5 @@
 import { createHash } from "node:crypto";
-import { z } from "zod";
+import { z } from "zod/v4";
 import { json, valueBounds } from "./json.js";
 const canonicalUuid = z.uuid().transform((value) => value.toLowerCase());
 const sha256Digest = z.string().regex(/^sha256:[0-9a-f]{64}$/);

package/dist/evals/checkpoint.d.ts

@@ -15,11 +15,15 @@ export declare class CheckpointIdentityError extends Error {
         captureContent?: unknown;
     } | undefined);
 }
-/** One owner per directory. A crash leaves .lock for explicit operator recovery; a forced exit of
- * `hue eval`, which stops its agents first, releases it. */
+/** One owner per directory. A forced exit of `hue eval`, which stops its agents first, releases
+ * it; a crash leaves .lock, which the next owner reclaims once the process that held it on this
+ * machine has exited. Any other lock is left for explicit operator recovery. */
 export declare class CheckpointStore {
     readonly directory: string;
     private untrack;
+    /** Whether this store took over the lock of a process that died here, so whatever that process
+     * was running never finished. */
+    reclaimed: boolean;
     private constructor();
     static acquire(directory: string, identity: unknown): Promise<CheckpointStore>;
     read<T>(key: string): Promise<T | undefined>;

package/dist/evals/checkpoint.js

@@ -1,7 +1,8 @@
 import { constants, rmSync } from "node:fs";
-import { lstat, mkdir, open, rename, rm } from "node:fs/promises";
+import { lstat, mkdir, open, readFile, rename, rm } from "node:fs/promises";
+import { hostname } from "node:os";
 import { join, resolve, sep } from "node:path";
-import { randomUUID } from "node:crypto";
+import { createHash, randomUUID } from "node:crypto";
 import { onForcedExit } from "./exit-cleanup.js";
 import { digest } from "./json.js";
 const CONTENT_POLICY = ["persistResultContent", "captureContent"];
@@ -46,11 +47,74 @@ function contentPolicyOnly(prior, identity) {
             return undefined;
     return Object.fromEntries(CONTENT_POLICY.map((key) => [key, before[key]]));
 }
-/** One owner per directory. A crash leaves .lock for explicit operator recovery; a forced exit of
- * `hue eval`, which stops its agents first, releases it. */
+async function createLock(lock) {
+    try {
+        await mkdir(lock, { mode: 0o700 });
+        return true;
+    }
+    catch (error) {
+        if (error.code === "EEXIST")
+            return false;
+        throw error;
+    }
+}
+/** Whether a process with this ID is running; one owned by another user still counts. */
+function running(pid) {
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch (error) {
+        return error.code === "EPERM";
+    }
+}
+/** Takes over a lock whose owner ran on this machine and has exited. The lock moves to a name
+ * derived from its unique owner record, which a rename can claim only once, so of the processes
+ * that saw the same dead owner one takes the lock and no live lock is ever moved; that name stays
+ * behind as the record. A lock without a readable owner, from another machine or from an SDK that
+ * did not record the machine is never reclaimed. */
+async function reclaimLock(root, lock) {
+    const owner = await readFile(join(lock, "owner.json"), "utf8").catch(() => undefined);
+    if (owner === undefined)
+        return false;
+    let pid;
+    let host;
+    try {
+        ({ pid, host } =
+            JSON.parse(owner).value ?? {});
+    }
+    catch {
+        return false;
+    }
+    if (typeof pid !== "number" ||
+        !Number.isInteger(pid) ||
+        pid <= 0 ||
+        pid === process.pid ||
+        host !== hostname() ||
+        running(pid))
+        return false;
+    const dead = join(root, `.lock-dead-${createHash("sha256").update(owner).digest("hex")}`);
+    try {
+        await rename(lock, dead);
+    }
+    catch {
+        return false;
+    }
+    if ((await readFile(join(dead, "owner.json"), "utf8").catch(() => undefined)) !== owner) {
+        await rename(dead, lock).catch(() => { });
+        return false;
+    }
+    return createLock(lock);
+}
+/** One owner per directory. A forced exit of `hue eval`, which stops its agents first, releases
+ * it; a crash leaves .lock, which the next owner reclaims once the process that held it on this
+ * machine has exited. Any other lock is left for explicit operator recovery. */
 export class CheckpointStore {
     directory;
     untrack = () => { };
+    /** Whether this store took over the lock of a process that died here, so whatever that process
+     * was running never finished. */
+    reclaimed = false;
     constructor(directory) {
         this.directory = directory;
     }
@@ -61,15 +125,14 @@ export class CheckpointStore {
         if (!info.isDirectory() || info.isSymbolicLink() || (info.mode & 0o077) !== 0)
             throw new Error("Use a private checkpoint directory (mode 0700, no symlink)");
         const store = new CheckpointStore(root);
+        const lock = join(root, ".lock");
+        const created = await createLock(lock);
+        if (!created && !(await reclaimLock(root, lock)))
+            throw new Error("Checkpoint directory is locked by a process that may still be running; confirm it stopped before explicitly removing .lock");
+        store.reclaimed = !created;
+        store.untrack = onForcedExit(() => rmSync(lock, { recursive: true, force: true }));
         try {
-            await mkdir(join(root, ".lock"), { mode: 0o700 });
-            store.untrack = onForcedExit(() => rmSync(join(root, ".lock"), { recursive: true, force: true }));
-        }
-        catch {
-            throw new Error("Checkpoint directory is locked; confirm its owner stopped before explicitly removing .lock");
-        }
-        try {
-            await store.write(".lock/owner", { pid: process.pid });
+            await store.write(".lock/owner", { pid: process.pid, host: hostname(), id: randomUUID() });
             const expected = { format: 1, identity, digest: digest(identity) };
             const prior = await store.read("manifest");
             if (prior && (prior.format !== 1 || prior.digest !== expected.digest))

package/dist/evals/runner.d.ts

@@ -105,6 +105,15 @@ export interface RunExperimentTargetContext {
 export interface RunExperimentOptions extends RunnerOptions {
     /** Hue tracing client; each case runs inside a `hue.experiment.case` span. */
     hue: HueClient;
+    /** Caller cancellation. Once it aborts, no further case starts, a case whose target stops with
+     * `TargetCancelledError` is saved as interrupted rather than completed, and the experiment is
+     * left unfinished: the call rejects with `TargetCancelledError`, and calling it again with the
+     * same checkpoint directory keeps the finished cases and runs the interrupted ones anew. */
+    signal?: AbortSignal;
+    /** Rerun, as a new attempt, a case that a process which died on this machine left running.
+     * Only for targets that give every attempt a fresh world: elsewhere the dead run's agent may
+     * still be acting, so such a case stays uncertain. */
+    rerunCrashedCases?: boolean;
     /** Experiment to run; its dataset version must be frozen. */
     experimentId: string;
     /** Whether each case waits for acknowledged trace export or explicitly omits evidence. Required. */

package/dist/evals/runner.js

@@ -213,6 +213,10 @@ async function allPages(page, maximum = 5000) {
  * world that cannot be sealed), telemetry not accepted by the one transport the cases share, or
  * a target outcome left uncertain because its world could not be sealed or read back. Starting
  * more cases into it only piles up the same failure; the run is resumed later instead. */
+/** A new attempt replacing an interrupted one, which Hue still holds as started. */
+function retry(previousExecutionId) {
+    return previousExecutionId ? { previousExecutionId, allowUncertainRetry: true } : {};
+}
 function systemic(error) {
     // A saved outcome still waiting for its trace (`TraceExportUnacknowledgedError`) is not
     // systemic: the flush failed on an earlier attempt, telemetry may have recovered since, and
@@ -230,6 +234,7 @@ async function pool(items, concurrency, execute) {
     let position = 0;
     const failures = [];
     let stop = false;
+    let cancelled;
     await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, async () => {
         while (position < items.length && !stop) {
             const item = items[position++];
@@ -237,12 +242,20 @@ async function pool(items, concurrency, execute) {
                 await execute(item);
             }
             catch (error) {
+                // The caller's stop: start nothing more, and report the stop once the cases in flight settle.
+                if (error instanceof TargetCancelledError) {
+                    cancelled = error;
+                    stop = true;
+                    continue;
+                }
                 failures.push(error);
                 if (systemic(error))
                     stop = true;
             }
         }
     }));
+    if (cancelled)
+        throw cancelled;
     if (failures.length === 1)
         throw failures[0];
     if (failures.length) {
@@ -498,17 +511,19 @@ export async function runExperiment(options) {
                 scores,
                 exportState: "pending",
                 ...(saved.trace ? { trace: saved.trace } : {}),
+                ...(saved.attempt ? { attempt: saved.attempt } : {}),
             };
             await store.write(file, prepared);
             return prepared;
         }
         // A case pinned to a world keeps its files apart and removes them when it ends: the agent's
         // copies of its inputs, its work, the evaluator's downloads and the staged outputs.
-        const worldDirectoryOf = (itemId) => join(filesRoot, `world-case-${uuid(itemId)}`);
+        // A retry gets its own, so an agent a dead run left behind cannot write into it.
+        const worldDirectoryOf = (itemId, attempt) => join(filesRoot, `world-case-${uuid(itemId)}${attempt ? `-${uuid(attempt)}` : ""}`);
         const runCase = async (item, files) => {
             const file = `case-${uuid(item.id)}`;
-            const worldDirectory = worldDirectoryOf(item.id);
-            const caseDirectoryOf = (frozen) => frozen.environmentVersionId ? worldDirectory : join(filesRoot, `case-${uuid(item.id)}`);
+            const directDirectory = join(filesRoot, `case-${uuid(item.id)}`);
+            const caseDirectoryOf = (frozen) => frozen.environmentVersionId ? files.directory : directDirectory;
             let checkpoint;
             try {
                 checkpoint = await store.read(file);
@@ -521,6 +536,24 @@ export async function runExperiment(options) {
                 throw error;
             }
             files.keepOutputs = checkpoint?.stage === "uploading";
+            if (checkpoint && "attempt" in checkpoint)
+                files.directory = worldDirectoryOf(item.id, checkpoint.attempt);
+            let retryOf;
+            // A case left running by a process that died here is as interrupted as one the caller
+            // stopped; a fresh world isolates it from anything the dead run's agent still does.
+            if (checkpoint?.stage === "running" && store.reclaimed && options.rerunCrashedCases) {
+                checkpoint = { ...checkpoint, stage: "interrupted" };
+                await store.write(file, checkpoint);
+            }
+            if (checkpoint?.stage === "interrupted") {
+                retryOf = checkpoint.executionId;
+                checkpoint = undefined;
+                // A direct case's directory goes too, so the retry finds no input copy or output of the
+                // stopped attempt.
+                await removeCaseFiles(files.directory);
+                await removeCaseFiles(directDirectory);
+                files.directory = worldDirectoryOf(item.id, retryOf);
+            }
             if (checkpoint && checkpoint.stage !== "prepared" && checkpoint.stage !== "uploading") {
                 if (checkpoint.stage === "serialization_failed")
                     throw new OutcomeSerializationError(checkpoint.executionId);
@@ -528,6 +561,7 @@ export async function runExperiment(options) {
                     ? await options.client.startExecution(experiment.id, item.id, {
                         idempotencyKey: checkpoint.startKey,
                         traceExternalId: checkpoint.traceExternalId,
+                        ...retry(checkpoint.previousExecutionId),
                     })
                     : await options.client.getExecution(checkpoint.executionId);
                 throw new UncertainExecutionError(item.id, execution.id);
@@ -542,7 +576,9 @@ export async function runExperiment(options) {
                 files.keepOutputs = false;
             }
             if (!checkpoint) {
-                if (item.execution)
+                if (options.signal?.aborted)
+                    throw new TargetCancelledError();
+                if (item.execution && item.execution.id !== retryOf)
                     throw new UncertainExecutionError(item.id, item.execution.id);
                 const frozenCase = await options.client.getExperimentCase(experiment.id, item.id);
                 if (frozenCase.datasetVersionId !== version.id)
@@ -574,13 +610,16 @@ export async function runExperiment(options) {
                         stage: "starting",
                         startKey: randomUUID(),
                         traceExternalId: span.traceId,
+                        ...(retryOf ? { previousExecutionId: retryOf } : {}),
                     };
                     await store.write(file, start);
                     const execution = await options.client.startExecution(experiment.id, item.id, {
                         idempotencyKey: start.startKey,
                         traceExternalId: span.traceId,
+                        ...retry(retryOf),
                     });
-                    await store.write(file, { stage: "running", executionId: execution.id });
+                    const attempt = retryOf ? { attempt: retryOf } : {};
+                    await store.write(file, { stage: "running", executionId: execution.id, ...attempt });
                     let state = "succeeded";
                     let output;
                     let generated;
@@ -604,6 +643,17 @@ export async function runExperiment(options) {
                     catch (error) {
                         if (error instanceof TargetOutcomeUncertainError)
                             throw error;
+                        // A stop the caller asked for is no outcome: the execution stays open for the
+                        // resumed run to replace instead of being scored as cancelled.
+                        if (error instanceof TargetCancelledError && options.signal?.aborted) {
+                            options.hue.recordError(span.span, error);
+                            await store.write(file, {
+                                stage: "interrupted",
+                                executionId: execution.id,
+                                ...attempt,
+                            });
+                            throw error;
+                        }
                         state = error instanceof TargetCancelledError ? "cancelled" : "error";
                         targetError = error;
                         options.hue.recordError(span.span, error);
@@ -659,6 +709,7 @@ export async function runExperiment(options) {
                         ...(options.persistResultContent && output !== undefined ? { output } : {}),
                         ...(state === "error" ? { error: errorPayload(targetError) } : {}),
                         files: staged,
+                        ...attempt,
                     };
                     // Without persisted result content a restart cannot reconstruct the outcome; the
                     // saved stage then reports the execution as uncertain instead of guessing.
@@ -838,21 +889,23 @@ export async function runExperiment(options) {
             // A world case's directory goes however the case ends, a forced exit included. An
             // interrupted upload keeps only the staged outputs it resumes from; inputs are downloaded
             // anew for scoring. A direct case has no such directory, so nothing is removed for it.
-            const worldDirectory = worldDirectoryOf(item.id);
-            const files = { keepOutputs: true };
-            const untrack = onForcedExit(() => releaseWorldFilesSync(worldDirectory, files.keepOutputs));
+            const files = { keepOutputs: true, directory: worldDirectoryOf(item.id) };
+            const untrack = onForcedExit(() => releaseWorldFilesSync(files.directory, files.keepOutputs));
             try {
                 await runCase(item, files);
-                await releaseWorldFiles(worldDirectory, false);
+                await releaseWorldFiles(files.directory, false);
             }
             catch (error) {
-                await releaseWorldFiles(worldDirectory, files.keepOutputs);
+                await releaseWorldFiles(files.directory, files.keepOutputs);
                 throw error;
             }
             finally {
                 untrack();
             }
         });
+        // A stop that came while the last cases saved their outcomes still leaves the run open.
+        if (options.signal?.aborted)
+            throw new TargetCancelledError();
         let finish = await store.read("finish");
         if (!finish) {
             finish = { key: randomUUID() };

package/dist/evals/scorer-publication.js

@@ -1,4 +1,4 @@
-import { z } from "zod";
+import { z } from "zod/v4";
 import { json, digest } from "./json.js";
 const metricName = z
     .string()

package/dist/evals/simulation.js

@@ -391,7 +391,7 @@ export async function runSimulation(options) {
             attempt.stage !== "completed" &&
             attempt.scenarioDigest !== scenarioDigest &&
             attempt.scenarioDigest !== previousDigest)
-            throw new Error("Recover the unfinished simulation before running a changed scenario or agent revision");
+            throw new Error("Recover the unfinished simulation before running a changed scenario or agent revision: run the unchanged one again to finish it, or use another checkpoint directory");
         if (!attempt || attempt.stage === "completed") {
             attempt = {
                 scenarioDigest,
@@ -417,12 +417,14 @@ export async function runSimulation(options) {
             ];
         }
         const experimentId = checkpointSegment(attempt.experimentId, "experiment id");
-        const runUrl = new URL(`/experiments/${experimentId}`, options.client.baseUrl).toString();
+        const runUrl = new URL(`/runs/${experimentId}`, options.client.baseUrl).toString();
         await options.onProgress?.({ type: "run_created", experimentId, runUrl });
         const requested = requestedConfiguration
             ? pinRequestedAttemptV2(requestedConfiguration, (await options.client.getExperiment(experimentId)).config)
             : undefined;
         const report = await runExperiment({
+            signal: options.signal,
+            rerunCrashedCases: true,
             client: options.client,
             hue: options.hue,
             experimentId,

package/dist/version.d.ts

@@ -1,2 +1,2 @@
 /** Package version shared by the instrumentation scope and the export User-Agent. */
-export declare const sdkVersion = "0.15.2";
+export declare const sdkVersion = "0.16.0";

package/dist/version.js

@@ -1,3 +1,3 @@
 // Generated by scripts/write-version.mjs from package.json; do not edit by hand.
 /** Package version shared by the instrumentation scope and the export User-Agent. */
-export const sdkVersion = "0.15.2";
+export const sdkVersion = "0.16.0";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hue-run/sdk",
-  "version": "0.15.2",
+  "version": "0.16.0",
   "private": false,
   "license": "MIT",
   "publishConfig": {
@@ -91,7 +91,8 @@
     "@opentelemetry/otlp-transformer": "0.222.0",
     "@opentelemetry/resources": "2.11.0",
     "@opentelemetry/sdk-logs": "0.222.0",
-    "@opentelemetry/sdk-trace": "2.11.0"
+    "@opentelemetry/sdk-trace": "2.11.0",
+    "zod": "^3.25.76 || ^4.6.5"
   },
   "devDependencies": {
     "@ai-sdk/otel": "1.0.99",
@@ -102,15 +103,13 @@
     "ai": "7.0.99",
     "ajv": "8.20.0",
     "protobufjs": "8.8.0",
-    "typescript": "7.0.2",
-    "zod": "4.6.5"
+    "typescript": "7.0.2"
   },
   "peerDependencies": {
     "@ai-sdk/otel": "^1.0.99",
     "@opentelemetry/api": "^1.9.1",
     "ai": "^6.0.0 || ^7.0.99",
-    "ajv": "^8.17.0",
-    "zod": "^4.6.5"
+    "ajv": "^8.17.0"
   },
   "peerDependenciesMeta": {
     "@ai-sdk/otel": {
@@ -121,9 +120,6 @@
     },
     "ajv": {
       "optional": true
-    },
-    "zod": {
-      "optional": true
     }
   },
   "engines": {