@hue-run/sdk 0.9.0 → 0.10.0

package/CLI.md

@@ -42,7 +42,7 @@ project manifest are refused because managers can update ancestor locks; Python
 The generated `hue.setup.mjs` or `hue_setup.py` always selects `captureContent: false` /
 `capture_content=False`. For a supported application, setup installs the dependency and adds the
 managed import and middleware registration to the existing entrypoint; an unreferenced helper is
-not a completed integration. TypeScript uses `@hue-run/sdk@0.9.0`, `@opentelemetry/api@1.9.1` and
+not a completed integration. TypeScript uses `@hue-run/sdk@0.10.0`, `@opentelemetry/api@1.9.1` and
 `@opentelemetry/context-async-hooks@2.11.0`; Python setup uses its separately tested package pin.
 Content capture requires an ordinary account-managed key and a later explicit application decision.
 
@@ -305,7 +305,7 @@ node packages/sdk-typescript/scripts/verify-package.mjs --artifacts-dir .artifac
 # Set project to an existing supported fixture; use the same directory on resume.
 project=/absolute/path/to/supported-fixture
 node packages/sdk-typescript/scripts/verify-setup-live.mjs \
-  --archive .artifacts/typescript/hue-run-sdk-0.9.0.tgz \
+  --archive .artifacts/typescript/hue-run-sdk-0.10.0.tgz \
   --origin https://STAGING_ORIGIN \
   --project "$project" --command setup \
   --evidence .context/setup-staging-before-claim.json
@@ -316,7 +316,7 @@ the private local handoff and finish the real browser claim, then reconcile the
 
 ```sh
 node packages/sdk-typescript/scripts/verify-setup-live.mjs \
-  --archive .artifacts/typescript/hue-run-sdk-0.9.0.tgz \
+  --archive .artifacts/typescript/hue-run-sdk-0.10.0.tgz \
   --origin https://STAGING_ORIGIN \
   --project "$project" --command claim \
   --evidence .context/setup-staging-after-claim.json
@@ -351,8 +351,11 @@ the CLI never prints it. The optional `zod` peer of `@hue-run/sdk/evals` must be
 Write an adapter module that hands the case inputs and the world's tools or MCP connection to
 the real agent. The module exports `default` or `runMyAgent`; `context` is the SDK's
 `SimulationTargetContext` (`world`, `mcp`, `tools`, `config`, `item`, `executionId`,
-`environmentRunId`, `signal`; `world` carries the provider mirror URLs, the world token, `env` and
-`mcpConfig` where Hue's simulation gateway serves the world):
+`environmentRunId`, `files`, `outputDirectory`, `signal`; `world` carries the provider mirror URLs,
+the world token, `env` and `mcpConfig` where Hue's simulation gateway serves the world). When the
+case also pins input files, `files` holds verified copies of its agent-visible ones (see
+[Evaluate a document eval set](#evaluate-a-document-eval-set)); return
+`withFiles(output, files)` from `@hue-run/sdk/evals` to upload documents the agent produced:
 
 ```ts
 // hue-agent.ts: erasable TypeScript only; Node.js 24 and Bun strip the types natively.
@@ -368,27 +371,35 @@ export default function runMyAgent(inputs: JsonValue, context: SimulationTargetC
 ```
 
 ```sh
-hue eval --case "Refund an eligible charge" ./hue-agent.ts --env-file .env.hue
-hue eval --case https://app.hue.run/projects/demo/scenarios/<id> ./hue-agent.ts --baseline <run id>
-hue eval --set "Billing regressions" --scorer-version <id> ./hue-agent.ts --save-version
+hue eval --case "Refund an eligible charge" ./hue-agent.ts --content --env-file .env.hue
+hue eval --case https://app.hue.run/projects/demo/scenarios/<id> ./hue-agent.ts --content --baseline <run id>
+hue eval --set "Billing regressions" --scorer-version <id> ./hue-agent.ts --content --save-version
 hue eval --case "Refund an eligible charge" --command "python agent.py" --timeout 120 --content
-hue eval --worker ./hue-agent.ts --agent-key support-agent --env-file .env.hue
+hue eval --worker ./hue-agent.ts --agent-key support-agent --content --env-file .env.hue
 ```
 
 `--scenario` remains an alias for `--case` for existing scripts. Pass one selection flag.
 
-The one-shot mode resolves the selection (`--case` by name, ID or URL; `--set` by name, ID or
-URL with explicit `--scorer-version` pins; or `--dataset-version` with `--scorer-version`),
+The one-shot mode resolves the selection (`--case` by name, or by the ID or URL of the published
+case or of the case conversion that published it, among the project's first 1,000 Scenarios;
+`--set` by name, ID or URL with explicit `--scorer-version` pins; or `--dataset-version` with `--scorer-version`),
 creates a fresh run from those immutable pins named `<agent key> @ <revision>` (a commit hash is
 shortened to 7 characters)
 (`--name` overrides), prints `Run: <url>` and `Experiment: <id>` as soon as the experiment exists,
 one line per case event (world created, agent started, world sealed), then
 `Waiting for Hue checks...` and a table with one row per case: boolean metrics as `PASS`/`FAIL`,
 numbers as values, text and category metrics as-is, an overall result per case and a pass count.
-Failing cases print the scorer explanation. `--baseline <experiment id|url>` adds improvement,
+Failing cases print the scorer explanation. An evaluator Hue records as not applicable to a case
+(its outcome scoring has no outcome criteria or conversion rubric to grade there) shows `n/a` in
+that case's columns and neither passes nor fails it, so in a mixed eval set a case is decided by
+the evaluators that apply to it; the pass line counts the not-applicable results. A case that no
+pinned evaluator applies to is an error that names what they need. `--baseline <experiment id|url>` adds improvement,
 regression and unchanged counts with per-case deltas. `--json` prints one JSON document
 (`experimentId`, `runId`, `runUrl`, `complete`, `cases`, `totals`, optional `baseline`) on stdout
-and sends progress to stderr. `--wait <seconds>` (default 300) bounds the verdict wait because
+and sends progress to stderr; each case lists its not-applicable evaluator versions in
+`notApplicable`, and `totals.notApplicable` counts them; a case failed for its telemetry has `state: "error"`,
+`passed: false` and
+`telemetry: { code: "telemetry_not_accepted", issues: [{ signal, kind, status?, count }] }`. `--wait <seconds>` (default 300) bounds the verdict wait because
 Hue-owned `world_outcome` checks are graded after the world seals. An experiment always covers
 every case of the saved version; there is no case subset.
 
@@ -399,35 +410,83 @@ every case of the saved version; there is no case subset.
 `HUE_EXECUTION_ID`, `HUE_ENVIRONMENT_RUN_ID`, `HUE_CASE_ID` and `HUE_CASE_KEY`, and
 `{"inputs": ..., "config": ...}` on stdin. The child does not receive `HUE_API_KEY`, `HUE_MCP_KEY`
 or any other Hue control-plane credential unless `--allow-hue-credentials` is passed; the rest of
-the parent environment (model keys, application settings) is inherited. Its stdout is the answer
+the parent environment (model keys, application settings) is inherited. `HUE_CASE_DIR`,
+`HUE_CASE_INPUTS` and `HUE_CASE_OUTPUT_DIR` name a private case directory with the layout direct
+cases use: the case inputs, verified copies of the case's agent-visible pinned files under
+`files/<role>/`, and `output/`, whose files are uploaded after the world seals (the command still
+starts in the current directory). The case directory never holds the world token and is removed
+after the case. Its stdout is the answer
 (JSON when it parses, otherwise trimmed text; empty means no output); a non-zero exit or the
-per-case `--timeout` (default 600 seconds) is a target failure. The world token is never logged.
+per-case `--timeout` (default 600 seconds) is a target failure. A timed-out or interrupted command
+is stopped as a whole process group on macOS and Linux: SIGTERM, then SIGKILL for anything still
+running 5 seconds later, including an agent a compound command started after its shell exited;
+the case settles only once nothing in the group is left. A command that exits normally but leaves
+processes in its group has them stopped the same way before its answer and files are read. A
+second Ctrl+C during that grace kills the group at once, removes the world case's files, the MCP
+configuration file and the checkpoint locks, and exits with 130. The world token is never logged.
 A world created while the deployment's simulation gateway is off gets the legacy `hue_sim_`
 capability under the same `HUE_MCP_*` names. The agent key defaults to the slug of the command's
 script name; `--revision` is sent to Hue as the agent revision of every world.
 
 `--worker` registers the adapter through `runLocalAgent()` with key `--agent-key` (default: the
 adapter filename slug), name `--agent-name` (default: the key), revision `--revision` (default:
-`AGENT_REVISION`, then the Git `HEAD` short hash, then `dev`) and capability `environment:v1`,
-prints the registration and each claimed run with its URL, executes runs launched from Hue until
-Ctrl+C or `--max-runs <n>`, and prints the verdict table after each run. Selection flags do not
-apply; Hue chooses the pinned experiment. The worker exits 0 when it stops normally.
+`AGENT_REVISION`, then the Git `HEAD` short hash, then `dev`) and capability `environment:v1`
+plus any repeated `--capability <value>`, prints the registration and each claimed run with its
+URL, executes runs launched from Hue until Ctrl+C or `--max-runs <n>`, and prints the verdict table
+after each run. Selection flags do not apply; Hue chooses the pinned experiment. The worker exits 0
+when it stops normally. Hue offers a case whose world comes with agent-visible files only to a
+registration that declares `environment-files:v1` and `input:<extension>` for each file type, for
+example `--capability environment-files:v1 --capability input:pdf`; the adapter or command then
+receives the files as described above. Hue keeps each registered revision's capabilities fixed,
+so declare new ones under a new `--revision`. A world pinned to provider profiles, a Gmail world
+for example, needs a reviewed `attemptBaselineV2` (see
+[Pinned provider-profile preflight](ENVIRONMENTS.md#pinned-provider-profile-preflight)); `--worker`
+registers none, so put `"attemptBaselineV2"` in the run's configuration JSON when you launch the
+run from Hue.
 
 A Scenario or eval set whose dataset version is not saved cannot back an experiment: the command
 exits 1 and asks for **Save eval-set version** in Hue or `--save-version`, which freezes that
 version at its current revision. Connection settings are `HUE_API_KEY` and `HUE_BASE_URL`
 (default `https://app.hue.run`), loaded from `--env-file <path>` (or its alias `--env-path`) first when given; `--origin`
-overrides the origin. Telemetry content capture stays off unless `--content` is passed; in
-one-shot mode it also decides whether case outputs, error messages and explanations are persisted
-to Hue. `--worker` always persists them, because a run launched from Hue is read on its run page:
-that is `runLocalAgent()`'s contract and `--content` does not change it. Trace evidence is required
-for every case.
+overrides the origin. Telemetry content capture stays off unless `--content` is passed; the
+examples pass it so the run's case spans carry content. Model and tool spans inside the agent come
+only from the agent's own instrumentation. Case outputs, error messages and explanations are
+stored in Hue whether or not `--content` is passed, so a case's answer can be graded and read on
+its run page. A `--command`'s stdout is its answer unless it writes a result file, and is stored,
+so it must not print credentials or debug logs there; before storing, `hue eval` replaces the
+credentials it handed the case (the world token and MCP headers, a legacy MCP token, attempt
+bearers) and every Hue control-plane credential in its environment with `[redacted]`, in the
+answer, an adapter's thrown message and name, generated file names and the local checkpoint.
+`--no-output` keeps outputs, error messages and explanations out of a one-shot run's stored
+results; with `--content` the case span still carries the output. `--worker` always stores them, because a run launched from Hue is read
+on its run page (that is `runLocalAgent()`'s contract), and refuses `--no-output`. An interrupted
+one-shot run keeps the choice it started with, so one run never mixes stored and unstored outputs:
+rerunning it with other `--no-output` or `--content` flags is refused with the flags it started
+with. Resume a run that an earlier SDK started without `--content` by passing `--no-output`.
+
+Files the agent writes to `output/` are uploaded as generated documents with or without
+`--no-output`, and they are not redacted the way the answer is: `hue eval` replaces the same
+credentials only in the UTF-8 `.txt`, `.csv` and `.json` documents it collects, while PDF, Office
+and image documents, and files an adapter returns by `path`, are uploaded as written. A direct
+case also keeps the agent's raw `output/` files on disk under its checkpoint directory
+(`.hue/eval/<agent-key>/<project>/direct/<experiment>/files/`). Never write credentials to
+`output/`.
+
+Trace evidence is required for every case: when Hue does not accept a case's traces or logs, the
+case is completed as failed (error `TelemetryNotAccepted`, evidence omitted as
+`telemetry_not_accepted`, no output or generated files attached) instead of being left started,
+and the run goes on. Stderr names the case as it completes, with the export
+issue counts, for example
+`[refund] telemetry not accepted, case failed: telemetry_not_accepted: traces failed 1 (HTTP 400)`;
+the case counts as an error in the table and JSON whatever its scores, and the command exits 1.
+Counts carry signals, kinds, HTTP statuses and record numbers only, never content or credentials.
 Resumable checkpoints live in `.hue/eval/<agent-key>/<project id>/` (a `.gitignore` is written
 inside `.hue/eval/`); `--checkpoint-dir` overrides the root. Rerunning the same selection resumes
 an interrupted run without invoking the agent again; a different selection is refused until the
 unfinished one is resumed or its directory is removed.
 
-Exit codes: `0` every case passed, `1` a case failed, errored, was skipped or Hue's checks were
+Exit codes: `0` every case passed the evaluators that apply to it, `1` a case failed, errored
+(including one no pinned evaluator applies to), was skipped or Hue's checks were
 still pending at `--wait`, `2` usage or configuration error (including a missing key), `130`
 interrupted. On Node.js 22, load TypeScript adapters with `NODE_OPTIONS=--experimental-strip-types`;
 non-erasable syntax (enums, parameter properties, namespaces) needs a loader such as `--import tsx`
@@ -445,8 +504,8 @@ published version, beside or instead of explicit `--scorer-version` IDs.
 ```sh
 hue eval --set gia-d1-citation --scorer gia-d1-citation \
   --command "pnpm --filter @august/frontend run hue:gia-agent" \
-  --revision prompt-v10 --wait 1800 --json --env-file .env.hue
-hue eval --set gia-d1-citation --set-version 1 --scorer gia-d1-citation ./hue-agent.ts --baseline <experiment id>
+  --revision prompt-v10 --wait 1800 --content --json --env-file .env.hue
+hue eval --set gia-d1-citation --set-version 1 --scorer gia-d1-citation ./hue-agent.ts --content --baseline <experiment id>
 ```
 
 For each case the command is spawned once **inside a private case directory** with
@@ -469,6 +528,17 @@ the command's stdout is the output (JSON when it parses). A single generated fil
 document by default. Model and provider credentials stay in the command's own environment; the
 scoped case files are copies under `--checkpoint-dir` (default `.hue/eval/<agent-key>/<project>/direct/<experiment>`).
 
+The same collection serves world cases, and it does not trust paths the agent controls. `output/`
+must be a real directory, and a helper name, when present, a regular file; a symlink, FIFO or
+directory in either place is the case's error. Symlinks and other special files among the
+documents are skipped, as are hidden and lock entries. Every file is opened without following a
+final symlink or blocking, must be the regular file the listing saw and within its limit (4 MiB
+for a helper, 25 MiB for a document, checked before reading), and its upload is staged from the
+bytes read. A file or directory that vanished or changed while it was collected, more than 32
+documents or more than 1024 entries is the case's error too. A hard link is an ordinary file and
+is collected like one: an agent running as your user could copy the file just as well, so run an
+agent that must not reach your files under another account or in a sandbox.
+
 An adapter file works too: it is called with `(inputs, context)` where `context.mode` is
 `"direct"` and `context` carries `config`, `item`, `executionId`, `files` (agent-visible pinned
 files on disk), `outputDirectory` and `signal`. Return a JSON value, or `withFiles(output, files)`
@@ -478,5 +548,6 @@ Code evaluators pinned to the run are **not** executed on your machine: the CLI
 deferred (`deferUnboundLocalScorers`) and Hue's grading executor scores the uploaded documents;
 evaluator-only pinned files such as a legal corpus are never downloaded. `Waiting for Hue
 checks...` then covers that grading, so size `--wait` to the evaluator's runtime. The
-`--json` document gains `"mode": "direct"` and `"deferredScorerVersionIds"`. Exit codes, `--baseline`
-and checkpoints behave as for Scenarios.
+`--json` document gains `"mode": "direct"` and `"deferredScorerVersionIds"`. Exit codes, `--baseline`,
+`--content`, `--no-output` and checkpoints behave as for Scenarios; generated documents are
+uploaded with or without them.

package/ENVIRONMENTS.md

@@ -62,7 +62,7 @@ to the callback and is never written to checkpoints.
 
 ### Published pins and verdicts
 
-`scenario: { kind: "pins", datasetVersionId, scorerVersionIds, config?, name? }` runs already
+`definition: { kind: "pins", datasetVersionId, scorerVersionIds, config?, name? }` runs already
 published immutable pins, such as a Scenario's frozen case and Hue-owned outcome checks or a saved
 eval set with explicitly chosen scorer versions. `resolveScenarioPins(client, selector)` reads
 those pins from a Scenario ID, its Hue URL or its name (`listScenarios` and `getScenario` expose
@@ -76,9 +76,11 @@ bound `local_code` pins.
 Hue grades `world_outcome` pins after the world seals, so the runner's report precedes the
 verdicts. `waitForResults(client, { runId, scorerVersionIds, subjectIds, timeoutMillis })` polls
 until every item has a terminal result for every pin or the budget elapses (`complete: false`);
-`summarizeVerdicts` turns results into per-case rows with `passed` and totals; `compareVerdicts`
-diffs two summaries by case key; and `collectExperimentVerdicts` combines those reads for one
-experiment. The `hue eval` command uses the same path; see
+`summarizeVerdicts` turns results into per-case rows with `passed` and totals; a result Hue marks
+`notApplicable` (the evaluator has no outcome criteria or conversion rubric to grade in that case)
+neither passes nor fails its case and is listed in the row's `notApplicable`, and a case no pinned
+evaluator applies to is an error. `compareVerdicts` diffs two summaries by case key, and
+`collectExperimentVerdicts` combines those reads for one experiment. The `hue eval` command uses the same path; see
 [Evaluate an agent against a case](CLI.md#evaluate-an-agent-against-a-case).
 
 ### Pinned provider-profile preflight
@@ -223,6 +225,8 @@ its own Gmail MCP or REST client, the mirror URL, the world token where the Goog
 import {
   agentEnvironment,
   createEnvironmentClient,
+  HueEnvironmentError,
+  isTransientEnvironmentError,
   worldHandoff,
   writeMcpConfig,
 } from "@hue-run/sdk/environment";
@@ -252,12 +256,22 @@ try {
   );
   const sealDeadline = Date.now() + 30_000;
   let sealed = false;
-  while (Date.now() < sealDeadline) {
-    if ((await environmentClient.getRun(run.id)).status !== "open") {
-      sealed = true;
-      break;
+  while (!sealed && Date.now() < sealDeadline) {
+    let wait = 250;
+    try {
+      // The deadline also ends a read that hangs, with the client's retries inside it.
+      const state = await environmentClient.getRun(run.id, {
+        signal: AbortSignal.timeout(Math.max(0, sealDeadline - Date.now())),
+      });
+      sealed = state.status !== "open";
+    } catch (error) {
+      if (!isTransientEnvironmentError(error)) throw error; // a refusal such as 404 is final
+      wait = Math.max(wait, (error as HueEnvironmentError).retryAfterMs ?? 0);
     }
-    await new Promise((resolve) => setTimeout(resolve, 250));
+    if (!sealed)
+      await new Promise((resolve) =>
+        setTimeout(resolve, Math.min(wait, Math.max(0, sealDeadline - Date.now()))),
+      );
   }
   if (!sealed) throw new Error("World was not sealed after its completion grace");
 }
@@ -272,6 +286,27 @@ world token, so an adapter that read `HUE_MCP_URL` and `HUE_MCP_TOKEN` keeps wor
 no Hue-native `tools` (Hue refuses them); a world created while the gateway is off keeps its tools
 and the `hue_sim_` capability and emits a one-time `DeprecationWarning`.
 
+A case pinned to a world may also pin input files. Before the case's execution starts, the
+helpers download each agent-visible file (`source`, `attached_template`, `attached_reference`,
+`original`), verify its size and SHA-256 against the case manifest and hand the verified copies to
+the callback as `context.files` beside `context.world`, in a private directory (mode 0700, files
+0600), with a private `context.outputDirectory`. Evaluator-only files (`org_template`,
+`evaluator_reference`) never reach the agent; a local code evaluator's copies are checked before
+the execution starts but saved only after the agent finished. Bytes that differ from the manifest raise
+`CaseFileError` with code `case_file_mismatch`; a name that is not one safe file name (a path
+separator, `.` or `..`, a C0 or C1 control character, a character Windows reserves such as `:` or
+`?`, a Windows device name such as `CON`, a trailing dot or space, more than 200 bytes) raises
+`case_file_name_refused`. Either stops the run before an
+execution or world exists for the case. Return `withFiles(output, files)` to upload what the agent
+produced: the files are published and linked to the execution as `artifactIds` and
+`primaryArtifactId`, within the limits of direct cases (at most 32 files, 25 MiB each, the
+accepted document types). The world token is never written into the case directory, which is
+removed when the case ends; only staged outputs an interrupted upload resumes from are kept until
+it does. `hue eval --command` receives the same files through
+`HUE_CASE_DIR`, `HUE_CASE_INPUTS` and `HUE_CASE_OUTPUT_DIR`. A `runLocalAgent` worker is offered
+such cases only when it declares `environment-files:v1` and `input:<extension>` for each file
+type; see [EVALUATIONS.md](EVALUATIONS.md#direct-cases-and-files).
+
 A refusal such as 409 `simulation_gateway_required` exposes its validated server code as `HueEnvironmentError.diagnostic`.
 
 `agentEnvironment` removes Hue control-plane credentials from the child by default: `HUE_API_KEY`,

package/EVALUATIONS.md

@@ -18,7 +18,8 @@ import { builtins, createEvaluationClient, runExperiment, rescore } from "@hue-r
 
 const connection = { apiKey: process.env.HUE_API_KEY! };
 const client = createEvaluationClient(connection);
-const hue = createHue({ ...connection, serviceName: "evaluation-demo", captureContent: false });
+// captureContent: false sends metadata only.
+const hue = createHue({ ...connection, serviceName: "evaluation-demo", captureContent: true });
 
 // Registry writes use optimistic revisions; they do not have automatic retries.
 const dataset = await client.createDataset({ name: "Greetings", slug: "greetings" });
@@ -115,7 +116,7 @@ Both choices are required and independent:
 
 Frozen dataset inputs/references already exist on Hue. Their presence is independent of these switches. No output is inferred from telemetry. JavaScript `undefined` means unavailable output; JSON `null`, false, zero and empty string remain present values. Historical subjects with unavailable output produce skipped results without executing a scorer callback. A failed quality metric remains `state:"scored"` with `passed:false`. Target errors and scorer errors remain separate. Target error types are generalized to `TargetError`; when content is enabled, bounded messages may be stored. Never put credentials in error messages, output, custom metric values, names or metadata.
 
-`traceEvidence:{mode:"required"}` waits for trace and log export acknowledgement after the root span ends. The explicit alternative `{mode:"omit",reason:"..."}` stores the omission reason and declared trace ID without a fake snapshot. It can complete despite an export failure; export diagnostics remain available on the Hue client. There is no automatic fallback to omission.
+`traceEvidence:{mode:"required"}` waits for trace and log export acknowledgement after the root span ends. The explicit alternative `{mode:"omit",reason:"..."}` stores the omission reason and declared trace ID without a fake snapshot. It can complete despite an export failure; export diagnostics remain available on the Hue client. There is no automatic fallback to omission. When required telemetry is not accepted, the runner by default keeps the saved outcome, leaves the execution started and rejects (`traceNotAccepted: "stop"`). With `traceNotAccepted: "fail_case"` it instead completes that case as failed (a `TelemetryNotAccepted` error, the evidence omitted with a reason starting `telemetry_not_accepted`, and no output or generated files attached, so no scorer can pass it), calls `onTelemetryNotAccepted(entry)` once, as soon as it completes the case (a resumed call only lists it), lists it in `report.telemetryNotAccepted` with sanitized issue counts, and goes on; `hue eval` uses this. A scorer that grades only the world can still score such a case, so treat a listed case as failed whatever its results say, as `hue eval` does.
 
 ## Local scorers
 
@@ -192,7 +193,7 @@ The runner stops scheduling more cases after an operational failure and waits fo
 
 ## Verification boundaries
 
-`scripts/verify-package.mjs` installs a real packed tarball outside the monorepo and runs HTTP contract tests against a synthetic service plus actual OpenTelemetry exporters. It checks two configurations, rescoring without target invocation, absent/null output, upload resume, uncertain execution, exclusive checkpoints, source/metric contracts, content policy and terminating schema workers. It also exercises the local worker's ready, incomplete and uncertain provider-attempt control-plane paths, but does not call an issued provider facade. `scripts/verify-file-cases.mjs` runs against the same installed consumer: pinned input download and verification, artifact publication of a generated document, file-aware local scoring during the run and again on a later rescore, and a declared file the target could not deliver recorded as the target's own error. `scripts/verify-evaluation-api.mjs` is a separate opt-in acceptance against a real Hue receiver/API; it creates synthetic datasets/scorers/experiments in the project associated with the supplied development key.
+`scripts/verify-package.mjs` installs a real packed tarball outside the monorepo and runs HTTP contract tests against a synthetic service plus actual OpenTelemetry exporters. It checks two configurations, rescoring without target invocation, absent/null output, upload resume, uncertain execution, exclusive checkpoints, source/metric contracts, content policy and terminating schema workers. It also exercises the local worker's ready, incomplete and uncertain provider-attempt control-plane paths, but does not call an issued provider facade. `scripts/verify-file-cases.mjs` runs against the same installed consumer: pinned input download and verification, artifact publication of a generated document, file-aware local scoring during the run and again on a later rescore, a declared file the target could not deliver recorded as the target's own error, and a worker world case that receives its agent-visible files with the world, uploads the returned document and refuses altered bytes as `case_file_mismatch`. `scripts/verify-evaluation-api.mjs` is a separate opt-in acceptance against a real Hue receiver/API; it creates synthetic datasets/scorers/experiments in the project associated with the supplied development key.
 
 ## Outbound local agent worker
 
@@ -215,7 +216,8 @@ import { createEvaluationClient, runLocalAgent } from "@hue-run/sdk/evals";
 import { runMyAgent } from "./agent.js"; // Your existing application entry point.
 
 const connection = { apiKey: process.env.HUE_API_KEY! };
-const hue = createHue({ ...connection, serviceName: "local-worker", captureContent: false });
+// captureContent: false sends metadata only.
+const hue = createHue({ ...connection, serviceName: "local-worker", captureContent: true });
 try {
   await runLocalAgent({
     client: createEvaluationClient(connection),
@@ -233,7 +235,9 @@ try {
 
 The callback receives cloned inputs, local tools, and an allowlisted context containing
 `config`, `item: {id, externalKey}`, `executionId`, `environmentRunId`,
-`trace: {traceId,spanId}` and a short-lived `mcp` capability. Expected outcomes, case metadata
+`trace: {traceId,spanId}`, a short-lived `mcp` capability, `files` (verified copies of the case's
+agent-visible input files, empty when it has none) and a private `outputDirectory`; return
+`withFiles(output, files)` to upload generated files, as for direct cases. Expected outcomes, case metadata
 and original source pins remain private to grading. Pass the tools or scoped MCP capability into
 the agent's actual tool boundary; their presence does not redirect provider calls. Capabilities
 are not written to checkpoints. `maxRuns` limits completed runs for one-shot workers, while
@@ -260,8 +264,8 @@ installed-registry-package to hosted-facade acceptance remains a post-publicatio
 ## Direct cases and files
 
 Cases without a simulated world — document workflows whose inputs are a task plus pinned files —
-run on the same runner under the same checkpoint rules. This is unreleased and ships in the next
-`@hue-run/sdk` release. It requires a Hue deployment that serves case `inputFiles` on experiment
+run on the same runner under the same checkpoint rules. They shipped in `@hue-run/sdk` `0.5.0`
+and require a Hue deployment that serves case `inputFiles` on experiment
 items, subject `files`, and the artifact reservation, upload, completion and download APIs. The
 Python SDK has no equivalent.
 
@@ -270,7 +274,14 @@ On the outbound worker, supply `directTarget` beside or instead of `target`. Sup
 `environment:v1` (both strings are exported as `localAgentCapabilities`). Declare the file
 capabilities the agent accepts and returns — `input:docx`, `input:pdf`, `output:docx` and so on — so
 Hue matches them against each case's `hue.requiredCapabilities` metadata and offers only matching
-cases. `registeredCapabilities` refuses a registration that names a capability without its callback.
+cases. A case pinned to a world whose manifest also holds agent-visible files is offered only to a
+registration that declares `environment-files:v1` (`localAgentCapabilities.environmentFiles`)
+and `input:<extension>` for each file; `target` then receives them as `context.files` with the
+world, as described in
+[ENVIRONMENTS.md](ENVIRONMENTS.md#worlds-served-by-the-simulation-gateway). The worker never adds
+`environment-files:v1` itself: Hue keeps each registered revision's capabilities fixed, so declare
+it under a new revision. `registeredCapabilities` refuses a registration that names a capability
+without its callback.
 
 ```ts
 import { basename } from "node:path";
@@ -312,16 +323,23 @@ The direct callback receives cloned inputs and an allowlisted context of `config
 plus the worker's `signal` when one was supplied. Expected outcomes and case metadata stay with
 grading, as they do for world cases.
 
-Before an execution exists, `runExperiment` downloads every pinned input file named by the frozen
+Before an execution exists, `runExperiment` downloads the pinned input files named by the frozen
 case's `inputFiles` and verifies byte count and SHA-256. A download failure is an SDK failure and
-consumes no execution slot; a file already saved with the pinned identity is reused instead of
-downloaded again. The target sees only the agent-visible roles — `source`, `attached_template`,
-`attached_reference` and `original` — as `context.files`; evaluator-only
+consumes no execution slot; bytes that differ from the manifest raise `CaseFileError` with the
+stable code `case_file_mismatch`, and a download stops one byte past the pinned size. A file
+already saved with the pinned identity is reused instead of downloaded again. The target sees only the agent-visible roles — `source`,
+`attached_template`, `attached_reference` and `original` — as `context.files`; evaluator-only
 `org_template` and `evaluator_reference` files (an organization's template, a legal corpus, an
-answer key) reach scorers but not the agent, and are not even downloaded when no bound code
-evaluator runs in this process. Verified copies live under `filesDirectory` (default
-`<checkpointDirectory>/files`, created mode 0700), and each case gets its own private
-`context.outputDirectory` to write into.
+answer key) reach scorers but not the agent. They are fetched only when a bound code evaluator
+runs in this process: checked with the others before the execution starts but not kept, then
+downloaded again and saved after the target finished, so they are not on disk while the agent
+runs. An agent running as the same operating-system user
+as the runner can still read whatever the runner can; run it under another account when local
+evaluator material must stay out of its reach. Verified copies live under `filesDirectory` (default
+`<checkpointDirectory>/files`, created mode 0700), evaluator-only files apart from the agent's, and
+each case gets its own private `context.outputDirectory` to write into. The files of a case pinned
+to a world are removed when it ends, except staged outputs an interrupted upload resumes from;
+those of a direct case stay for inspection.
 
 Return generated documents by wrapping the output in `withFiles(output, files)`. Each entry names a
 `path` or in-memory `bytes`, a `filename`, a Hue-accepted `contentType` and at most one `primary`.
@@ -329,9 +347,11 @@ The runner copies them next to its checkpoint, publishes them through the artifa
 upload and verified completion APIs with stable per-execution keys, and completes the execution with
 `artifactIds` and `primaryArtifactId`, so the subject's frozen manifest holds inputs and outputs
 together. Generated files are always uploaded regardless of `persistResultContent`: they are the
-execution's evidence. A declared file that cannot be read, exceeds 25 MiB, repeats a filename or has
-an unsupported content type is saved as the target's error (`TargetError`), not as an uncertain
-execution. A crash after the target finished resumes from the staged files without invoking the
+execution's evidence. A declared `path` is read once, as a regular file (a symlink, FIFO or device
+is refused) whose size is checked before reading, and staged owner-only from those bytes. A
+declared file that cannot be read, exceeds 25 MiB, repeats a filename or has an unsupported
+content type is saved as the target's error (`TargetError`), not as an uncertain execution. A
+files directory must be owned by the current user and closed to everyone else (mode 0700). A crash after the target finished resumes from the staged files without invoking the
 target again; if result content is not persisted, the JSON output cannot be reconstructed and the
 case is reported as uncertain.
 

package/README.md

@@ -35,7 +35,7 @@ import { createHue, HueExportError } from "@hue-run/sdk";
 const hue = createHue({
   apiKey: process.env.HUE_API_KEY!, // a project service key, on the server only
   serviceName: "my-agent",
-  captureContent: false, // required: explicitly choose true or false
+  captureContent: true, // required: true records content; false sends metadata only
   onExportIssue: (issue) => console.error(issue), // sanitized counts, never server bodies
 });
 
@@ -58,6 +58,11 @@ try {
 await hue.shutdown(); // flushes and releases providers owned by this client
 ```
 
+`captureContent` is required. The example sets `true`, the recommended choice: trace inspection,
+evaluations and judges read the prompts, responses and tool inputs/outputs it records. Choose
+`false` for metadata only when a policy forbids sending that content; see
+[Privacy and content](#privacy-and-content).
+
 The default destination is `https://app.hue.run`. Data goes to
 `/api/v1/otlp/v1/traces` and `/api/v1/otlp/v1/logs` with a Bearer project key.
 Set `baseUrl` only for another Hue deployment. It must be an origin without an API path; a trailing slash is accepted.
@@ -287,6 +292,12 @@ outputs and exports through Hue's attached processors keeps the label.
 
 ## Privacy and content
 
+Set `captureContent: true` for full traces: prompts and messages, responses and tool inputs/outputs,
+alongside model, usage, timing and errors. Trace inspection, evaluations and judges read that
+content. Choose `false`, metadata-only mode, when your users decline or an application or data
+policy forbids sending that content to another service. `redact` and the credential filtering
+described below apply in both modes.
+
 `captureContent: false` disables manual input/output/messages/tool content and
 removes recognized GenAI, Vercel, OpenInference and OpenLLMetry content attributes,
 legacy GenAI content events, log bodies, status messages and exception text before
@@ -306,12 +317,18 @@ an OpenAI hosted MCP tool. Before export, and before `redact`, Hue replaces the
 credential-like fields including `authorization`, `authorization_token`, `headers`, `api_key`,
 `access_token`, `x-api-key`, and keys ending in `token`, `secret`, `password`, `apikey` or
 `credential` (case-insensitively, ignoring `-` and `_`) with `"[redacted]"` in recorded tool definitions
-(`gen_ai.tool.definitions`, `ai.prompt.tools`, `llm.tools.*.tool.json_schema`) and in the `tools`
-and `mcp_servers` entries of a raw provider request or response recorded as `input.value`,
-`output.value` or `llm.invocation_parameters`. Parameters named in a JSON Schema `properties`
+(`gen_ai.tool.definitions`, `ai.prompt.tools`, `llm.tools.*.tool.json_schema`) and in the
+top-level `tools` and `mcp_servers` entries of any JSON object recorded as `input.value`,
+`output.value` or `llm.invocation_parameters`, whether or not it is a provider request; nothing
+else in that object changes. Parameters named in a JSON Schema `properties`
 object keep their schemas, so a tool that takes a `headers` argument is still described. A
 definition nested more than 256 levels deep rejects its record. Sensitive `default`, `const`,
-`examples` and `enum` values under credential-named schema parameters are redacted too.
+`examples` and `enum` values under credential-named schema parameters are redacted too. A `url`
+or `server_url` field (named like the credential fields, so `serverUrl` too) that has userinfo, a
+query or a fragment loses its userinfo and fragment, and each query value becomes `[redacted]`
+under its original name; the result is serialized as WHATWG `URL` does (lowercase scheme and
+host, IDN hosts in Punycode, default ports dropped, dot segments resolved). A URL with none of
+them is exported as recorded, and a value `URL` cannot parse becomes `"[redacted]"`.
 
 With `captureContent: false`, export removes tool definitions but keeps a summary of them on the
 same record: `hue.tool.names` lists each definition's `name` (Chat Completions `function.name`,
@@ -324,12 +341,18 @@ rotates. Only definitions another integration recorded can be summarized: `hueTe
 inputs and exports through Hue's attached processors gets the summary.
 Recorded messages can inline files: GenAI `blob` parts in `gen_ai.input.messages` /
 `gen_ai.output.messages` (what the AI SDK 7 adapter records for a file part) and AI SDK 6 `file`
-parts in `ai.prompt.messages`. A span whose messages exceed 256 KiB would be rejected, so before
-export Hue replaces the `content`/`data` of any such part longer than 64 KiB with the file's
-`sha256` (of the decoded bytes for base64 and `data:` URLs, of the UTF-8 text otherwise) and
-`size`, keeping the part's other fields such as `type`, `mime_type` and `mediaType`. Smaller inline
+parts in `ai.prompt.messages`. A span whose messages exceed 256 KiB would be rejected, so when a
+record is queued Hue replaces the `content`/`data` of any such part longer than 64 KiB with the
+file's `sha256` and `size`, both of the file's own bytes whatever its media type (base64 content
+and `;base64` `data:` URLs decoded, other `data:` URLs percent-decoded, anything else as UTF-8 text), keeping the part's other fields such as `type`, `mime_type` and `mediaType`. Smaller inline
 files are exported as recorded. The digest matches `hue.recordFile`'s `hue.file.sha256` for the same
-bytes, so a file can be recognized wherever it appears.
+bytes, so a file can be recognized wherever it appears. The replacement happens before the record is
+charged to the queue budget, so a large file does not drop its span. It is bounded: a message
+attribute longer than 8 MiB of text is left unchanged, and the default 8 MiB `maxQueueBytes` then
+drops the span. Base64 grows a file by a third, so that ceiling is an inline file of about 6 MiB.
+The same message text is hashed once per record, and a record inspects at most 16 MiB of message
+text in all; messages past that are charged as recorded. With `captureContent: false` the messages,
+which export removes, are neither hashed nor charged.
 
 Manual helpers encode JSON values without converting null into absence. Unknown
 outputs and usage remain absent. This SDK does not estimate tokens or cost. A thrown
@@ -361,7 +384,7 @@ import { createHue, createHueTransport } from "@hue-run/sdk";
 const transport = createHueTransport({
   apiKey: process.env.HUE_API_KEY!,
   serviceName: "existing-app",
-  captureContent: false,
+  captureContent: true, // false sends metadata only
 });
 const tracerProvider = new TracerProvider({ spanProcessors: [transport.spanProcessor] });
 const loggerProvider = new LoggerProvider({ processors: [transport.logRecordProcessor] });
@@ -576,9 +599,9 @@ an uncertain preparation. See the
 an issued facade endpoint or the official Gmail service, and do not claim universal provider
 parity.
 
-An unreleased `directTarget` callback extends the same worker to cases without a world: the runner
-verifies the case's pinned input files, hands them to the agent, and uploads the documents it
-returns as verified Hue artifacts for scoring. See
+The `directTarget` callback, shipped in TypeScript `0.5.0`, extends the same worker to cases
+without a world: the runner verifies the case's pinned input files, hands them to the agent, and
+uploads the documents it returns as verified Hue artifacts for scoring. See
 [direct cases and files](EVALUATIONS.md#direct-cases-and-files).
 
 Scorer deferral shipped in TypeScript `0.3.1`. Only built-ins
@@ -591,17 +614,23 @@ their agents with `runLocalAgent()`; setup does not register workers or launch s
 
 ### Command-line evaluation
 
-The unreleased `hue eval` command wraps `runSimulation()` and `runLocalAgent()` for an adapter
-file or a shell command: `hue eval --case "<name>" ./hue-agent.ts` creates a fresh run
-from a published case's immutable pins, runs the agent in one isolated world per case, waits
+The `hue eval` command, shipped in TypeScript `0.5.0` (`--case` since `0.6.0`, `--scenario`
+before), wraps `runSimulation()` and `runLocalAgent()` for an adapter file or a shell command:
+`hue eval --case "<name>" ./hue-agent.ts --content` creates a fresh run from a published case's
+immutable pins, runs the agent in one isolated world per case, waits
 for Hue's outcome checks and prints the run URL and per-case PASS/FAIL verdicts with an exit code;
 `--worker` registers the same adapter for runs launched from Hue. It needs a Read and
-write key in `HUE_API_KEY` (never printed) and keeps content capture off unless `--content`
-is passed. See [Evaluate an agent against a case](CLI.md#evaluate-an-agent-against-a-case).
+write key in `HUE_API_KEY` (never printed). Content capture stays off unless `--content` is
+passed; the example passes it so case spans carry content. Case outputs, error messages and
+explanations are stored in Hue either way (`--no-output` keeps them out of a one-shot run's
+results); a command's stdout is its stored answer, with the credentials `hue eval` handed the case
+replaced by `[redacted]`.
+Instrument the agent itself for its model and tool spans. See
+[Evaluate an agent against a case](CLI.md#evaluate-an-agent-against-a-case).
 Eval sets whose cases pin files instead of a world run as direct cases through `runExperiment()`:
-`hue eval --set <slug> --scorer <slug> --command "…"` hands the agent each case's pinned files in a
-private directory, uploads the documents it writes and waits for Hue's grading executor to score
-them. See [Evaluate a document eval set](CLI.md#evaluate-a-document-eval-set).
+`hue eval --set <slug> --scorer <slug> --command "…" --content` hands the agent each case's
+pinned files in a private directory, uploads the documents it writes and waits for Hue's grading
+executor to score them. See [Evaluate a document eval set](CLI.md#evaluate-a-document-eval-set).
 
 ## Managed targets
 

package/dist/cli/eval-direct.d.ts

@@ -38,10 +38,21 @@ export declare function stageDirectCase(root: string, input: {
     executionId: string;
     files: LocalFile[];
 }): Promise<DirectCaseLayout>;
+/** Test seams: `beforeEntry` runs before each listed entry is examined, `afterListing` between
+ * the listing and the reads. */
+export interface CollectionSeams {
+    beforeEntry?: (name: string) => Promise<void>;
+    afterListing?: () => Promise<void>;
+}
 /**
  * Turns the output folder into the target's result. Every regular file, in subdirectories too,
  * becomes a generated file; an unsupported extension is the agent's error rather than a silently
  * dropped document. `fallbackOutput` (for example the command's stdout) is used when no JSON
  * output or summary file was written.
+ *
+ * Nothing here trusts a path the agent could have changed: the output directory must be a real
+ * directory, every file is opened without following a final symlink or blocking on a FIFO, must
+ * be the regular file the listing saw and within its size limit, and the result carries the bytes
+ * read from that open file. `seams` are for tests only.
  */
-export declare function collectDirectOutputs(outputDirectory: string, fallbackOutput?: JsonValue): Promise<TargetResult>;
+export declare function collectDirectOutputs(outputDirectory: string, fallbackOutput?: JsonValue, seams?: CollectionSeams): Promise<TargetResult>;