@hue-run/sdk 0.10.0 → 0.11.0

package/CLI.md

@@ -42,7 +42,7 @@ project manifest are refused because managers can update ancestor locks; Python
 The generated `hue.setup.mjs` or `hue_setup.py` always selects `captureContent: false` /
 `capture_content=False`. For a supported application, setup installs the dependency and adds the
 managed import and middleware registration to the existing entrypoint; an unreferenced helper is
-not a completed integration. TypeScript uses `@hue-run/sdk@0.10.0`, `@opentelemetry/api@1.9.1` and
+not a completed integration. TypeScript uses `@hue-run/sdk@0.11.0`, `@opentelemetry/api@1.9.1` and
 `@opentelemetry/context-async-hooks@2.11.0`; Python setup uses its separately tested package pin.
 Content capture requires an ordinary account-managed key and a later explicit application decision.
 
@@ -113,6 +113,7 @@ hue setup --format human  # explicit append-only terminal rendering
 hue setup --origin http://127.0.0.1:PORT # isolated loopback tests only
 hue login                 # validate keys created in Hue and store them in ./.env.hue
 hue mcp install --client claude-code # write the Hue MCP configuration for a coding agent
+hue listen --subscription <id> --forward-to http://localhost:3000/slack/events # deliver events to a local bot
 ```
 
 Hosted setup accepts HTTPS origins only. HTTP is accepted solely for `localhost`, `127.0.0.1` and
@@ -176,30 +177,130 @@ warns; `--gitignore` appends the file name to the `.gitignore` next to it. Exit
 
 ## Install the MCP for your coding agent
 
-`hue mcp install --client <name>` writes, runs or prints the configuration for Hue's MCP server.
-Every shape matches the snippet Hue shows in Settings: server name `hue`, the endpoint (default
-`https://mcp.hue.run/mcp`; `--url https://mcp.staging.hue.run/mcp` for staging) and a reference to
-the `HUE_MCP_KEY` environment variable. A key value is never written.
-
-| Client | Result |
-| --- | --- |
-| `claude-code` | Merges `mcpServers.hue` into `./.mcp.json`. `--scope user` runs `claude mcp add --transport http --scope user hue URL --header 'Authorization: Bearer ${HUE_MCP_KEY}'` when `claude` is on `PATH`, otherwise prints it. |
-| `cursor` | Merges `mcpServers.hue` into `./.cursor/mcp.json` with `${env:HUE_MCP_KEY}`. |
-| `codex` | Runs `codex mcp add hue --url URL --bearer-token-env-var HUE_MCP_KEY`, or prints the `[mcp_servers.hue]` TOML block for `~/.codex/config.toml`. |
-| `vscode` | Merges `servers.hue` and the `hue-mcp-key` password input into `./.vscode/mcp.json`. |
-| `windsurf` | Prints the `serverUrl` snippet for `~/.codeium/windsurf/mcp_config.json`; nothing is written to the home directory. |
-| `gemini` | Runs `gemini mcp add --transport http hue URL -H 'Authorization: Bearer $HUE_MCP_KEY'`, or prints it. |
+`hue mcp install --client <name>` writes, runs or prints the configuration for Hue's MCP server,
+in the shapes of the [connection guide](https://docs.hue.run/agents/mcp-server): server name
+`hue` and the endpoint (default `https://mcp.hue.run/mcp`; `--url https://mcp.staging.hue.run/mcp`
+for staging). Hue's Settings page does not show these snippets; this command keeps its own copy.
+
+`--auth` chooses how the client authenticates:
+
+- `key` (the default, except for `conductor`) references the `HUE_MCP_KEY` environment variable,
+  which `hue login` stores in `.env.hue`. A key value is never written.
+- `oauth` configures the URL only. The client opens Hue in a browser, where you sign in and
+  approve the connection; no key or header is involved. A new sign-in connection has **Read and
+  write** access to every active project in the organization you choose, within your role, so an
+  agent first calls `list_projects` and passes the chosen project's id as `project_id`. It is available
+  for `claude-code`, `codex` and `conductor`. Hue does not yet accept Cursor's sign-in callback, so
+  `cursor` and the other clients use a key.
+
+`--read-only` adds `?read_only=true` to a key configuration's URL, so Hue hides and rejects write
+tools whatever the key allows. A project key, with or without it, keeps reaching its one project.
+With `--auth oauth` it is refused: a sign-in connection has **Read and write** access, so use a
+**Read** project key for read-only access.
+
+`--toolsets <names>` lists only the named tools, for clients with a tool-count limit or agents that
+only read production: `observe` (the production reads: projects, traces, spans and their recorded
+content, sessions, trace checks, intents and documentation), `all`, or the groups `project`, `traces`, `evals`, `environments`,
+`intents` and `docs`, comma-separated. A key configuration adds `?toolsets=<names>` to the URL. A
+sign-in configuration keeps the URL bare and sends the `X-Hue-MCP-Toolsets` header instead,
+because toolsets are not part of the connection's identity: Claude Code stores the header, and for
+Codex the command prints the `http_headers` line to add to `~/.codex/config.toml`, since
+`codex mcp add` stores none. `--toolsets` replaces a selection already in `--url`, and a selection in `--url` is kept without
+it; otherwise `cursor` selects `observe`. `--toolsets all` lists every tool. Unknown names are refused rather than passed on, because Hue
+ignores them and would list every tool. `get_project_context` is always listed and reports the
+selection.
+
+| Client | Key (`--auth key`) | Sign-in (`--auth oauth`) |
+| --- | --- | --- |
+| `claude-code` | Merges `mcpServers.hue` into `./.mcp.json`. `--scope user` runs `claude mcp add --transport http --scope user hue URL --header 'Authorization: Bearer ${HUE_MCP_KEY}'` when `claude` is on `PATH`, otherwise prints it. | Merges `mcpServers.hue` with only `type` and `url` into `./.mcp.json`; `--scope user` runs `claude mcp add --transport http --scope user hue URL`. Then run `/mcp` in Claude Code, select `hue` and choose **Authenticate**. |
+| `codex` | Runs `codex mcp add hue --url URL --bearer-token-env-var HUE_MCP_KEY`, or prints the `[mcp_servers.hue]` TOML block for `~/.codex/config.toml`. | Runs `codex mcp add hue --url URL`, which can start sign-in right away; `codex mcp login hue` starts or repeats it. |
+| `conductor` | Runs the Claude Code user-scope and Codex key commands above. | The default: runs `claude mcp add --transport http --scope user hue URL` and `codex mcp add hue --url URL`. Then use `hue`'s authentication action in Conductor's MCP status (the plug icon or `/mcp-status`). |
+| `cursor` | Merges `mcpServers.hue` into `./.cursor/mcp.json` with `${env:HUE_MCP_KEY}`. | Refused. |
+| `vscode` | Merges `servers.hue` and the `hue-mcp-key` password input into `./.vscode/mcp.json`. | Refused. |
+| `windsurf` | Prints the `serverUrl` snippet for `~/.codeium/windsurf/mcp_config.json`; nothing is written to the home directory. | Refused. |
+| `gemini` | Runs `gemini mcp add --scope user --transport http hue URL --header 'Authorization: Bearer ${HUE_MCP_KEY}'`, or prints it. | Refused. |
+
+Conductor has no MCP configuration of its own: its Claude Code and Codex agents read their user
+configuration in every workspace, so `conductor` registers the server at user scope with each CLI
+on `PATH` and prints the command for a CLI that is missing. A failing CLI does not stop the other;
+the command then exits `1`. Hue has not yet verified the complete signed-in Conductor flow (consent
+through a trace read) end to end; if Hue's tools do not load after signing in, run the command
+again with `--auth key`.
 
 JSON files are parsed and merged: other servers, inputs and top-level fields are kept, only the
 `hue` entry is replaced, and invalid JSON (including comments) is refused together with the snippet
 to add by hand. Files are written with mode `0644` through a temporary file and an atomic rename;
-symlinks are refused. `--dry-run` prints the resulting file content or command without writing or
+symlinks are refused. `--dry-run` prints the resulting file content or commands without writing or
 running; `--print` prints only the snippet. Client CLIs run without a shell, so the
-`${HUE_MCP_KEY}` and `$HUE_MCP_KEY` references reach them literally; the printed commands use
-single quotes for the same reason. After installation the command reminds you to export
-`HUE_MCP_KEY` in the shell that starts the client (VS Code prompts for the key instead) and prints
-the verification prompt: `Use the Hue MCP: call get_project_context, then show my 5 most recent
-error traces with links.` Exit codes: `0` done or printed, `1` failed, `2` usage error.
+`${HUE_MCP_KEY}` reference reaches them literally; the printed commands use single quotes for the
+same reason, and quote a URL with a query such as `?read_only=true`, whose `?` is a zsh glob.
+
+After a key installation the command reminds you to export `HUE_MCP_KEY` in the shell that starts
+the client (VS Code prompts for the key instead). A desktop app started from the Dock or a launcher
+does not see that shell's variables, and Conductor's agents read the login-shell environment that
+Conductor captures, so sign-in is the simpler choice there. After a sign-in installation it names
+the client's authentication action. Both end with the prompt that verifies the connection:
+`Use the Hue MCP: call list_projects and confirm which project to inspect. Then call get_project_context and show that project's traces from the last 24 hours that need attention or have errors, with links; if there are none, show its 5 most recent traces. For an organization connection, pass the project's id as project_id on each call after list_projects.` `list_projects` returns a project key's one project too. Exit codes:
+`0` done or printed, `1` failed, `2` usage error.
+
+## Deliver simulated events to a local bot
+
+`hue listen` delivers a simulated world's events to a receiver on your machine, so an event-driven
+agent such as a Slack bot can be tested without a public URL, the way `stripe listen --forward-to`
+delivers webhooks. Hue signs each request as the provider does (Slack Events API requests today);
+the command pulls them, forwards each one unchanged to `--forward-to` and reports the bot's answer
+to Hue, which settles or retries the event on the provider's schedule. It needs a **listen** event
+subscription on the world or on a connection key; the subscription's id and signing secret are shown
+once when it is created. Configure the bot with that signing secret where `SLACK_SIGNING_SECRET`
+went, then start the command: Hue offers the URL verification first, and events follow once the bot
+has answered it. Event subscriptions are not yet available on `https://app.hue.run`; until the
+origin offers them, the command stops with Hue's refusal.
+
+```sh
+hue listen --subscription <id> --forward-to http://localhost:3000/slack/events --env-path .env.world
+```
+The credential is the subscription's own, read from the environment (or `--env-path`, loaded first;
+a variable already set keeps its value): `HUE_WORLD_TOKEN` for a world's subscription or
+`HUE_CONNECTION_KEY` for a connection key's, chosen with `--credential world-token|connection-key`
+when both are set. It is never taken from the command line, where other local processes and shell
+history can read it. A project key is refused: a value equal to `HUE_API_KEY` or `HUE_MCP_KEY` exits
+2 before any request, and Hue answers any credential that is not the subscription's with `401`.
+`--origin` (default `HUE_BASE_URL`, then `https://app.hue.run`) must be HTTPS, or plain HTTP on a
+loopback test server.
+
+Each request reaches the bot with its body bytes and headers unchanged, so the provider's signature
+verifies as it would in production; only `Host`, `Content-Length` and connection headers are the
+local request's own, and Hue's credential is never forwarded. No redirect is followed: a `3xx` is
+reported as the answer. An answer counts only within three seconds, Slack's acknowledgement window;
+a later one is a timeout. A failing answer with `x-slack-no-retry: 1` is reported so Hue stops
+retrying it. Only a successful URL verification answer's body (at most 4 KiB, for its challenge) is
+sent to Hue; any other answer body stays on this machine. Each pull waits for at most 20 seconds and
+leases up to `--max` deliveries (1 to 10, default 10), which are forwarded concurrently; the next
+pull starts once they are acknowledged, and a delivery beyond `--max` is never forwarded. A provider
+retry of an event is a new delivery and is forwarded again with its `X-Slack-Retry-Num` and
+`X-Slack-Retry-Reason`, so the bot deduplicates by `event_id` as it does with Slack. A delivery Hue
+hands out a second time is not sent again; its recorded answer is repeated. An acknowledgement Hue
+did not answer is repeated until the delivery's lease ends, as Hue stated it (30 seconds from the
+lease).
+
+`--forward-to` must name this machine: `localhost`, a `.localhost` name or a loopback address, and a
+name is refused unless every address it resolves to is a loopback address. `--allow-remote-forward`
+permits another host over HTTPS only, because every request carries the subscription's verification
+token and a valid signature; plain HTTP still reaches only loopback addresses. Credentials and
+fragments in the URL are refused. Output is one line per delivery (time, event id, retry number, the
+bot's status and duration, and the state Hue recorded), never a body, a signature or a credential;
+credentials in error messages are replaced with `[redacted]`.
+
+Ctrl+C or `SIGTERM` stops pulling (a waiting pull is cancelled), finishes the deliveries in flight
+and acknowledges them, then exits `0`; a second Ctrl+C abandons the forwards and acknowledgements
+still in flight and exits `130`. Nothing is acknowledged before the bot answered or its window
+closed, and a lease left unacknowledged lapses into a timeout that Hue retries on the provider's
+schedule, so no event is lost and none is marked delivered unanswered. Network errors, `429`, `5xx`
+and another open pull for the same subscription (a second `hue listen`) are retried with backoff up
+to 30 seconds; a `Retry-After` can lengthen a wait to at most 60 seconds, never shorten it. Exit
+codes: `0` stopped, `1` Hue refused the credential, the subscription (unknown, revoked, or one that
+delivers to a request URL) or the pull itself (a redirect, another refusal or an unreadable answer),
+`2` usage error, `130` interrupted twice.
 
 ## Local state and conflicts
 
@@ -305,7 +406,7 @@ node packages/sdk-typescript/scripts/verify-package.mjs --artifacts-dir .artifac
 # Set project to an existing supported fixture; use the same directory on resume.
 project=/absolute/path/to/supported-fixture
 node packages/sdk-typescript/scripts/verify-setup-live.mjs \
-  --archive .artifacts/typescript/hue-run-sdk-0.10.0.tgz \
+  --archive .artifacts/typescript/hue-run-sdk-0.11.0.tgz \
   --origin https://STAGING_ORIGIN \
   --project "$project" --command setup \
   --evidence .context/setup-staging-before-claim.json
@@ -316,7 +417,7 @@ the private local handoff and finish the real browser claim, then reconcile the
 
 ```sh
 node packages/sdk-typescript/scripts/verify-setup-live.mjs \
-  --archive .artifacts/typescript/hue-run-sdk-0.10.0.tgz \
+  --archive .artifacts/typescript/hue-run-sdk-0.11.0.tgz \
   --origin https://STAGING_ORIGIN \
   --project "$project" --command claim \
   --evidence .context/setup-staging-after-claim.json
@@ -393,11 +494,19 @@ Failing cases print the scorer explanation. An evaluator Hue records as not appl
 (its outcome scoring has no outcome criteria or conversion rubric to grade there) shows `n/a` in
 that case's columns and neither passes nor fails it, so in a mixed eval set a case is decided by
 the evaluators that apply to it; the pass line counts the not-applicable results. A case that no
-pinned evaluator applies to is an error that names what they need. `--baseline <experiment id|url>` adds improvement,
+pinned evaluator applies to is an error that names what they need. A result of a pinned Hue judge
+(an evaluator of kind `world_judge`) that Hue marks advisory, as it does every judge's today, is
+shown in its column, marked `(advisory)`, but never decides a case or the exit code; the pass line
+counts advisory failures as not counted, and a case only advisory results ran for is an error. Any
+other evaluator's result, an error, or a metric that carries `passed` is never advisory, whatever
+its evidence says. A metric name that
+several evaluators report, such as two judges' `verdict`, gets a column per evaluator, labelled
+with the version's first eight characters. `--baseline <experiment id|url>` adds improvement,
 regression and unchanged counts with per-case deltas. `--json` prints one JSON document
 (`experimentId`, `runId`, `runUrl`, `complete`, `cases`, `totals`, optional `baseline`) on stdout
 and sends progress to stderr; each case lists its not-applicable evaluator versions in
-`notApplicable`, and `totals.notApplicable` counts them; a case failed for its telemetry has `state: "error"`,
+`notApplicable` and its advisory ones in `advisory`, and `totals.notApplicable` counts the
+former; a case failed for its telemetry has `state: "error"`,
 `passed: false` and
 `telemetry: { code: "telemetry_not_accepted", issues: [{ signal, kind, status?, count }] }`. `--wait <seconds>` (default 300) bounds the verdict wait because
 Hue-owned `world_outcome` checks are graded after the world seals. An experiment always covers
@@ -406,7 +515,9 @@ every case of the saved version; there is no case subset.
 `--command "<shell command>"` spawns the command once per case with the world's environment
 (`HUE_WORLD_ID`, `HUE_WORLD_TOKEN`, one `HUE_SIM_<SURFACE ID>_URL` per provider mirror,
 `HUE_MCP_CONFIG` naming an owner-only `mcpServers` file that is removed after the case, and
-`HUE_MCP_URL`, `HUE_MCP_TOKEN`, `HUE_MCP_EXPIRES_AT` for the first MCP mirror), plus
+`HUE_MCP_URL`, `HUE_MCP_TOKEN`, `HUE_MCP_EXPIRES_AT` for the first MCP mirror; these name the
+world's simulated MCP server and replace a `HUE_MCP_URL` that `hue login` stored for Hue's project
+MCP server), plus
 `HUE_EXECUTION_ID`, `HUE_ENVIRONMENT_RUN_ID`, `HUE_CASE_ID` and `HUE_CASE_KEY`, and
 `{"inputs": ..., "config": ...}` on stdin. The child does not receive `HUE_API_KEY`, `HUE_MCP_KEY`
 or any other Hue control-plane credential unless `--allow-hue-credentials` is passed; the rest of
@@ -475,7 +586,10 @@ case also keeps the agent's raw `output/` files on disk under its checkpoint dir
 Trace evidence is required for every case: when Hue does not accept a case's traces or logs, the
 case is completed as failed (error `TelemetryNotAccepted`, evidence omitted as
 `telemetry_not_accepted`, no output or generated files attached) instead of being left started,
-and the run goes on. Stderr names the case as it completes, with the export
+and the run goes on. Each export of the CLI's own telemetry may take 30 seconds, retries
+included: a slow acknowledgement is waited for, and a 429, 502, 503, 504 or network error is
+retried within that time, honouring `Retry-After`; any other response, or rejected records, fails
+the case at once. Stderr names the case as it completes, with the export
 issue counts, for example
 `[refund] telemetry not accepted, case failed: telemetry_not_accepted: traces failed 1 (HTTP 400)`;
 the case counts as an error in the table and JSON whatever its scores, and the command exits 1.
@@ -485,8 +599,9 @@ inside `.hue/eval/`); `--checkpoint-dir` overrides the root. Rerunning the same
 an interrupted run without invoking the agent again; a different selection is refused until the
 unfinished one is resumed or its directory is removed.
 
-Exit codes: `0` every case passed the evaluators that apply to it, `1` a case failed, errored
-(including one no pinned evaluator applies to), was skipped or Hue's checks were
+Exit codes: `0` every case passed the evaluators that apply to it (advisory ones never count),
+`1` a case failed, errored (including one no pinned evaluator applies to or decides), was skipped
+or Hue's checks were
 still pending at `--wait`, `2` usage or configuration error (including a missing key), `130`
 interrupted. On Node.js 22, load TypeScript adapters with `NODE_OPTIONS=--experimental-strip-types`;
 non-erasable syntax (enums, parameter properties, namespaces) needs a loader such as `--import tsx`
@@ -519,6 +634,11 @@ For each case the command is spawned once **inside a private case directory** wi
 <case dir>/output/                write the generated documents here
 ```
 
+Hue verifies each generated document before the case completes. Once a document is reserved and
+its bytes are uploaded, a verification that outlasts a request is waited out for up to three
+minutes, so a completion that cannot reach Hue takes that long to fail; a reservation or upload
+that cannot reach Hue still fails at once.
+
 Every regular file the command leaves under `output/` is uploaded as a generated document
 (accepted: `.pdf .docx .pptx .xlsx .json .txt .csv .png .jpg .jpeg .webp`; another extension or an
 empty file is the case's error). Optional helpers: `manifest.json` (`{"primary": "<filename>",

package/ENVIRONMENTS.md

@@ -79,7 +79,11 @@ until every item has a terminal result for every pin or the budget elapses (`com
 `summarizeVerdicts` turns results into per-case rows with `passed` and totals; a result Hue marks
 `notApplicable` (the evaluator has no outcome criteria or conversion rubric to grade in that case)
 neither passes nor fails its case and is listed in the row's `notApplicable`, and a case no pinned
-evaluator applies to is an error. `compareVerdicts` diffs two summaries by case key, and
+evaluator applies to is an error. A result of a Hue judge listed in `waitForResults`'
+`judgeScorerVersionIds` (`collectExperimentVerdicts` passes the experiment's `world_judge` pins)
+whose evidence says `advisory: true` keeps its metrics in the row and is listed in its
+`advisory`, but never decides the case; a case only advisory results ran for is an error. Any
+other result, an error, or one with a metric carrying `passed` is never advisory. `compareVerdicts` diffs two summaries by case key, and
 `collectExperimentVerdicts` combines those reads for one experiment. The `hue eval` command uses the same path; see
 [Evaluate an agent against a case](CLI.md#evaluate-an-agent-against-a-case).
 

package/EVALUATIONS.md

@@ -159,7 +159,7 @@ built-ins keep their usual missing-output behavior. See
 
 Scorer deferral shipped in TypeScript `0.3.1`: the local runner executes only the three known built-in entries and bound `local_code` scorers. It leaves every other pin pending and reports its ID in `deferredScorerVersionIds`, including kinds and built-in entries introduced by a newer server. It never uploads a placeholder result that would occupy the immutable result slot, including placeholders already saved in an older SDK's checkpoint. Direct `scoreLocally()` calls reject pins that require another executor.
 
-`world_outcome` pins run inside Hue and need no local callback or executable source digest. A supporting server owns their execution from saved world evidence. Legacy `local_code` pins still require the exact registered callback; changing the worker cannot convert those immutable pins into hosted ones.
+`world_outcome` pins (`hue.conversion_outcome.v1` and `.v2`, and the `hue.outcome_assertions.v2` and `.v3` case-outcome evaluators, `.v3` with its pinned judge in `config.judge`) run inside Hue and need no local callback or executable source digest. A supporting server owns their execution from saved world evidence. Legacy `local_code` pins still require the exact registered callback; changing the worker cannot convert those immutable pins into hosted ones.
 
 Manual results require a human session. Hosted model-judge dispatch is an explicit separate API operation: inspect `getJudgeBudget()`, then call `createJudgeJobs(runId,{idempotencyKey,jobs:[{evaluationItemId,scorerVersionId}]})`. `listJudgeJobs`, `getJudgeJob` and `cancelJudgeJob` expose job progress and cancellation requests. These methods never claim that local execution has hosted provenance. Hosted job endpoints are covered by HTTP contract tests here; live hosted model execution is a separate platform acceptance phase. `listResults` and `getResult` read recorded local or hosted results.
 
@@ -183,7 +183,7 @@ An exclusive `.lock` prevents two processes from invoking targets through the sa
 
 The runner saves a starting marker before `start`, and a running marker before invoking a target. If an outcome is not durably saved, resume throws `UncertainExecutionError` and never reruns the target. Replaying an original start key only recovers its execution ID. The low-level `startExecution` API requires an explicit `previousExecutionId`; replacing a still-started attempt additionally requires `allowUncertainRetry:true`. The convenience runner does not automatically adopt externally created attempts. Create a fresh experiment for a fresh target run after investigating side effects.
 
-After a target completes and local scoring finishes, the runner saves the allowed completion/result payloads before uploading. Network/API errors leave those payloads and stable keys available for another call to `runExperiment` with the same directory. Saved receipt IDs prevent duplicate result writes. Serialization failure raises `OutcomeSerializationError` with the execution ID; it does not relabel the target as failed or invoke it again. Scoring/upload failures never change target state.
+After a target completes and local scoring finishes, the runner saves the allowed completion/result payloads before uploading. Network/API errors leave those payloads and stable keys available for another call to `runExperiment` with the same directory. Saved receipt IDs prevent duplicate result writes. An output beyond what Hue stores for one case (200,000 bytes of JSON, 20,000 values or 32 levels of nesting) is that case's own failure: the case completes as `error` with the error type `OutputTooLarge` (and, when result content is persisted, a message naming the bound), no output is stored and the other cases keep running; return a large result as a generated file instead. Output that is not JSON (a cycle, a class instance, a non-finite number) raises `OutcomeSerializationError` with the execution ID. The output is checked as it is read, and the first value that is not JSON or past the value or nesting bound decides which; the byte bound is checked last, so an output past it that also holds a value that is not JSON raises `OutcomeSerializationError`, and one too long for the runtime to serialize is refused before it is serialized. `OutcomeSerializationError` never relabels the target as failed or invokes it again. Scoring/upload failures never change target state.
 
 A crash between target completion and saving its permitted result still leaves an uncertain outcome; metadata-only mode intentionally cannot reconstruct discarded output. Export acknowledgement is saved separately. If required telemetry export was not acknowledged, a fresh empty exporter is not evidence of prior receipt: automatic completion is refused. Inspect/export the original trace or use the low-level completion API with an explicit omission policy; do not rerun a known completed target to manufacture telemetry. API completion/results retain their own idempotency guarantees for explicit recovery.
 

package/README.md

@@ -83,8 +83,9 @@ claim. Setup never enables content capture or creates a simulation, Hue Run, eva
 or remote execution. See the [setup CLI contract](./CLI.md) for the supported shapes and release gates.
 
 For an existing account, `hue login` validates keys created in Hue Settings and stores them in
-`.env.hue` without printing them, and `hue mcp install --client claude-code` (or `cursor`, `codex`,
-`vscode`, `windsurf`, `gemini`) writes the Hue MCP configuration that references `HUE_MCP_KEY`. See
+`.env.hue` without printing them, and `hue mcp install --client claude-code` (or `codex`,
+`conductor`, `cursor`, `vscode`, `windsurf`, `gemini`) writes the Hue MCP configuration that
+references `HUE_MCP_KEY`, or with `--auth oauth` only the URL, for signing in with Hue. See
 [Sign in and store keys](./CLI.md#sign-in-and-store-keys) and
 [Install the MCP for your coding agent](./CLI.md#install-the-mcp-for-your-coding-agent).
 
@@ -155,12 +156,22 @@ await hue.model(
 
 Each OpenAI Responses `mcp_call`, `web_search_call`, `file_search_call` or `code_interpreter_call`
 item, and each Anthropic `mcp_tool_use` or `server_tool_use` block with its result, becomes an
-`execute_tool {name}` child span with `gen_ai.tool.type` `extension`, `gen_ai.tool.call.id` and,
-for MCP calls, `mcp.server.name` (the provider's label). Arguments and results follow
-`captureContent`; a failed call carries `error.type` (`mcp_error`, the provider's status or error
-code) and ERROR status. An `mcp_list_tools` item becomes a `tools/list` child span with that
-server's `gen_ai.tool.definitions`. Pass the request so each server's host is recorded as
-`server.address` (only server URLs are read, never credentials), and `servers` to record a
+`execute_tool {name}` child span with `gen_ai.tool.type` `extension`, `gen_ai.tool.call.id`,
+`hue.tool.call.position` (the 0-based position of its item in the response: the OpenAI `output`
+index or the Anthropic `content` block index, in both capture modes) and, for MCP calls,
+`mcp.server.name` (the provider's label). Arguments and results follow `captureContent`; a failed
+call carries `error.type` (`mcp_error`, the provider's status or error code) and ERROR status. With
+`captureContent: true`, a failed OpenAI MCP call's status description is the provider's error
+text, credentials scrubbed (URL userinfo, query values and fragments, or the whole URL for a scheme
+other than `http(s)`, `ws(s)` and `ftp`; tokens with a known credential prefix such as `hue_sk_`,
+`sk-` or `xoxb-`; authorization-scheme credentials and an `Authorization` header's whole value;
+the value of a credential-named `key=value` or `key: value` pair, quoted or not) and cut to 1,024
+characters. An `mcp_list_tools`
+item becomes a `tools/list` child span with that server's `gen_ai.tool.definitions`; with
+`captureContent: false` it carries only `hue.tool.names` and `hue.tool.definitions.sha256`, the
+summary described below. Pass the request so each server's host is recorded as `server.address`
+(only server URLs are read, never credentials; a host name keeps its underscores, as WHATWG URL
+parsing does), and `servers` to record a
 server's real `name`, `version`, Hue `provider` and `surface` under its label. `provider` defaults
 to the enclosing `model()` call's provider (`openai` or `anthropic`). The spans have no duration of
 their own: the provider ran the tools inside the model request.

package/dist/cli/eval.js

@@ -446,6 +446,10 @@ function parseAnswer(text) {
     }
 }
 const MIN_SECRET_LENGTH = 16;
+/** Each export of the CLI's own telemetry, retries included, may take this long. A case's trace
+ * is required evidence, and Hue can acknowledge it slowly or ask for a retry after a few seconds,
+ * which the 10-second default leaves no room for. A refusal still fails at once. */
+const CASE_TRACE_EXPORT_MILLIS = 30_000;
 function caseSecrets(context) {
     const values = new Set();
     // Hue issues long credentials; a short value would redact ordinary text around it.
@@ -696,27 +700,38 @@ const STATE_LABEL = {
     pending: "PENDING",
 };
 export function renderTable(verdicts, output) {
-    const names = [];
-    // The evaluators that report each metric: a case none of them applies to shows n/a there, and
-    // one an evaluator applies to without reporting the metric shows "-".
-    const reportedBy = new Map();
-    for (const item of verdicts.summary.cases)
+    const cases = verdicts.summary.cases;
+    // A column per metric name; a name several evaluators report, such as two judges' `verdict`,
+    // gets a column per evaluator so each one's value shows.
+    const reporters = new Map();
+    for (const item of cases)
         for (const metric of item.metrics) {
-            if (!names.includes(metric.name))
-                names.push(metric.name);
-            reportedBy.set(metric.name, (reportedBy.get(metric.name) ?? new Set()).add(metric.scorerVersionId));
+            const versions = reporters.get(metric.name) ?? [];
+            if (!versions.includes(metric.scorerVersionId))
+                versions.push(metric.scorerVersionId);
+            reporters.set(metric.name, versions);
         }
-    const header = ["Case", ...names, "Result"];
-    const rows = verdicts.summary.cases.map((item) => [
+    const columns = [...reporters].flatMap(([name, versions]) => versions.map((version) => ({
+        name,
+        version,
+        label: versions.length === 1 ? name : `${name} ${version.slice(0, 8)}`,
+    })));
+    // An advisory evaluator's values are shown, marked, and never decide a case.
+    const advisory = new Set(cases.flatMap((item) => item.advisory ?? []));
+    const header = [
+        "Case",
+        ...columns.map((column) => advisory.has(column.version) ? `${column.label} (advisory)` : column.label),
+        "Result",
+    ];
+    const rows = cases.map((item) => [
         item.externalKey,
-        ...names.map((name) => {
-            const metric = item.metrics.find((candidate) => candidate.name === name);
+        ...columns.map((column) => {
+            const metric = item.metrics.find((candidate) => candidate.name === column.name && candidate.scorerVersionId === column.version);
             if (metric)
                 return metricText(metric);
-            const notApplicable = new Set(item.notApplicable);
-            return [...reportedBy.get(name)].every((version) => notApplicable.has(version))
-                ? "n/a"
-                : "-";
+            // A case the column's evaluator does not apply to shows n/a; one it applies to without
+            // reporting the metric shows "-".
+            return item.notApplicable?.includes(column.version) ? "n/a" : "-";
         }),
         STATE_LABEL[item.state],
     ]);
@@ -737,6 +752,8 @@ export function renderTable(verdicts, output) {
             output.log(`  ${item.externalKey}: ${detail}`);
     }
     const totals = verdicts.summary.totals;
+    const advisoryFailures = cases.reduce((count, item) => count +
+        item.metrics.filter((metric) => item.advisory?.includes(metric.scorerVersionId) && !metricPassed(metric)).length, 0);
     const extra = [
         totals.error ? `${totals.error} error` : "",
         totals.skipped ? `${totals.skipped} skipped` : "",
@@ -744,6 +761,9 @@ export function renderTable(verdicts, output) {
         totals.notApplicable
             ? `${totals.notApplicable} evaluator result${totals.notApplicable === 1 ? "" : "s"} not applicable`
             : "",
+        advisoryFailures
+            ? `${advisoryFailures} advisory failure${advisoryFailures === 1 ? "" : "s"} not counted`
+            : "",
     ].filter(Boolean);
     output.log(`${totals.passed} of ${totals.cases} case${totals.cases === 1 ? "" : "s"} passed${extra.length ? ` (${extra.join(", ")})` : ""}`);
     if (!verdicts.results.complete)
@@ -1426,7 +1446,13 @@ export async function runEvalCommand(argv) {
                     allowHueCredentials: values["allow-hue-credentials"],
                 })),
         };
-        hue = createHue({ apiKey, baseUrl, serviceName: key, captureContent: values.content });
+        hue = createHue({
+            apiKey,
+            baseUrl,
+            serviceName: key,
+            captureContent: values.content,
+            timeoutMillis: CASE_TRACE_EXPORT_MILLIS,
+        });
         return values.worker
             ? await runWorker(values, connection, agents.simulation, agent, hue, output, controller.signal)
             : await runOnce(values, connection, agents, agent, hue, output, controller.signal);

package/dist/cli/listen.d.ts

@@ -0,0 +1,76 @@
+import { lookup as dnsLookup } from "node:dns";
+/**
+ * `hue listen`: delivers a simulated world's events to a receiver on this machine, the way a
+ * provider would post them to a public request URL. The client pulls leased deliveries from Hue
+ * with the subscription's own credential (a world token or a connection key, never a project
+ * key), forwards each stored request unchanged to the local URL, and acknowledges it with the
+ * local answer. Every pull is one bounded request; nothing holds a connection open. A delivery is
+ * acknowledged only after the receiver answered or its window closed, so stopping never marks an
+ * event delivered that was not: an unacknowledged lease lapses and Hue retries it on the
+ * provider's schedule.
+ */
+/** Streams, environment, network and signal seams for {@link runListenCommand}; tests inject these. */
+export interface ListenCommandIo {
+    stdout?: NodeJS.WritableStream;
+    stderr?: NodeJS.WritableStream;
+    env?: NodeJS.ProcessEnv;
+    cwd?: string;
+    fetch?: typeof fetch;
+    /** Registers the stop request (SIGINT and SIGTERM by default) and returns its removal. */
+    onInterrupt?: (listener: () => void) => () => void;
+    /** The acknowledgement window a forward waits for; the provider's three seconds by default. */
+    forwardTimeoutMs?: number;
+    /** Delays between retries of a failed pull. */
+    retryDelaysMs?: readonly number[];
+}
+export declare const LISTEN_USAGE = "Usage: hue listen --subscription ID --forward-to URL [--origin URL] [--env-path PATH]\n                  [--credential world-token|connection-key] [--max N] [--allow-remote-forward]\n\nDeliver a simulated world's events to a receiver on this machine. Hue's signed requests (Slack\nEvents API requests today) are pulled with the subscription's own credential, forwarded unchanged\nto --forward-to, and acknowledged with the receiver's answer. Press Ctrl+C to stop: pulls stop,\ndeliveries in flight are finished and acknowledged, and nothing unanswered is acknowledged.\n\nThe credential is read from the environment, never from the command line:\n  HUE_WORLD_TOKEN     the world token, for a subscription on one world\n  HUE_CONNECTION_KEY  the connection key, for a subscription on a connection key\nA project key (HUE_API_KEY, HUE_MCP_KEY) is refused.\n\nOptions:\n  --subscription ID       The listen subscription to deliver (its id from Hue)\n  --forward-to URL        The local receiver, for example http://localhost:3000/slack/events\n  --origin URL            Hue origin (default HUE_BASE_URL, then https://app.hue.run)\n  --env-path PATH         Load a dotenv file first (--env-file also works when it exists)\n  --credential KIND       world-token or connection-key, when both variables are set\n  --max N                 Deliveries leased per pull and forwarded at once, 1 to 10 (default 10)\n  --allow-remote-forward  Allow a --forward-to host other than this machine, over https only\n  -h, --help              Show this help";
+/** A leased delivery: the exact request the provider would send. */
+interface Delivery {
+    deliveryId: string;
+    eventId: string | null;
+    kind: "url_verification" | "event_callback";
+    retryNum: number;
+    /** The lease's end as the server stated it, or null when absent. */
+    leaseExpiresAt: number | null;
+    request: {
+        headers: Record<string, string>;
+        body: string;
+    };
+}
+/** The local answer, in the acknowledgement's own shape. */
+interface LocalAnswer {
+    outcome: "response" | "timeout" | "connection_failed" | "tls_error";
+    status?: number;
+    durationMs: number;
+    noRetry?: boolean;
+    body?: string;
+}
+/** A loopback address: 127.0.0.0/8, ::1, or 127.0.0.0/8 carried in IPv4-mapped IPv6. */
+export declare function isLoopbackAddress(address: string): boolean;
+/**
+ * The receiver URL, or why it is refused: HTTP or HTTPS without credentials or fragment, on this
+ * machine (`localhost`, a `.localhost` name or a loopback address) unless `allowRemote`, and then
+ * HTTPS only off this machine.
+ */
+export declare function parseForwardTarget(value: string, allowRemote: boolean): URL | string;
+/**
+ * Sends one delivery to the receiver and reports its answer, or null when `signal` abandoned it.
+ * The body bytes and every header go out unchanged (the provider's signature covers them), no
+ * redirect is followed, and the answer counts only inside `timeoutMs`. Only a URL verification's
+ * successful answer body is read.
+ */
+export declare function forwardDelivery(target: URL, delivery: Delivery, options?: {
+    timeoutMs?: number;
+    allowRemote?: boolean;
+    signal?: AbortSignal;
+    /** Resolves `--forward-to` names; tests stand in for DNS. */
+    resolver?: typeof dnsLookup;
+}): Promise<LocalAnswer | null>;
+/** One delivery from a pull answer, or null when it is not one this client can forward safely. */
+export declare function parseDelivery(value: unknown, subscriptionId: string): Delivery | null;
+/**
+ * Runs `hue listen` and returns the process exit code: 0 stopped, 1 failed, 2 usage error, 130
+ * interrupted twice (acknowledgements abandoned). `argv` may start with the `listen` command word.
+ */
+export declare function runListenCommand(argv: string[], io?: ListenCommandIo): Promise<number>;
+export {};