@hue-run/sdk 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLI.md +22 -13
- package/ENVIRONMENTS.md +61 -0
- package/README.md +47 -1
- package/dist/cli/eval.js +79 -24
- package/dist/client.js +3 -2
- package/dist/config.d.ts +3 -1
- package/dist/config.js +8 -1
- package/dist/environment/client.d.ts +16 -2
- package/dist/environment/client.js +46 -3
- package/dist/environment/types.d.ts +131 -2
- package/dist/environment/world.d.ts +50 -0
- package/dist/environment/world.js +105 -0
- package/dist/environment.d.ts +2 -0
- package/dist/environment.js +1 -0
- package/dist/evals/environment-target.d.ts +53 -2
- package/dist/evals/environment-target.js +114 -10
- package/dist/evals/local-worker.d.ts +12 -5
- package/dist/evals/local-worker.js +13 -5
- package/dist/evals/runner.d.ts +1 -1
- package/dist/evals/runner.js +2 -2
- package/dist/evals/simulation.d.ts +14 -5
- package/dist/evals/simulation.js +26 -8
- package/dist/live-spans.d.ts +25 -0
- package/dist/live-spans.js +115 -0
- package/dist/transport.d.ts +17 -0
- package/dist/transport.js +390 -61
- package/dist/types.d.ts +7 -1
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +1 -1
package/CLI.md
CHANGED
|
@@ -42,8 +42,8 @@ project manifest are refused because managers can update ancestor locks; Python
|
|
|
42
42
|
The generated `hue.setup.mjs` or `hue_setup.py` always selects `captureContent: false` /
|
|
43
43
|
`capture_content=False`. For a supported application, setup installs the dependency and adds the
|
|
44
44
|
managed import and middleware registration to the existing entrypoint; an unreferenced helper is
|
|
45
|
-
not a completed integration. TypeScript uses `@hue-run/sdk@0.
|
|
46
|
-
`@opentelemetry/context-async-hooks@2.11.0`; Python uses
|
|
45
|
+
not a completed integration. TypeScript uses `@hue-run/sdk@0.8.0`, `@opentelemetry/api@1.9.1` and
|
|
46
|
+
`@opentelemetry/context-async-hooks@2.11.0`; Python setup uses its separately tested package pin.
|
|
47
47
|
Content capture requires an ordinary account-managed key and a later explicit application decision.
|
|
48
48
|
|
|
49
49
|
The generated bootstrap supplies standard active SERVER-span context across asynchronous/streaming
|
|
@@ -231,7 +231,8 @@ capability `setup_telemetry_write`.
|
|
|
231
231
|
Normal project credentials and unknown token shapes are refused in setup responses and private
|
|
232
232
|
managed state. These credentials authorize metadata-only OTLP at `/api/v1/otlp/v1/traces` and exact
|
|
233
233
|
content-free receipt verification at `/api/v1/setup/traces/{traceId}/receipt`. They do not authorize
|
|
234
|
-
generic project, receipt, evaluation, log or browsing APIs.
|
|
234
|
+
generic project, receipt, evaluation, log or browsing APIs. The Python package version
|
|
235
|
+
pinned by setup can export with the setup
|
|
235
236
|
credential; the CLI verifies the dedicated setup receipt instead of Python's generic receipt helper.
|
|
236
237
|
|
|
237
238
|
Each command uses bounded timeouts and retries. Provisioning records at most five attempts per local
|
|
@@ -300,7 +301,7 @@ node packages/sdk-typescript/scripts/verify-package.mjs --artifacts-dir .artifac
|
|
|
300
301
|
# Set project to an existing supported fixture; use the same directory on resume.
|
|
301
302
|
project=/absolute/path/to/supported-fixture
|
|
302
303
|
node packages/sdk-typescript/scripts/verify-setup-live.mjs \
|
|
303
|
-
--archive .artifacts/typescript/hue-run-sdk-0.
|
|
304
|
+
--archive .artifacts/typescript/hue-run-sdk-0.8.0.tgz \
|
|
304
305
|
--origin https://STAGING_ORIGIN \
|
|
305
306
|
--project "$project" --command setup \
|
|
306
307
|
--evidence .context/setup-staging-before-claim.json
|
|
@@ -311,7 +312,7 @@ the private local handoff and finish the real browser claim, then reconcile the
|
|
|
311
312
|
|
|
312
313
|
```sh
|
|
313
314
|
node packages/sdk-typescript/scripts/verify-setup-live.mjs \
|
|
314
|
-
--archive .artifacts/typescript/hue-run-sdk-0.
|
|
315
|
+
--archive .artifacts/typescript/hue-run-sdk-0.8.0.tgz \
|
|
315
316
|
--origin https://STAGING_ORIGIN \
|
|
316
317
|
--project "$project" --command claim \
|
|
317
318
|
--evidence .context/setup-staging-after-claim.json
|
|
@@ -345,8 +346,9 @@ the CLI never prints it. The optional `zod` peer of `@hue-run/sdk/evals` must be
|
|
|
345
346
|
|
|
346
347
|
Write an adapter module that hands the case inputs and the world's tools or MCP connection to
|
|
347
348
|
the real agent. The module exports `default` or `runMyAgent`; `context` is the SDK's
|
|
348
|
-
`SimulationTargetContext` (`
|
|
349
|
-
`signal`
|
|
349
|
+
`SimulationTargetContext` (`world`, `mcp`, `tools`, `config`, `item`, `executionId`,
|
|
350
|
+
`environmentRunId`, `signal`; `world` carries the provider mirror URLs, the world token, `env` and
|
|
351
|
+
`mcpConfig` where Hue's simulation gateway serves the world):
|
|
350
352
|
|
|
351
353
|
```ts
|
|
352
354
|
// hue-agent.ts: erasable TypeScript only; Node.js 24 and Bun strip the types natively.
|
|
@@ -385,12 +387,19 @@ and sends progress to stderr. `--wait <seconds>` (default 300) bounds the verdic
|
|
|
385
387
|
Hue-owned `world_outcome` checks are graded after the world seals. An experiment always covers
|
|
386
388
|
every case of the saved version; there is no case subset.
|
|
387
389
|
|
|
388
|
-
`--command "<shell command>"` spawns the command once per case with
|
|
389
|
-
`
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
390
|
+
`--command "<shell command>"` spawns the command once per case with the world's environment
|
|
391
|
+
(`HUE_WORLD_ID`, `HUE_WORLD_TOKEN`, one `HUE_SIM_<SURFACE ID>_URL` per provider mirror,
|
|
392
|
+
`HUE_MCP_CONFIG` naming an owner-only `mcpServers` file that is removed after the case, and
|
|
393
|
+
`HUE_MCP_URL`, `HUE_MCP_TOKEN`, `HUE_MCP_EXPIRES_AT` for the first MCP mirror), plus
|
|
394
|
+
`HUE_EXECUTION_ID`, `HUE_ENVIRONMENT_RUN_ID`, `HUE_CASE_ID` and `HUE_CASE_KEY`, and
|
|
395
|
+
`{"inputs": ..., "config": ...}` on stdin. The child does not receive `HUE_API_KEY`, `HUE_MCP_KEY`
|
|
396
|
+
or any other Hue control-plane credential unless `--allow-hue-credentials` is passed; the rest of
|
|
397
|
+
the parent environment (model keys, application settings) is inherited. Its stdout is the answer
|
|
398
|
+
(JSON when it parses, otherwise trimmed text; empty means no output); a non-zero exit or the
|
|
399
|
+
per-case `--timeout` (default 600 seconds) is a target failure. The world token is never logged.
|
|
400
|
+
A world created while the deployment's simulation gateway is off gets the legacy `hue_sim_`
|
|
401
|
+
capability under the same `HUE_MCP_*` names. The agent key defaults to the slug of the command's
|
|
402
|
+
script name; `--revision` is sent to Hue as the agent revision of every world.
|
|
394
403
|
|
|
395
404
|
`--worker` registers the adapter through `runLocalAgent()` with key `--agent-key` (default: the
|
|
396
405
|
adapter filename slug), name `--agent-name` (default: the key), revision `--revision` (default:
|
package/ENVIRONMENTS.md
CHANGED
|
@@ -196,6 +196,67 @@ An observation with `status: "error"` is a recorded world answer, not a transpor
|
|
|
196
196
|
Run mutations retry with stable invocation/idempotency identities. Registry writes do not retry
|
|
197
197
|
automatically because identity creation and publication have no request key.
|
|
198
198
|
|
|
199
|
+
## Worlds served by the simulation gateway
|
|
200
|
+
|
|
201
|
+
Where a Hue deployment has the simulation gateway on, `createRun` returns the World API handoff
|
|
202
|
+
beside the run: `token` (a `hue_world_…` credential that lives exactly as long as the world),
|
|
203
|
+
`surfaces[]` (one mirror URL per pinned provider surface, such as the Gmail MCP and REST mirrors),
|
|
204
|
+
`env` (`HUE_WORLD_ID`, `HUE_WORLD_TOKEN`, `BAGGAGE`, `TRACEPARENT` and one
|
|
205
|
+
`HUE_SIM_<SURFACE ID>_URL` per surface) and `mcpConfig` (the common `mcpServers` shape with the
|
|
206
|
+
token in the `Authorization` header). The agent is pointed at the mirrors by configuration only:
|
|
207
|
+
its own Gmail MCP or REST client, the mirror URL, the world token where the Google credential went.
|
|
208
|
+
|
|
209
|
+
```ts
|
|
210
|
+
import {
|
|
211
|
+
agentEnvironment,
|
|
212
|
+
createEnvironmentClient,
|
|
213
|
+
worldHandoff,
|
|
214
|
+
writeMcpConfig,
|
|
215
|
+
} from "@hue-run/sdk/environment";
|
|
216
|
+
|
|
217
|
+
const run = await environmentClient.createRun({
|
|
218
|
+
idempotencyKey: `execution:${executionId}`,
|
|
219
|
+
environmentVersionId,
|
|
220
|
+
executionId,
|
|
221
|
+
ttlSeconds: 600,
|
|
222
|
+
traceparent: `00-${span.traceId}-${span.spanId}-01`, // parents the world span on the case span
|
|
223
|
+
agentRevision: "my-agent@1.4.2", // joins the world's fingerprint
|
|
224
|
+
});
|
|
225
|
+
const world = worldHandoff(run); // null for a world created while the gateway is off
|
|
226
|
+
const child = agentEnvironment(world!, { parent: process.env }); // no HUE_API_KEY in the agent
|
|
227
|
+
const config = await writeMcpConfig(world!); // owner-only mcp.json; dispose after the run
|
|
228
|
+
try {
|
|
229
|
+
await spawnAgent({ env: child, mcpConfigPath: config.path });
|
|
230
|
+
} finally {
|
|
231
|
+
await config.dispose();
|
|
232
|
+
await environmentClient.finishRun(run.id, {
|
|
233
|
+
idempotencyKey: `execution:${executionId}:completed`,
|
|
234
|
+
status: "completed",
|
|
235
|
+
});
|
|
236
|
+
}
|
|
237
|
+
```
|
|
238
|
+
|
|
239
|
+
`runSimulation`, `runLocalAgent` and `hue eval` do this for you: they create the world with the
|
|
240
|
+
execution, the stable key, the case span's context and the agent revision, pass the handoff as
|
|
241
|
+
`context.world`, and finish before returning so telemetry is flushed and the execution completed
|
|
242
|
+
afterwards. For one compatibility release `context.mcp` is the world's first MCP mirror with the
|
|
243
|
+
world token, so an adapter that read `HUE_MCP_URL` and `HUE_MCP_TOKEN` keeps working;
|
|
244
|
+
`agentEnvironment` sets those names too unless `legacyMcpVariables: false`. A gateway world binds
|
|
245
|
+
no Hue-native `tools` (Hue refuses them); a world created while the gateway is off keeps its tools
|
|
246
|
+
and the `hue_sim_` capability and emits a one-time `DeprecationWarning`.
|
|
247
|
+
|
|
248
|
+
`agentEnvironment` removes Hue control-plane credentials from the child by default: `HUE_API_KEY`,
|
|
249
|
+
`HUE_MCP_KEY` and any variable whose value is a `hue_sk_`, `hue_mcp_` or `hue_attempt_`
|
|
250
|
+
credential. Pass `includeHueCredentials: true` only for an agent that must call Hue's own API.
|
|
251
|
+
Nothing in these helpers logs the token; keep it out of your own logs and checkpoints.
|
|
252
|
+
|
|
253
|
+
Finish answers `lifecycle: "completing"` with `sealedAt: null` for a gateway world: the seal
|
|
254
|
+
follows a 5 s grace so in-flight writes land, and a late finish answers 409, which the helpers
|
|
255
|
+
treat as the seal they can no longer change. `getEvidence(runId, { section, bodies })` reads the
|
|
256
|
+
sealed world's evaluator-only evidence (start and end state, the diff, the call ledger, coverage,
|
|
257
|
+
fingerprint) with the project key; a world token can never read it. The client honors Hue's
|
|
258
|
+
`Retry-After` on 429 and 503.
|
|
259
|
+
|
|
199
260
|
## Coverage gaps
|
|
200
261
|
|
|
201
262
|
A provider adapter can record a known valid provider request that the environment cannot
|
package/README.md
CHANGED
|
@@ -329,7 +329,8 @@ const hue = createHue({
|
|
|
329
329
|
|
|
330
330
|
## Delivery behavior
|
|
331
331
|
|
|
332
|
-
Exports
|
|
332
|
+
Exports retry temporary HTTP/network failures (429, 502, 503, 504 and connection errors,
|
|
333
|
+
honoring `Retry-After`) within the export timeout, by OpenTelemetry's OTLP/HTTP exporter rules. Each
|
|
333
334
|
request is limited to 1 MiB before gzip (with space reserved for gzip overhead) and each content value to 256 KiB. Batches
|
|
334
335
|
split at record boundaries. Each signal queues at most 2,048 records, including
|
|
335
336
|
exports in flight; overflow is reported through the callback, counters and next
|
|
@@ -349,6 +350,51 @@ drain, including records emitted before its call. Stop request production
|
|
|
349
350
|
before shutdown so late spans cannot race it. A client does not own instrumented
|
|
350
351
|
operations still running in the application.
|
|
351
352
|
|
|
353
|
+
## Live spans
|
|
354
|
+
|
|
355
|
+
OpenTelemetry exports a span only when it ends, so a long streamed turn would otherwise stay
|
|
356
|
+
invisible until it finishes. When a Hue or AI span is still open at the transport's next 500 ms
|
|
357
|
+
tick, the transport queues a placeholder: an ordinary OTLP span whose parent is the running span,
|
|
358
|
+
with its name, kind, start time and current attributes, an end time of 0,
|
|
359
|
+
`hue.span_type = "pending_span"` and `hue.pending_parent_id` (the running span's own parent,
|
|
360
|
+
omitted for a root). Hue shows the span as running and replaces the placeholder when the real span
|
|
361
|
+
arrives.
|
|
362
|
+
|
|
363
|
+
The placeholder then waits for the next batch export like any queued span: up to 500 ms until the
|
|
364
|
+
next tick, then the 1 s batch delay when nothing else is queued, so it usually reaches Hue within
|
|
365
|
+
about 1.5 s of its span starting. It goes out sooner when a batch is already scheduled, for example
|
|
366
|
+
because another span has just ended, and at once for a full batch or `flush()`. It goes out later
|
|
367
|
+
while an earlier export is still in flight, because the next batch is scheduled only after that
|
|
368
|
+
export finishes. A placeholder whose span has ended by the time it is exported is not sent, so a
|
|
369
|
+
short span may send none and appear in Hue only when it finishes.
|
|
370
|
+
|
|
371
|
+
- Only spans from the client's tracer (`withSpan`, `tool`, `model`, `hue.tracer` and the AI SDK
|
|
372
|
+
adapters) and spans with a `gen_ai.`, `ai.`, `llm.` or `traceloop.` attribute at start, or a
|
|
373
|
+
name starting with `ai.`, are announced. HTTP, database and other framework spans are not.
|
|
374
|
+
- Placeholder attributes follow `captureContent` and `redact` like the real span. Tool
|
|
375
|
+
definitions, system instructions and any value over 64 KiB are left out.
|
|
376
|
+
- The transport builds each placeholder from the running span itself, so code in a wrapping
|
|
377
|
+
processor's `onEnd` never runs on it. If a wrapper forwards `onStart` to Hue but scrubs
|
|
378
|
+
attributes, renames the span or drops it in `onEnd`, a placeholder exported while the span is
|
|
379
|
+
still open is sent anyway: it has the span's original name, and its attributes as set on the span
|
|
380
|
+
with `captureContent` and `redact` applied. Scrub with `redact`, which applies to placeholders
|
|
381
|
+
too, or before the value is set on the span; do not forward `onStart` for spans you rename or
|
|
382
|
+
drop; or turn live spans off with `liveSpans: false`.
|
|
383
|
+
- Placeholders are advisory. They are queued only while the queue is under a quarter of its
|
|
384
|
+
record and byte budgets, and skipped silently otherwise. While queued they count in
|
|
385
|
+
`pendingSpans` and `pendingBytes`, but never as accepted, rejected, failed or dropped records.
|
|
386
|
+
Losing only placeholders records a warning and does not make `flush()` throw.
|
|
387
|
+
- A Hue server that accepts placeholders sends `Hue-Pending-Spans: 1` on trace acknowledgements.
|
|
388
|
+
When a response to a request carrying placeholders lacks it, the receiver predates them: the
|
|
389
|
+
transport attributes up to one rejection per placeholder to them, records one warning and stops
|
|
390
|
+
sending placeholders for that client. Other rejections count against real spans as usual.
|
|
391
|
+
- Opt out with `liveSpans: false`. Setup credentials never send placeholders.
|
|
392
|
+
- With an existing provider, announcements start in `spanProcessor.onStart`. A wrapping processor
|
|
393
|
+
that forwards `onStart` should forward `onEnd` for the same spans: a span that ends without
|
|
394
|
+
reaching Hue is forgotten at the next tick, but a placeholder already sent keeps it showing as
|
|
395
|
+
running until Hue marks the trace stalled. The filtering wrapper above forwards no starts, so it
|
|
396
|
+
sends no placeholders.
|
|
397
|
+
|
|
352
398
|
## Verify a stored application trace
|
|
353
399
|
|
|
354
400
|
After exercising a real application request and finishing its stream, flush the
|
package/dist/cli/eval.js
CHANGED
|
@@ -6,6 +6,7 @@ import { parseArgs } from "node:util";
|
|
|
6
6
|
import { randomUUID } from "node:crypto";
|
|
7
7
|
import { createHue } from "../client.js";
|
|
8
8
|
import { createEnvironmentClient } from "../environment/client.js";
|
|
9
|
+
import { agentEnvironment, stripHueControlPlaneCredentials, writeMcpConfig, } from "../environment/world.js";
|
|
9
10
|
import { EvaluationClient, HueApiError } from "../evals/client.js";
|
|
10
11
|
import { TargetResult } from "../evals/types.js";
|
|
11
12
|
import { CheckpointStore } from "../evals/checkpoint.js";
|
|
@@ -32,9 +33,15 @@ Selection (exactly one, not used with --worker):
|
|
|
32
33
|
|
|
33
34
|
Agent (exactly one):
|
|
34
35
|
<adapter-file> Module exporting default or runMyAgent(inputs, context)
|
|
35
|
-
--command "<shell command>" Simulation: spawned per case with
|
|
36
|
-
|
|
37
|
-
|
|
36
|
+
--command "<shell command>" Simulation: spawned per case with the world's environment
|
|
37
|
+
(HUE_WORLD_ID, HUE_WORLD_TOKEN, one HUE_SIM_<SURFACE>_URL per
|
|
38
|
+
provider mirror, HUE_MCP_CONFIG, plus HUE_MCP_URL, HUE_MCP_TOKEN
|
|
39
|
+
and HUE_MCP_EXPIRES_AT for the first MCP mirror), HUE_EXECUTION_ID,
|
|
40
|
+
HUE_ENVIRONMENT_RUN_ID, HUE_CASE_ID and HUE_CASE_KEY set;
|
|
41
|
+
{"inputs","config"} on stdin. HUE_API_KEY and other Hue
|
|
42
|
+
control-plane credentials are removed from the child.
|
|
43
|
+
--allow-hue-credentials Keep HUE_API_KEY and other Hue control-plane credentials in
|
|
44
|
+
the --command child (off by default)
|
|
38
45
|
Direct: spawned in a private case directory with HUE_CASE_DIR,
|
|
39
46
|
HUE_CASE_INPUTS, HUE_CASE_OUTPUT_DIR, HUE_CASE_ID, HUE_CASE_KEY
|
|
40
47
|
and HUE_EXECUTION_ID set; files/<role>/ hold the pinned inputs
|
|
@@ -59,7 +66,7 @@ Output and limits:
|
|
|
59
66
|
outputs/explanations (--worker always persists them)
|
|
60
67
|
--save-version Freeze an unsaved eval-set version before running
|
|
61
68
|
--checkpoint-dir <path> Private checkpoint directory (default: .hue/eval/<agent-key>)
|
|
62
|
-
--concurrency <n> Cases in flight, 1-
|
|
69
|
+
--concurrency <n> Cases in flight, 1-64 (default: 1)
|
|
63
70
|
--timeout <seconds> Per-case --command timeout (default: 600)
|
|
64
71
|
--wait <seconds> Verdict wait after the run finishes (default: 300)
|
|
65
72
|
-h, --help Show this help
|
|
@@ -92,6 +99,7 @@ function parse(argv) {
|
|
|
92
99
|
"scorer-version": { type: "string", multiple: true },
|
|
93
100
|
mode: { type: "string" },
|
|
94
101
|
command: { type: "string" },
|
|
102
|
+
"allow-hue-credentials": { type: "boolean", default: false },
|
|
95
103
|
worker: { type: "boolean", default: false },
|
|
96
104
|
"max-runs": { type: "string" },
|
|
97
105
|
"agent-key": { type: "string" },
|
|
@@ -296,29 +304,65 @@ function parseAnswer(text) {
|
|
|
296
304
|
return text;
|
|
297
305
|
}
|
|
298
306
|
}
|
|
299
|
-
/**
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
+
/**
|
|
308
|
+
* Runs the shell command once per case. The world token and mirror URLs travel only through the
|
|
309
|
+
* child's environment and an owner-only MCP configuration file that is removed after the run;
|
|
310
|
+
* Hue control-plane credentials (`HUE_API_KEY`, `HUE_MCP_KEY`, any `hue_sk_` value) stay with
|
|
311
|
+
* the CLI unless `--allow-hue-credentials` is passed. A world created while the gateway is off
|
|
312
|
+
* still gets the `hue_sim_` capability under the same `HUE_MCP_*` names.
|
|
313
|
+
*/
|
|
314
|
+
/** The parent environment an agent child starts from: without Hue control-plane credentials
|
|
315
|
+
* unless `--allow-hue-credentials` was passed. */
|
|
316
|
+
function parentEnvironment(allowHueCredentials) {
|
|
317
|
+
return allowHueCredentials
|
|
318
|
+
? Object.fromEntries(Object.entries(process.env).filter((entry) => entry[1] !== undefined))
|
|
319
|
+
: stripHueControlPlaneCredentials(process.env);
|
|
320
|
+
}
|
|
321
|
+
function commandAdapter(command, timeoutSeconds, options) {
|
|
322
|
+
return async (inputs, context) => {
|
|
323
|
+
const parent = parentEnvironment(options.allowHueCredentials);
|
|
324
|
+
const identity = {
|
|
307
325
|
HUE_EXECUTION_ID: context.executionId,
|
|
308
326
|
HUE_ENVIRONMENT_RUN_ID: context.environmentRunId,
|
|
309
327
|
HUE_CASE_ID: context.item.id,
|
|
310
328
|
HUE_CASE_KEY: context.item.externalKey,
|
|
311
|
-
}
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
329
|
+
};
|
|
330
|
+
const configFile = context.world ? await writeMcpConfig(context.world) : undefined;
|
|
331
|
+
try {
|
|
332
|
+
const env = context.world
|
|
333
|
+
? {
|
|
334
|
+
...agentEnvironment(context.world, { parent, includeHueCredentials: true }),
|
|
335
|
+
...identity,
|
|
336
|
+
HUE_MCP_CONFIG: configFile.path,
|
|
337
|
+
}
|
|
338
|
+
: {
|
|
339
|
+
...parent,
|
|
340
|
+
...(context.mcp
|
|
341
|
+
? {
|
|
342
|
+
HUE_MCP_URL: context.mcp.url,
|
|
343
|
+
HUE_MCP_TOKEN: context.mcp.token,
|
|
344
|
+
HUE_MCP_EXPIRES_AT: context.mcp.expiresAt,
|
|
345
|
+
}
|
|
346
|
+
: {}),
|
|
347
|
+
...identity,
|
|
348
|
+
};
|
|
349
|
+
return parseAnswer(await spawnAgentCommand(command, {
|
|
350
|
+
env,
|
|
351
|
+
stdin: JSON.stringify({ inputs, config: context.config }),
|
|
352
|
+
timeoutSeconds,
|
|
353
|
+
...(context.signal ? { signal: context.signal } : {}),
|
|
354
|
+
}));
|
|
355
|
+
}
|
|
356
|
+
finally {
|
|
357
|
+
await configFile?.dispose();
|
|
358
|
+
}
|
|
359
|
+
};
|
|
316
360
|
}
|
|
317
361
|
/**
|
|
318
362
|
* Direct cases: the command works in a private case directory and writes its documents to
|
|
319
363
|
* `output/`. Its stdout is only used as the JSON output when it wrote no result or summary file.
|
|
320
364
|
*/
|
|
321
|
-
function directCommandAdapter(command, timeoutSeconds) {
|
|
365
|
+
function directCommandAdapter(command, timeoutSeconds, options) {
|
|
322
366
|
return async (inputs, context) => {
|
|
323
367
|
const layout = await stageDirectCase(context.outputDirectory, {
|
|
324
368
|
inputs,
|
|
@@ -330,7 +374,7 @@ function directCommandAdapter(command, timeoutSeconds) {
|
|
|
330
374
|
const stdout = await spawnAgentCommand(command, {
|
|
331
375
|
cwd: layout.caseDirectory,
|
|
332
376
|
env: {
|
|
333
|
-
...
|
|
377
|
+
...parentEnvironment(options.allowHueCredentials),
|
|
334
378
|
HUE_CASE_DIR: layout.caseDirectory,
|
|
335
379
|
HUE_CASE_INPUTS: layout.inputsPath,
|
|
336
380
|
HUE_CASE_OUTPUT_DIR: layout.outputDirectory,
|
|
@@ -636,7 +680,7 @@ function parseBaseline(value) {
|
|
|
636
680
|
}
|
|
637
681
|
async function runOnce(values, connection, agents, agent, hue, output, signal) {
|
|
638
682
|
const client = new EvaluationClient(connection);
|
|
639
|
-
const concurrency = integer("concurrency", values.concurrency, 1, 1,
|
|
683
|
+
const concurrency = integer("concurrency", values.concurrency, 1, 1, 64);
|
|
640
684
|
const baselineId = parseBaseline(values.baseline);
|
|
641
685
|
const mode = parseMode(values.mode);
|
|
642
686
|
const pins = await resolveSelection(client, values);
|
|
@@ -683,6 +727,10 @@ async function runOnce(values, connection, agents, agent, hue, output, signal) {
|
|
|
683
727
|
persistResultContent: values.content,
|
|
684
728
|
traceEvidence: { mode: "required" },
|
|
685
729
|
concurrency,
|
|
730
|
+
agentRevision: agent.revision,
|
|
731
|
+
// The CLI adapts to whatever the deployment serves; the library warning is for code that
|
|
732
|
+
// still reads the legacy capability itself.
|
|
733
|
+
deprecationWarnings: false,
|
|
686
734
|
signal,
|
|
687
735
|
target: agents.simulation,
|
|
688
736
|
async onProgress(event) {
|
|
@@ -820,7 +868,7 @@ async function runWorker(values, connection, adapter, agent, hue, output, signal
|
|
|
820
868
|
const client = new ObservedClient(connection);
|
|
821
869
|
const environmentClient = createEnvironmentClient(connection);
|
|
822
870
|
const wait = integer("wait", values.wait, 300, 0, 86_400);
|
|
823
|
-
const concurrency = integer("concurrency", values.concurrency, 1, 1,
|
|
871
|
+
const concurrency = integer("concurrency", values.concurrency, 1, 1, 64);
|
|
824
872
|
const maxRuns = values["max-runs"] === undefined
|
|
825
873
|
? undefined
|
|
826
874
|
: integer("max-runs", values["max-runs"], 1, 1, 1_000_000);
|
|
@@ -847,6 +895,7 @@ async function runWorker(values, connection, adapter, agent, hue, output, signal
|
|
|
847
895
|
},
|
|
848
896
|
scorers: [],
|
|
849
897
|
concurrency,
|
|
898
|
+
deprecationWarnings: false,
|
|
850
899
|
signal,
|
|
851
900
|
...(maxRuns === undefined ? {} : { maxRuns }),
|
|
852
901
|
target(inputs, tools, context) {
|
|
@@ -857,7 +906,8 @@ async function runWorker(values, connection, adapter, agent, hue, output, signal
|
|
|
857
906
|
executionId: context.executionId,
|
|
858
907
|
environmentRunId: context.environmentRunId,
|
|
859
908
|
tools,
|
|
860
|
-
|
|
909
|
+
...(context.world ? { world: context.world } : {}),
|
|
910
|
+
...(context.mcp ? { mcp: context.mcp } : {}),
|
|
861
911
|
...(context.connectionBundle ? { connectionBundle: context.connectionBundle } : {}),
|
|
862
912
|
signal,
|
|
863
913
|
});
|
|
@@ -959,8 +1009,13 @@ export async function runEvalCommand(argv) {
|
|
|
959
1009
|
throw new Error("The adapter returned generated files for a simulated-world case");
|
|
960
1010
|
return answer;
|
|
961
1011
|
}
|
|
962
|
-
: commandAdapter(values.command, timeout
|
|
963
|
-
|
|
1012
|
+
: commandAdapter(values.command, timeout, {
|
|
1013
|
+
allowHueCredentials: values["allow-hue-credentials"],
|
|
1014
|
+
}),
|
|
1015
|
+
direct: loaded ??
|
|
1016
|
+
directCommandAdapter(values.command, timeout, {
|
|
1017
|
+
allowHueCredentials: values["allow-hue-credentials"],
|
|
1018
|
+
}),
|
|
964
1019
|
};
|
|
965
1020
|
hue = createHue({ apiKey, baseUrl, serviceName: key, captureContent: values.content });
|
|
966
1021
|
return values.worker
|
package/dist/client.js
CHANGED
|
@@ -6,6 +6,7 @@ import { W3CTraceContextPropagator } from "@opentelemetry/core";
|
|
|
6
6
|
import { LoggerProvider } from "@opentelemetry/sdk-logs";
|
|
7
7
|
import { TracerProvider } from "@opentelemetry/sdk-trace";
|
|
8
8
|
import { defaultResource, resourceFromAttributes } from "@opentelemetry/resources";
|
|
9
|
+
import { HUE_SCOPE } from "./config.js";
|
|
9
10
|
import { encodeContent, noopSpan, safeSpan } from "./safety.js";
|
|
10
11
|
import { createHueTransport, HueExportError } from "./transport.js";
|
|
11
12
|
import { verifyTrace } from "./receipt.js";
|
|
@@ -197,8 +198,8 @@ export class HueClient {
|
|
|
197
198
|
}
|
|
198
199
|
this.captureContent = this.transport.options.captureContent;
|
|
199
200
|
this.enabled = this.transport.options.enabled !== false;
|
|
200
|
-
this.tracer = new ContextualTracer(this.tracerProvider.getTracer(
|
|
201
|
-
this.logger = this.loggerProvider.getLogger(
|
|
201
|
+
this.tracer = new ContextualTracer(this.tracerProvider.getTracer(HUE_SCOPE, sdkVersion), this.storage, () => this.enabled && !this.closed, () => this.transport.instrumentationFailure());
|
|
202
|
+
this.logger = this.loggerProvider.getLogger(HUE_SCOPE, sdkVersion);
|
|
202
203
|
}
|
|
203
204
|
/**
|
|
204
205
|
* Verifies that Hue stored a trace by ID, optionally waiting for expected span IDs and normalized
|
package/dist/config.d.ts
CHANGED
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
import type { HueOptions, SharedHueOptions } from "./types.js";
|
|
2
2
|
export declare const MAX_BODY_BYTES: number;
|
|
3
3
|
export declare const MAX_CONTENT_BYTES: number;
|
|
4
|
+
/** Instrumentation scope of the client's own tracer and logger. */
|
|
5
|
+
export declare const HUE_SCOPE = "@hue-run/sdk";
|
|
4
6
|
/** Loopback hostnames that may use plain HTTP without opting in. */
|
|
5
7
|
export declare function isLoopbackHost(hostname: string): boolean;
|
|
6
8
|
/** True when a validated origin exports over plain HTTP to a host other than loopback. */
|
|
7
9
|
export declare function isInsecureOrigin(baseUrl: string): boolean;
|
|
8
|
-
export declare function validateOptions(options: HueOptions): HueOptions & Required<Pick<SharedHueOptions, "captureContent" | "baseUrl" | "timeoutMillis" | "maxQueueBytes">> & {
|
|
10
|
+
export declare function validateOptions(options: HueOptions): HueOptions & Required<Pick<SharedHueOptions, "captureContent" | "baseUrl" | "timeoutMillis" | "maxQueueBytes" | "liveSpans">> & {
|
|
9
11
|
/** Project key after validation; empty for a disabled client. */
|
|
10
12
|
apiKey: string;
|
|
11
13
|
/** Service name after validation; `hue-disabled` for a disabled client. */
|
package/dist/config.js
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
export const MAX_BODY_BYTES = 1024 * 1024;
|
|
2
2
|
export const MAX_CONTENT_BYTES = 256 * 1024;
|
|
3
|
+
/** Instrumentation scope of the client's own tracer and logger. */
|
|
4
|
+
export const HUE_SCOPE = "@hue-run/sdk";
|
|
3
5
|
/** Loopback hostnames that may use plain HTTP without opting in. */
|
|
4
6
|
export function isLoopbackHost(hostname) {
|
|
5
7
|
return hostname === "localhost" || hostname === "127.0.0.1" || hostname === "[::1]";
|
|
@@ -30,6 +32,7 @@ export function validateOptions(options) {
|
|
|
30
32
|
baseUrl: "https://app.hue.run",
|
|
31
33
|
timeoutMillis: 10000,
|
|
32
34
|
maxQueueBytes: 8 * 1024 * 1024,
|
|
35
|
+
liveSpans: false,
|
|
33
36
|
};
|
|
34
37
|
}
|
|
35
38
|
if (typeof options.captureContent !== "boolean")
|
|
@@ -74,5 +77,9 @@ export function validateOptions(options) {
|
|
|
74
77
|
maxQueueBytes < 1024 ||
|
|
75
78
|
maxQueueBytes > 64 * 1024 * 1024)
|
|
76
79
|
throw new TypeError("maxQueueBytes must be 1024–67108864");
|
|
77
|
-
|
|
80
|
+
if (options.liveSpans !== undefined && typeof options.liveSpans !== "boolean")
|
|
81
|
+
throw new TypeError("liveSpans must be a boolean");
|
|
82
|
+
// Setup credentials send installer telemetry only, never in-progress placeholders.
|
|
83
|
+
const liveSpans = options.liveSpans !== false && !options.apiKey.startsWith("hue_setup_");
|
|
84
|
+
return { ...options, baseUrl: url.origin, timeoutMillis, maxQueueBytes, liveSpans };
|
|
78
85
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { ActionInput, ActionResult, CoverageGapInput, CoverageGapResult, CreateRunInput, Environment, PublishableEnvironmentDefinition, EnvironmentIdentity, EnvironmentPage, EnvironmentPageOptions, EnvironmentRun, EnvironmentSummary, EnvironmentVersion, EnvironmentVersionSummary, FinishRunInput, SealedRun, StepPage, StepPageOptions } from "./types.js";
|
|
1
|
+
import type { ActionInput, ActionResult, CoverageGapInput, CoverageGapResult, CreateRunInput, Environment, PublishableEnvironmentDefinition, EnvironmentIdentity, EnvironmentPage, EnvironmentPageOptions, EnvironmentRun, EnvironmentSummary, EnvironmentVersion, EnvironmentVersionSummary, FinishRunInput, SealedRun, StepPage, StepPageOptions, WorldEvidence, WorldEvidenceOptions } from "./types.js";
|
|
2
2
|
/** Connection and retry options for {@link createEnvironmentClient}. */
|
|
3
3
|
export interface EnvironmentClientOptions {
|
|
4
4
|
/** Project service key sent as a bearer token; server-side only. */
|
|
@@ -14,9 +14,13 @@ export interface EnvironmentClientOptions {
|
|
|
14
14
|
export declare class HueEnvironmentError extends Error {
|
|
15
15
|
/** HTTP status when Hue answered; absent for transport, timeout or parse failure. */
|
|
16
16
|
readonly status?: number | undefined;
|
|
17
|
+
/** Hue's `Retry-After` in milliseconds, bounded, when a 429 or 503 carried one. */
|
|
18
|
+
readonly retryAfterMs?: number | undefined;
|
|
17
19
|
constructor(
|
|
18
20
|
/** HTTP status when Hue answered; absent for transport, timeout or parse failure. */
|
|
19
|
-
status?: number | undefined
|
|
21
|
+
status?: number | undefined,
|
|
22
|
+
/** Hue's `Retry-After` in milliseconds, bounded, when a 429 or 503 carried one. */
|
|
23
|
+
retryAfterMs?: number | undefined);
|
|
20
24
|
}
|
|
21
25
|
/** Typed client for authored environments, isolated runs and immutable journals. */
|
|
22
26
|
export declare class EnvironmentClient {
|
|
@@ -57,6 +61,13 @@ export declare class EnvironmentClient {
|
|
|
57
61
|
sealedAt: string | null;
|
|
58
62
|
stateDigest: string;
|
|
59
63
|
finalState?: import("./types.js").JsonValue;
|
|
64
|
+
worldId?: string;
|
|
65
|
+
lifecycle?: import("./types.js").WorldLifecycle;
|
|
66
|
+
completingUntil?: string | null;
|
|
67
|
+
traceExternalId?: string | null;
|
|
68
|
+
surfaces?: import("./types.js").WorldSurface[];
|
|
69
|
+
flags?: import("./types.js").WorldFlag[];
|
|
70
|
+
connection?: null;
|
|
60
71
|
validity: "environment_incomplete" | "not_assessed";
|
|
61
72
|
coverageGap: import("./types.js").CoverageGap | null;
|
|
62
73
|
}>;
|
|
@@ -68,6 +79,9 @@ export declare class EnvironmentClient {
|
|
|
68
79
|
listSteps(runId: string, page?: StepPageOptions): Promise<StepPage>;
|
|
69
80
|
/** Seals a world as completed or abandoned and freezes its evidence. */
|
|
70
81
|
finishRun(runId: string, input: FinishRunInput): Promise<SealedRun>;
|
|
82
|
+
/** Reads a sealed world's evaluator-only evidence with the project key; a world token never
|
|
83
|
+
* can. An open world answers 409 until it is sealed. */
|
|
84
|
+
getEvidence(runId: string, options?: WorldEvidenceOptions): Promise<WorldEvidence>;
|
|
71
85
|
}
|
|
72
86
|
/** Creates a typed simulated-environment client. */
|
|
73
87
|
export declare function createEnvironmentClient(options: EnvironmentClientOptions): EnvironmentClient;
|
|
@@ -3,17 +3,38 @@ import { aggregateBounds, json, uuid, valueBounds } from "../evals/json.js";
|
|
|
3
3
|
/** Sanitized environment API failure that never includes response text or credentials. */
|
|
4
4
|
export class HueEnvironmentError extends Error {
|
|
5
5
|
status;
|
|
6
|
+
retryAfterMs;
|
|
6
7
|
constructor(
|
|
7
8
|
/** HTTP status when Hue answered; absent for transport, timeout or parse failure. */
|
|
8
|
-
status
|
|
9
|
+
status,
|
|
10
|
+
/** Hue's `Retry-After` in milliseconds, bounded, when a 429 or 503 carried one. */
|
|
11
|
+
retryAfterMs) {
|
|
9
12
|
super(status
|
|
10
13
|
? `Hue environment request failed (HTTP ${status})`
|
|
11
14
|
: "Hue environment connection or response failed");
|
|
12
15
|
this.status = status;
|
|
16
|
+
this.retryAfterMs = retryAfterMs;
|
|
13
17
|
this.name = "HueEnvironmentError";
|
|
14
18
|
}
|
|
15
19
|
}
|
|
16
20
|
const RETRYABLE = new Set([408, 429, 500, 502, 503, 504]);
|
|
21
|
+
/** A `Retry-After` longer than this waits this long: Hue asks for a second, never minutes. */
|
|
22
|
+
const MAX_RETRY_AFTER_MS = 10_000;
|
|
23
|
+
const TRACEPARENT = /^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$/;
|
|
24
|
+
/** Version 00 with real identifiers: an all-zero trace or span ID is invalid, and Hue refuses it. */
|
|
25
|
+
function validTraceparent(value) {
|
|
26
|
+
if (!TRACEPARENT.test(value))
|
|
27
|
+
return false;
|
|
28
|
+
const [, traceId, spanId] = value.split("-");
|
|
29
|
+
return !/^0+$/.test(traceId) && !/^0+$/.test(spanId);
|
|
30
|
+
}
|
|
31
|
+
/** Whole seconds only, as Hue sends them; a date or garbage is ignored. */
|
|
32
|
+
function retryAfterMillis(response) {
|
|
33
|
+
const header = response.headers.get("retry-after");
|
|
34
|
+
if (header === null || !/^\d{1,6}$/.test(header.trim()))
|
|
35
|
+
return undefined;
|
|
36
|
+
return Math.min(Number(header.trim()) * 1000, MAX_RETRY_AFTER_MS);
|
|
37
|
+
}
|
|
17
38
|
const MAX_RESPONSE_BYTES = 4 * 1024 * 1024;
|
|
18
39
|
const REQUEST_BOUNDS = { ...valueBounds, bytes: 1024 * 1024 };
|
|
19
40
|
/** The server bounds JSON inside each entity independently, then permits the parsed
|
|
@@ -62,7 +83,7 @@ export class EnvironmentClient {
|
|
|
62
83
|
}
|
|
63
84
|
if (!response.ok) {
|
|
64
85
|
await response.body?.cancel();
|
|
65
|
-
throw new HueEnvironmentError(response.status);
|
|
86
|
+
throw new HueEnvironmentError(response.status, response.status === 429 || response.status === 503 ? retryAfterMillis(response) : undefined);
|
|
66
87
|
}
|
|
67
88
|
try {
|
|
68
89
|
const reader = response.body?.getReader();
|
|
@@ -105,8 +126,10 @@ export class EnvironmentClient {
|
|
|
105
126
|
const recoverable = error.status === undefined || RETRYABLE.has(error.status);
|
|
106
127
|
if (!recoverable || attempt >= this.maxAttempts)
|
|
107
128
|
throw error;
|
|
129
|
+
// Hue's admission refusals say how long to wait; anything else backs off.
|
|
108
130
|
const backoff = Math.min(100 * 2 ** (attempt - 1), 2000);
|
|
109
|
-
|
|
131
|
+
const wait = error.retryAfterMs ?? backoff + Math.random() * backoff;
|
|
132
|
+
await new Promise((resolve) => setTimeout(resolve, wait));
|
|
110
133
|
}
|
|
111
134
|
}
|
|
112
135
|
}
|
|
@@ -157,6 +180,13 @@ export class EnvironmentClient {
|
|
|
157
180
|
throw new RangeError("ttlSeconds must be 1–86400");
|
|
158
181
|
if (input.seed !== undefined && !/^[a-f0-9]{32}$/.test(input.seed))
|
|
159
182
|
throw new TypeError("Seed must be 32 lowercase hexadecimal characters");
|
|
183
|
+
if (input.traceparent !== undefined && !validTraceparent(input.traceparent))
|
|
184
|
+
throw new TypeError("traceparent must be a version-00 W3C trace context");
|
|
185
|
+
if (input.agentRevision !== undefined &&
|
|
186
|
+
(typeof input.agentRevision !== "string" ||
|
|
187
|
+
input.agentRevision.length < 1 ||
|
|
188
|
+
input.agentRevision.length > 256))
|
|
189
|
+
throw new RangeError("agentRevision must be 1–256 characters");
|
|
160
190
|
return this.request("POST", "/environment-runs", {
|
|
161
191
|
...input,
|
|
162
192
|
environmentVersionId: uuid(input.environmentVersionId),
|
|
@@ -202,6 +232,19 @@ export class EnvironmentClient {
|
|
|
202
232
|
finishRun(runId, input) {
|
|
203
233
|
return this.request("POST", `/environment-runs/${uuid(runId)}/finish`, input);
|
|
204
234
|
}
|
|
235
|
+
/** Reads a sealed world's evaluator-only evidence with the project key; a world token never
|
|
236
|
+
* can. An open world answers 409 until it is sealed. */
|
|
237
|
+
getEvidence(runId, options = {}) {
|
|
238
|
+
const query = new URLSearchParams();
|
|
239
|
+
if (options.section !== undefined) {
|
|
240
|
+
if (!["all", "start", "end", "diff", "ledger"].includes(options.section))
|
|
241
|
+
throw new TypeError("Evidence section must be all, start, end, diff or ledger");
|
|
242
|
+
query.set("section", options.section);
|
|
243
|
+
}
|
|
244
|
+
if (options.bodies !== undefined)
|
|
245
|
+
query.set("bodies", options.bodies ? "true" : "false");
|
|
246
|
+
return this.request("GET", `/environment-runs/${uuid(runId)}/evidence${query.size ? `?${query}` : ""}`);
|
|
247
|
+
}
|
|
205
248
|
}
|
|
206
249
|
/** Creates a typed simulated-environment client. */
|
|
207
250
|
export function createEnvironmentClient(options) {
|