@hue-run/sdk 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLI.md +19 -11
- package/ENVIRONMENTS.md +61 -0
- package/README.md +16 -1
- package/dist/cli/eval.js +79 -24
- package/dist/environment/client.d.ts +16 -2
- package/dist/environment/client.js +46 -3
- package/dist/environment/types.d.ts +131 -2
- package/dist/environment/world.d.ts +50 -0
- package/dist/environment/world.js +105 -0
- package/dist/environment.d.ts +2 -0
- package/dist/environment.js +1 -0
- package/dist/evals/environment-target.d.ts +53 -2
- package/dist/evals/environment-target.js +114 -10
- package/dist/evals/local-worker.d.ts +12 -5
- package/dist/evals/local-worker.js +13 -5
- package/dist/evals/runner.d.ts +1 -1
- package/dist/evals/runner.js +2 -2
- package/dist/evals/simulation.d.ts +14 -5
- package/dist/evals/simulation.js +26 -8
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +1 -1
package/CLI.md
CHANGED
|
@@ -42,7 +42,7 @@ project manifest are refused because managers can update ancestor locks; Python
|
|
|
42
42
|
The generated `hue.setup.mjs` or `hue_setup.py` always selects `captureContent: false` /
|
|
43
43
|
`capture_content=False`. For a supported application, setup installs the dependency and adds the
|
|
44
44
|
managed import and middleware registration to the existing entrypoint; an unreferenced helper is
|
|
45
|
-
not a completed integration. TypeScript uses `@hue-run/sdk@0.
|
|
45
|
+
not a completed integration. TypeScript uses `@hue-run/sdk@0.8.0`, `@opentelemetry/api@1.9.1` and
|
|
46
46
|
`@opentelemetry/context-async-hooks@2.11.0`; Python setup uses its separately tested package pin.
|
|
47
47
|
Content capture requires an ordinary account-managed key and a later explicit application decision.
|
|
48
48
|
|
|
@@ -301,7 +301,7 @@ node packages/sdk-typescript/scripts/verify-package.mjs --artifacts-dir .artifac
|
|
|
301
301
|
# Set project to an existing supported fixture; use the same directory on resume.
|
|
302
302
|
project=/absolute/path/to/supported-fixture
|
|
303
303
|
node packages/sdk-typescript/scripts/verify-setup-live.mjs \
|
|
304
|
-
--archive .artifacts/typescript/hue-run-sdk-0.
|
|
304
|
+
--archive .artifacts/typescript/hue-run-sdk-0.8.0.tgz \
|
|
305
305
|
--origin https://STAGING_ORIGIN \
|
|
306
306
|
--project "$project" --command setup \
|
|
307
307
|
--evidence .context/setup-staging-before-claim.json
|
|
@@ -312,7 +312,7 @@ the private local handoff and finish the real browser claim, then reconcile the
|
|
|
312
312
|
|
|
313
313
|
```sh
|
|
314
314
|
node packages/sdk-typescript/scripts/verify-setup-live.mjs \
|
|
315
|
-
--archive .artifacts/typescript/hue-run-sdk-0.
|
|
315
|
+
--archive .artifacts/typescript/hue-run-sdk-0.8.0.tgz \
|
|
316
316
|
--origin https://STAGING_ORIGIN \
|
|
317
317
|
--project "$project" --command claim \
|
|
318
318
|
--evidence .context/setup-staging-after-claim.json
|
|
@@ -346,8 +346,9 @@ the CLI never prints it. The optional `zod` peer of `@hue-run/sdk/evals` must be
|
|
|
346
346
|
|
|
347
347
|
Write an adapter module that hands the case inputs and the world's tools or MCP connection to
|
|
348
348
|
the real agent. The module exports `default` or `runMyAgent`; `context` is the SDK's
|
|
349
|
-
`SimulationTargetContext` (`
|
|
350
|
-
`signal`
|
|
349
|
+
`SimulationTargetContext` (`world`, `mcp`, `tools`, `config`, `item`, `executionId`,
|
|
350
|
+
`environmentRunId`, `signal`; `world` carries the provider mirror URLs, the world token, `env` and
|
|
351
|
+
`mcpConfig` where Hue's simulation gateway serves the world):
|
|
351
352
|
|
|
352
353
|
```ts
|
|
353
354
|
// hue-agent.ts: erasable TypeScript only; Node.js 24 and Bun strip the types natively.
|
|
@@ -386,12 +387,19 @@ and sends progress to stderr. `--wait <seconds>` (default 300) bounds the verdic
|
|
|
386
387
|
Hue-owned `world_outcome` checks are graded after the world seals. An experiment always covers
|
|
387
388
|
every case of the saved version; there is no case subset.
|
|
388
389
|
|
|
389
|
-
`--command "<shell command>"` spawns the command once per case with
|
|
390
|
-
`
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
390
|
+
`--command "<shell command>"` spawns the command once per case with the world's environment
|
|
391
|
+
(`HUE_WORLD_ID`, `HUE_WORLD_TOKEN`, one `HUE_SIM_<SURFACE ID>_URL` per provider mirror,
|
|
392
|
+
`HUE_MCP_CONFIG` naming an owner-only `mcpServers` file that is removed after the case, and
|
|
393
|
+
`HUE_MCP_URL`, `HUE_MCP_TOKEN`, `HUE_MCP_EXPIRES_AT` for the first MCP mirror), plus
|
|
394
|
+
`HUE_EXECUTION_ID`, `HUE_ENVIRONMENT_RUN_ID`, `HUE_CASE_ID` and `HUE_CASE_KEY`, and
|
|
395
|
+
`{"inputs": ..., "config": ...}` on stdin. The child does not receive `HUE_API_KEY`, `HUE_MCP_KEY`
|
|
396
|
+
or any other Hue control-plane credential unless `--allow-hue-credentials` is passed; the rest of
|
|
397
|
+
the parent environment (model keys, application settings) is inherited. Its stdout is the answer
|
|
398
|
+
(JSON when it parses, otherwise trimmed text; empty means no output); a non-zero exit or the
|
|
399
|
+
per-case `--timeout` (default 600 seconds) is a target failure. The world token is never logged.
|
|
400
|
+
A world created while the deployment's simulation gateway is off gets the legacy `hue_sim_`
|
|
401
|
+
capability under the same `HUE_MCP_*` names. The agent key defaults to the slug of the command's
|
|
402
|
+
script name; `--revision` is sent to Hue as the agent revision of every world.
|
|
395
403
|
|
|
396
404
|
`--worker` registers the adapter through `runLocalAgent()` with key `--agent-key` (default: the
|
|
397
405
|
adapter filename slug), name `--agent-name` (default: the key), revision `--revision` (default:
|
package/ENVIRONMENTS.md
CHANGED
|
@@ -196,6 +196,67 @@ An observation with `status: "error"` is a recorded world answer, not a transpor
|
|
|
196
196
|
Run mutations retry with stable invocation/idempotency identities. Registry writes do not retry
|
|
197
197
|
automatically because identity creation and publication have no request key.
|
|
198
198
|
|
|
199
|
+
## Worlds served by the simulation gateway
|
|
200
|
+
|
|
201
|
+
Where a Hue deployment has the simulation gateway on, `createRun` returns the World API handoff
|
|
202
|
+
beside the run: `token` (a `hue_world_…` credential that lives exactly as long as the world),
|
|
203
|
+
`surfaces[]` (one mirror URL per pinned provider surface, such as the Gmail MCP and REST mirrors),
|
|
204
|
+
`env` (`HUE_WORLD_ID`, `HUE_WORLD_TOKEN`, `BAGGAGE`, `TRACEPARENT` and one
|
|
205
|
+
`HUE_SIM_<SURFACE ID>_URL` per surface) and `mcpConfig` (the common `mcpServers` shape with the
|
|
206
|
+
token in the `Authorization` header). The agent is pointed at the mirrors by configuration only:
|
|
207
|
+
its own Gmail MCP or REST client, the mirror URL, the world token where the Google credential went.
|
|
208
|
+
|
|
209
|
+
```ts
|
|
210
|
+
import {
|
|
211
|
+
agentEnvironment,
|
|
212
|
+
createEnvironmentClient,
|
|
213
|
+
worldHandoff,
|
|
214
|
+
writeMcpConfig,
|
|
215
|
+
} from "@hue-run/sdk/environment";
|
|
216
|
+
|
|
217
|
+
const run = await environmentClient.createRun({
|
|
218
|
+
idempotencyKey: `execution:${executionId}`,
|
|
219
|
+
environmentVersionId,
|
|
220
|
+
executionId,
|
|
221
|
+
ttlSeconds: 600,
|
|
222
|
+
traceparent: `00-${span.traceId}-${span.spanId}-01`, // parents the world span on the case span
|
|
223
|
+
agentRevision: "my-agent@1.4.2", // joins the world's fingerprint
|
|
224
|
+
});
|
|
225
|
+
const world = worldHandoff(run); // null for a world created while the gateway is off
|
|
226
|
+
const child = agentEnvironment(world!, { parent: process.env }); // no HUE_API_KEY in the agent
|
|
227
|
+
const config = await writeMcpConfig(world!); // owner-only mcp.json; dispose after the run
|
|
228
|
+
try {
|
|
229
|
+
await spawnAgent({ env: child, mcpConfigPath: config.path });
|
|
230
|
+
} finally {
|
|
231
|
+
await config.dispose();
|
|
232
|
+
await environmentClient.finishRun(run.id, {
|
|
233
|
+
idempotencyKey: `execution:${executionId}:completed`,
|
|
234
|
+
status: "completed",
|
|
235
|
+
});
|
|
236
|
+
}
|
|
237
|
+
```
|
|
238
|
+
|
|
239
|
+
`runSimulation`, `runLocalAgent` and `hue eval` do this for you: they create the world with the
|
|
240
|
+
execution, the stable key, the case span's context and the agent revision, pass the handoff as
|
|
241
|
+
`context.world`, and finish before returning so telemetry is flushed and the execution completed
|
|
242
|
+
afterwards. For one compatibility release `context.mcp` is the world's first MCP mirror with the
|
|
243
|
+
world token, so an adapter that read `HUE_MCP_URL` and `HUE_MCP_TOKEN` keeps working;
|
|
244
|
+
`agentEnvironment` sets those names too unless `legacyMcpVariables: false`. A gateway world binds
|
|
245
|
+
no Hue-native `tools` (Hue refuses them); a world created while the gateway is off keeps its tools
|
|
246
|
+
and the `hue_sim_` capability and emits a one-time `DeprecationWarning`.
|
|
247
|
+
|
|
248
|
+
`agentEnvironment` removes Hue control-plane credentials from the child by default: `HUE_API_KEY`,
|
|
249
|
+
`HUE_MCP_KEY` and any variable whose value is a `hue_sk_`, `hue_mcp_` or `hue_attempt_`
|
|
250
|
+
credential. Pass `includeHueCredentials: true` only for an agent that must call Hue's own API.
|
|
251
|
+
Nothing in these helpers logs the token; keep it out of your own logs and checkpoints.
|
|
252
|
+
|
|
253
|
+
Finish answers `lifecycle: "completing"` with `sealedAt: null` for a gateway world: the seal
|
|
254
|
+
follows a 5 s grace so in-flight writes land, and a late finish answers 409, which the helpers
|
|
255
|
+
treat as the seal they can no longer change. `getEvidence(runId, { section, bodies })` reads the
|
|
256
|
+
sealed world's evaluator-only evidence (start and end state, the diff, the call ledger, coverage,
|
|
257
|
+
fingerprint) with the project key; a world token can never read it. The client honors Hue's
|
|
258
|
+
`Retry-After` on 429 and 503.
|
|
259
|
+
|
|
199
260
|
## Coverage gaps
|
|
200
261
|
|
|
201
262
|
A provider adapter can record a known valid provider request that the environment cannot
|
package/README.md
CHANGED
|
@@ -358,13 +358,28 @@ tick, the transport queues a placeholder: an ordinary OTLP span whose parent is
|
|
|
358
358
|
with its name, kind, start time and current attributes, an end time of 0,
|
|
359
359
|
`hue.span_type = "pending_span"` and `hue.pending_parent_id` (the running span's own parent,
|
|
360
360
|
omitted for a root). Hue shows the span as running and replaces the placeholder when the real span
|
|
361
|
-
arrives.
|
|
361
|
+
arrives.
|
|
362
|
+
|
|
363
|
+
The placeholder then waits for the next batch export like any queued span: up to 500 ms until the
|
|
364
|
+
next tick, then the 1 s batch delay when nothing else is queued, so it usually reaches Hue within
|
|
365
|
+
about 1.5 s of its span starting. It goes out sooner when a batch is already scheduled, for example
|
|
366
|
+
because another span has just ended, and at once for a full batch or `flush()`. It goes out later
|
|
367
|
+
while an earlier export is still in flight, because the next batch is scheduled only after that
|
|
368
|
+
export finishes. A placeholder whose span has ended by the time it is exported is not sent, so a
|
|
369
|
+
short span may send none and appear in Hue only when it finishes.
|
|
362
370
|
|
|
363
371
|
- Only spans from the client's tracer (`withSpan`, `tool`, `model`, `hue.tracer` and the AI SDK
|
|
364
372
|
adapters) and spans with a `gen_ai.`, `ai.`, `llm.` or `traceloop.` attribute at start, or a
|
|
365
373
|
name starting with `ai.`, are announced. HTTP, database and other framework spans are not.
|
|
366
374
|
- Placeholder attributes follow `captureContent` and `redact` like the real span. Tool
|
|
367
375
|
definitions, system instructions and any value over 64 KiB are left out.
|
|
376
|
+
- The transport builds each placeholder from the running span itself, so code in a wrapping
|
|
377
|
+
processor's `onEnd` never runs on it. If a wrapper forwards `onStart` to Hue but scrubs
|
|
378
|
+
attributes, renames the span or drops it in `onEnd`, a placeholder exported while the span is
|
|
379
|
+
still open is sent anyway: it has the span's original name, and its attributes as set on the span
|
|
380
|
+
with `captureContent` and `redact` applied. Scrub with `redact`, which applies to placeholders
|
|
381
|
+
too, or before the value is set on the span; do not forward `onStart` for spans you rename or
|
|
382
|
+
drop; or turn live spans off with `liveSpans: false`.
|
|
368
383
|
- Placeholders are advisory. They are queued only while the queue is under a quarter of its
|
|
369
384
|
record and byte budgets, and skipped silently otherwise. While queued they count in
|
|
370
385
|
`pendingSpans` and `pendingBytes`, but never as accepted, rejected, failed or dropped records.
|
package/dist/cli/eval.js
CHANGED
|
@@ -6,6 +6,7 @@ import { parseArgs } from "node:util";
|
|
|
6
6
|
import { randomUUID } from "node:crypto";
|
|
7
7
|
import { createHue } from "../client.js";
|
|
8
8
|
import { createEnvironmentClient } from "../environment/client.js";
|
|
9
|
+
import { agentEnvironment, stripHueControlPlaneCredentials, writeMcpConfig, } from "../environment/world.js";
|
|
9
10
|
import { EvaluationClient, HueApiError } from "../evals/client.js";
|
|
10
11
|
import { TargetResult } from "../evals/types.js";
|
|
11
12
|
import { CheckpointStore } from "../evals/checkpoint.js";
|
|
@@ -32,9 +33,15 @@ Selection (exactly one, not used with --worker):
|
|
|
32
33
|
|
|
33
34
|
Agent (exactly one):
|
|
34
35
|
<adapter-file> Module exporting default or runMyAgent(inputs, context)
|
|
35
|
-
--command "<shell command>" Simulation: spawned per case with
|
|
36
|
-
|
|
37
|
-
|
|
36
|
+
--command "<shell command>" Simulation: spawned per case with the world's environment
|
|
37
|
+
(HUE_WORLD_ID, HUE_WORLD_TOKEN, one HUE_SIM_<SURFACE>_URL per
|
|
38
|
+
provider mirror, HUE_MCP_CONFIG, plus HUE_MCP_URL, HUE_MCP_TOKEN
|
|
39
|
+
and HUE_MCP_EXPIRES_AT for the first MCP mirror), HUE_EXECUTION_ID,
|
|
40
|
+
HUE_ENVIRONMENT_RUN_ID, HUE_CASE_ID and HUE_CASE_KEY set;
|
|
41
|
+
{"inputs","config"} on stdin. HUE_API_KEY and other Hue
|
|
42
|
+
control-plane credentials are removed from the child.
|
|
43
|
+
--allow-hue-credentials Keep HUE_API_KEY and other Hue control-plane credentials in
|
|
44
|
+
the --command child (off by default)
|
|
38
45
|
Direct: spawned in a private case directory with HUE_CASE_DIR,
|
|
39
46
|
HUE_CASE_INPUTS, HUE_CASE_OUTPUT_DIR, HUE_CASE_ID, HUE_CASE_KEY
|
|
40
47
|
and HUE_EXECUTION_ID set; files/<role>/ hold the pinned inputs
|
|
@@ -59,7 +66,7 @@ Output and limits:
|
|
|
59
66
|
outputs/explanations (--worker always persists them)
|
|
60
67
|
--save-version Freeze an unsaved eval-set version before running
|
|
61
68
|
--checkpoint-dir <path> Private checkpoint directory (default: .hue/eval/<agent-key>)
|
|
62
|
-
--concurrency <n> Cases in flight, 1-
|
|
69
|
+
--concurrency <n> Cases in flight, 1-64 (default: 1)
|
|
63
70
|
--timeout <seconds> Per-case --command timeout (default: 600)
|
|
64
71
|
--wait <seconds> Verdict wait after the run finishes (default: 300)
|
|
65
72
|
-h, --help Show this help
|
|
@@ -92,6 +99,7 @@ function parse(argv) {
|
|
|
92
99
|
"scorer-version": { type: "string", multiple: true },
|
|
93
100
|
mode: { type: "string" },
|
|
94
101
|
command: { type: "string" },
|
|
102
|
+
"allow-hue-credentials": { type: "boolean", default: false },
|
|
95
103
|
worker: { type: "boolean", default: false },
|
|
96
104
|
"max-runs": { type: "string" },
|
|
97
105
|
"agent-key": { type: "string" },
|
|
@@ -296,29 +304,65 @@ function parseAnswer(text) {
|
|
|
296
304
|
return text;
|
|
297
305
|
}
|
|
298
306
|
}
|
|
299
|
-
/**
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
+
/**
|
|
308
|
+
* Runs the shell command once per case. The world token and mirror URLs travel only through the
|
|
309
|
+
* child's environment and an owner-only MCP configuration file that is removed after the run;
|
|
310
|
+
* Hue control-plane credentials (`HUE_API_KEY`, `HUE_MCP_KEY`, any `hue_sk_` value) stay with
|
|
311
|
+
* the CLI unless `--allow-hue-credentials` is passed. A world created while the gateway is off
|
|
312
|
+
* still gets the `hue_sim_` capability under the same `HUE_MCP_*` names.
|
|
313
|
+
*/
|
|
314
|
+
/** The parent environment an agent child starts from: without Hue control-plane credentials
|
|
315
|
+
* unless `--allow-hue-credentials` was passed. */
|
|
316
|
+
function parentEnvironment(allowHueCredentials) {
|
|
317
|
+
return allowHueCredentials
|
|
318
|
+
? Object.fromEntries(Object.entries(process.env).filter((entry) => entry[1] !== undefined))
|
|
319
|
+
: stripHueControlPlaneCredentials(process.env);
|
|
320
|
+
}
|
|
321
|
+
function commandAdapter(command, timeoutSeconds, options) {
|
|
322
|
+
return async (inputs, context) => {
|
|
323
|
+
const parent = parentEnvironment(options.allowHueCredentials);
|
|
324
|
+
const identity = {
|
|
307
325
|
HUE_EXECUTION_ID: context.executionId,
|
|
308
326
|
HUE_ENVIRONMENT_RUN_ID: context.environmentRunId,
|
|
309
327
|
HUE_CASE_ID: context.item.id,
|
|
310
328
|
HUE_CASE_KEY: context.item.externalKey,
|
|
311
|
-
}
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
329
|
+
};
|
|
330
|
+
const configFile = context.world ? await writeMcpConfig(context.world) : undefined;
|
|
331
|
+
try {
|
|
332
|
+
const env = context.world
|
|
333
|
+
? {
|
|
334
|
+
...agentEnvironment(context.world, { parent, includeHueCredentials: true }),
|
|
335
|
+
...identity,
|
|
336
|
+
HUE_MCP_CONFIG: configFile.path,
|
|
337
|
+
}
|
|
338
|
+
: {
|
|
339
|
+
...parent,
|
|
340
|
+
...(context.mcp
|
|
341
|
+
? {
|
|
342
|
+
HUE_MCP_URL: context.mcp.url,
|
|
343
|
+
HUE_MCP_TOKEN: context.mcp.token,
|
|
344
|
+
HUE_MCP_EXPIRES_AT: context.mcp.expiresAt,
|
|
345
|
+
}
|
|
346
|
+
: {}),
|
|
347
|
+
...identity,
|
|
348
|
+
};
|
|
349
|
+
return parseAnswer(await spawnAgentCommand(command, {
|
|
350
|
+
env,
|
|
351
|
+
stdin: JSON.stringify({ inputs, config: context.config }),
|
|
352
|
+
timeoutSeconds,
|
|
353
|
+
...(context.signal ? { signal: context.signal } : {}),
|
|
354
|
+
}));
|
|
355
|
+
}
|
|
356
|
+
finally {
|
|
357
|
+
await configFile?.dispose();
|
|
358
|
+
}
|
|
359
|
+
};
|
|
316
360
|
}
|
|
317
361
|
/**
|
|
318
362
|
* Direct cases: the command works in a private case directory and writes its documents to
|
|
319
363
|
* `output/`. Its stdout is only used as the JSON output when it wrote no result or summary file.
|
|
320
364
|
*/
|
|
321
|
-
function directCommandAdapter(command, timeoutSeconds) {
|
|
365
|
+
function directCommandAdapter(command, timeoutSeconds, options) {
|
|
322
366
|
return async (inputs, context) => {
|
|
323
367
|
const layout = await stageDirectCase(context.outputDirectory, {
|
|
324
368
|
inputs,
|
|
@@ -330,7 +374,7 @@ function directCommandAdapter(command, timeoutSeconds) {
|
|
|
330
374
|
const stdout = await spawnAgentCommand(command, {
|
|
331
375
|
cwd: layout.caseDirectory,
|
|
332
376
|
env: {
|
|
333
|
-
...
|
|
377
|
+
...parentEnvironment(options.allowHueCredentials),
|
|
334
378
|
HUE_CASE_DIR: layout.caseDirectory,
|
|
335
379
|
HUE_CASE_INPUTS: layout.inputsPath,
|
|
336
380
|
HUE_CASE_OUTPUT_DIR: layout.outputDirectory,
|
|
@@ -636,7 +680,7 @@ function parseBaseline(value) {
|
|
|
636
680
|
}
|
|
637
681
|
async function runOnce(values, connection, agents, agent, hue, output, signal) {
|
|
638
682
|
const client = new EvaluationClient(connection);
|
|
639
|
-
const concurrency = integer("concurrency", values.concurrency, 1, 1,
|
|
683
|
+
const concurrency = integer("concurrency", values.concurrency, 1, 1, 64);
|
|
640
684
|
const baselineId = parseBaseline(values.baseline);
|
|
641
685
|
const mode = parseMode(values.mode);
|
|
642
686
|
const pins = await resolveSelection(client, values);
|
|
@@ -683,6 +727,10 @@ async function runOnce(values, connection, agents, agent, hue, output, signal) {
|
|
|
683
727
|
persistResultContent: values.content,
|
|
684
728
|
traceEvidence: { mode: "required" },
|
|
685
729
|
concurrency,
|
|
730
|
+
agentRevision: agent.revision,
|
|
731
|
+
// The CLI adapts to whatever the deployment serves; the library warning is for code that
|
|
732
|
+
// still reads the legacy capability itself.
|
|
733
|
+
deprecationWarnings: false,
|
|
686
734
|
signal,
|
|
687
735
|
target: agents.simulation,
|
|
688
736
|
async onProgress(event) {
|
|
@@ -820,7 +868,7 @@ async function runWorker(values, connection, adapter, agent, hue, output, signal
|
|
|
820
868
|
const client = new ObservedClient(connection);
|
|
821
869
|
const environmentClient = createEnvironmentClient(connection);
|
|
822
870
|
const wait = integer("wait", values.wait, 300, 0, 86_400);
|
|
823
|
-
const concurrency = integer("concurrency", values.concurrency, 1, 1,
|
|
871
|
+
const concurrency = integer("concurrency", values.concurrency, 1, 1, 64);
|
|
824
872
|
const maxRuns = values["max-runs"] === undefined
|
|
825
873
|
? undefined
|
|
826
874
|
: integer("max-runs", values["max-runs"], 1, 1, 1_000_000);
|
|
@@ -847,6 +895,7 @@ async function runWorker(values, connection, adapter, agent, hue, output, signal
|
|
|
847
895
|
},
|
|
848
896
|
scorers: [],
|
|
849
897
|
concurrency,
|
|
898
|
+
deprecationWarnings: false,
|
|
850
899
|
signal,
|
|
851
900
|
...(maxRuns === undefined ? {} : { maxRuns }),
|
|
852
901
|
target(inputs, tools, context) {
|
|
@@ -857,7 +906,8 @@ async function runWorker(values, connection, adapter, agent, hue, output, signal
|
|
|
857
906
|
executionId: context.executionId,
|
|
858
907
|
environmentRunId: context.environmentRunId,
|
|
859
908
|
tools,
|
|
860
|
-
|
|
909
|
+
...(context.world ? { world: context.world } : {}),
|
|
910
|
+
...(context.mcp ? { mcp: context.mcp } : {}),
|
|
861
911
|
...(context.connectionBundle ? { connectionBundle: context.connectionBundle } : {}),
|
|
862
912
|
signal,
|
|
863
913
|
});
|
|
@@ -959,8 +1009,13 @@ export async function runEvalCommand(argv) {
|
|
|
959
1009
|
throw new Error("The adapter returned generated files for a simulated-world case");
|
|
960
1010
|
return answer;
|
|
961
1011
|
}
|
|
962
|
-
: commandAdapter(values.command, timeout
|
|
963
|
-
|
|
1012
|
+
: commandAdapter(values.command, timeout, {
|
|
1013
|
+
allowHueCredentials: values["allow-hue-credentials"],
|
|
1014
|
+
}),
|
|
1015
|
+
direct: loaded ??
|
|
1016
|
+
directCommandAdapter(values.command, timeout, {
|
|
1017
|
+
allowHueCredentials: values["allow-hue-credentials"],
|
|
1018
|
+
}),
|
|
964
1019
|
};
|
|
965
1020
|
hue = createHue({ apiKey, baseUrl, serviceName: key, captureContent: values.content });
|
|
966
1021
|
return values.worker
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { ActionInput, ActionResult, CoverageGapInput, CoverageGapResult, CreateRunInput, Environment, PublishableEnvironmentDefinition, EnvironmentIdentity, EnvironmentPage, EnvironmentPageOptions, EnvironmentRun, EnvironmentSummary, EnvironmentVersion, EnvironmentVersionSummary, FinishRunInput, SealedRun, StepPage, StepPageOptions } from "./types.js";
|
|
1
|
+
import type { ActionInput, ActionResult, CoverageGapInput, CoverageGapResult, CreateRunInput, Environment, PublishableEnvironmentDefinition, EnvironmentIdentity, EnvironmentPage, EnvironmentPageOptions, EnvironmentRun, EnvironmentSummary, EnvironmentVersion, EnvironmentVersionSummary, FinishRunInput, SealedRun, StepPage, StepPageOptions, WorldEvidence, WorldEvidenceOptions } from "./types.js";
|
|
2
2
|
/** Connection and retry options for {@link createEnvironmentClient}. */
|
|
3
3
|
export interface EnvironmentClientOptions {
|
|
4
4
|
/** Project service key sent as a bearer token; server-side only. */
|
|
@@ -14,9 +14,13 @@ export interface EnvironmentClientOptions {
|
|
|
14
14
|
export declare class HueEnvironmentError extends Error {
|
|
15
15
|
/** HTTP status when Hue answered; absent for transport, timeout or parse failure. */
|
|
16
16
|
readonly status?: number | undefined;
|
|
17
|
+
/** Hue's `Retry-After` in milliseconds, bounded, when a 429 or 503 carried one. */
|
|
18
|
+
readonly retryAfterMs?: number | undefined;
|
|
17
19
|
constructor(
|
|
18
20
|
/** HTTP status when Hue answered; absent for transport, timeout or parse failure. */
|
|
19
|
-
status?: number | undefined
|
|
21
|
+
status?: number | undefined,
|
|
22
|
+
/** Hue's `Retry-After` in milliseconds, bounded, when a 429 or 503 carried one. */
|
|
23
|
+
retryAfterMs?: number | undefined);
|
|
20
24
|
}
|
|
21
25
|
/** Typed client for authored environments, isolated runs and immutable journals. */
|
|
22
26
|
export declare class EnvironmentClient {
|
|
@@ -57,6 +61,13 @@ export declare class EnvironmentClient {
|
|
|
57
61
|
sealedAt: string | null;
|
|
58
62
|
stateDigest: string;
|
|
59
63
|
finalState?: import("./types.js").JsonValue;
|
|
64
|
+
worldId?: string;
|
|
65
|
+
lifecycle?: import("./types.js").WorldLifecycle;
|
|
66
|
+
completingUntil?: string | null;
|
|
67
|
+
traceExternalId?: string | null;
|
|
68
|
+
surfaces?: import("./types.js").WorldSurface[];
|
|
69
|
+
flags?: import("./types.js").WorldFlag[];
|
|
70
|
+
connection?: null;
|
|
60
71
|
validity: "environment_incomplete" | "not_assessed";
|
|
61
72
|
coverageGap: import("./types.js").CoverageGap | null;
|
|
62
73
|
}>;
|
|
@@ -68,6 +79,9 @@ export declare class EnvironmentClient {
|
|
|
68
79
|
listSteps(runId: string, page?: StepPageOptions): Promise<StepPage>;
|
|
69
80
|
/** Seals a world as completed or abandoned and freezes its evidence. */
|
|
70
81
|
finishRun(runId: string, input: FinishRunInput): Promise<SealedRun>;
|
|
82
|
+
/** Reads a sealed world's evaluator-only evidence with the project key; a world token never
|
|
83
|
+
* can. An open world answers 409 until it is sealed. */
|
|
84
|
+
getEvidence(runId: string, options?: WorldEvidenceOptions): Promise<WorldEvidence>;
|
|
71
85
|
}
|
|
72
86
|
/** Creates a typed simulated-environment client. */
|
|
73
87
|
export declare function createEnvironmentClient(options: EnvironmentClientOptions): EnvironmentClient;
|
|
@@ -3,17 +3,38 @@ import { aggregateBounds, json, uuid, valueBounds } from "../evals/json.js";
|
|
|
3
3
|
/** Sanitized environment API failure that never includes response text or credentials. */
|
|
4
4
|
export class HueEnvironmentError extends Error {
|
|
5
5
|
status;
|
|
6
|
+
retryAfterMs;
|
|
6
7
|
constructor(
|
|
7
8
|
/** HTTP status when Hue answered; absent for transport, timeout or parse failure. */
|
|
8
|
-
status
|
|
9
|
+
status,
|
|
10
|
+
/** Hue's `Retry-After` in milliseconds, bounded, when a 429 or 503 carried one. */
|
|
11
|
+
retryAfterMs) {
|
|
9
12
|
super(status
|
|
10
13
|
? `Hue environment request failed (HTTP ${status})`
|
|
11
14
|
: "Hue environment connection or response failed");
|
|
12
15
|
this.status = status;
|
|
16
|
+
this.retryAfterMs = retryAfterMs;
|
|
13
17
|
this.name = "HueEnvironmentError";
|
|
14
18
|
}
|
|
15
19
|
}
|
|
16
20
|
const RETRYABLE = new Set([408, 429, 500, 502, 503, 504]);
|
|
21
|
+
/** A `Retry-After` longer than this waits this long: Hue asks for a second, never minutes. */
|
|
22
|
+
const MAX_RETRY_AFTER_MS = 10_000;
|
|
23
|
+
const TRACEPARENT = /^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$/;
|
|
24
|
+
/** Version 00 with real identifiers: an all-zero trace or span ID is invalid, and Hue refuses it. */
|
|
25
|
+
function validTraceparent(value) {
|
|
26
|
+
if (!TRACEPARENT.test(value))
|
|
27
|
+
return false;
|
|
28
|
+
const [, traceId, spanId] = value.split("-");
|
|
29
|
+
return !/^0+$/.test(traceId) && !/^0+$/.test(spanId);
|
|
30
|
+
}
|
|
31
|
+
/** Whole seconds only, as Hue sends them; a date or garbage is ignored. */
|
|
32
|
+
function retryAfterMillis(response) {
|
|
33
|
+
const header = response.headers.get("retry-after");
|
|
34
|
+
if (header === null || !/^\d{1,6}$/.test(header.trim()))
|
|
35
|
+
return undefined;
|
|
36
|
+
return Math.min(Number(header.trim()) * 1000, MAX_RETRY_AFTER_MS);
|
|
37
|
+
}
|
|
17
38
|
const MAX_RESPONSE_BYTES = 4 * 1024 * 1024;
|
|
18
39
|
const REQUEST_BOUNDS = { ...valueBounds, bytes: 1024 * 1024 };
|
|
19
40
|
/** The server bounds JSON inside each entity independently, then permits the parsed
|
|
@@ -62,7 +83,7 @@ export class EnvironmentClient {
|
|
|
62
83
|
}
|
|
63
84
|
if (!response.ok) {
|
|
64
85
|
await response.body?.cancel();
|
|
65
|
-
throw new HueEnvironmentError(response.status);
|
|
86
|
+
throw new HueEnvironmentError(response.status, response.status === 429 || response.status === 503 ? retryAfterMillis(response) : undefined);
|
|
66
87
|
}
|
|
67
88
|
try {
|
|
68
89
|
const reader = response.body?.getReader();
|
|
@@ -105,8 +126,10 @@ export class EnvironmentClient {
|
|
|
105
126
|
const recoverable = error.status === undefined || RETRYABLE.has(error.status);
|
|
106
127
|
if (!recoverable || attempt >= this.maxAttempts)
|
|
107
128
|
throw error;
|
|
129
|
+
// Hue's admission refusals say how long to wait; anything else backs off.
|
|
108
130
|
const backoff = Math.min(100 * 2 ** (attempt - 1), 2000);
|
|
109
|
-
|
|
131
|
+
const wait = error.retryAfterMs ?? backoff + Math.random() * backoff;
|
|
132
|
+
await new Promise((resolve) => setTimeout(resolve, wait));
|
|
110
133
|
}
|
|
111
134
|
}
|
|
112
135
|
}
|
|
@@ -157,6 +180,13 @@ export class EnvironmentClient {
|
|
|
157
180
|
throw new RangeError("ttlSeconds must be 1–86400");
|
|
158
181
|
if (input.seed !== undefined && !/^[a-f0-9]{32}$/.test(input.seed))
|
|
159
182
|
throw new TypeError("Seed must be 32 lowercase hexadecimal characters");
|
|
183
|
+
if (input.traceparent !== undefined && !validTraceparent(input.traceparent))
|
|
184
|
+
throw new TypeError("traceparent must be a version-00 W3C trace context");
|
|
185
|
+
if (input.agentRevision !== undefined &&
|
|
186
|
+
(typeof input.agentRevision !== "string" ||
|
|
187
|
+
input.agentRevision.length < 1 ||
|
|
188
|
+
input.agentRevision.length > 256))
|
|
189
|
+
throw new RangeError("agentRevision must be 1–256 characters");
|
|
160
190
|
return this.request("POST", "/environment-runs", {
|
|
161
191
|
...input,
|
|
162
192
|
environmentVersionId: uuid(input.environmentVersionId),
|
|
@@ -202,6 +232,19 @@ export class EnvironmentClient {
|
|
|
202
232
|
finishRun(runId, input) {
|
|
203
233
|
return this.request("POST", `/environment-runs/${uuid(runId)}/finish`, input);
|
|
204
234
|
}
|
|
235
|
+
/** Reads a sealed world's evaluator-only evidence with the project key; a world token never
|
|
236
|
+
* can. An open world answers 409 until it is sealed. */
|
|
237
|
+
getEvidence(runId, options = {}) {
|
|
238
|
+
const query = new URLSearchParams();
|
|
239
|
+
if (options.section !== undefined) {
|
|
240
|
+
if (!["all", "start", "end", "diff", "ledger"].includes(options.section))
|
|
241
|
+
throw new TypeError("Evidence section must be all, start, end, diff or ledger");
|
|
242
|
+
query.set("section", options.section);
|
|
243
|
+
}
|
|
244
|
+
if (options.bodies !== undefined)
|
|
245
|
+
query.set("bodies", options.bodies ? "true" : "false");
|
|
246
|
+
return this.request("GET", `/environment-runs/${uuid(runId)}/evidence${query.size ? `?${query}` : ""}`);
|
|
247
|
+
}
|
|
205
248
|
}
|
|
206
249
|
/** Creates a typed simulated-environment client. */
|
|
207
250
|
export function createEnvironmentClient(options) {
|