@hue-run/sdk 0.12.1 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLI.md +6 -4
- package/ENVIRONMENTS.md +18 -4
- package/README.md +29 -13
- package/dist/client.d.ts +8 -0
- package/dist/client.js +23 -2
- package/dist/environment/types.d.ts +8 -1
- package/dist/environment/world.d.ts +8 -0
- package/dist/environment/world.js +17 -0
- package/dist/environment.d.ts +1 -1
- package/dist/environment.js +1 -1
- package/dist/evals/failure.d.ts +17 -0
- package/dist/evals/failure.js +102 -0
- package/dist/evals/runner.js +21 -12
- package/dist/evals/types.d.ts +6 -0
- package/dist/privacy.d.ts +18 -0
- package/dist/privacy.js +147 -57
- package/dist/safety.d.ts +18 -0
- package/dist/safety.js +76 -23
- package/dist/transport.d.ts +5 -0
- package/dist/transport.js +277 -57
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +1 -1
package/CLI.md
CHANGED
|
@@ -42,7 +42,7 @@ project manifest are refused because managers can update ancestor locks; Python
|
|
|
42
42
|
The generated `hue.setup.mjs` or `hue_setup.py` always selects `captureContent: false` /
|
|
43
43
|
`capture_content=False`. For a supported application, setup installs the dependency and adds the
|
|
44
44
|
managed import and middleware registration to the existing entrypoint; an unreferenced helper is
|
|
45
|
-
not a completed integration. TypeScript uses `@hue-run/sdk@0.
|
|
45
|
+
not a completed integration. TypeScript uses `@hue-run/sdk@0.13.0`, `@opentelemetry/api@1.9.1` and
|
|
46
46
|
`@opentelemetry/context-async-hooks@2.11.0`; Python setup uses its separately tested package pin.
|
|
47
47
|
Content capture requires an ordinary account-managed key and a later explicit application decision.
|
|
48
48
|
|
|
@@ -469,7 +469,7 @@ node packages/sdk-typescript/scripts/verify-package.mjs --artifacts-dir .artifac
|
|
|
469
469
|
# Set project to an existing supported fixture; use the same directory on resume.
|
|
470
470
|
project=/absolute/path/to/supported-fixture
|
|
471
471
|
node packages/sdk-typescript/scripts/verify-setup-live.mjs \
|
|
472
|
-
--archive .artifacts/typescript/hue-run-sdk-0.
|
|
472
|
+
--archive .artifacts/typescript/hue-run-sdk-0.13.0.tgz \
|
|
473
473
|
--origin https://STAGING_ORIGIN \
|
|
474
474
|
--project "$project" --command setup \
|
|
475
475
|
--evidence .context/setup-staging-before-claim.json
|
|
@@ -480,7 +480,7 @@ the private local handoff and finish the real browser claim, then reconcile the
|
|
|
480
480
|
|
|
481
481
|
```sh
|
|
482
482
|
node packages/sdk-typescript/scripts/verify-setup-live.mjs \
|
|
483
|
-
--archive .artifacts/typescript/hue-run-sdk-0.
|
|
483
|
+
--archive .artifacts/typescript/hue-run-sdk-0.13.0.tgz \
|
|
484
484
|
--origin https://STAGING_ORIGIN \
|
|
485
485
|
--project "$project" --command claim \
|
|
486
486
|
--evidence .context/setup-staging-after-claim.json
|
|
@@ -576,7 +576,9 @@ Hue-owned `world_outcome` checks are graded after the world seals. An experiment
|
|
|
576
576
|
every case of the saved version; there is no case subset.
|
|
577
577
|
|
|
578
578
|
`--command "<shell command>"` spawns the command once per case with the world's environment
|
|
579
|
-
(`HUE_WORLD_ID`, `HUE_WORLD_TOKEN`,
|
|
579
|
+
(`HUE_WORLD_ID`, `HUE_WORLD_TOKEN`, `HUE_WORLD_NOW` (the world's clock at creation, RFC 3339: read
|
|
580
|
+
today's date from it, never from the wall clock), one `HUE_SIM_<SURFACE ID>_URL` per provider
|
|
581
|
+
mirror,
|
|
580
582
|
`HUE_MCP_CONFIG` naming an owner-only `mcpServers` file that is removed after the case, and
|
|
581
583
|
`HUE_MCP_URL`, `HUE_MCP_TOKEN`, `HUE_MCP_EXPIRES_AT` for the first MCP mirror; these name the
|
|
582
584
|
world's simulated MCP server and replace a `HUE_MCP_URL` that `hue login` stored for Hue's project
|
package/ENVIRONMENTS.md
CHANGED
|
@@ -220,10 +220,24 @@ automatically because identity creation and publication have no request key.
|
|
|
220
220
|
Where a Hue deployment has the simulation gateway on, `createRun` returns the World API handoff
|
|
221
221
|
beside the run: `token` (a `hue_world_…` credential that lives exactly as long as the world),
|
|
222
222
|
`surfaces[]` (one mirror URL per pinned provider surface, such as the Gmail MCP and REST mirrors),
|
|
223
|
-
`
|
|
224
|
-
`
|
|
225
|
-
|
|
226
|
-
|
|
223
|
+
`now` (the world's clock at creation as an RFC 3339 timestamp), `env` (`HUE_WORLD_ID`,
|
|
224
|
+
`HUE_WORLD_TOKEN`, `HUE_WORLD_NOW`, `BAGGAGE`, `TRACEPARENT` and one `HUE_SIM_<SURFACE ID>_URL`
|
|
225
|
+
per surface) and `mcpConfig` (the common `mcpServers` shape with the token in the `Authorization`
|
|
226
|
+
header). The agent is pointed at the mirrors by configuration only: its own Gmail MCP or REST
|
|
227
|
+
client, the mirror URL, the world token where the Google credential went.
|
|
228
|
+
|
|
229
|
+
The world has its own date. A trace-built world's clock is frozen at the recorded start, so a
|
|
230
|
+
run that begins days or months after the recording still finds the recorded emails, events and
|
|
231
|
+
records at their recorded times. An agent that reads the wall clock for "today", "tomorrow" or
|
|
232
|
+
`newer_than:7d` asks the world about dates it holds nothing for; one that reads the world's date
|
|
233
|
+
lands on the recorded ones. Read it with `worldNow`: from the handoff, or from the environment of
|
|
234
|
+
an agent child, where `agentEnvironment` and `hue eval --command` set `HUE_WORLD_NOW`.
|
|
235
|
+
|
|
236
|
+
```ts
|
|
237
|
+
import { worldNow } from "@hue-run/sdk/environment";
|
|
238
|
+
|
|
239
|
+
const today = worldNow() ?? new Date(); // HUE_WORLD_NOW in this process, else the wall clock
|
|
240
|
+
```
|
|
227
241
|
|
|
228
242
|
```ts
|
|
229
243
|
import {
|
package/README.md
CHANGED
|
@@ -45,7 +45,6 @@ await hue.withSpan(
|
|
|
45
45
|
async (span) => {
|
|
46
46
|
const output = await hue.tool("uppercase", "hello", () => "hello".toUpperCase());
|
|
47
47
|
span.setOutput(output);
|
|
48
|
-
hue.recordMessages({ output: [{ role: "assistant", content: output }] });
|
|
49
48
|
},
|
|
50
49
|
{ sessionId: "session-123", userId: "user-123", input: "hello" },
|
|
51
50
|
);
|
|
@@ -107,7 +106,8 @@ by the semantic conventions'
|
|
|
107
106
|
[input messages](https://github.com/open-telemetry/semantic-conventions/blob/v1.41.0/docs/gen-ai/gen-ai-input-messages.json)
|
|
108
107
|
and
|
|
109
108
|
[output messages](https://github.com/open-telemetry/semantic-conventions/blob/v1.41.0/docs/gen-ai/gen-ai-output-messages.json)
|
|
110
|
-
JSON schemas
|
|
109
|
+
JSON schemas. GenAI conventions are in Development; Hue supports the fields described here
|
|
110
|
+
and accepts caller-supplied JSON without validating the full draft. Convert provider-native
|
|
111
111
|
messages before recording them:
|
|
112
112
|
|
|
113
113
|
```ts
|
|
@@ -313,7 +313,7 @@ policy forbids sending that content to another service. `redact` and the credent
|
|
|
313
313
|
described below apply in both modes.
|
|
314
314
|
|
|
315
315
|
`captureContent: false` disables manual input/output/messages/tool content and
|
|
316
|
-
removes recognized GenAI, Vercel, OpenInference and
|
|
316
|
+
removes recognized GenAI, Vercel, OpenInference, OpenLLMetry and Langfuse content attributes,
|
|
317
317
|
legacy GenAI content events, log bodies, status messages and exception text before
|
|
318
318
|
export. The exported `contentPrefixes` array lists the attribute keys (and their dotted
|
|
319
319
|
children) that are removed. Model/provider/token metadata remains available. Generic custom attribute
|
|
@@ -323,7 +323,13 @@ names cannot be classified automatically; use them deliberately.
|
|
|
323
323
|
there is no SDK retention timer or automatic content expiry. To redact strings
|
|
324
324
|
before export, supply `redact(value, path)`; it applies to supported strings in
|
|
325
325
|
attributes, resources, event/link attributes and log bodies. Return a string.
|
|
326
|
-
Invalid
|
|
326
|
+
Invalid helper content is omitted with an instrumentation failure; the span can still be delivered.
|
|
327
|
+
A content value over Hue's 256 KiB value cap is not omitted: text is cut to a UTF-8 prefix and
|
|
328
|
+
bytes or a structured log body are replaced by the receiver's own marker
|
|
329
|
+
(`{ "hue.truncated": true, "hue.truncated_bytes": <size> }`), and the value's key is listed in
|
|
330
|
+
the record's `hue.truncated` attribute, where Hue's receiver lists the values it cuts itself,
|
|
331
|
+
so the span still carries the call, the recorded part of its value and the rest of its evidence.
|
|
332
|
+
Export-time redactor failures reject the affected record and are reported by flush. Shared resources are redacted once per
|
|
327
333
|
export batch. Do not put user content or secrets in span names or scope names.
|
|
328
334
|
|
|
329
335
|
Hosted tools carry credentials in their definitions, such as the `authorization` and `headers` of
|
|
@@ -355,7 +361,7 @@ rotates. Only definitions another integration recorded can be summarized: `hueTe
|
|
|
355
361
|
inputs and exports through Hue's attached processors gets the summary.
|
|
356
362
|
Recorded messages can inline files: GenAI `blob` parts in `gen_ai.input.messages` /
|
|
357
363
|
`gen_ai.output.messages` (what the AI SDK 7 adapter records for a file part) and AI SDK 6 `file`
|
|
358
|
-
parts in `ai.prompt.messages`. A span whose messages exceed 256 KiB would be
|
|
364
|
+
parts in `ai.prompt.messages`. A span whose messages exceed 256 KiB would be cut, so when a
|
|
359
365
|
record is queued Hue replaces the `content`/`data` of any such part longer than 64 KiB with the
|
|
360
366
|
file's `sha256` and `size`, both of the file's own bytes whatever its media type (base64 content
|
|
361
367
|
and `;base64` `data:` URLs decoded, other `data:` URLs percent-decoded, anything else as UTF-8 text), keeping the part's other fields such as `type`, `mime_type` and `mediaType`. Smaller inline
|
|
@@ -379,8 +385,8 @@ span in `finally`.
|
|
|
379
385
|
the active span, with the messages in its body. The record also carries `gen_ai.operation.name`,
|
|
380
386
|
`gen_ai.provider.name` and `gen_ai.request.model` as attributes, copied from the enclosing
|
|
381
387
|
`hue.model()` span or passed as `operation`, `provider` and `model`, and `gen_ai.conversation.id`
|
|
382
|
-
from the active session,
|
|
383
|
-
context.
|
|
388
|
+
from the active session, when those values are known. Use `recordMessages` inside `hue.model()` or
|
|
389
|
+
supply `operation` and `provider` explicitly to include the required inference context.
|
|
384
390
|
|
|
385
391
|
## Existing OpenTelemetry providers
|
|
386
392
|
|
|
@@ -447,7 +453,8 @@ that accepts `/api/v1/otlp/v1/traces` and `/api/v1/otlp/v1/logs` (for example an
|
|
|
447
453
|
Collector `otlp` receiver with `http.traces_url_path` and `logs_url_path` set to those paths,
|
|
448
454
|
forwarding to Jaeger or the debug exporter) and pass any placeholder `apiKey`; HTTP is allowed
|
|
449
455
|
for loopback origins. `checkConnection()` and `verifyTrace()` are Hue-only diagnostics and are
|
|
450
|
-
not available against a generic collector.
|
|
456
|
+
not available against a generic collector. Set `liveSpans: false` so Hue-specific placeholders
|
|
457
|
+
do not reach the collector's other destinations.
|
|
451
458
|
|
|
452
459
|
A collector on a private network is not loopback: a docker-compose sibling such as
|
|
453
460
|
`http://otel-collector:4318` or an in-cluster service requires the explicit opt-in
|
|
@@ -461,6 +468,7 @@ const hue = createHue({
|
|
|
461
468
|
serviceName: "my-agent",
|
|
462
469
|
captureContent: true,
|
|
463
470
|
baseUrl: "http://otel-collector:4318",
|
|
471
|
+
liveSpans: false,
|
|
464
472
|
allowInsecureHttp: true,
|
|
465
473
|
});
|
|
466
474
|
```
|
|
@@ -470,7 +478,11 @@ const hue = createHue({
|
|
|
470
478
|
Exports retry temporary HTTP/network failures (429, 502, 503, 504 and connection errors,
|
|
471
479
|
honoring `Retry-After`) within the export timeout, by OpenTelemetry's OTLP/HTTP exporter rules. Each
|
|
472
480
|
request is limited to 1 MiB before gzip (with space reserved for gzip overhead) and each content value to 256 KiB. Batches
|
|
473
|
-
split at record boundaries.
|
|
481
|
+
split at record boundaries. A record over the request limit on its own sheds its content values,
|
|
482
|
+
largest first, each replaced by the receiver's marker and listed under `hue.truncated`, until it
|
|
483
|
+
fits; only a record too large without any content value is lost, reported as an `invalid` issue,
|
|
484
|
+
and counted on its trace's root span as `hue.sdk.dropped_records` when the root is exported, so
|
|
485
|
+
Hue reads the trace as incomplete by that many records. Each signal queues at most 2,048 records, including
|
|
474
486
|
exports in flight; overflow is reported through the callback, counters and next
|
|
475
487
|
flush. This is an in-memory queue, not durable storage.
|
|
476
488
|
|
|
@@ -522,10 +534,14 @@ short span may send none and appear in Hue only when it finishes.
|
|
|
522
534
|
record and byte budgets, and skipped silently otherwise. While queued they count in
|
|
523
535
|
`pendingSpans` and `pendingBytes`, but never as accepted, rejected, failed or dropped records.
|
|
524
536
|
Losing only placeholders records a warning and does not make `flush()` throw.
|
|
525
|
-
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
537
|
+
- Placeholders travel in their own requests, after an export's completed spans, so a rejection
|
|
538
|
+
count is always one kind of record's: a completed span's rejection is never credited to a
|
|
539
|
+
placeholder, and a rejected request of placeholders is a warning. A Hue server that accepts
|
|
540
|
+
placeholders sends `Hue-Pending-Spans: 1` on every trace acknowledgement. The first trace
|
|
541
|
+
acknowledgement without it, whatever its request carried, shows the receiver predates them: the
|
|
542
|
+
transport records one warning (counting the placeholders the receiver rejected in that request, if any) and stops
|
|
543
|
+
sending placeholders for that client, so a receiver that acknowledged completed spans first
|
|
544
|
+
never receives one.
|
|
529
545
|
- Opt out with `liveSpans: false`. Setup credentials never send placeholders.
|
|
530
546
|
- With an existing provider, announcements start in `spanProcessor.onStart`. A wrapping processor
|
|
531
547
|
that forwards `onStart` should forward `onEnd` for the same spans: a span that ends without
|
package/dist/client.d.ts
CHANGED
|
@@ -154,6 +154,14 @@ export declare class HueClient {
|
|
|
154
154
|
* alone is omitted and counted while the rest is recorded.
|
|
155
155
|
*/
|
|
156
156
|
recordFile(file: FileRecord, explicitContext?: Context): void;
|
|
157
|
+
/** The keys of each span's content values that were cut to Hue's value cap, for `hue.truncated`. */
|
|
158
|
+
private truncatedKeys;
|
|
159
|
+
/**
|
|
160
|
+
* A content value as a JSON attribute within Hue's 256 KiB value cap. A larger value is cut to
|
|
161
|
+
* a UTF-8 prefix and its key listed under `hue.truncated`, where Hue's receiver lists the
|
|
162
|
+
* values it cuts itself: the span carries the call and the recorded part of its value instead
|
|
163
|
+
* of losing the value whole. Only a value that cannot be encoded is omitted and counted.
|
|
164
|
+
*/
|
|
157
165
|
private setContent;
|
|
158
166
|
/**
|
|
159
167
|
* Confirms the key and origin by reading the current project; a setup and CI diagnostic, not a
|
package/dist/client.js
CHANGED
|
@@ -10,7 +10,8 @@ import { defaultResource, resourceFromAttributes } from "@opentelemetry/resource
|
|
|
10
10
|
import { HUE_SCOPE } from "./config.js";
|
|
11
11
|
import { hostedServerAddresses, hostedToolActivity, hostedToolProvider, providerErrorDescription, } from "./provider-tools.js";
|
|
12
12
|
import { toolCatalogSummary } from "./tool-definitions.js";
|
|
13
|
-
import { encodeContent, noopSpan, safeSpan } from "./safety.js";
|
|
13
|
+
import { encodeBoundedContent, encodeContent, noopSpan, safeSpan, } from "./safety.js";
|
|
14
|
+
import { TRUNCATED_KEY, withTruncatedKeys } from "./privacy.js";
|
|
14
15
|
import { createHueTransport, HueExportError } from "./transport.js";
|
|
15
16
|
import { verifyTrace } from "./receipt.js";
|
|
16
17
|
import { sdkVersion } from "./version.js";
|
|
@@ -712,11 +713,31 @@ export class HueClient {
|
|
|
712
713
|
this.instrumentationFailed();
|
|
713
714
|
}
|
|
714
715
|
}
|
|
716
|
+
/** The keys of each span's content values that were cut to Hue's value cap, for `hue.truncated`. */
|
|
717
|
+
truncatedKeys = new WeakMap();
|
|
718
|
+
/**
|
|
719
|
+
* A content value as a JSON attribute within Hue's 256 KiB value cap. A larger value is cut to
|
|
720
|
+
* a UTF-8 prefix and its key listed under `hue.truncated`, where Hue's receiver lists the
|
|
721
|
+
* values it cuts itself: the span carries the call and the recorded part of its value instead
|
|
722
|
+
* of losing the value whole. Only a value that cannot be encoded is omitted and counted.
|
|
723
|
+
*/
|
|
715
724
|
setContent(span, key, value) {
|
|
716
725
|
if (!this.enabled || this.closed || !this.captureContent)
|
|
717
726
|
return;
|
|
718
727
|
try {
|
|
719
|
-
|
|
728
|
+
const { text, truncated } = encodeBoundedContent(value);
|
|
729
|
+
span.setAttribute(key, text);
|
|
730
|
+
// The span's list as it stands: the application's own entries (read from the SDK span,
|
|
731
|
+
// which carries its attributes; the API alone shows none) and this helper's earlier ones.
|
|
732
|
+
const listed = withTruncatedKeys(span.attributes?.[TRUNCATED_KEY], this.truncatedKeys.get(span) ?? []);
|
|
733
|
+
const keys = truncated
|
|
734
|
+
? withTruncatedKeys(listed, [key])
|
|
735
|
+
: listed.filter((entry) => entry !== key);
|
|
736
|
+
if (truncated || listed.includes(key)) {
|
|
737
|
+
// A whole value replacing a cut one unmarks its key; an untouched list is left alone.
|
|
738
|
+
this.truncatedKeys.set(span, keys);
|
|
739
|
+
span.setAttribute(TRUNCATED_KEY, keys);
|
|
740
|
+
}
|
|
720
741
|
}
|
|
721
742
|
catch {
|
|
722
743
|
this.instrumentationFailed();
|
|
@@ -364,7 +364,11 @@ export interface WorldHandoff {
|
|
|
364
364
|
baggage: string;
|
|
365
365
|
/** Mirror surfaces of every provider instance the environment version binds. */
|
|
366
366
|
surfaces: WorldSurface[];
|
|
367
|
-
/**
|
|
367
|
+
/** The world's clock at creation (RFC 3339): the date an agent computes its dates from in
|
|
368
|
+
* place of the wall clock, a trace-built world's recorded start. Also `env.HUE_WORLD_NOW`;
|
|
369
|
+
* null for a Hue that names neither. */
|
|
370
|
+
now: string | null;
|
|
371
|
+
/** `HUE_WORLD_ID`, `HUE_WORLD_TOKEN`, `HUE_WORLD_NOW`, `BAGGAGE`, `TRACEPARENT` and one
|
|
368
372
|
* `HUE_SIM_<SURFACE ID>_URL` per surface. */
|
|
369
373
|
env: Record<string, string>;
|
|
370
374
|
/** The `mcpServers` configuration for the MCP surfaces. */
|
|
@@ -378,6 +382,9 @@ export interface EnvironmentRun {
|
|
|
378
382
|
environmentVersionId: string;
|
|
379
383
|
/** Current virtual nanoseconds as a decimal string. */
|
|
380
384
|
clockNs: string;
|
|
385
|
+
/** The world's clock at creation as an RFC 3339 timestamp (`clockNs` as a date): a
|
|
386
|
+
* trace-built world's recorded start. Absent from a Hue that predates it. */
|
|
387
|
+
now?: string;
|
|
381
388
|
/** Digest of current world state. */
|
|
382
389
|
stateDigest: string;
|
|
383
390
|
/** Maximum recorded actions. */
|
|
@@ -5,6 +5,14 @@ import type { EnvironmentRun, LegacyMcpCapability, WorldHandoff } from "./types.
|
|
|
5
5
|
* mirror URLs. The handoff carries the world token; keep it out of logs and checkpoints.
|
|
6
6
|
*/
|
|
7
7
|
export declare function worldHandoff(run: EnvironmentRun): WorldHandoff | null;
|
|
8
|
+
/**
|
|
9
|
+
* The world's clock: what an agent reads for today's date in place of the wall clock, so a
|
|
10
|
+
* date-relative request (tomorrow, `newer_than:7d`, this month) lands on the dates the world
|
|
11
|
+
* holds however long after the recording the run starts. From the handoff's `now`, else from
|
|
12
|
+
* `HUE_WORLD_NOW` in the given environment (this process's by default, where `hue eval
|
|
13
|
+
* --command` and `agentEnvironment` set it). Null when neither names it.
|
|
14
|
+
*/
|
|
15
|
+
export declare function worldNow(source?: WorldHandoff | Record<string, string | undefined>): Date | null;
|
|
8
16
|
/** Variables that authenticate against Hue's control plane rather than a simulated provider. */
|
|
9
17
|
export declare const HUE_CONTROL_PLANE_VARIABLES: readonly string[];
|
|
10
18
|
/** True for a control-plane variable by name, or for any variable holding a Hue credential. */
|
|
@@ -18,6 +18,7 @@ export function worldHandoff(run) {
|
|
|
18
18
|
completingUntil: run.completingUntil ?? null,
|
|
19
19
|
traceparent: run.traceparent ?? null,
|
|
20
20
|
baggage: run.baggage ?? `hue-world=${id}`,
|
|
21
|
+
now: run.now ?? run.env.HUE_WORLD_NOW ?? null,
|
|
21
22
|
surfaces: run.surfaces.map((surface) => ({ ...surface })),
|
|
22
23
|
env: { ...run.env },
|
|
23
24
|
mcpConfig: {
|
|
@@ -28,6 +29,22 @@ export function worldHandoff(run) {
|
|
|
28
29
|
},
|
|
29
30
|
};
|
|
30
31
|
}
|
|
32
|
+
/**
|
|
33
|
+
* The world's clock: what an agent reads for today's date in place of the wall clock, so a
|
|
34
|
+
* date-relative request (tomorrow, `newer_than:7d`, this month) lands on the dates the world
|
|
35
|
+
* holds however long after the recording the run starts. From the handoff's `now`, else from
|
|
36
|
+
* `HUE_WORLD_NOW` in the given environment (this process's by default, where `hue eval
|
|
37
|
+
* --command` and `agentEnvironment` set it). Null when neither names it.
|
|
38
|
+
*/
|
|
39
|
+
export function worldNow(source = process.env) {
|
|
40
|
+
const text = "env" in source && typeof source.env === "object" && source.env !== null
|
|
41
|
+
? (source.now ?? source.env.HUE_WORLD_NOW)
|
|
42
|
+
: source.HUE_WORLD_NOW;
|
|
43
|
+
if (!text)
|
|
44
|
+
return null;
|
|
45
|
+
const date = new Date(text);
|
|
46
|
+
return Number.isNaN(date.getTime()) ? null : date;
|
|
47
|
+
}
|
|
31
48
|
/** Variables that authenticate against Hue's control plane rather than a simulated provider. */
|
|
32
49
|
export const HUE_CONTROL_PLANE_VARIABLES = [
|
|
33
50
|
"HUE_API_KEY",
|
package/dist/environment.d.ts
CHANGED
|
@@ -3,5 +3,5 @@ export type { EnvironmentClientOptions } from "./environment/client.js";
|
|
|
3
3
|
export { bindEnvironmentTools } from "./environment/tools.js";
|
|
4
4
|
export type { BindEnvironmentToolsOptions, EnvironmentTool } from "./environment/tools.js";
|
|
5
5
|
export type * from "./environment/types.js";
|
|
6
|
-
export { agentEnvironment, HUE_CONTROL_PLANE_VARIABLES, isHueControlPlaneCredential, legacyMcpCapability, stripHueControlPlaneCredentials, worldHandoff, writeMcpConfig, } from "./environment/world.js";
|
|
6
|
+
export { agentEnvironment, HUE_CONTROL_PLANE_VARIABLES, isHueControlPlaneCredential, legacyMcpCapability, stripHueControlPlaneCredentials, worldHandoff, worldNow, writeMcpConfig, } from "./environment/world.js";
|
|
7
7
|
export type { AgentEnvironmentOptions, McpConfigFile } from "./environment/world.js";
|
package/dist/environment.js
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
export { createEnvironmentClient, EnvironmentClient, HueEnvironmentError, isTransientEnvironmentError, } from "./environment/client.js";
|
|
2
2
|
export { bindEnvironmentTools } from "./environment/tools.js";
|
|
3
|
-
export { agentEnvironment, HUE_CONTROL_PLANE_VARIABLES, isHueControlPlaneCredential, legacyMcpCapability, stripHueControlPlaneCredentials, worldHandoff, writeMcpConfig, } from "./environment/world.js";
|
|
3
|
+
export { agentEnvironment, HUE_CONTROL_PLANE_VARIABLES, isHueControlPlaneCredential, legacyMcpCapability, stripHueControlPlaneCredentials, worldHandoff, worldNow, writeMcpConfig, } from "./environment/world.js";
|
package/dist/evals/failure.d.ts
CHANGED
|
@@ -10,9 +10,26 @@
|
|
|
10
10
|
* The runner records the type on the execution in place of `TargetError`; Hue classes the first
|
|
11
11
|
* three as infrastructure and the last as configuration, so an error from Hue's own clients is
|
|
12
12
|
* never filed as the agent's. Anything else is the agent's own error.
|
|
13
|
+
*
|
|
14
|
+
* A simulated world the agent reaches through its own client (an MCP client, a REST client, a
|
|
15
|
+
* fetch) answers Hue's gateway refusals with a status and a diagnostic: `503
|
|
16
|
+
* authorization_unavailable` when Hue could not authorize the world token because its database
|
|
17
|
+
* could not answer, `503 gateway_failure` when the gateway failed. The MCP SDK's transport throws
|
|
18
|
+
* these as `StreamableHTTPError` or `SseError` with the status as `code` and no response, so the
|
|
19
|
+
* status is read from there too; the diagnostic is the error's cause (`serviceFailureCause`),
|
|
20
|
+
* recorded beside the type so a run's reader sees what refused it.
|
|
13
21
|
*/
|
|
14
22
|
export type ServiceFailureType = "ServiceRefused" | "ConnectionFailed" | "TimedOut" | "ConfigurationRejected";
|
|
15
23
|
/** The service failure a target's throw shows, by the errors linked from it, in precedence
|
|
16
24
|
* order: a timeout, a failed connection, a retryable refusal, a status Hue's own client was
|
|
17
25
|
* refused with; null for the agent's own error. */
|
|
18
26
|
export declare function serviceFailureType(error: unknown): ServiceFailureType | null;
|
|
27
|
+
/**
|
|
28
|
+
* The service's own word for what failed, when the answer a target's throw carries names one:
|
|
29
|
+
* Hue's gateway diagnostic (`authorization_unavailable`, `gateway_failure`), read from the
|
|
30
|
+
* `x-hue-diagnostic` header or the JSON body's `diagnostic` of any error linked from the throw,
|
|
31
|
+
* nearest first. Undefined when no answer names one, or the name is not a diagnostic's token.
|
|
32
|
+
* Read for a service failure (`serviceFailureType`), where the runner records it as the error's
|
|
33
|
+
* `cause`.
|
|
34
|
+
*/
|
|
35
|
+
export declare function serviceFailureCause(error: unknown): string | undefined;
|
package/dist/evals/failure.js
CHANGED
|
@@ -69,6 +69,17 @@ function names(error) {
|
|
|
69
69
|
}
|
|
70
70
|
return found;
|
|
71
71
|
}
|
|
72
|
+
/** The MCP TypeScript SDK's transport errors, which carry the HTTP status as `code` and no
|
|
73
|
+
* response: a POST the server answered with a failing status (`StreamableHTTPError`) or an SSE
|
|
74
|
+
* stream it refused (`SseError`). Recognized by class name, never imported. */
|
|
75
|
+
const MCP_TRANSPORT_NAMES = new Set(["StreamableHTTPError", "SseError"]);
|
|
76
|
+
/** The status an MCP transport error carries, or undefined for any other error. */
|
|
77
|
+
function transportStatus(error) {
|
|
78
|
+
const { code } = error;
|
|
79
|
+
return typeof code === "number" && names(error).some((name) => MCP_TRANSPORT_NAMES.has(name))
|
|
80
|
+
? code
|
|
81
|
+
: undefined;
|
|
82
|
+
}
|
|
72
83
|
const retryable = (status) => typeof status === "number" &&
|
|
73
84
|
(status === 408 || status === 429 || (status >= 500 && status < 600));
|
|
74
85
|
function timedOut(error) {
|
|
@@ -99,6 +110,9 @@ function disconnected(error) {
|
|
|
99
110
|
function refused(error) {
|
|
100
111
|
if (hueClientError(error))
|
|
101
112
|
return retryable(error.status);
|
|
113
|
+
// An MCP transport error is the exchange itself: the server answered the status it carries.
|
|
114
|
+
if (retryable(transportStatus(error)))
|
|
115
|
+
return true;
|
|
102
116
|
const { status, statusCode, headers, response, url, responseHeaders } = error;
|
|
103
117
|
const exchanged = [headers, response, url, responseHeaders].some((item) => item !== undefined && item !== null);
|
|
104
118
|
// A fetch `Response` carries the status itself.
|
|
@@ -148,3 +162,91 @@ export function serviceFailureType(error) {
|
|
|
148
162
|
return "ConfigurationRejected";
|
|
149
163
|
return null;
|
|
150
164
|
}
|
|
165
|
+
/** The header Hue's gateway names its diagnostic in, beside the JSON body's `diagnostic`. */
|
|
166
|
+
const DIAGNOSTIC_HEADER = "x-hue-diagnostic";
|
|
167
|
+
/** A diagnostic's spelling: a short lower-case token (`authorization_unavailable`). */
|
|
168
|
+
const DIAGNOSTIC = /^[a-z][a-z0-9_]{0,63}$/;
|
|
169
|
+
/** The most of a body or message read for a diagnostic. */
|
|
170
|
+
const MAX_BODY = 8192;
|
|
171
|
+
/** A header's value from a `Headers`, a record of names to values (case-insensitive), or a
|
|
172
|
+
* record of names to arrays, as the HTTP clients spell them; undefined otherwise. */
|
|
173
|
+
function headerValue(headers, name) {
|
|
174
|
+
if (headers === null || typeof headers !== "object")
|
|
175
|
+
return undefined;
|
|
176
|
+
const get = headers.get;
|
|
177
|
+
if (typeof get === "function") {
|
|
178
|
+
const value = get.call(headers, name);
|
|
179
|
+
return typeof value === "string" ? value : undefined;
|
|
180
|
+
}
|
|
181
|
+
for (const [key, value] of Object.entries(headers)) {
|
|
182
|
+
if (key.toLowerCase() !== name)
|
|
183
|
+
continue;
|
|
184
|
+
const first = Array.isArray(value) ? value[0] : value;
|
|
185
|
+
return typeof first === "string" ? first : undefined;
|
|
186
|
+
}
|
|
187
|
+
return undefined;
|
|
188
|
+
}
|
|
189
|
+
/** The `diagnostic` a JSON body names: an object's, at the top or in a JSON-RPC error's `data`,
|
|
190
|
+
* or the object a string carries from its first brace (an MCP transport error's message quotes
|
|
191
|
+
* the body it was answered). */
|
|
192
|
+
function bodyDiagnostic(body) {
|
|
193
|
+
if (typeof body === "string") {
|
|
194
|
+
const start = body.indexOf("{");
|
|
195
|
+
if (start < 0 || body.length - start > MAX_BODY)
|
|
196
|
+
return undefined;
|
|
197
|
+
try {
|
|
198
|
+
return bodyDiagnostic(JSON.parse(body.slice(start)));
|
|
199
|
+
}
|
|
200
|
+
catch {
|
|
201
|
+
return undefined;
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
if (body === null || typeof body !== "object")
|
|
205
|
+
return undefined;
|
|
206
|
+
// Hue's REST refusal names it at the top; its MCP refusal, a JSON-RPC error, in `error.data`.
|
|
207
|
+
const { diagnostic, error } = body;
|
|
208
|
+
if (typeof diagnostic === "string")
|
|
209
|
+
return diagnostic;
|
|
210
|
+
const data = error !== null && typeof error === "object" ? error.data : undefined;
|
|
211
|
+
const named = data !== null && typeof data === "object"
|
|
212
|
+
? data.diagnostic
|
|
213
|
+
: undefined;
|
|
214
|
+
return typeof named === "string" ? named : undefined;
|
|
215
|
+
}
|
|
216
|
+
/** The diagnostic one error carries: the one it names itself (Hue's own clients read the header
|
|
217
|
+
* into `diagnostic`), else the header on its `headers`, `responseHeaders` or `response.headers`,
|
|
218
|
+
* else the body on its `responseBody`, `body` or, for an MCP transport error, its message. */
|
|
219
|
+
function diagnosticOf(error) {
|
|
220
|
+
const { diagnostic, headers, responseHeaders, response, responseBody, body, message } = error;
|
|
221
|
+
const answered = response !== null && typeof response === "object"
|
|
222
|
+
? response.headers
|
|
223
|
+
: undefined;
|
|
224
|
+
return ((typeof diagnostic === "string" ? diagnostic : undefined) ??
|
|
225
|
+
headerValue(headers, DIAGNOSTIC_HEADER) ??
|
|
226
|
+
headerValue(responseHeaders, DIAGNOSTIC_HEADER) ??
|
|
227
|
+
headerValue(answered, DIAGNOSTIC_HEADER) ??
|
|
228
|
+
bodyDiagnostic(responseBody) ??
|
|
229
|
+
bodyDiagnostic(body) ??
|
|
230
|
+
(transportStatus(error) !== undefined ? bodyDiagnostic(message) : undefined));
|
|
231
|
+
}
|
|
232
|
+
/**
|
|
233
|
+
* The service's own word for what failed, when the answer a target's throw carries names one:
|
|
234
|
+
* Hue's gateway diagnostic (`authorization_unavailable`, `gateway_failure`), read from the
|
|
235
|
+
* `x-hue-diagnostic` header or the JSON body's `diagnostic` of any error linked from the throw,
|
|
236
|
+
* nearest first. Undefined when no answer names one, or the name is not a diagnostic's token.
|
|
237
|
+
* Read for a service failure (`serviceFailureType`), where the runner records it as the error's
|
|
238
|
+
* `cause`.
|
|
239
|
+
*/
|
|
240
|
+
export function serviceFailureCause(error) {
|
|
241
|
+
for (const item of linked(error)) {
|
|
242
|
+
try {
|
|
243
|
+
const diagnostic = diagnosticOf(item);
|
|
244
|
+
if (diagnostic !== undefined && DIAGNOSTIC.test(diagnostic))
|
|
245
|
+
return diagnostic;
|
|
246
|
+
}
|
|
247
|
+
catch {
|
|
248
|
+
// A property that throws when read names nothing.
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
return undefined;
|
|
252
|
+
}
|
package/dist/evals/runner.js
CHANGED
|
@@ -6,7 +6,7 @@ import { ROOT_CONTEXT } from "@opentelemetry/api";
|
|
|
6
6
|
import { HueExportError } from "../transport.js";
|
|
7
7
|
import { HueApiError } from "./client.js";
|
|
8
8
|
import { loadEnvironmentEvidence } from "./environment-evidence.js";
|
|
9
|
-
import { serviceFailureType } from "./failure.js";
|
|
9
|
+
import { serviceFailureCause, serviceFailureType } from "./failure.js";
|
|
10
10
|
import { CheckpointStore } from "./checkpoint.js";
|
|
11
11
|
import { onForcedExit } from "./exit-cleanup.js";
|
|
12
12
|
import { assertSafeFileNames, downloadCaseFiles, downloadCaseInputs, localOutputFiles, verifyCaseFiles, OutputFileError, stageOutputFiles, targetFileRoles, uploadOutputFiles, } from "./files.js";
|
|
@@ -434,17 +434,26 @@ export async function runExperiment(options) {
|
|
|
434
434
|
// A target stopped at its world's deadline is a `TargetTimeout`, told apart from an error the
|
|
435
435
|
// target raised itself: the agent did not finish, rather than finishing wrongly. A service the
|
|
436
436
|
// agent called that refused, dropped the connection or timed out is named apart from the
|
|
437
|
-
// agent's own error (`failure.ts`), so Hue counts it as infrastructure, not as the agent
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
:
|
|
447
|
-
|
|
437
|
+
// agent's own error (`failure.ts`), so Hue counts it as infrastructure, not as the agent, with
|
|
438
|
+
// the service's own word for what failed as the error's `cause` when its answer names one
|
|
439
|
+
// (Hue's gateway diagnostic).
|
|
440
|
+
const errorPayload = (error) => {
|
|
441
|
+
const service = error instanceof OutputTooLargeError || error instanceof TargetTimeoutError
|
|
442
|
+
? null
|
|
443
|
+
: serviceFailureType(error);
|
|
444
|
+
const cause = service ? serviceFailureCause(error) : undefined;
|
|
445
|
+
return {
|
|
446
|
+
type: error instanceof OutputTooLargeError
|
|
447
|
+
? "OutputTooLarge"
|
|
448
|
+
: error instanceof TargetTimeoutError
|
|
449
|
+
? "TargetTimeout"
|
|
450
|
+
: (service ?? "TargetError"),
|
|
451
|
+
...(cause === undefined ? {} : { cause }),
|
|
452
|
+
...(options.persistResultContent && error instanceof Error
|
|
453
|
+
? { message: sanitize(error.message) }
|
|
454
|
+
: {}),
|
|
455
|
+
};
|
|
456
|
+
};
|
|
448
457
|
/** Publish staged files, score with every verified file on disk and save the completion. */
|
|
449
458
|
async function prepare(file, saved, frozenCase, caseDirectory,
|
|
450
459
|
/** The target's output, in memory when result content is not persisted. */
|
package/dist/evals/types.d.ts
CHANGED
|
@@ -546,6 +546,12 @@ export interface TypedError {
|
|
|
546
546
|
* (`agent`, `infrastructure`, `configuration`, `telemetry` or `unsupported`), which Hue reads
|
|
547
547
|
* only for a name it does not know. The SDK's own types need none. */
|
|
548
548
|
class?: "agent" | "infrastructure" | "configuration" | "telemetry" | "unsupported";
|
|
549
|
+
/** For a service failure (`ServiceRefused`, `ConnectionFailed`, `TimedOut`,
|
|
550
|
+
* `ConfigurationRejected`): the service's own word for what failed, when its answer names one,
|
|
551
|
+
* such as Hue's gateway diagnostic (`authorization_unavailable`) from the answer's
|
|
552
|
+
* `x-hue-diagnostic` header or its JSON body's `diagnostic`. A short lower-case token; absent
|
|
553
|
+
* when the answer names none. */
|
|
554
|
+
cause?: string;
|
|
549
555
|
}
|
|
550
556
|
/** Input for {@link EvaluationClient.startExecution}. */
|
|
551
557
|
export interface StartExecution {
|
package/dist/privacy.d.ts
CHANGED
|
@@ -2,8 +2,26 @@ import type { ReadableSpan } from "@opentelemetry/sdk-trace";
|
|
|
2
2
|
import type { ReadableLogRecord } from "@opentelemetry/sdk-logs";
|
|
3
3
|
import { type Resource } from "@opentelemetry/resources";
|
|
4
4
|
import type { HueOptions } from "./types.js";
|
|
5
|
+
import { truncateUtf8 } from "./safety.js";
|
|
6
|
+
export { truncateUtf8 };
|
|
5
7
|
/** Attribute keys (and their dotted children) removed in metadata-only mode. */
|
|
6
8
|
export declare const contentPrefixes: string[];
|
|
9
|
+
export declare function isContentKey(key: string): boolean;
|
|
10
|
+
/** The record attribute listing the keys whose values were cut to fit Hue's 256 KiB value cap:
|
|
11
|
+
* Hue's receiver writes it for the values it cuts, and the SDK writes it for the values it cuts
|
|
12
|
+
* before export, under the same name, so a reader learns of a cut value from one place. */
|
|
13
|
+
export declare const TRUNCATED_KEY = "hue.truncated";
|
|
14
|
+
/** The size of a structured value the marker replaced. */
|
|
15
|
+
export declare const TRUNCATED_BYTES_KEY = "hue.truncated_bytes";
|
|
16
|
+
/** The root span attribute counting this trace's records the SDK could not export at all (a
|
|
17
|
+
* record over the request limit even with every content value shed, or one its queue had no
|
|
18
|
+
* room for): what Hue stores of the trace is incomplete by that many records. */
|
|
19
|
+
export declare const DROPPED_RECORDS_KEY = "hue.sdk.dropped_records";
|
|
20
|
+
/** Hue's receiver's own marker for a structured value it replaced: nothing of the value, its
|
|
21
|
+
* size, and the flag a reader tests. */
|
|
22
|
+
export declare function truncatedMarker(bytes: number): Record<string, unknown>;
|
|
23
|
+
/** `listed` with `keys` added, each once, in order. */
|
|
24
|
+
export declare function withTruncatedKeys(listed: unknown, keys: readonly string[]): string[];
|
|
7
25
|
export type ResourceCache = WeakMap<Resource, Resource>;
|
|
8
26
|
export declare function redactSpan(span: ReadableSpan, options: HueOptions, cache: ResourceCache): ReadableSpan;
|
|
9
27
|
export declare function redactLog(log: ReadableLogRecord, options: HueOptions, cache: ResourceCache): ReadableLogRecord;
|