@hue-run/sdk 0.12.1 → 0.13.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLI.md +44 -19
- package/ENVIRONMENTS.md +23 -6
- package/EVALUATIONS.md +19 -1
- package/README.md +29 -13
- package/dist/client.d.ts +8 -0
- package/dist/client.js +23 -2
- package/dist/environment/types.d.ts +8 -1
- package/dist/environment/world.d.ts +14 -1
- package/dist/environment/world.js +61 -4
- package/dist/environment.d.ts +1 -1
- package/dist/environment.js +1 -1
- package/dist/evals/client.d.ts +37 -6
- package/dist/evals/client.js +44 -5
- package/dist/evals/failure.d.ts +17 -0
- package/dist/evals/failure.js +102 -0
- package/dist/evals/runner.js +21 -12
- package/dist/evals/scorer-publication.js +9 -2
- package/dist/evals/scorers.js +3 -1
- package/dist/evals/types.d.ts +58 -0
- package/dist/privacy.d.ts +18 -0
- package/dist/privacy.js +147 -57
- package/dist/safety.d.ts +18 -0
- package/dist/safety.js +76 -23
- package/dist/transport.d.ts +5 -0
- package/dist/transport.js +277 -57
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +1 -1
package/CLI.md
CHANGED
|
@@ -42,7 +42,7 @@ project manifest are refused because managers can update ancestor locks; Python
|
|
|
42
42
|
The generated `hue.setup.mjs` or `hue_setup.py` always selects `captureContent: false` /
|
|
43
43
|
`capture_content=False`. For a supported application, setup installs the dependency and adds the
|
|
44
44
|
managed import and middleware registration to the existing entrypoint; an unreferenced helper is
|
|
45
|
-
not a completed integration. TypeScript uses `@hue-run/sdk@0.
|
|
45
|
+
not a completed integration. TypeScript uses `@hue-run/sdk@0.13.1`, `@opentelemetry/api@1.9.1` and
|
|
46
46
|
`@opentelemetry/context-async-hooks@2.11.0`; Python setup uses its separately tested package pin.
|
|
47
47
|
Content capture requires an ordinary account-managed key and a later explicit application decision.
|
|
48
48
|
|
|
@@ -469,7 +469,7 @@ node packages/sdk-typescript/scripts/verify-package.mjs --artifacts-dir .artifac
|
|
|
469
469
|
# Set project to an existing supported fixture; use the same directory on resume.
|
|
470
470
|
project=/absolute/path/to/supported-fixture
|
|
471
471
|
node packages/sdk-typescript/scripts/verify-setup-live.mjs \
|
|
472
|
-
--archive .artifacts/typescript/hue-run-sdk-0.
|
|
472
|
+
--archive .artifacts/typescript/hue-run-sdk-0.13.1.tgz \
|
|
473
473
|
--origin https://STAGING_ORIGIN \
|
|
474
474
|
--project "$project" --command setup \
|
|
475
475
|
--evidence .context/setup-staging-before-claim.json
|
|
@@ -480,7 +480,7 @@ the private local handoff and finish the real browser claim, then reconcile the
|
|
|
480
480
|
|
|
481
481
|
```sh
|
|
482
482
|
node packages/sdk-typescript/scripts/verify-setup-live.mjs \
|
|
483
|
-
--archive .artifacts/typescript/hue-run-sdk-0.
|
|
483
|
+
--archive .artifacts/typescript/hue-run-sdk-0.13.1.tgz \
|
|
484
484
|
--origin https://STAGING_ORIGIN \
|
|
485
485
|
--project "$project" --command claim \
|
|
486
486
|
--evidence .context/setup-staging-after-claim.json
|
|
@@ -512,12 +512,38 @@ the sealed world. Hue never executes the agent. `HUE_API_KEY` must be a **Read a
|
|
|
512
512
|
write** project key (a **Read** or **Tracing only** key cannot read cases or create runs);
|
|
513
513
|
the CLI never prints it. The optional `zod` peer of `@hue-run/sdk/evals` must be installed.
|
|
514
514
|
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
515
|
+
Run the agent's own start command with `--command`. `hue eval` spawns it once per case with the
|
|
516
|
+
world's environment (below) and `{"inputs", "config"}` on stdin, and stores its stdout as the
|
|
517
|
+
answer. The agent reaches each app through Hue's mirror with its own production client: the one
|
|
518
|
+
committed helper described in [Make your agent eval-ready](https://docs.hue.run/evaluations/eval-ready-agent)
|
|
519
|
+
reads `HUE_SIM_<SURFACE ID>_URL` and `HUE_WORLD_TOKEN` when `HUE_WORLD_TOKEN` is set, fails
|
|
520
|
+
closed when a needed variable is missing, and returns the production URL and credential
|
|
521
|
+
otherwise. Never hand the agent Hue-native tools (`context.tools`, `context.mcp`) or a tool its
|
|
522
|
+
production agent lacks. Pin the SDK and bring its `zod` peer along, so the agent's repository
|
|
523
|
+
gains no dependency:
|
|
524
|
+
|
|
525
|
+
```sh
|
|
526
|
+
npx --yes --package @hue-run/sdk@0.13.1 --package "zod@^4.6.5" hue eval --case "Refund an eligible charge" --command "node agent.js" --env-file .env.hue
|
|
527
|
+
npx --yes --package @hue-run/sdk@0.13.1 --package "zod@^4.6.5" hue eval --case https://app.hue.run/projects/demo/scenarios/<id> --command "python agent.py" --timeout 120 --baseline <experiment id>
|
|
528
|
+
npx --yes --package @hue-run/sdk@0.13.1 --package "zod@^4.6.5" hue eval --set "Billing regressions" --scorer-version <id> --command "node agent.js" --save-version
|
|
529
|
+
npx --yes --package @hue-run/sdk@0.13.1 --package "zod@^4.6.5" hue eval --worker --command "node agent.js" --agent-key support-agent --env-file .env.hue
|
|
530
|
+
```
|
|
531
|
+
|
|
532
|
+
Without the second `--package`, `hue eval` exits with `hue eval needs zod`, even for `--help`;
|
|
533
|
+
installing zod in the project does not help when the SDK itself is not installed there. An
|
|
534
|
+
unpinned `--package @hue-run/sdk` runs a local, possibly older, copy in a project that already
|
|
535
|
+
depends on it.
|
|
536
|
+
|
|
537
|
+
An adapter module is the in-process alternative for a TypeScript agent. It exports `default` or
|
|
538
|
+
`runMyAgent`; `context` is the SDK's `SimulationTargetContext` (`world`, `mcp`, `tools`, `config`,
|
|
539
|
+
`item`, `executionId`, `environmentRunId`, `files`, `outputDirectory`, `signal`; `world` carries
|
|
540
|
+
the provider mirror URLs, the world token, `env` and `mcpConfig` where Hue's simulation gateway
|
|
541
|
+
serves the world). An adapter runs inside `hue eval`'s own process, where `HUE_WORLD_TOKEN` and
|
|
542
|
+
`HUE_EXECUTION_ID` are not set, so a helper that reads the process environment would return
|
|
543
|
+
production values: throw when `context.world` is missing, then pass `context.world.env` as the
|
|
544
|
+
helper's environment argument or spawn the agent with `agentEnvironment(context.world)`. Never
|
|
545
|
+
set `process.env` yourself; with `--concurrency` above 1 that leaks one case's world into another.
|
|
546
|
+
When the case also pins input files, `files` holds verified copies of its agent-visible ones (see
|
|
521
547
|
[Evaluate a document eval set](#evaluate-a-document-eval-set)); return
|
|
522
548
|
`withFiles(output, files)` from `@hue-run/sdk/evals` to upload documents the agent produced:
|
|
523
549
|
|
|
@@ -527,19 +553,16 @@ import type { JsonValue, SimulationTargetContext } from "@hue-run/sdk/evals";
|
|
|
527
553
|
import { runAgent } from "./src/agent.js"; // The application's existing entry point.
|
|
528
554
|
|
|
529
555
|
export default function runMyAgent(inputs: JsonValue, context: SimulationTargetContext) {
|
|
530
|
-
|
|
531
|
-
//
|
|
532
|
-
|
|
533
|
-
|
|
556
|
+
if (!context.world) throw new Error("This case has no world handoff; refusing to run against the real app");
|
|
557
|
+
// The agent's own app clients read the mirrors through the committed helper, given the
|
|
558
|
+
// world's environment; never hand it context.tools or context.mcp. Pass context.signal for
|
|
559
|
+
// cooperative cancellation.
|
|
560
|
+
return runAgent({ inputs, config: context.config, env: context.world.env, signal: context.signal });
|
|
534
561
|
}
|
|
535
562
|
```
|
|
536
563
|
|
|
537
564
|
```sh
|
|
538
|
-
hue eval --case "Refund an eligible charge" ./hue-agent.ts --content --env-file .env.hue
|
|
539
|
-
hue eval --case https://app.hue.run/projects/demo/scenarios/<id> ./hue-agent.ts --content --baseline <run id>
|
|
540
|
-
hue eval --set "Billing regressions" --scorer-version <id> ./hue-agent.ts --content --save-version
|
|
541
|
-
hue eval --case "Refund an eligible charge" --command "python agent.py" --timeout 120 --content
|
|
542
|
-
hue eval --worker ./hue-agent.ts --agent-key support-agent --content --env-file .env.hue
|
|
565
|
+
npx --yes --package @hue-run/sdk@0.13.1 --package "zod@^4.6.5" hue eval --case "Refund an eligible charge" ./hue-agent.ts --content --env-file .env.hue
|
|
543
566
|
```
|
|
544
567
|
|
|
545
568
|
`--scenario` remains an alias for `--case` for existing scripts. Pass one selection flag.
|
|
@@ -576,7 +599,9 @@ Hue-owned `world_outcome` checks are graded after the world seals. An experiment
|
|
|
576
599
|
every case of the saved version; there is no case subset.
|
|
577
600
|
|
|
578
601
|
`--command "<shell command>"` spawns the command once per case with the world's environment
|
|
579
|
-
(`HUE_WORLD_ID`, `HUE_WORLD_TOKEN`,
|
|
602
|
+
(`HUE_WORLD_ID`, `HUE_WORLD_TOKEN`, `HUE_WORLD_NOW` (the world's clock at creation, RFC 3339; `worldNow()` reads it and returns
|
|
603
|
+
`null` without it, so fall back explicitly with `worldNow() ?? new Date()`), one `HUE_SIM_<SURFACE ID>_URL` per provider
|
|
604
|
+
mirror,
|
|
580
605
|
`HUE_MCP_CONFIG` naming an owner-only `mcpServers` file that is removed after the case, and
|
|
581
606
|
`HUE_MCP_URL`, `HUE_MCP_TOKEN`, `HUE_MCP_EXPIRES_AT` for the first MCP mirror; these name the
|
|
582
607
|
world's simulated MCP server and replace a `HUE_MCP_URL` that `hue login` stored for Hue's project
|
package/ENVIRONMENTS.md
CHANGED
|
@@ -220,10 +220,24 @@ automatically because identity creation and publication have no request key.
|
|
|
220
220
|
Where a Hue deployment has the simulation gateway on, `createRun` returns the World API handoff
|
|
221
221
|
beside the run: `token` (a `hue_world_…` credential that lives exactly as long as the world),
|
|
222
222
|
`surfaces[]` (one mirror URL per pinned provider surface, such as the Gmail MCP and REST mirrors),
|
|
223
|
-
`
|
|
224
|
-
`
|
|
225
|
-
|
|
226
|
-
|
|
223
|
+
`now` (the world's clock at creation as an RFC 3339 timestamp), `env` (`HUE_WORLD_ID`,
|
|
224
|
+
`HUE_WORLD_TOKEN`, `HUE_WORLD_NOW`, `BAGGAGE`, `TRACEPARENT` and one `HUE_SIM_<SURFACE ID>_URL`
|
|
225
|
+
per surface) and `mcpConfig` (the common `mcpServers` shape with the token in the `Authorization`
|
|
226
|
+
header). The agent is pointed at the mirrors by configuration only: its own Gmail MCP or REST
|
|
227
|
+
client, the mirror URL, the world token where the Google credential went.
|
|
228
|
+
|
|
229
|
+
The world has its own date. A trace-built world's clock is frozen at the recorded start, so a
|
|
230
|
+
run that begins days or months after the recording still finds the recorded emails, events and
|
|
231
|
+
records at their recorded times. An agent that reads the wall clock for "today", "tomorrow" or
|
|
232
|
+
`newer_than:7d` asks the world about dates it holds nothing for; one that reads the world's date
|
|
233
|
+
lands on the recorded ones. Read it with `worldNow`: from the handoff, or from the environment of
|
|
234
|
+
an agent child, where `agentEnvironment` and `hue eval --command` set `HUE_WORLD_NOW`.
|
|
235
|
+
|
|
236
|
+
```ts
|
|
237
|
+
import { worldNow } from "@hue-run/sdk/environment";
|
|
238
|
+
|
|
239
|
+
const today = worldNow() ?? new Date(); // HUE_WORLD_NOW in this process, else the wall clock
|
|
240
|
+
```
|
|
227
241
|
|
|
228
242
|
```ts
|
|
229
243
|
import {
|
|
@@ -314,8 +328,11 @@ type; see [EVALUATIONS.md](EVALUATIONS.md#direct-cases-and-files).
|
|
|
314
328
|
A refusal such as 409 `simulation_gateway_required` exposes its validated server code as `HueEnvironmentError.diagnostic`.
|
|
315
329
|
|
|
316
330
|
`agentEnvironment` removes Hue control-plane credentials from the child by default: `HUE_API_KEY`,
|
|
317
|
-
`HUE_MCP_KEY
|
|
318
|
-
|
|
331
|
+
`HUE_MCP_KEY`, and any value carrying a Hue control-plane credential (`hue_sk_`, `hue_setup_`,
|
|
332
|
+
`hue_attempt_`, `hue_mcp_`, `hue_inv_`, `hue_install_`, `hue_at_`, `hue_rt_`, `hue_oauth_`)
|
|
333
|
+
anywhere in it, including wrapped, percent-encoded or JSON-Unicode-escaped values. World tokens and
|
|
334
|
+
`hue_sim_` capabilities pass. Pass `includeHueCredentials: true` only for an agent that must call
|
|
335
|
+
Hue's own API.
|
|
319
336
|
Nothing in these helpers logs the token; keep it out of your own logs and checkpoints.
|
|
320
337
|
|
|
321
338
|
Finish answers `lifecycle: "completing"` with `sealedAt: null` for a gateway world: the seal
|
package/EVALUATIONS.md
CHANGED
|
@@ -107,6 +107,24 @@ evidence and scoring while retaining this runner's checkpoint guarantees. See
|
|
|
107
107
|
Cases whose inputs are a task plus pinned documents, and rescoring over documents a run already
|
|
108
108
|
saved, use the same runner without a world. See [Direct cases and files](#direct-cases-and-files).
|
|
109
109
|
|
|
110
|
+
## Tags
|
|
111
|
+
|
|
112
|
+
Tags group datasets, scorers and experiments, for example by the area of the agent they test.
|
|
113
|
+
Methods take tag names: a name the project has no tag for creates one, and names match without
|
|
114
|
+
case. `createDataset`, `createScorer` and `createExperiment` accept `tags`; `updateDataset`,
|
|
115
|
+
`updateScorer` and `updateExperiment` replace an item's tags; `listDatasets` and `listScorers`
|
|
116
|
+
filter by `tags`, matching any of them; and `listTags()` returns the project's tags. An experiment
|
|
117
|
+
also shows its dataset's tags, which experiment filters match as well. The product-named methods
|
|
118
|
+
(`createEvalSet`, `updateEvalSet`, `listEvalSets`, `createEvaluator`, `updateEvaluator`,
|
|
119
|
+
`listEvaluators`, `createRun`, `updateRun`) take the same `tags`. Tags require a Hue server that
|
|
120
|
+
supports them.
|
|
121
|
+
|
|
122
|
+
```ts
|
|
123
|
+
await client.createEvalSet({ name: "Refunds", slug: "refunds", tags: ["billing"] });
|
|
124
|
+
await client.updateEvaluator(evaluatorId, { tags: ["billing", "regression"] });
|
|
125
|
+
const billing = await client.listEvalSets({ tags: ["billing"] });
|
|
126
|
+
```
|
|
127
|
+
|
|
110
128
|
## Content and result states
|
|
111
129
|
|
|
112
130
|
Both choices are required and independent:
|
|
@@ -146,7 +164,7 @@ A callback receives `{inputs,hasOutput,output?,hasExpected,expected?,metadata,ex
|
|
|
146
164
|
- `{state:"error",error:{type,message?}}`.
|
|
147
165
|
- `{state:"skipped",explanation:"reason"}`.
|
|
148
166
|
|
|
149
|
-
All declared metrics must appear exactly once and satisfy pinned types, bounds and categories. The binding must match language, entrypoint, SHA-256 source digest and metric definitions. The digest is an authenticated caller declaration; it does not attest closures, dependency versions or actual execution. Callback source is never downloaded or evaluated. Callbacks are trusted local code; they have **no execution timeout or side-effect cancellation**. Concurrency limits active cases to 1–16 (default 1), with scorers evaluated sequentially within each case.
|
|
167
|
+
All declared metrics must appear exactly once and satisfy pinned types, bounds and categories. Each metric may include an optional `description` (1–500 characters) shown with its check in Hue's run view. The binding must match language, entrypoint, SHA-256 source digest and metric definitions. The digest is an authenticated caller declaration; it does not attest closures, dependency versions or actual execution. Callback source is never downloaded or evaluated. Callbacks are trusted local code; they have **no execution timeout or side-effect cancellation**. Concurrency limits active cases to 1–16 (default 1), with scorers evaluated sequentially within each case.
|
|
150
168
|
|
|
151
169
|
`files` is present only when the runner handled files for that execution: every pinned input file
|
|
152
170
|
(all roles, including the evaluator-only `org_template`) and every generated output, each with its
|
package/README.md
CHANGED
|
@@ -45,7 +45,6 @@ await hue.withSpan(
|
|
|
45
45
|
async (span) => {
|
|
46
46
|
const output = await hue.tool("uppercase", "hello", () => "hello".toUpperCase());
|
|
47
47
|
span.setOutput(output);
|
|
48
|
-
hue.recordMessages({ output: [{ role: "assistant", content: output }] });
|
|
49
48
|
},
|
|
50
49
|
{ sessionId: "session-123", userId: "user-123", input: "hello" },
|
|
51
50
|
);
|
|
@@ -107,7 +106,8 @@ by the semantic conventions'
|
|
|
107
106
|
[input messages](https://github.com/open-telemetry/semantic-conventions/blob/v1.41.0/docs/gen-ai/gen-ai-input-messages.json)
|
|
108
107
|
and
|
|
109
108
|
[output messages](https://github.com/open-telemetry/semantic-conventions/blob/v1.41.0/docs/gen-ai/gen-ai-output-messages.json)
|
|
110
|
-
JSON schemas
|
|
109
|
+
JSON schemas. GenAI conventions are in Development; Hue supports the fields described here
|
|
110
|
+
and accepts caller-supplied JSON without validating the full draft. Convert provider-native
|
|
111
111
|
messages before recording them:
|
|
112
112
|
|
|
113
113
|
```ts
|
|
@@ -313,7 +313,7 @@ policy forbids sending that content to another service. `redact` and the credent
|
|
|
313
313
|
described below apply in both modes.
|
|
314
314
|
|
|
315
315
|
`captureContent: false` disables manual input/output/messages/tool content and
|
|
316
|
-
removes recognized GenAI, Vercel, OpenInference and
|
|
316
|
+
removes recognized GenAI, Vercel, OpenInference, OpenLLMetry and Langfuse content attributes,
|
|
317
317
|
legacy GenAI content events, log bodies, status messages and exception text before
|
|
318
318
|
export. The exported `contentPrefixes` array lists the attribute keys (and their dotted
|
|
319
319
|
children) that are removed. Model/provider/token metadata remains available. Generic custom attribute
|
|
@@ -323,7 +323,13 @@ names cannot be classified automatically; use them deliberately.
|
|
|
323
323
|
there is no SDK retention timer or automatic content expiry. To redact strings
|
|
324
324
|
before export, supply `redact(value, path)`; it applies to supported strings in
|
|
325
325
|
attributes, resources, event/link attributes and log bodies. Return a string.
|
|
326
|
-
Invalid
|
|
326
|
+
Invalid helper content is omitted with an instrumentation failure; the span can still be delivered.
|
|
327
|
+
A content value over Hue's 256 KiB value cap is not omitted: text is cut to a UTF-8 prefix and
|
|
328
|
+
bytes or a structured log body are replaced by the receiver's own marker
|
|
329
|
+
(`{ "hue.truncated": true, "hue.truncated_bytes": <size> }`), and the value's key is listed in
|
|
330
|
+
the record's `hue.truncated` attribute, where Hue's receiver lists the values it cuts itself,
|
|
331
|
+
so the span still carries the call, the recorded part of its value and the rest of its evidence.
|
|
332
|
+
Export-time redactor failures reject the affected record and are reported by flush. Shared resources are redacted once per
|
|
327
333
|
export batch. Do not put user content or secrets in span names or scope names.
|
|
328
334
|
|
|
329
335
|
Hosted tools carry credentials in their definitions, such as the `authorization` and `headers` of
|
|
@@ -355,7 +361,7 @@ rotates. Only definitions another integration recorded can be summarized: `hueTe
|
|
|
355
361
|
inputs and exports through Hue's attached processors gets the summary.
|
|
356
362
|
Recorded messages can inline files: GenAI `blob` parts in `gen_ai.input.messages` /
|
|
357
363
|
`gen_ai.output.messages` (what the AI SDK 7 adapter records for a file part) and AI SDK 6 `file`
|
|
358
|
-
parts in `ai.prompt.messages`. A span whose messages exceed 256 KiB would be
|
|
364
|
+
parts in `ai.prompt.messages`. A span whose messages exceed 256 KiB would be cut, so when a
|
|
359
365
|
record is queued Hue replaces the `content`/`data` of any such part longer than 64 KiB with the
|
|
360
366
|
file's `sha256` and `size`, both of the file's own bytes whatever its media type (base64 content
|
|
361
367
|
and `;base64` `data:` URLs decoded, other `data:` URLs percent-decoded, anything else as UTF-8 text), keeping the part's other fields such as `type`, `mime_type` and `mediaType`. Smaller inline
|
|
@@ -379,8 +385,8 @@ span in `finally`.
|
|
|
379
385
|
the active span, with the messages in its body. The record also carries `gen_ai.operation.name`,
|
|
380
386
|
`gen_ai.provider.name` and `gen_ai.request.model` as attributes, copied from the enclosing
|
|
381
387
|
`hue.model()` span or passed as `operation`, `provider` and `model`, and `gen_ai.conversation.id`
|
|
382
|
-
from the active session,
|
|
383
|
-
context.
|
|
388
|
+
from the active session, when those values are known. Use `recordMessages` inside `hue.model()` or
|
|
389
|
+
supply `operation` and `provider` explicitly to include the required inference context.
|
|
384
390
|
|
|
385
391
|
## Existing OpenTelemetry providers
|
|
386
392
|
|
|
@@ -447,7 +453,8 @@ that accepts `/api/v1/otlp/v1/traces` and `/api/v1/otlp/v1/logs` (for example an
|
|
|
447
453
|
Collector `otlp` receiver with `http.traces_url_path` and `logs_url_path` set to those paths,
|
|
448
454
|
forwarding to Jaeger or the debug exporter) and pass any placeholder `apiKey`; HTTP is allowed
|
|
449
455
|
for loopback origins. `checkConnection()` and `verifyTrace()` are Hue-only diagnostics and are
|
|
450
|
-
not available against a generic collector.
|
|
456
|
+
not available against a generic collector. Set `liveSpans: false` so Hue-specific placeholders
|
|
457
|
+
do not reach the collector's other destinations.
|
|
451
458
|
|
|
452
459
|
A collector on a private network is not loopback: a docker-compose sibling such as
|
|
453
460
|
`http://otel-collector:4318` or an in-cluster service requires the explicit opt-in
|
|
@@ -461,6 +468,7 @@ const hue = createHue({
|
|
|
461
468
|
serviceName: "my-agent",
|
|
462
469
|
captureContent: true,
|
|
463
470
|
baseUrl: "http://otel-collector:4318",
|
|
471
|
+
liveSpans: false,
|
|
464
472
|
allowInsecureHttp: true,
|
|
465
473
|
});
|
|
466
474
|
```
|
|
@@ -470,7 +478,11 @@ const hue = createHue({
|
|
|
470
478
|
Exports retry temporary HTTP/network failures (429, 502, 503, 504 and connection errors,
|
|
471
479
|
honoring `Retry-After`) within the export timeout, by OpenTelemetry's OTLP/HTTP exporter rules. Each
|
|
472
480
|
request is limited to 1 MiB before gzip (with space reserved for gzip overhead) and each content value to 256 KiB. Batches
|
|
473
|
-
split at record boundaries.
|
|
481
|
+
split at record boundaries. A record over the request limit on its own sheds its content values,
|
|
482
|
+
largest first, each replaced by the receiver's marker and listed under `hue.truncated`, until it
|
|
483
|
+
fits; only a record too large without any content value is lost, reported as an `invalid` issue,
|
|
484
|
+
and counted on its trace's root span as `hue.sdk.dropped_records` when the root is exported, so
|
|
485
|
+
Hue reads the trace as incomplete by that many records. Each signal queues at most 2,048 records, including
|
|
474
486
|
exports in flight; overflow is reported through the callback, counters and next
|
|
475
487
|
flush. This is an in-memory queue, not durable storage.
|
|
476
488
|
|
|
@@ -522,10 +534,14 @@ short span may send none and appear in Hue only when it finishes.
|
|
|
522
534
|
record and byte budgets, and skipped silently otherwise. While queued they count in
|
|
523
535
|
`pendingSpans` and `pendingBytes`, but never as accepted, rejected, failed or dropped records.
|
|
524
536
|
Losing only placeholders records a warning and does not make `flush()` throw.
|
|
525
|
-
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
537
|
+
- Placeholders travel in their own requests, after an export's completed spans, so a rejection
|
|
538
|
+
count is always one kind of record's: a completed span's rejection is never credited to a
|
|
539
|
+
placeholder, and a rejected request of placeholders is a warning. A Hue server that accepts
|
|
540
|
+
placeholders sends `Hue-Pending-Spans: 1` on every trace acknowledgement. The first trace
|
|
541
|
+
acknowledgement without it, whatever its request carried, shows the receiver predates them: the
|
|
542
|
+
transport records one warning (counting the placeholders the receiver rejected in that request, if any) and stops
|
|
543
|
+
sending placeholders for that client, so a receiver that acknowledged completed spans first
|
|
544
|
+
never receives one.
|
|
529
545
|
- Opt out with `liveSpans: false`. Setup credentials never send placeholders.
|
|
530
546
|
- With an existing provider, announcements start in `spanProcessor.onStart`. A wrapping processor
|
|
531
547
|
that forwards `onStart` should forward `onEnd` for the same spans: a span that ends without
|
package/dist/client.d.ts
CHANGED
|
@@ -154,6 +154,14 @@ export declare class HueClient {
|
|
|
154
154
|
* alone is omitted and counted while the rest is recorded.
|
|
155
155
|
*/
|
|
156
156
|
recordFile(file: FileRecord, explicitContext?: Context): void;
|
|
157
|
+
/** The keys of each span's content values that were cut to Hue's value cap, for `hue.truncated`. */
|
|
158
|
+
private truncatedKeys;
|
|
159
|
+
/**
|
|
160
|
+
* A content value as a JSON attribute within Hue's 256 KiB value cap. A larger value is cut to
|
|
161
|
+
* a UTF-8 prefix and its key listed under `hue.truncated`, where Hue's receiver lists the
|
|
162
|
+
* values it cuts itself: the span carries the call and the recorded part of its value instead
|
|
163
|
+
* of losing the value whole. Only a value that cannot be encoded is omitted and counted.
|
|
164
|
+
*/
|
|
157
165
|
private setContent;
|
|
158
166
|
/**
|
|
159
167
|
* Confirms the key and origin by reading the current project; a setup and CI diagnostic, not a
|
package/dist/client.js
CHANGED
|
@@ -10,7 +10,8 @@ import { defaultResource, resourceFromAttributes } from "@opentelemetry/resource
|
|
|
10
10
|
import { HUE_SCOPE } from "./config.js";
|
|
11
11
|
import { hostedServerAddresses, hostedToolActivity, hostedToolProvider, providerErrorDescription, } from "./provider-tools.js";
|
|
12
12
|
import { toolCatalogSummary } from "./tool-definitions.js";
|
|
13
|
-
import { encodeContent, noopSpan, safeSpan } from "./safety.js";
|
|
13
|
+
import { encodeBoundedContent, encodeContent, noopSpan, safeSpan, } from "./safety.js";
|
|
14
|
+
import { TRUNCATED_KEY, withTruncatedKeys } from "./privacy.js";
|
|
14
15
|
import { createHueTransport, HueExportError } from "./transport.js";
|
|
15
16
|
import { verifyTrace } from "./receipt.js";
|
|
16
17
|
import { sdkVersion } from "./version.js";
|
|
@@ -712,11 +713,31 @@ export class HueClient {
|
|
|
712
713
|
this.instrumentationFailed();
|
|
713
714
|
}
|
|
714
715
|
}
|
|
716
|
+
/** The keys of each span's content values that were cut to Hue's value cap, for `hue.truncated`. */
|
|
717
|
+
truncatedKeys = new WeakMap();
|
|
718
|
+
/**
|
|
719
|
+
* A content value as a JSON attribute within Hue's 256 KiB value cap. A larger value is cut to
|
|
720
|
+
* a UTF-8 prefix and its key listed under `hue.truncated`, where Hue's receiver lists the
|
|
721
|
+
* values it cuts itself: the span carries the call and the recorded part of its value instead
|
|
722
|
+
* of losing the value whole. Only a value that cannot be encoded is omitted and counted.
|
|
723
|
+
*/
|
|
715
724
|
setContent(span, key, value) {
|
|
716
725
|
if (!this.enabled || this.closed || !this.captureContent)
|
|
717
726
|
return;
|
|
718
727
|
try {
|
|
719
|
-
|
|
728
|
+
const { text, truncated } = encodeBoundedContent(value);
|
|
729
|
+
span.setAttribute(key, text);
|
|
730
|
+
// The span's list as it stands: the application's own entries (read from the SDK span,
|
|
731
|
+
// which carries its attributes; the API alone shows none) and this helper's earlier ones.
|
|
732
|
+
const listed = withTruncatedKeys(span.attributes?.[TRUNCATED_KEY], this.truncatedKeys.get(span) ?? []);
|
|
733
|
+
const keys = truncated
|
|
734
|
+
? withTruncatedKeys(listed, [key])
|
|
735
|
+
: listed.filter((entry) => entry !== key);
|
|
736
|
+
if (truncated || listed.includes(key)) {
|
|
737
|
+
// A whole value replacing a cut one unmarks its key; an untouched list is left alone.
|
|
738
|
+
this.truncatedKeys.set(span, keys);
|
|
739
|
+
span.setAttribute(TRUNCATED_KEY, keys);
|
|
740
|
+
}
|
|
720
741
|
}
|
|
721
742
|
catch {
|
|
722
743
|
this.instrumentationFailed();
|
|
@@ -364,7 +364,11 @@ export interface WorldHandoff {
|
|
|
364
364
|
baggage: string;
|
|
365
365
|
/** Mirror surfaces of every provider instance the environment version binds. */
|
|
366
366
|
surfaces: WorldSurface[];
|
|
367
|
-
/**
|
|
367
|
+
/** The world's clock at creation (RFC 3339): the date an agent computes its dates from in
|
|
368
|
+
* place of the wall clock, a trace-built world's recorded start. Also `env.HUE_WORLD_NOW`;
|
|
369
|
+
* null for a Hue that names neither. */
|
|
370
|
+
now: string | null;
|
|
371
|
+
/** `HUE_WORLD_ID`, `HUE_WORLD_TOKEN`, `HUE_WORLD_NOW`, `BAGGAGE`, `TRACEPARENT` and one
|
|
368
372
|
* `HUE_SIM_<SURFACE ID>_URL` per surface. */
|
|
369
373
|
env: Record<string, string>;
|
|
370
374
|
/** The `mcpServers` configuration for the MCP surfaces. */
|
|
@@ -378,6 +382,9 @@ export interface EnvironmentRun {
|
|
|
378
382
|
environmentVersionId: string;
|
|
379
383
|
/** Current virtual nanoseconds as a decimal string. */
|
|
380
384
|
clockNs: string;
|
|
385
|
+
/** The world's clock at creation as an RFC 3339 timestamp (`clockNs` as a date): a
|
|
386
|
+
* trace-built world's recorded start. Absent from a Hue that predates it. */
|
|
387
|
+
now?: string;
|
|
381
388
|
/** Digest of current world state. */
|
|
382
389
|
stateDigest: string;
|
|
383
390
|
/** Maximum recorded actions. */
|
|
@@ -5,9 +5,22 @@ import type { EnvironmentRun, LegacyMcpCapability, WorldHandoff } from "./types.
|
|
|
5
5
|
* mirror URLs. The handoff carries the world token; keep it out of logs and checkpoints.
|
|
6
6
|
*/
|
|
7
7
|
export declare function worldHandoff(run: EnvironmentRun): WorldHandoff | null;
|
|
8
|
+
/**
|
|
9
|
+
* The world's clock: what an agent reads for today's date in place of the wall clock, so a
|
|
10
|
+
* date-relative request (tomorrow, `newer_than:7d`, this month) lands on the dates the world
|
|
11
|
+
* holds however long after the recording the run starts. From the handoff's `now`, else from
|
|
12
|
+
* `HUE_WORLD_NOW` in the given environment (this process's by default, where `hue eval
|
|
13
|
+
* --command` and `agentEnvironment` set it). Null when neither names it.
|
|
14
|
+
*/
|
|
15
|
+
export declare function worldNow(source?: WorldHandoff | Record<string, string | undefined>): Date | null;
|
|
8
16
|
/** Variables that authenticate against Hue's control plane rather than a simulated provider. */
|
|
9
17
|
export declare const HUE_CONTROL_PLANE_VARIABLES: readonly string[];
|
|
10
|
-
/**
|
|
18
|
+
/** Prefixes of the Hue control-plane credentials the server issues: project and setup keys,
|
|
19
|
+
* attempt grants, the project MCP key, invocation and installation tokens, and OAuth tokens and
|
|
20
|
+
* secrets. World tokens (`hue_world_`) and MCP capabilities (`hue_sim_`) are what the agent holds. */
|
|
21
|
+
export declare const HUE_CONTROL_PLANE_CREDENTIAL_PREFIXES: readonly string[];
|
|
22
|
+
/** True for a control-plane variable by name, or for a value carrying a Hue control-plane
|
|
23
|
+
* credential anywhere in it, including wrapped, percent-encoded or JSON-Unicode-escaped values. */
|
|
11
24
|
export declare function isHueControlPlaneCredential(name: string, value: string | undefined): boolean;
|
|
12
25
|
/** The parent's variables without Hue control-plane credentials; `undefined` values are dropped. */
|
|
13
26
|
export declare function stripHueControlPlaneCredentials(parent: Record<string, string | undefined>): Record<string, string>;
|
|
@@ -18,6 +18,7 @@ export function worldHandoff(run) {
|
|
|
18
18
|
completingUntil: run.completingUntil ?? null,
|
|
19
19
|
traceparent: run.traceparent ?? null,
|
|
20
20
|
baggage: run.baggage ?? `hue-world=${id}`,
|
|
21
|
+
now: run.now ?? run.env.HUE_WORLD_NOW ?? null,
|
|
21
22
|
surfaces: run.surfaces.map((surface) => ({ ...surface })),
|
|
22
23
|
env: { ...run.env },
|
|
23
24
|
mcpConfig: {
|
|
@@ -28,6 +29,22 @@ export function worldHandoff(run) {
|
|
|
28
29
|
},
|
|
29
30
|
};
|
|
30
31
|
}
|
|
32
|
+
/**
|
|
33
|
+
* The world's clock: what an agent reads for today's date in place of the wall clock, so a
|
|
34
|
+
* date-relative request (tomorrow, `newer_than:7d`, this month) lands on the dates the world
|
|
35
|
+
* holds however long after the recording the run starts. From the handoff's `now`, else from
|
|
36
|
+
* `HUE_WORLD_NOW` in the given environment (this process's by default, where `hue eval
|
|
37
|
+
* --command` and `agentEnvironment` set it). Null when neither names it.
|
|
38
|
+
*/
|
|
39
|
+
export function worldNow(source = process.env) {
|
|
40
|
+
const text = "env" in source && typeof source.env === "object" && source.env !== null
|
|
41
|
+
? (source.now ?? source.env.HUE_WORLD_NOW)
|
|
42
|
+
: source.HUE_WORLD_NOW;
|
|
43
|
+
if (!text)
|
|
44
|
+
return null;
|
|
45
|
+
const date = new Date(text);
|
|
46
|
+
return Number.isNaN(date.getTime()) ? null : date;
|
|
47
|
+
}
|
|
31
48
|
/** Variables that authenticate against Hue's control plane rather than a simulated provider. */
|
|
32
49
|
export const HUE_CONTROL_PLANE_VARIABLES = [
|
|
33
50
|
"HUE_API_KEY",
|
|
@@ -35,13 +52,53 @@ export const HUE_CONTROL_PLANE_VARIABLES = [
|
|
|
35
52
|
"HUE_PROJECT_KEY",
|
|
36
53
|
"HUE_SERVICE_KEY",
|
|
37
54
|
];
|
|
38
|
-
/**
|
|
39
|
-
|
|
40
|
-
|
|
55
|
+
/** Prefixes of the Hue control-plane credentials the server issues: project and setup keys,
|
|
56
|
+
* attempt grants, the project MCP key, invocation and installation tokens, and OAuth tokens and
|
|
57
|
+
* secrets. World tokens (`hue_world_`) and MCP capabilities (`hue_sim_`) are what the agent holds. */
|
|
58
|
+
export const HUE_CONTROL_PLANE_CREDENTIAL_PREFIXES = [
|
|
59
|
+
"hue_sk_",
|
|
60
|
+
"hue_attempt_",
|
|
61
|
+
"hue_mcp_",
|
|
62
|
+
"hue_setup_",
|
|
63
|
+
"hue_inv_",
|
|
64
|
+
"hue_install_",
|
|
65
|
+
"hue_at_",
|
|
66
|
+
"hue_rt_",
|
|
67
|
+
"hue_oauth_",
|
|
68
|
+
];
|
|
69
|
+
/** A Hue credential anywhere in a value, including JSON Unicode escapes, not inside a longer word. */
|
|
70
|
+
const HUE_CREDENTIAL_SHAPE = new RegExp(`(?:^|[^A-Za-z0-9_])(?:${HUE_CONTROL_PLANE_CREDENTIAL_PREFIXES.join("|")})`);
|
|
71
|
+
const ASCII_PERCENT_ESCAPE = /%[0-7][0-9a-f]/i;
|
|
72
|
+
const ASCII_JSON_UNICODE_ESCAPE = /\\u00([0-7][0-9a-fA-F])/g;
|
|
73
|
+
function decodeAsciiJsonUnicodeEscapes(value) {
|
|
74
|
+
return value.replace(ASCII_JSON_UNICODE_ESCAPE, (_, hex) => String.fromCharCode(Number.parseInt(hex, 16)));
|
|
75
|
+
}
|
|
76
|
+
function hasHueControlPlaneCredential(value) {
|
|
77
|
+
let candidates = new Set([value]);
|
|
78
|
+
for (let layer = 0; layer <= 4; layer++) {
|
|
79
|
+
const jsonDecodedCandidates = new Set();
|
|
80
|
+
for (const candidate of candidates) {
|
|
81
|
+
jsonDecodedCandidates.add(candidate);
|
|
82
|
+
jsonDecodedCandidates.add(decodeAsciiJsonUnicodeEscapes(candidate));
|
|
83
|
+
}
|
|
84
|
+
if ([...jsonDecodedCandidates].some((candidate) => HUE_CREDENTIAL_SHAPE.test(candidate)))
|
|
85
|
+
return true;
|
|
86
|
+
if (layer === 4)
|
|
87
|
+
return [...jsonDecodedCandidates].some((candidate) => ASCII_PERCENT_ESCAPE.test(candidate));
|
|
88
|
+
candidates = new Set([...jsonDecodedCandidates]
|
|
89
|
+
.filter((candidate) => ASCII_PERCENT_ESCAPE.test(candidate))
|
|
90
|
+
.map((candidate) => candidate.replace(/%([0-7][0-9a-f])/gi, (_, hex) => String.fromCharCode(Number.parseInt(hex, 16)))));
|
|
91
|
+
if (candidates.size === 0)
|
|
92
|
+
return false;
|
|
93
|
+
}
|
|
94
|
+
return false;
|
|
95
|
+
}
|
|
96
|
+
/** True for a control-plane variable by name, or for a value carrying a Hue control-plane
|
|
97
|
+
* credential anywhere in it, including wrapped, percent-encoded or JSON-Unicode-escaped values. */
|
|
41
98
|
export function isHueControlPlaneCredential(name, value) {
|
|
42
99
|
if (HUE_CONTROL_PLANE_VARIABLES.includes(name))
|
|
43
100
|
return true;
|
|
44
|
-
return typeof value === "string" &&
|
|
101
|
+
return typeof value === "string" && hasHueControlPlaneCredential(value);
|
|
45
102
|
}
|
|
46
103
|
/** The parent's variables without Hue control-plane credentials; `undefined` values are dropped. */
|
|
47
104
|
export function stripHueControlPlaneCredentials(parent) {
|
package/dist/environment.d.ts
CHANGED
|
@@ -3,5 +3,5 @@ export type { EnvironmentClientOptions } from "./environment/client.js";
|
|
|
3
3
|
export { bindEnvironmentTools } from "./environment/tools.js";
|
|
4
4
|
export type { BindEnvironmentToolsOptions, EnvironmentTool } from "./environment/tools.js";
|
|
5
5
|
export type * from "./environment/types.js";
|
|
6
|
-
export { agentEnvironment, HUE_CONTROL_PLANE_VARIABLES, isHueControlPlaneCredential, legacyMcpCapability, stripHueControlPlaneCredentials, worldHandoff, writeMcpConfig, } from "./environment/world.js";
|
|
6
|
+
export { agentEnvironment, HUE_CONTROL_PLANE_CREDENTIAL_PREFIXES, HUE_CONTROL_PLANE_VARIABLES, isHueControlPlaneCredential, legacyMcpCapability, stripHueControlPlaneCredentials, worldHandoff, worldNow, writeMcpConfig, } from "./environment/world.js";
|
|
7
7
|
export type { AgentEnvironmentOptions, McpConfigFile } from "./environment/world.js";
|
package/dist/environment.js
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
export { createEnvironmentClient, EnvironmentClient, HueEnvironmentError, isTransientEnvironmentError, } from "./environment/client.js";
|
|
2
2
|
export { bindEnvironmentTools } from "./environment/tools.js";
|
|
3
|
-
export { agentEnvironment, HUE_CONTROL_PLANE_VARIABLES, isHueControlPlaneCredential, legacyMcpCapability, stripHueControlPlaneCredentials, worldHandoff, writeMcpConfig, } from "./environment/world.js";
|
|
3
|
+
export { agentEnvironment, HUE_CONTROL_PLANE_CREDENTIAL_PREFIXES, HUE_CONTROL_PLANE_VARIABLES, isHueControlPlaneCredential, legacyMcpCapability, stripHueControlPlaneCredentials, worldHandoff, worldNow, writeMcpConfig, } from "./environment/world.js";
|
package/dist/evals/client.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { ProjectConnection } from "../types.js";
|
|
2
2
|
import { type AttemptConnectionBundleV2, type PrepareAttemptRequestV2 } from "./attempt.js";
|
|
3
|
-
import type { ArtifactReservation, ArtifactUpload, CaseConversion, CaseConversionSummary, CaseWrite, CompleteExecution, Completion, Dataset, DatasetCase, DatasetVersion, EvalSet, EvalSetCase, EvalSetVersion, EvaluationItem, EvaluationRun, EvaluationRunSummary, EnvironmentEvidenceSnapshot, Execution, Experiment, ExperimentCase, ExperimentItem, Evaluator, EvaluatorVersion,
|
|
3
|
+
import type { ArtifactReservation, ArtifactUpload, CaseConversion, CaseConversionSummary, CaseWrite, CompleteExecution, Completion, Dataset, DatasetCase, DatasetVersion, EvalSet, EvalSetCase, EvalSetVersion, EvaluationItem, EvaluationRun, EvaluationRunSummary, EnvironmentEvidenceSnapshot, Execution, Experiment, ExperimentCase, ExperimentItem, Evaluator, EvaluatorVersion, JsonValue, LocalAgentClaim, LocalAgentRegistration, RegisteredLocalAgent, JudgeBudget, JudgeJob, Page, PageOptions, RegistryPageOptions, Result, ResultSummary, Run, RunCase, Scorer, ScorerDefinition, ScorerVersion, Scoring, ScoringResultInput, ScoringResultSummary, ScoringSubject, ScoringSummary, SimulationMcpCapability, StartExecution, Subject, StoredResult, StoredScoringResult, IdentityUpdate, Tag, TaggedIdentity, UpdatedRun } from "./types.js";
|
|
4
4
|
/** Connection options for {@link createEvaluationClient}. */
|
|
5
5
|
export interface EvaluationClientOptions {
|
|
6
6
|
/** Project service key sent as a Bearer token; server side only. */
|
|
@@ -79,8 +79,12 @@ export declare class EvaluationClient {
|
|
|
79
79
|
private registryPage;
|
|
80
80
|
/** Reads the current project to confirm the key and origin. */
|
|
81
81
|
checkConnection(): Promise<ProjectConnection>;
|
|
82
|
-
/** Creates a dataset with an initial draft version. */
|
|
83
|
-
createDataset(input:
|
|
82
|
+
/** Creates a dataset with an initial draft version; `tags` names the tags it starts with. */
|
|
83
|
+
createDataset(input: TaggedIdentity): Promise<Dataset>;
|
|
84
|
+
/** Renames a dataset or replaces its tags by name; the same values again change nothing. */
|
|
85
|
+
updateDataset(id: string, input: IdentityUpdate & {
|
|
86
|
+
slug?: string;
|
|
87
|
+
}): Promise<Omit<Dataset, "versions">>;
|
|
84
88
|
/** Reads a dataset and its versions. */
|
|
85
89
|
getDataset(id: string): Promise<Dataset>;
|
|
86
90
|
/** Lists datasets without their versions. */
|
|
@@ -103,7 +107,14 @@ export declare class EvaluationClient {
|
|
|
103
107
|
/** Freezes a draft version at the given revision; frozen versions are immutable. */
|
|
104
108
|
freezeDatasetVersion(id: string, expectedRevision: number): Promise<DatasetVersion>;
|
|
105
109
|
/** Creates a scorer identity; publish definitions with {@link publishScorerVersion}. */
|
|
106
|
-
createScorer(input:
|
|
110
|
+
createScorer(input: TaggedIdentity): Promise<Scorer>;
|
|
111
|
+
/** Renames a scorer or replaces its tags by name; the same values again change nothing. */
|
|
112
|
+
updateScorer(id: string, input: IdentityUpdate): Promise<Scorer>;
|
|
113
|
+
/** Lists the project's tags in the order they were created. */
|
|
114
|
+
listTags(): Promise<{
|
|
115
|
+
/** The project's tags. */
|
|
116
|
+
items: Tag[];
|
|
117
|
+
}>;
|
|
107
118
|
/** Reads a scorer and its published versions. */
|
|
108
119
|
getScorer(id: string): Promise<Scorer>;
|
|
109
120
|
/** Lists scorers. */
|
|
@@ -113,7 +124,11 @@ export declare class EvaluationClient {
|
|
|
113
124
|
/** Reads a published scorer version. */
|
|
114
125
|
getScorerVersion(id: string): Promise<ScorerVersion>;
|
|
115
126
|
/** Creates an eval set using the existing v1 registry path. */
|
|
116
|
-
createEvalSet(input:
|
|
127
|
+
createEvalSet(input: TaggedIdentity): Promise<EvalSet>;
|
|
128
|
+
/** Renames an eval set or replaces its tags by name. */
|
|
129
|
+
updateEvalSet(id: string, input: IdentityUpdate & {
|
|
130
|
+
slug?: string;
|
|
131
|
+
}): Promise<Omit<EvalSet, "versions">>;
|
|
117
132
|
/** Reads an eval set and its versions. */
|
|
118
133
|
getEvalSet(id: string): Promise<EvalSet>;
|
|
119
134
|
/** Lists eval sets. */
|
|
@@ -136,7 +151,9 @@ export declare class EvaluationClient {
|
|
|
136
151
|
/** Freezes a draft eval set version at its expected revision. */
|
|
137
152
|
freezeEvalSetVersion(id: string, expectedRevision: number): Promise<EvalSetVersion>;
|
|
138
153
|
/** Creates an evaluator identity. */
|
|
139
|
-
createEvaluator(input:
|
|
154
|
+
createEvaluator(input: TaggedIdentity): Promise<Evaluator>;
|
|
155
|
+
/** Renames an evaluator or replaces its tags by name. */
|
|
156
|
+
updateEvaluator(id: string, input: IdentityUpdate): Promise<Evaluator>;
|
|
140
157
|
/** Reads an evaluator and its published versions. */
|
|
141
158
|
getEvaluator(id: string): Promise<Evaluator>;
|
|
142
159
|
/** Lists evaluators. */
|
|
@@ -152,6 +169,8 @@ export declare class EvaluationClient {
|
|
|
152
169
|
datasetVersionId: string;
|
|
153
170
|
scorerVersionIds: string[];
|
|
154
171
|
config: JsonValue;
|
|
172
|
+
/** The experiment's own tag names; it also shows its dataset's tags. */
|
|
173
|
+
tags?: string[];
|
|
155
174
|
}): Promise<{
|
|
156
175
|
/** Experiment ID. */
|
|
157
176
|
id: string;
|
|
@@ -160,6 +179,11 @@ export declare class EvaluationClient {
|
|
|
160
179
|
}>;
|
|
161
180
|
/** Reads an experiment with its evaluation run and execution counts. */
|
|
162
181
|
getExperiment(id: string): Promise<Experiment>;
|
|
182
|
+
/** Renames an experiment or replaces its own tags by name. */
|
|
183
|
+
updateExperiment(id: string, input: {
|
|
184
|
+
name?: string;
|
|
185
|
+
tags?: string[];
|
|
186
|
+
}): Promise<UpdatedRun>;
|
|
163
187
|
/** Lists an experiment's cases with their latest executions. */
|
|
164
188
|
listExperimentItems(id: string, page?: PageOptions): Promise<Page<ExperimentItem>>;
|
|
165
189
|
/** Reads one frozen case of an experiment. */
|
|
@@ -261,6 +285,8 @@ export declare class EvaluationClient {
|
|
|
261
285
|
evalSetVersionId: string;
|
|
262
286
|
evaluatorVersionIds: string[];
|
|
263
287
|
config: JsonValue;
|
|
288
|
+
/** The run's own tag names; it also shows its eval set's tags. */
|
|
289
|
+
tags?: string[];
|
|
264
290
|
}): Promise<{
|
|
265
291
|
/** Run ID. */
|
|
266
292
|
id: string;
|
|
@@ -271,6 +297,11 @@ export declare class EvaluationClient {
|
|
|
271
297
|
}>;
|
|
272
298
|
/** Reads a run with its scoring progress. */
|
|
273
299
|
getRun(id: string): Promise<Run>;
|
|
300
|
+
/** Renames a run or replaces its own tags by name; its eval set's tags are not its own. */
|
|
301
|
+
updateRun(id: string, input: {
|
|
302
|
+
name?: string;
|
|
303
|
+
tags?: string[];
|
|
304
|
+
}): Promise<UpdatedRun>;
|
|
274
305
|
/** Lists a run's cases with their latest executions. */
|
|
275
306
|
listRunItems(id: string, page?: PageOptions): Promise<Page<ExperimentItem>>;
|
|
276
307
|
/** Reads one frozen case of a run. */
|