@hue-run/sdk 0.15.2 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLI.md +30 -21
- package/EVALUATIONS.md +1 -1
- package/README.md +71 -49
- package/dist/blobs.d.ts +35 -17
- package/dist/blobs.js +141 -75
- package/dist/cli/env-file.d.ts +2 -0
- package/dist/cli/env-file.js +2 -0
- package/dist/cli/eval-direct.d.ts +3 -2
- package/dist/cli/eval-direct.js +49 -6
- package/dist/cli/eval.js +109 -22
- package/dist/cli/login.js +2 -3
- package/dist/client.d.ts +3 -1
- package/dist/client.js +70 -20
- package/dist/evals/attempt.d.ts +1 -1
- package/dist/evals/attempt.js +1 -1
- package/dist/evals/checkpoint.d.ts +6 -2
- package/dist/evals/checkpoint.js +75 -12
- package/dist/evals/files.js +12 -0
- package/dist/evals/runner.d.ts +9 -0
- package/dist/evals/runner.js +63 -10
- package/dist/evals/scorer-publication.js +1 -1
- package/dist/evals/simulation.js +4 -2
- package/dist/inline-files.d.ts +38 -29
- package/dist/inline-files.js +288 -83
- package/dist/privacy.d.ts +1 -1
- package/dist/privacy.js +4 -10
- package/dist/safety.d.ts +1 -0
- package/dist/safety.js +7 -1
- package/dist/snapshot.d.ts +6 -3
- package/dist/snapshot.js +70 -25
- package/dist/transport.d.ts +11 -0
- package/dist/transport.js +296 -20
- package/dist/types.d.ts +7 -5
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +6 -10
package/CLI.md
CHANGED
|
@@ -42,7 +42,7 @@ project manifest are refused because managers can update ancestor locks; Python
|
|
|
42
42
|
The generated `hue.setup.mjs` or `hue_setup.py` always selects `captureContent: false` /
|
|
43
43
|
`capture_content=False`. For a supported application, setup installs the dependency and adds the
|
|
44
44
|
managed import and middleware registration to the existing entrypoint; an unreferenced helper is
|
|
45
|
-
not a completed integration. TypeScript uses `@hue-run/sdk@0.
|
|
45
|
+
not a completed integration. TypeScript uses `@hue-run/sdk@0.17.0`, `@opentelemetry/api@1.9.1` and
|
|
46
46
|
`@opentelemetry/context-async-hooks@2.11.0`; Python setup uses its separately tested package pin.
|
|
47
47
|
Content capture requires an ordinary account-managed key and a later explicit application decision.
|
|
48
48
|
|
|
@@ -469,7 +469,7 @@ node packages/sdk-typescript/scripts/verify-package.mjs --artifacts-dir .artifac
|
|
|
469
469
|
# Set project to an existing supported fixture; use the same directory on resume.
|
|
470
470
|
project=/absolute/path/to/supported-fixture
|
|
471
471
|
node packages/sdk-typescript/scripts/verify-setup-live.mjs \
|
|
472
|
-
--archive .artifacts/typescript/hue-run-sdk-0.
|
|
472
|
+
--archive .artifacts/typescript/hue-run-sdk-0.17.0.tgz \
|
|
473
473
|
--origin https://STAGING_ORIGIN \
|
|
474
474
|
--project "$project" --command setup \
|
|
475
475
|
--evidence .context/setup-staging-before-claim.json
|
|
@@ -480,7 +480,7 @@ the private local handoff and finish the real browser claim, then reconcile the
|
|
|
480
480
|
|
|
481
481
|
```sh
|
|
482
482
|
node packages/sdk-typescript/scripts/verify-setup-live.mjs \
|
|
483
|
-
--archive .artifacts/typescript/hue-run-sdk-0.
|
|
483
|
+
--archive .artifacts/typescript/hue-run-sdk-0.17.0.tgz \
|
|
484
484
|
--origin https://STAGING_ORIGIN \
|
|
485
485
|
--project "$project" --command claim \
|
|
486
486
|
--evidence .context/setup-staging-after-claim.json
|
|
@@ -510,7 +510,7 @@ and prints Hue's verdicts. It is the command-line form of `runSimulation()` (one
|
|
|
510
510
|
process, Hue creates one isolated simulated world per case, and Hue-owned outcome checks grade
|
|
511
511
|
the sealed world. Hue never executes the agent. `HUE_API_KEY` must be a **Read and
|
|
512
512
|
write** project key (a **Read** or **Tracing only** key cannot read cases or create runs);
|
|
513
|
-
the CLI never prints it.
|
|
513
|
+
the CLI never prints it.
|
|
514
514
|
|
|
515
515
|
Run the agent's own start command with `--command`. `hue eval` spawns it once per case with the
|
|
516
516
|
world's environment (below) and `{"inputs", "config"}` on stdin, and stores its stdout as the
|
|
@@ -519,19 +519,20 @@ committed helper described in [Make your agent eval-ready](https://docs.hue.run/
|
|
|
519
519
|
reads `HUE_SIM_<SURFACE ID>_URL` and `HUE_WORLD_TOKEN` when `HUE_WORLD_TOKEN` is set, fails
|
|
520
520
|
closed when a needed variable is missing, and returns the production URL and credential
|
|
521
521
|
otherwise. Never hand the agent Hue-native tools (`context.tools`, `context.mcp`) or a tool its
|
|
522
|
-
production agent lacks. Pin the SDK
|
|
523
|
-
gains no dependency:
|
|
522
|
+
production agent lacks. Pin the SDK, so the agent's repository gains no dependency:
|
|
524
523
|
|
|
525
524
|
```sh
|
|
526
|
-
npx
|
|
527
|
-
npx
|
|
528
|
-
npx
|
|
529
|
-
npx
|
|
525
|
+
npx -y @hue-run/sdk@0.17.0 eval --case "Refund an eligible charge" --command "node agent.js"
|
|
526
|
+
npx -y @hue-run/sdk@0.17.0 eval --case https://app.hue.run/projects/demo/scenarios/<id> --command "python agent.py" --timeout 120 --baseline <run id>
|
|
527
|
+
npx -y @hue-run/sdk@0.17.0 eval --set "Billing regressions" --scorer-version <id> --command "node agent.js" --save-version
|
|
528
|
+
npx -y @hue-run/sdk@0.17.0 eval --worker --command "node agent.js" --agent-key support-agent --revision <new-agent-revision>
|
|
530
529
|
```
|
|
531
530
|
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
531
|
+
One invocation creates one run holding every selected case; it prints the run URL
|
|
532
|
+
(`https://app.hue.run/runs/<id>`) once the run exists. Add `--check` to confirm the key, the
|
|
533
|
+
selection, the case kind and the agent first: it prints what the run would do and exits without
|
|
534
|
+
creating a run, a world or a checkpoint. An
|
|
535
|
+
unpinned `npx @hue-run/sdk` runs a local, possibly older, copy in a project that already
|
|
535
536
|
depends on it.
|
|
536
537
|
|
|
537
538
|
An adapter module is the in-process alternative for a TypeScript agent. It exports `default` or
|
|
@@ -562,7 +563,7 @@ export default function runMyAgent(inputs: JsonValue, context: SimulationTargetC
|
|
|
562
563
|
```
|
|
563
564
|
|
|
564
565
|
```sh
|
|
565
|
-
npx
|
|
566
|
+
npx -y @hue-run/sdk@0.17.0 eval --case "Refund an eligible charge" ./hue-agent.ts --content
|
|
566
567
|
```
|
|
567
568
|
|
|
568
569
|
`--scenario` remains an alias for `--case` for existing scripts. Pass one selection flag.
|
|
@@ -593,7 +594,7 @@ required judge to decide answer-only cases. Any
|
|
|
593
594
|
other evaluator's result, an error, or a metric that carries `passed` is never advisory, whatever
|
|
594
595
|
its evidence says. A metric name that
|
|
595
596
|
several evaluators report, such as two judges' `verdict`, gets a column per evaluator, labelled
|
|
596
|
-
with the version's first eight characters. `--baseline <
|
|
597
|
+
with the version's first eight characters. `--baseline <run id|url>` adds improvement,
|
|
597
598
|
regression and unchanged counts with per-case deltas. `--json` prints one JSON document
|
|
598
599
|
(`experimentId`, `runId`, `runUrl`, `complete`, `cases`, `totals`, optional `baseline`) on stdout
|
|
599
600
|
and sends progress to stderr; each case lists its not-applicable evaluator versions in
|
|
@@ -670,7 +671,8 @@ the sealed world at once and attaches the trace once its export is complete. The
|
|
|
670
671
|
A Scenario or eval set whose dataset version is not saved cannot back an experiment: the command
|
|
671
672
|
exits 1 and asks for **Save eval-set version** in Hue or `--save-version`, which freezes that
|
|
672
673
|
version at its current revision. Connection settings are `HUE_API_KEY` and `HUE_BASE_URL`
|
|
673
|
-
(default `https://app.hue.run`), loaded from `--env-file <path>` (or its alias `--env-path`) first when given
|
|
674
|
+
(default `https://app.hue.run`), loaded from `--env-file <path>` (or its alias `--env-path`) first when given, otherwise from
|
|
675
|
+
`./.env.hue` when the shell has no `HUE_API_KEY` (a key in the shell keeps its own origin); `--origin`
|
|
674
676
|
overrides the origin. Since TypeScript 0.14.0, telemetry content capture defaults on; `--no-content` opts out.
|
|
675
677
|
Earlier releases require `--content`; the examples include it for compatibility. Model and tool spans inside the agent come
|
|
676
678
|
only from the agent's own instrumentation. Case outputs, error messages and explanations are
|
|
@@ -734,8 +736,8 @@ published version, beside or instead of explicit `--scorer-version` IDs.
|
|
|
734
736
|
```sh
|
|
735
737
|
hue eval --set gia-d1-citation --scorer gia-d1-citation \
|
|
736
738
|
--command "pnpm --filter @august/frontend run hue:gia-agent" \
|
|
737
|
-
--revision prompt-v10 --wait 1800 --content --json
|
|
738
|
-
hue eval --set gia-d1-citation --set-version 1 --scorer gia-d1-citation ./hue-agent.ts --content --baseline <
|
|
739
|
+
--revision prompt-v10 --wait 1800 --content --json
|
|
740
|
+
hue eval --set gia-d1-citation --set-version 1 --scorer gia-d1-citation ./hue-agent.ts --content --baseline <run id>
|
|
739
741
|
```
|
|
740
742
|
|
|
741
743
|
For each case the command is spawned once **inside a private case directory** with
|
|
@@ -755,14 +757,21 @@ minutes, so a completion that cannot reach Hue takes that long to fail; a reserv
|
|
|
755
757
|
that cannot reach Hue still fails at once.
|
|
756
758
|
|
|
757
759
|
Every regular file the command leaves under `output/` is uploaded as a generated document
|
|
758
|
-
(accepted: `.pdf .docx .pptx .xlsx .json .txt .csv .png .jpg .jpeg .webp
|
|
759
|
-
empty file is the case's
|
|
760
|
-
|
|
760
|
+
(accepted: `.pdf .docx .pptx .xlsx .json .txt .csv .png .jpg .jpeg .webp .gif .avif .mp3 .mpeg
|
|
761
|
+
.wav .ogg .m4a .flac .opus .aac .mp4 .webm`; another extension or an empty file is the case's
|
|
762
|
+
error). Optional helpers: `manifest.json` (`{"primary": "<filename>", "output": <json>,
|
|
763
|
+
"contentTypes": {"<path>": "<MIME>"}}`) names the primary document, the JSON output and declared
|
|
764
|
+
content types; `result.json` is the JSON
|
|
761
765
|
output; `summary.txt` or `summary.md` is recorded as `{"summary": "..."}`. When none is written,
|
|
762
766
|
the command's stdout is the output (JSON when it parses). A single generated file is the primary
|
|
763
767
|
document by default. Model and provider credentials stay in the command's own environment; the
|
|
764
768
|
scoped case files are copies under `--checkpoint-dir` (default `.hue/eval/<agent-key>/<project>/direct/<experiment>`).
|
|
765
769
|
|
|
770
|
+
For `.mp4` and `.webm` files, `manifest.json.contentTypes` must declare the matching audio or
|
|
771
|
+
video MIME type: `audio/mp4` or `video/mp4`, and `audio/webm` or `video/webm`. Each key must match
|
|
772
|
+
the actual relative path under `output/`, including case. For example,
|
|
773
|
+
`{"contentTypes":{"nested/clip.MP4":"video/mp4"}}` declares `output/nested/clip.MP4`.
|
|
774
|
+
|
|
766
775
|
The same collection serves world cases, and it does not trust paths the agent controls. `output/`
|
|
767
776
|
must be a real directory, and a helper name, when present, a regular file; a symlink, FIFO or
|
|
768
777
|
directory in either place is the case's error. Symlinks and other special files among the
|
package/EVALUATIONS.md
CHANGED
|
@@ -7,7 +7,7 @@ Install the optional runtime-contract peer with the SDK before importing
|
|
|
7
7
|
npm install @hue-run/sdk zod
|
|
8
8
|
```
|
|
9
9
|
|
|
10
|
-
The SDK executes targets and scorers on your machine. Hue stores pinned definitions, experiment progress and results. It does not execute uploaded source code. Follow the
|
|
10
|
+
The SDK executes targets and scorers on your machine. Hue stores pinned definitions, experiment progress and results. It does not execute uploaded source code. Follow [Install the SDK](https://docs.hue.run/quickstart#2-install-the-sdk) to add `@hue-run/sdk` to your application.
|
|
11
11
|
|
|
12
12
|
Create a **Read and write** project service key under **Settings → Integrations & API keys** and expose it to this server-side process as `HUE_API_KEY`. A **Tracing only** key cannot author datasets or evaluation runs.
|
|
13
13
|
|
package/README.md
CHANGED
|
@@ -205,26 +205,33 @@ while the callback result is returned unchanged.
|
|
|
205
205
|
## Files
|
|
206
206
|
|
|
207
207
|
Record a file the work read, received or produced with `hue.recordFile`. It adds a `hue.file`
|
|
208
|
-
event to the active span
|
|
209
|
-
SHA-256
|
|
208
|
+
event to the active span, or to the span of an explicit context passed second. The event identifies
|
|
209
|
+
the file by SHA-256. Pass `upload: true` to store its bytes in Hue as well:
|
|
210
210
|
|
|
211
211
|
```ts
|
|
212
212
|
import { readFile } from "node:fs/promises";
|
|
213
213
|
|
|
214
214
|
await hue.withSpan("review contract", async () => {
|
|
215
215
|
const pdf = await readFile("contract.pdf");
|
|
216
|
-
hue.recordFile({
|
|
217
|
-
|
|
216
|
+
hue.recordFile({
|
|
217
|
+
role: "input", mediaType: "application/pdf", data: pdf, name: "contract.pdf", upload: true,
|
|
218
|
+
});
|
|
218
219
|
});
|
|
219
220
|
```
|
|
220
221
|
|
|
221
|
-
`
|
|
222
|
-
|
|
223
|
-
have them.
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
222
|
+
`upload` defaults to `false`, which hashes and measures `data` locally without exporting bytes.
|
|
223
|
+
`data` is limited to 25 MiB. Larger data is omitted and counted as an instrumentation failure;
|
|
224
|
+
pass `sha256` and `byteSize` when you already have them. Upload requires supplied bytes, content
|
|
225
|
+
capture and an ordinary project key. It runs during export and adds a `hue.blobs` entry to the
|
|
226
|
+
same span, keyed `event:hue.file:<sha256>`, while preserving the event.
|
|
227
|
+
|
|
228
|
+
The event carries `hue.file.sha256`, `hue.file.role` (`input`, `attachment` or `output`),
|
|
229
|
+
`hue.file.media_type` and `hue.file.size` in both capture modes. `hue.file.name` requires
|
|
230
|
+
`captureContent: true`. Names must be nonblank after trimming, contain at most 256 UTF-16 code
|
|
231
|
+
units, be well-formed Unicode and contain no C0 characters (U+0000–U+001F) or DEL (U+007F).
|
|
232
|
+
Tabs, newlines, DEL and lone surrogates make a name invalid. An invalid name is omitted and
|
|
233
|
+
counted as an instrumentation failure; the otherwise valid event remains. Invalid records and
|
|
234
|
+
records without an active span count as instrumentation failures.
|
|
228
235
|
|
|
229
236
|
## Vercel AI SDK 6
|
|
230
237
|
|
|
@@ -369,26 +376,34 @@ definition list. The digest is the same in both SDKs and does not change when a
|
|
|
369
376
|
rotates. Only definitions another integration recorded can be summarized: `hueTelemetry(hue)` with
|
|
370
377
|
`captureContent: false` records none, whereas an application whose AI SDK integration records
|
|
371
378
|
inputs and exports through Hue's attached processors gets the summary.
|
|
372
|
-
Recorded messages can
|
|
373
|
-
`gen_ai.output.messages`
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
span's
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
`
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
379
|
+
Recorded messages can contain images, documents and audio. Hue inspects
|
|
380
|
+
`gen_ai.input.messages`, `gen_ai.output.messages` and `ai.prompt.messages`, including `parts`
|
|
381
|
+
and OpenAI `content` arrays. Supported byte shapes include GenAI `blob`, AI SDK `file` and
|
|
382
|
+
`image`, OpenAI Chat and Responses image, file and audio parts, Anthropic base64 sources, Gemini
|
|
383
|
+
`inlineData` and `inline_data`, Vertex `media`, and Bedrock image and document sources.
|
|
384
|
+
Remote URLs, provider file IDs and S3 references stay unchanged. Hue never fetches them.
|
|
385
|
+
|
|
386
|
+
Nonempty binary media in a span's own message attributes is eligible for upload at every size
|
|
387
|
+
within the inspection limits. Empty byte fields stay recorded and do not upload. The byte field is
|
|
388
|
+
removed and the part carries `sha256` and `size`; the span's `hue.blobs` entry points to the part,
|
|
389
|
+
for example `gen_ai.input.messages#/0/parts/1`. Text media over 64 KiB of decoded bytes uploads
|
|
390
|
+
with a UTF-8-safe 16 KiB prefix in its content field. Its entry points to that field, for example
|
|
391
|
+
`gen_ai.input.messages#/0/parts/1/content`. Smaller text stays inline. Hue reads both these forms
|
|
392
|
+
and earlier binary prefixes. If upload fails, the part keeps its digest with no blob entry.
|
|
393
|
+
In resources, events, links and log records, inline media up to 64 KiB stays verbatim; larger
|
|
394
|
+
inline media is replaced by its digest, as events, links and log records were in 0.16. These
|
|
395
|
+
nested values never upload. Ordinary hold, cut or drop rules still apply to the resulting record.
|
|
396
|
+
|
|
397
|
+
Entries use the recorded media type and, when valid, the filename, name or title. A valid name
|
|
398
|
+
is nonblank after trimming, at most 256 UTF-16 code units and well-formed Unicode, with no C0
|
|
399
|
+
characters (U+0000–U+001F) or DEL (U+007F). Invalid names are omitted
|
|
400
|
+
from the entry without rewriting the original field. Data URLs and fields explicitly encoded as
|
|
401
|
+
base64 by the provider contract are decoded before hashing. Other non-data strings with a valid
|
|
402
|
+
declared textual media type remain raw UTF-8: `YWJj` hashes as those four characters. The digest
|
|
403
|
+
matches `hue.recordFile`'s digest for the same bytes. Inspection is bounded to 8 Mi UTF-16
|
|
404
|
+
code units of text per message attribute and 16 Mi per record. Longer attributes use the ordinary value limit.
|
|
405
|
+
With `captureContent: false`, messages are removed without inspection or uploads. See
|
|
406
|
+
[Values over the inline limit](#values-over-the-inline-limit).
|
|
392
407
|
|
|
393
408
|
Manual helpers encode JSON values without converting null into absence. Unknown
|
|
394
409
|
outputs and usage remain absent. This SDK does not estimate tokens or cost. A thrown
|
|
@@ -528,8 +543,7 @@ dropped event and link counts, rather than being dropped whole.
|
|
|
528
543
|
|
|
529
544
|
Hue keeps a value inline up to its value limit (1 MiB unless the receiver advertises another in
|
|
530
545
|
`Hue-Max-Value-Bytes`). A span's own string attribute larger than that (helper content such as
|
|
531
|
-
`input.value`, `output.value` and tool results, or another instrumentation's attribute), and
|
|
532
|
-
inline file over 64 KiB in a span's own recorded messages, is uploaded to Hue apart from the span,
|
|
546
|
+
`input.value`, `output.value` and tool results, or another instrumentation's attribute), and eligible media in a span's own recorded messages, are uploaded to Hue apart from the span,
|
|
533
547
|
up to Hue's upload limit of 1,000,000,000 bytes:
|
|
534
548
|
|
|
535
549
|
1. Export computes the SHA-256 of the value's bytes (a string as UTF-8, a file as its own bytes)
|
|
@@ -541,8 +555,9 @@ up to Hue's upload limit of 1,000,000,000 bytes:
|
|
|
541
555
|
with `allowInsecureHttp`), and
|
|
542
556
|
the upload is confirmed with `POST /api/v1/otlp/blobs/complete`. A PUT the store answers with
|
|
543
557
|
412 (the same value uploaded at the same time by another span) counts as stored.
|
|
544
|
-
3.
|
|
545
|
-
`hue.blobs
|
|
558
|
+
3. Whole string values and large text media keep a UTF-8-safe 16 KiB prefix. Binary media keeps
|
|
559
|
+
its digest on the part. The span lists stored values in `hue.blobs`, at most 64 compact JSON
|
|
560
|
+
strings with fields ordered `key`, `sha256`, `size`, `content_type` and optional `name`:
|
|
546
561
|
`{"key":"output.value","sha256":"<64 hex>","size":5242880,"content_type":"application/json"}`.
|
|
547
562
|
A whole value is listed under `hue.truncated` as well, since its inline text is a prefix.
|
|
548
563
|
|
|
@@ -555,17 +570,20 @@ work, and a `shutdownSafe()` whose budget runs out ends the uploads in flight.
|
|
|
555
570
|
|
|
556
571
|
A value that cannot be uploaded (Hue or the store refused it or failed, the network or a deadline
|
|
557
572
|
did, the receiver has no upload route, or the value is over 1 GB) is exported as before uploads
|
|
558
|
-
existed:
|
|
573
|
+
existed: whole attributes keep a prefix up to the limit and are listed under `hue.truncated`;
|
|
574
|
+
media parts keep their digest without a blob entry; failed opt-in `recordFile` uploads preserve
|
|
575
|
+
the file event. The explicit helper limit is 25 MiB; whole attributes retain their 1 GB limit. Each
|
|
559
576
|
export reports such values in a `warning` issue per cause, whose `count` is the values and
|
|
560
|
-
`traceIds` their traces, and counts them in `report.uploadFallbacks`;
|
|
561
|
-
in `report.uploadedValues`.
|
|
577
|
+
`traceIds` their traces, and counts them in `report.uploadFallbacks`; successful uploads are counted
|
|
578
|
+
in `report.uploadedValues`. Both counters include whole attributes, eligible media and opt-in
|
|
579
|
+
`recordFile` uploads. A warning never makes `flush()` throw. After a transient failure, or
|
|
562
580
|
a refusal that is not about the value alone, uploads pause for 30 seconds, and a receiver without
|
|
563
581
|
the upload route (a 404 that is not Hue's, a 405 or 501, or a 200 that is no reservation) is not
|
|
564
582
|
asked again for 10 minutes, so an unavailable store or another OTLP receiver meets no retry storm.
|
|
565
583
|
|
|
566
584
|
To upload a value whole, the queue holds it until export apart from `maxQueueBytes`, within a
|
|
567
585
|
128 MiB budget of held values (a string charged two bytes a code unit); a recorded message holding
|
|
568
|
-
|
|
586
|
+
eligible media is held the same way. `maxQueueBytes` is still charged what the value
|
|
569
587
|
would cost cut (a message, its copy with the files as their digest), so one that cannot be
|
|
570
588
|
uploaded is exported exactly as it was before. A value the budget cannot hold, or one queued while
|
|
571
589
|
the receiver is known to lack the upload route, is cut (its files digested) when it is queued, as
|
|
@@ -574,8 +592,9 @@ is held while nothing else is. While values can be uploaded, helpers encode cont
|
|
|
574
592
|
upload limit and 1,048,576 values, rather than cutting it to the value limit as they record it.
|
|
575
593
|
|
|
576
594
|
Nothing is uploaded with `captureContent: false` or a setup credential (`hue_setup_…`), nor from
|
|
577
|
-
log records (Hue does not read a log record's uploaded values yet),
|
|
578
|
-
or in-progress placeholders
|
|
595
|
+
log records (Hue does not read a log record's uploaded values yet), links, resources
|
|
596
|
+
or in-progress placeholders. Span events upload only through explicit `recordFile({ upload: true, ... })`;
|
|
597
|
+
other event values are cut as before. `redact` and the credential scrub of
|
|
579
598
|
tool definitions and recorded requests run on the whole value before it is hashed, so a value they
|
|
580
599
|
rewrite is uploaded as rewritten.
|
|
581
600
|
|
|
@@ -677,15 +696,12 @@ conditions only. Use Hue's UI to inspect captured values and redaction.
|
|
|
677
696
|
|
|
678
697
|
## Dependencies
|
|
679
698
|
|
|
680
|
-
The tracing core depends only on official `@opentelemetry/*` packages
|
|
681
|
-
`@hue-run/sdk/evals` entry point
|
|
682
|
-
|
|
683
|
-
loaded inside a worker only when `builtins.jsonSchema` scores a case; without it that scorer
|
|
699
|
+
The tracing core depends only on official `@opentelemetry/*` packages and `zod`, which the
|
|
700
|
+
`@hue-run/sdk/evals` entry point and `hue eval` use for their bounded runtime contracts and which
|
|
701
|
+
installs with the SDK. JSON Schema scoring also uses `ajv`, an optional peer loaded inside a worker only when `builtins.jsonSchema` scores a case; without it that scorer
|
|
684
702
|
reports `SchemaValidatorUnavailable`. Install it when you use that scorer:
|
|
685
703
|
|
|
686
704
|
```bash
|
|
687
|
-
npm install zod
|
|
688
|
-
# Add ajv too when using builtins.jsonSchema.
|
|
689
705
|
npm install ajv
|
|
690
706
|
```
|
|
691
707
|
|
|
@@ -782,7 +798,13 @@ the [managed-run guide](https://docs.hue.run/evaluations/managed-runs).
|
|
|
782
798
|
|
|
783
799
|
`runAgentForEvaluation` adapts your application result to `{ output, files? }`.
|
|
784
800
|
Files contain `filename`, `contentType`, actual `Uint8Array` data and an optional
|
|
785
|
-
`primary` flag.
|
|
801
|
+
`primary` flag. Generated files accept the same 22 content types as Python managed outputs:
|
|
802
|
+
PDF, DOCX, PPTX, XLSX, `application/json`, `text/plain`, `text/csv`, `image/png`, `image/jpeg`,
|
|
803
|
+
`image/webp`, `image/gif`, `image/avif`, `audio/mpeg`, `audio/wav`, `audio/ogg`, `audio/webm`,
|
|
804
|
+
`audio/mp4`, `audio/flac`, `audio/opus`, `audio/aac`, `video/mp4` and `video/webm`.
|
|
805
|
+
Declare `audio/mp4` or `video/mp4` for `.mp4` files, and `audio/webm` or `video/webm` for
|
|
806
|
+
`.webm` files; the extension alone does not distinguish audio from video.
|
|
807
|
+
The helper verifies input bytes, claims the invocation, saves the
|
|
786
808
|
outcome and correlates its span with Hue. It never retries the agent automatically.
|
|
787
809
|
Use a 120-second host request limit for the default 90-second execution and
|
|
788
810
|
30-second finalization budget; your target must honor `signal`.
|
|
@@ -795,6 +817,6 @@ existing-provider flush callbacks and recovery. Local/CI runners remain availabl
|
|
|
795
817
|
|
|
796
818
|
Use `createHueSafe(options)` for best-effort startup. Invalid initialization returns a disabled client that keeps your `onExportIssue` hook and records the reason as an instrumentation failure. Pass `enabled: false` to disable Hue without a key or a `captureContent` choice; disabled helpers still execute the application callback, and `inject()` keeps propagating the application's own trace context. `flushSafe({ timeoutMillis: 1000 })` and `shutdownSafe({ timeoutMillis: 1000 })` return `{ ok, timedOut, report }` without rejecting. Strict initialization, connection checks and `flush()` remain available for diagnostics; do not gate application readiness or responses on them.
|
|
797
819
|
|
|
798
|
-
Capture/serialization/redaction/provider failures omit unsafe telemetry, record failures, and preserve the original business result/error. Async diagnostic rejections are contained; diagnostics are rate-limited. The default `maxQueueBytes` is 8 MiB across traces/logs including in-flight work, alongside the existing record cap. `pendingBytes` is a current queue gauge; `droppedSpans`, `droppedLogs` and `instrumentationFailures` are cumulative failure counters; `uploadedValues` and `uploadFallbacks` count
|
|
820
|
+
Capture/serialization/redaction/provider failures omit unsafe telemetry, record failures, and preserve the original business result/error. Async diagnostic rejections are contained; diagnostics are rate-limited. The default `maxQueueBytes` is 8 MiB across traces/logs including in-flight work, alongside the existing record cap. `pendingBytes` is a current queue gauge; `droppedSpans`, `droppedLogs` and `instrumentationFailures` are cumulative failure counters; `uploadedValues` and `uploadFallbacks` count successful uploads and upload fallbacks for whole attributes, eligible binary or large text media, and opt-in `recordFile` uploads. Whole-attribute fallbacks keep a prefix; media fallbacks keep a digest; helper upload fallbacks keep the file event. This is a telemetry budget, not a total process memory ceiling. A timeout bounds the caller and does not cancel a borrowed provider. Never retry the business operation to recover telemetry. See [production setup](https://docs.hue.run/sdks/typescript#production-setup).
|
|
799
821
|
|
|
800
822
|
Queued records snapshot supported telemetry values when a span ends or a log is emitted; later caller mutations cannot change queued data. Resource attributes still awaiting detection are omitted with a sanitized warning. Later records include them after detection finishes; await resource detection before instrumentation when those attributes are required.
|
package/dist/blobs.d.ts
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
|
+
import { type HrTime } from "@opentelemetry/api";
|
|
2
|
+
import { type MediaDigestMemo, type FileDigest } from "./inline-files.js";
|
|
1
3
|
/**
|
|
2
|
-
*
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
* as it was before uploads existed: cut to the inline limit and listed under `hue.truncated` (an
|
|
7
|
-
* inline file as its digest), reported as a warning and counted.
|
|
4
|
+
* Large values and binary media travel apart from their span through reservation, presigned PUT,
|
|
5
|
+
* and completion. Text keeps a 16 KiB prefix. Binary media keeps digest metadata on its part.
|
|
6
|
+
* Successful uploads are listed in `hue.blobs`; failures keep inline fallback or file metadata,
|
|
7
|
+
* are reported as warnings, and are counted.
|
|
8
8
|
*/
|
|
9
9
|
/** The span attribute listing the values the SDK uploaded apart from the span: one compact JSON
|
|
10
|
-
* object per value, `{"key", "sha256", "size", "content_type"}`,
|
|
10
|
+
* object per value, `{"key", "sha256", "size", "content_type"}`, with an optional `name`. */
|
|
11
11
|
export declare const BLOBS_KEY = "hue.blobs";
|
|
12
12
|
/** The largest value one upload may hold (`Hue-Max-Blob-Bytes`): a larger value is cut inline. */
|
|
13
13
|
export declare const MAX_BLOB_BYTES = 1000000000;
|
|
@@ -24,7 +24,7 @@ export declare const MAX_BLOB_ENTRIES = 64;
|
|
|
24
24
|
export declare const MAX_HELD_BLOB_BYTES: number;
|
|
25
25
|
/** How long one export may spend uploading its values, on top of its requests. */
|
|
26
26
|
export declare function uploadBudgetMillis(timeoutMillis: number): number;
|
|
27
|
-
/** Why
|
|
27
|
+
/** Why content kept its fallback instead of being uploaded. */
|
|
28
28
|
export type FallbackReason = "unsupported" | "failed" | "tooLarge" | "budget";
|
|
29
29
|
/** The fixed description of each fallback, as an export issue reports it. */
|
|
30
30
|
export declare const FALLBACK_MESSAGES: Record<FallbackReason, string>;
|
|
@@ -38,6 +38,18 @@ export interface OffloadCandidate {
|
|
|
38
38
|
export interface OffloadCollector {
|
|
39
39
|
candidates: OffloadCandidate[];
|
|
40
40
|
cut: string[];
|
|
41
|
+
attachments?: MediaAttachment[];
|
|
42
|
+
/** The span's own message attributes admission inspected within the record's allowance:
|
|
43
|
+
* export inspects only these. Absent for a record admission did not copy, which export
|
|
44
|
+
* inspects within its own allowance. */
|
|
45
|
+
inspected?: ReadonlySet<string> | undefined;
|
|
46
|
+
}
|
|
47
|
+
export interface MediaAttachment {
|
|
48
|
+
bytes: Uint8Array;
|
|
49
|
+
digest: FileDigest;
|
|
50
|
+
contentType: string;
|
|
51
|
+
name?: string;
|
|
52
|
+
eventTime?: HrTime;
|
|
41
53
|
}
|
|
42
54
|
/** The outcome of one export's uploads, reported once the export's records are placed. */
|
|
43
55
|
export interface OffloadTally {
|
|
@@ -48,6 +60,8 @@ export interface OffloadTally {
|
|
|
48
60
|
}>;
|
|
49
61
|
}
|
|
50
62
|
export declare function newTally(): OffloadTally;
|
|
63
|
+
export declare function countFallback(tally: OffloadTally, reason: FallbackReason, traceId: string): void;
|
|
64
|
+
/** A media type accepted by the upload contract. */
|
|
51
65
|
export declare function isBlobContentType(value: unknown): value is string;
|
|
52
66
|
/** The media type an uploaded text is stored under: JSON when it reads as a JSON object or array
|
|
53
67
|
* (a structured value's text), plain UTF-8 text otherwise. Both are previewed as text. */
|
|
@@ -62,6 +76,8 @@ export declare function digestOf(value: string | Uint8Array): Promise<{
|
|
|
62
76
|
sha256: string;
|
|
63
77
|
size: number;
|
|
64
78
|
}>;
|
|
79
|
+
/** The `hue.blobs` element for one uploaded value. */
|
|
80
|
+
export declare function blobEntry(key: string, sha256: string, size: number, contentType: string, name?: string): string;
|
|
65
81
|
/** What one export's uploads share: their deadline, and the export's hold for a receiver that
|
|
66
82
|
* limited the rate (`wait` resolves false when the hold may not take `millis` more). */
|
|
67
83
|
export interface UploadContext {
|
|
@@ -100,6 +116,7 @@ export declare class BlobUploader {
|
|
|
100
116
|
private running;
|
|
101
117
|
private waiting;
|
|
102
118
|
private requests;
|
|
119
|
+
private outcomes;
|
|
103
120
|
constructor(options: UploaderOptions);
|
|
104
121
|
/** Whether a value over the inline limit is worth holding whole for upload now. */
|
|
105
122
|
available(): boolean;
|
|
@@ -114,12 +131,12 @@ export declare class BlobUploader {
|
|
|
114
131
|
private pause;
|
|
115
132
|
/**
|
|
116
133
|
* Stores `value` under `key` of the trace: reserved, uploaded unless Hue already holds it, and
|
|
117
|
-
* completed.
|
|
134
|
+
* completed. Only a fresh upload takes a {@link limit} slot: another reference to bytes the
|
|
135
|
+
* trace is already storing shares that upload's outcome without one, so repeated files never
|
|
136
|
+
* hold the slots new files need. The caller must not hold a slot.
|
|
118
137
|
*/
|
|
119
|
-
store(traceId: string, key: string, value: string | Uint8Array, contentType: string, digest:
|
|
120
|
-
|
|
121
|
-
size: number;
|
|
122
|
-
}, upload: UploadContext): Promise<Outcome>;
|
|
138
|
+
store(traceId: string, key: string, value: string | Uint8Array, contentType: string, digest: FileDigest, upload: UploadContext): Promise<Outcome>;
|
|
139
|
+
private storeFresh;
|
|
123
140
|
private jsonHeaders;
|
|
124
141
|
/** Reserves the value: Hue answers that it holds it already, or a presigned PUT. A rate limit
|
|
125
142
|
* is waited out within the export's hold, a transient refusal retried once. */
|
|
@@ -145,11 +162,12 @@ export interface OffloadOptions {
|
|
|
145
162
|
valueBytes: number;
|
|
146
163
|
upload: UploadContext;
|
|
147
164
|
tally: OffloadTally;
|
|
165
|
+
memo?: MediaDigestMemo;
|
|
148
166
|
}
|
|
149
167
|
/**
|
|
150
|
-
* A span's own attributes with each candidate placed:
|
|
151
|
-
*
|
|
152
|
-
*
|
|
168
|
+
* A span's own attributes with each candidate placed: binary media becomes a digest, while
|
|
169
|
+
* text media over 64 KiB retains a 16 KiB prefix after upload or a digest on failure.
|
|
170
|
+
* A value still over the inline limit is uploaded and replaced by its first
|
|
153
171
|
* 16 KiB (or cut to the limit), each upload listed under `hue.blobs`, each whole value cut or
|
|
154
172
|
* uploaded listed under `hue.truncated`. Every fallback is counted in the tally.
|
|
155
173
|
*/
|
|
@@ -160,5 +178,5 @@ export declare function offloadAttributes(attributes: Record<string, unknown>, c
|
|
|
160
178
|
* over the inline limit cut to it (bytes replaced by the receiver's marker) and listed under
|
|
161
179
|
* `hue.truncated`.
|
|
162
180
|
*/
|
|
163
|
-
export declare function fallbackAttributes(attributes: Record<string, unknown>, collector: OffloadCollector, valueBytes: number, tally: OffloadTally, traceId: string): Record<string, unknown>;
|
|
181
|
+
export declare function fallbackAttributes(attributes: Record<string, unknown>, collector: OffloadCollector, valueBytes: number, tally: OffloadTally, traceId: string, memo?: MediaDigestMemo): Record<string, unknown>;
|
|
164
182
|
export {};
|