@hue-run/sdk 0.20.8 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLI.md +25 -18
- package/EVALUATIONS.md +1 -1
- package/README.md +93 -3
- package/dist/cli/eval.js +31 -15
- package/dist/client.d.ts +12 -0
- package/dist/client.js +50 -0
- package/dist/tool-endpoint.d.ts +4 -0
- package/dist/tool-endpoint.js +46 -0
- package/dist/transport.js +3 -3
- package/dist/types.d.ts +14 -4
- package/dist/version.d.ts +3 -1
- package/dist/version.js +3 -1
- package/package.json +11 -11
package/CLI.md
CHANGED
|
@@ -42,7 +42,7 @@ project manifest are refused because managers can update ancestor locks; Python
|
|
|
42
42
|
The generated `hue.setup.mjs` or `hue_setup.py` always selects `captureContent: false` /
|
|
43
43
|
`capture_content=False`. For a supported application, setup installs the dependency and adds the
|
|
44
44
|
managed import and middleware registration to the existing entrypoint; an unreferenced helper is
|
|
45
|
-
not a completed integration. TypeScript uses `@hue-run/sdk@0.
|
|
45
|
+
not a completed integration. TypeScript uses `@hue-run/sdk@0.21.0`, `@opentelemetry/api@1.9.1` and
|
|
46
46
|
`@opentelemetry/context-async-hooks@2.11.0`; Python setup uses its separately tested package pin.
|
|
47
47
|
Content capture requires an ordinary account-managed key and a later explicit application decision.
|
|
48
48
|
|
|
@@ -381,8 +381,8 @@ and Hue never receives or stores them. The command is part of the `hue` executab
|
|
|
381
381
|
# LANGFUSE_PUBLIC_KEY=pk-lf-...
|
|
382
382
|
# LANGFUSE_SECRET_KEY=sk-lf-...
|
|
383
383
|
# LANGFUSE_BASE_URL=https://cloud.langfuse.com
|
|
384
|
-
npx -y @hue-run/sdk@0.
|
|
385
|
-
npx -y @hue-run/sdk@0.
|
|
384
|
+
npx -y @hue-run/sdk@0.21.0 import langfuse --env-path .env.langfuse --from 2026-01-01 --dry-run
|
|
385
|
+
npx -y @hue-run/sdk@0.21.0 import langfuse --env-path .env.langfuse --from 2026-01-01 --media
|
|
386
386
|
```
|
|
387
387
|
|
|
388
388
|
`LANGFUSE_PUBLIC_KEY` and `LANGFUSE_SECRET_KEY` come from the environment or `--env-path`
|
|
@@ -698,7 +698,7 @@ node packages/sdk-typescript/scripts/verify-package.mjs --artifacts-dir .artifac
|
|
|
698
698
|
# Set project to an existing supported fixture; use the same directory on resume.
|
|
699
699
|
project=/absolute/path/to/supported-fixture
|
|
700
700
|
node packages/sdk-typescript/scripts/verify-setup-live.mjs \
|
|
701
|
-
--archive .artifacts/typescript/hue-run-sdk-0.
|
|
701
|
+
--archive .artifacts/typescript/hue-run-sdk-0.21.0.tgz \
|
|
702
702
|
--origin https://STAGING_ORIGIN \
|
|
703
703
|
--project "$project" --command setup \
|
|
704
704
|
--evidence .context/setup-staging-before-claim.json
|
|
@@ -709,7 +709,7 @@ the private local handoff and finish the real browser claim, then reconcile the
|
|
|
709
709
|
|
|
710
710
|
```sh
|
|
711
711
|
node packages/sdk-typescript/scripts/verify-setup-live.mjs \
|
|
712
|
-
--archive .artifacts/typescript/hue-run-sdk-0.
|
|
712
|
+
--archive .artifacts/typescript/hue-run-sdk-0.21.0.tgz \
|
|
713
713
|
--origin https://STAGING_ORIGIN \
|
|
714
714
|
--project "$project" --command claim \
|
|
715
715
|
--evidence .context/setup-staging-after-claim.json
|
|
@@ -751,10 +751,10 @@ otherwise. Never hand the agent Hue-native tools (`context.tools`, `context.mcp`
|
|
|
751
751
|
production agent lacks. Pin the SDK, so the agent's repository gains no dependency:
|
|
752
752
|
|
|
753
753
|
```sh
|
|
754
|
-
npx -y @hue-run/sdk@0.
|
|
755
|
-
npx -y @hue-run/sdk@0.
|
|
756
|
-
npx -y @hue-run/sdk@0.
|
|
757
|
-
npx -y @hue-run/sdk@0.
|
|
754
|
+
npx -y @hue-run/sdk@0.21.0 eval --case "Refund an eligible charge" --command "node agent.js"
|
|
755
|
+
npx -y @hue-run/sdk@0.21.0 eval --case https://app.hue.run/projects/demo/scenarios/<id> --command "python agent.py" --timeout 120 --baseline <run id>
|
|
756
|
+
npx -y @hue-run/sdk@0.21.0 eval --set "Billing regressions" --scorer-version <id> --command "node agent.js" --save-version
|
|
757
|
+
npx -y @hue-run/sdk@0.21.0 eval --worker --command "node agent.js" --agent-key support-agent --revision <new-agent-revision>
|
|
758
758
|
```
|
|
759
759
|
|
|
760
760
|
One invocation creates one run holding every selected case; it prints the run URL
|
|
@@ -792,7 +792,7 @@ export default function runMyAgent(inputs: JsonValue, context: SimulationTargetC
|
|
|
792
792
|
```
|
|
793
793
|
|
|
794
794
|
```sh
|
|
795
|
-
npx -y @hue-run/sdk@0.
|
|
795
|
+
npx -y @hue-run/sdk@0.21.0 eval --case "Refund an eligible charge" ./hue-agent.ts --content
|
|
796
796
|
```
|
|
797
797
|
|
|
798
798
|
`--scenario` remains an alias for `--case` for existing scripts. Pass one selection flag.
|
|
@@ -834,21 +834,28 @@ former; a case failed for its telemetry has `state: "error"`,
|
|
|
834
834
|
Hue-owned `world_outcome` checks are graded after the world seals. An experiment always covers
|
|
835
835
|
every case of the saved version; there is no case subset.
|
|
836
836
|
|
|
837
|
-
|
|
838
|
-
`
|
|
839
|
-
|
|
837
|
+
`hue eval` runs up to four cases at a time by default in one-shot and worker modes.
|
|
838
|
+
`--concurrency 1` runs cases serially. Any explicit `--concurrency` value from 1 to 100 takes precedence.
|
|
839
|
+
Each case keeps its own files, environment and checkpoint.
|
|
840
|
+
|
|
841
|
+
The CLI requests the run's pinned `llm_judge` evaluators by default after outputs upload,
|
|
842
|
+
before verdict polling, in direct, simulation and worker modes. `--no-judges` skips these requests;
|
|
843
|
+
`--judges` remains accepted for existing commands. Requests use Hue's hosted judge budget;
|
|
840
844
|
Hue applies its admission controls to new jobs. An accepted request remains replayable if the
|
|
841
|
-
budget is later disabled. Cases without stored output are excluded
|
|
842
|
-
`--judges` with `--no-output` is refused.
|
|
845
|
+
budget is later disabled. Cases without stored output are excluded. `--no-output` skips judge
|
|
846
|
+
requests, and explicitly combining `--judges` with `--no-output` is refused. Without a pinned
|
|
847
|
+
`llm_judge`, the CLI requests no model jobs. Creating a judge alone does not start one.
|
|
848
|
+
`--check` creates no judge jobs.
|
|
843
849
|
|
|
844
|
-
With `--
|
|
850
|
+
With `--wait 0`, the CLI requests the jobs and returns their current verdict state;
|
|
845
851
|
pending jobs continue in Hue. Uploaded outputs are checkpointed before the attempt or worker
|
|
846
852
|
queue is marked completed. If this handoff or dispatch fails, or its acknowledgement is lost,
|
|
847
853
|
rerun the unchanged command with the same checkpoint directory. The CLI resumes judging the uploaded
|
|
848
|
-
outputs without invoking the agent, using the same idempotency keys.
|
|
854
|
+
outputs without invoking the agent, using the same idempotency keys. Changing to `--no-judges`
|
|
855
|
+
or `--no-output` while that handoff is pending stops before a new agent run. After dispatch succeeds,
|
|
849
856
|
the next ordinary invocation starts a new run as usual.
|
|
850
857
|
|
|
851
|
-
`--worker
|
|
858
|
+
`--worker` discards cancelled or released judge handoffs. Once judge requests succeed, a
|
|
852
859
|
verdict-read failure is logged and the worker keeps accepting runs.
|
|
853
860
|
|
|
854
861
|
`--command "<shell command>"` spawns the command once per case with the world's environment
|
package/EVALUATIONS.md
CHANGED
|
@@ -184,7 +184,7 @@ Scorer deferral shipped in TypeScript `0.3.1`: the local runner executes only th
|
|
|
184
184
|
|
|
185
185
|
`world_outcome` pins (`hue.conversion_outcome.v1` and `.v2`, and the `hue.outcome_assertions.v2` and `.v3` case-outcome evaluators, `.v3` with its pinned judge in `config.judge`) run inside Hue and need no local callback or executable source digest. A supporting server owns their execution from saved world evidence. Legacy `local_code` pins still require the exact registered callback; changing the worker cannot convert those immutable pins into hosted ones.
|
|
186
186
|
|
|
187
|
-
Manual results require a human session. Hosted model-judge dispatch is explicit: call `createJudgeJobs(runId,{idempotencyKey,jobs:[{evaluationItemId,scorerVersionId}]})`. Hue checks its judge budget when admitting new requests. `getJudgeBudget()` reports its current state, and on current servers the organization's monthly AI budget as `organizationBudget`, but replay an interrupted request with the same key and body even if that state changed.
|
|
187
|
+
Manual results require a human session. Hosted model-judge dispatch is explicit: call `createJudgeJobs(runId,{idempotencyKey,jobs:[{evaluationItemId,scorerVersionId}]})`. Hue checks its judge budget when admitting new requests. `getJudgeBudget()` reports its current state, and on current servers the organization's monthly AI budget as `organizationBudget`, but replay an interrupted request with the same key and body even if that state changed. [`hue eval`](./CLI.md#evaluate-an-agent-against-a-case) dispatches pinned model judges by default after outputs upload; `--no-judges` skips them. Creating or publishing a judge alone starts no model job. `listJudgeJobs`, `getJudgeJob` and `cancelJudgeJob` expose job progress and cancellation requests. These methods never claim that local execution has hosted provenance. Hosted job endpoints are covered by HTTP contract tests here; live hosted model execution is a separate platform acceptance phase. `listResults` and `getResult` read recorded local or hosted results.
|
|
188
188
|
|
|
189
189
|
When present, the budget's `authentication` reports credential resolution only. An
|
|
190
190
|
`available` status or `configured: true` does not prove that a provider accepted the
|
package/README.md
CHANGED
|
@@ -202,6 +202,22 @@ helpers (`setInput`, `setOutput`, `tool` arguments and results, `recordMessages`
|
|
|
202
202
|
that is not JSON, such as a `Date` or a class instance, is omitted with an instrumentation failure
|
|
203
203
|
while the callback result is returned unchanged.
|
|
204
204
|
|
|
205
|
+
### Tool endpoint evidence (unreleased)
|
|
206
|
+
|
|
207
|
+
This checkout adds optional `ToolOptions.endpoint`: `{ url, source, method?, captureQueryParameters? }`.
|
|
208
|
+
Use the actual HTTP(S) request URL, client config URL or receipt URL; set `source` to `request`,
|
|
209
|
+
`client_config` or `receipt`. Supply `method` only when known. With content capture enabled,
|
|
210
|
+
Hue records `url.full`, `hue.tool.endpoint.source` and optional `http.request.method` through the
|
|
211
|
+
existing redactor (`attributes.url.full` for the URL). URL userinfo and fragments are removed;
|
|
212
|
+
query values are omitted unless their names appear in `captureQueryParameters`. Credential names
|
|
213
|
+
are rejected even if allowlisted. Duplicate and empty allowed query values remain intact.
|
|
214
|
+
|
|
215
|
+
Metadata-only mode omits endpoint evidence. Invalid evidence is omitted and counted without
|
|
216
|
+
changing the tool's result or exception. This helper does not observe HTTP automatically or infer
|
|
217
|
+
account identifiers. Do not supply secret-bearing URL paths, and preserve existing masking of
|
|
218
|
+
account identifiers. See the [setup skill](../../skills/hue/SKILL.md#http-tool-endpoint-evidence)
|
|
219
|
+
for dispatcher integration and source checkout requirements.
|
|
220
|
+
|
|
205
221
|
## Images, documents and audio
|
|
206
222
|
|
|
207
223
|
Hue stores the bytes of images, documents and audio apart from the spans and log records that
|
|
@@ -536,8 +552,8 @@ the Langfuse project's own keys (`LANGFUSE_PUBLIC_KEY`, `LANGFUSE_SECRET_KEY`, a
|
|
|
536
552
|
`HUE_API_KEY`:
|
|
537
553
|
|
|
538
554
|
```sh
|
|
539
|
-
npx -y @hue-run/sdk@0.
|
|
540
|
-
npx -y @hue-run/sdk@0.
|
|
555
|
+
npx -y @hue-run/sdk@0.21.0 import langfuse --env-path .env.langfuse --from 2026-01-01 --dry-run
|
|
556
|
+
npx -y @hue-run/sdk@0.21.0 import langfuse --env-path .env.langfuse --from 2026-01-01 --media
|
|
541
557
|
```
|
|
542
558
|
|
|
543
559
|
- It reads Langfuse Cloud and self-hosted v4 through the Observations API v2, and self-hosted v3
|
|
@@ -588,6 +604,79 @@ npx -y @hue-run/sdk@latest traces import upload ./exports.hue-import --authoriza
|
|
|
588
604
|
See [Import a bundle of existing traces](CLI.md#import-a-bundle-of-existing-traces) for the
|
|
589
605
|
formats, the report, what Hue refuses and the exit codes.
|
|
590
606
|
|
|
607
|
+
## Sub-agents
|
|
608
|
+
|
|
609
|
+
Run a sub-agent with `hue.subagent(name, callback, options)`, inside the tool call that starts it:
|
|
610
|
+
|
|
611
|
+
```ts
|
|
612
|
+
const findings = await hue.tool("research", { topic }, () =>
|
|
613
|
+
hue.subagent(
|
|
614
|
+
"researcher",
|
|
615
|
+
async () => {
|
|
616
|
+
const notes = await hue.model("gpt-5-mini", () => callModel(topic), { provider: "openai" });
|
|
617
|
+
return hue.tool("summarize", notes, () => summarize(notes));
|
|
618
|
+
},
|
|
619
|
+
{ input: topic },
|
|
620
|
+
),
|
|
621
|
+
);
|
|
622
|
+
```
|
|
623
|
+
|
|
624
|
+
The sub-agent runs inside one `invoke_agent researcher` span with `gen_ai.operation.name`
|
|
625
|
+
`invoke_agent`, `gen_ai.agent.name` `researcher` and `hue.agent.role` `subagent`. That span is a
|
|
626
|
+
child of the active span, here the `execute_tool research` span, and the sub-agent's own model and
|
|
627
|
+
tool spans are its children, so Hue scopes them to the sub-agent. Sub-agents nest: a sub-agent's
|
|
628
|
+
tool can start another. The span is `INTERNAL` unless `options.kind` says otherwise; session, user
|
|
629
|
+
and workspace identifiers are inherited unless `options` sets them. `options.input` and the
|
|
630
|
+
callback's defined result are recorded as `input.value` and `output.value` when `captureContent`
|
|
631
|
+
is true (a callback that calls `setOutput` keeps its own output), and errors are recorded and
|
|
632
|
+
rethrown as in `withSpan`.
|
|
633
|
+
|
|
634
|
+
The calling agent's role is not set for you: Hue reads roles only from the spans that state them,
|
|
635
|
+
and the span a sub-agent starts under is usually a tool call, not the agent. To label the calling
|
|
636
|
+
agent, give its own span the role:
|
|
637
|
+
|
|
638
|
+
```ts
|
|
639
|
+
await hue.withSpan("invoke_agent planner", () => runPlanner(), {
|
|
640
|
+
attributes: {
|
|
641
|
+
"gen_ai.operation.name": "invoke_agent",
|
|
642
|
+
"gen_ai.agent.name": "planner",
|
|
643
|
+
"hue.agent.role": "orchestrator",
|
|
644
|
+
},
|
|
645
|
+
});
|
|
646
|
+
```
|
|
647
|
+
|
|
648
|
+
When the sub-agent runs in another process, send the caller's trace context with the request and
|
|
649
|
+
start the sub-agent under it:
|
|
650
|
+
|
|
651
|
+
```ts
|
|
652
|
+
// The calling process, inside the tool call that hands the task over.
|
|
653
|
+
await hue.tool("research", { topic }, async () => {
|
|
654
|
+
const headers: Record<string, string> = { "content-type": "application/json" };
|
|
655
|
+
hue.inject(headers); // adds `traceparent` only
|
|
656
|
+
const response = await fetch(researcherUrl, {
|
|
657
|
+
method: "POST",
|
|
658
|
+
headers,
|
|
659
|
+
body: JSON.stringify({ topic, sessionId }),
|
|
660
|
+
});
|
|
661
|
+
return response.json();
|
|
662
|
+
});
|
|
663
|
+
|
|
664
|
+
// The sub-agent's process, for example an Express handler.
|
|
665
|
+
const { topic, sessionId } = req.body;
|
|
666
|
+
const findings = await hue.subagent("researcher", () => research(topic), {
|
|
667
|
+
parentContext: hue.extract(req.headers),
|
|
668
|
+
sessionId,
|
|
669
|
+
});
|
|
670
|
+
```
|
|
671
|
+
|
|
672
|
+
The `invoke_agent researcher` span then belongs to the caller's trace, with the caller's
|
|
673
|
+
`execute_tool research` span as its parent. `hue.extract()` reads a plain header object such as
|
|
674
|
+
Node's `req.headers`; pass `Object.fromEntries(request.headers)` for a Fetch `Request`. Only
|
|
675
|
+
`traceparent` crosses the boundary, never the API key, baggage or the session, so pass the
|
|
676
|
+
session identifier yourself as above. Accept `traceparent` only from your own services: whoever
|
|
677
|
+
sets it chooses the trace the sub-agent's spans join. On an endpoint that also serves other
|
|
678
|
+
callers, extract it only from authenticated internal requests and start a new trace otherwise.
|
|
679
|
+
|
|
591
680
|
## Vercel AI SDK 6
|
|
592
681
|
|
|
593
682
|
AI SDK 6 accepts a per-call tracer through `experimental_telemetry`. Pass
|
|
@@ -782,7 +871,8 @@ and similar attributes on your own providers.
|
|
|
782
871
|
|
|
783
872
|
For external parent context pass `parentContext` to `withSpan`. Across processes, use
|
|
784
873
|
`hue.inject(carrier)` inside the producing span and `hue.extract(carrier)` in the worker; both
|
|
785
|
-
speak W3C `traceparent` only and never include the API key or baggage.
|
|
874
|
+
speak W3C `traceparent` only and never include the API key or baggage. Extract it only from
|
|
875
|
+
trusted internal callers, as described under [Sub-agents](#sub-agents). Hue registers no global
|
|
786
876
|
propagator, so `propagation.inject()` from `@opentelemetry/api` is a no-op unless your
|
|
787
877
|
application configured one. `getContext()` exposes the helper's current context for APIs taking
|
|
788
878
|
an explicit context. Session, user and workspace identifiers (`sessionId` as
|
package/dist/cli/eval.js
CHANGED
|
@@ -92,15 +92,16 @@ Output and limits:
|
|
|
92
92
|
--check One-shot: check the key, selection and agent, print what the
|
|
93
93
|
run would do and exit without creating a run or a world
|
|
94
94
|
--checkpoint-dir <path> Private checkpoint directory (default: .hue/eval/<agent-key>)
|
|
95
|
-
--concurrency <n> Cases in flight, 1-100 (default:
|
|
95
|
+
--concurrency <n> Cases in flight, 1-100 (default: 4)
|
|
96
96
|
--trace-not-accepted <policy> A case whose telemetry Hue did not accept in time: fail_case
|
|
97
97
|
(default) completes it as failed with the evidence omitted;
|
|
98
98
|
pending completes it in its true state with the trace still to
|
|
99
99
|
arrive, which Hue attaches once its export is complete
|
|
100
100
|
--timeout <seconds> Per-case --command timeout (default: 600)
|
|
101
101
|
--wait <seconds> Verdict wait after the run finishes (default: 300)
|
|
102
|
-
--judges
|
|
103
|
-
|
|
102
|
+
--no-judges Skip pinned hosted model judges (requested by default
|
|
103
|
+
after outputs upload; uses the project's judge budget)
|
|
104
|
+
--judges Request pinned judges (compatibility flag)
|
|
104
105
|
-h, --help Show this help
|
|
105
106
|
|
|
106
107
|
HUE_API_KEY must be a "Read and write" project key; it is never printed.
|
|
@@ -154,7 +155,8 @@ function parse(argv) {
|
|
|
154
155
|
content: { type: "boolean" },
|
|
155
156
|
"no-content": { type: "boolean", default: false },
|
|
156
157
|
"no-output": { type: "boolean", default: false },
|
|
157
|
-
judges: { type: "boolean"
|
|
158
|
+
judges: { type: "boolean" },
|
|
159
|
+
"no-judges": { type: "boolean", default: false },
|
|
158
160
|
"trace-not-accepted": { type: "string" },
|
|
159
161
|
"save-version": { type: "boolean", default: false },
|
|
160
162
|
check: { type: "boolean", default: false },
|
|
@@ -1123,6 +1125,19 @@ export async function dispatchPinnedJudges(client, experimentId, subjectIds, sig
|
|
|
1123
1125
|
await dispatch();
|
|
1124
1126
|
return count;
|
|
1125
1127
|
}
|
|
1128
|
+
/** Keep the judge lock through execution so an opt-out cannot race a pending dispatch. */
|
|
1129
|
+
async function acquireJudgeCheckpoint(directory, projectId, baseUrl, enabled) {
|
|
1130
|
+
const store = await CheckpointStore.acquire(checkpointPath(directory, ["judge-dispatch", "judge checkpoint"]), { kind: "cli-judge-dispatch", projectId, baseUrl });
|
|
1131
|
+
try {
|
|
1132
|
+
if (!enabled && (await store.read("pending")))
|
|
1133
|
+
throw new Error("Recover the pending hosted checks with the original flags before using --no-judges or --no-output");
|
|
1134
|
+
return store;
|
|
1135
|
+
}
|
|
1136
|
+
catch (error) {
|
|
1137
|
+
await store.release();
|
|
1138
|
+
throw error;
|
|
1139
|
+
}
|
|
1140
|
+
}
|
|
1126
1141
|
/** Durable handoff before the original attempt marker or queue acknowledgement. */
|
|
1127
1142
|
async function checkpointJudges(judging, run, completion) {
|
|
1128
1143
|
await judging.store.write("pending", {
|
|
@@ -1242,7 +1257,7 @@ function parseBaseline(value) {
|
|
|
1242
1257
|
}
|
|
1243
1258
|
async function runOnce(values, connection, agents, agent, hue, output, signal) {
|
|
1244
1259
|
const client = new EvaluationClient(connection);
|
|
1245
|
-
const concurrency = integer("concurrency", values.concurrency,
|
|
1260
|
+
const concurrency = integer("concurrency", values.concurrency, 4, 1, 100);
|
|
1246
1261
|
const baselineId = parseBaseline(values.baseline);
|
|
1247
1262
|
const mode = parseMode(values.mode);
|
|
1248
1263
|
integer("wait", values.wait, 300, 0, 86_400);
|
|
@@ -1267,9 +1282,10 @@ async function runOnce(values, connection, agents, agent, hue, output, signal) {
|
|
|
1267
1282
|
// Each kind of run has its own checkpoint directory and identity: a mixed run never resumes
|
|
1268
1283
|
// from, or is resumed by, a direct or simulation run's checkpoint.
|
|
1269
1284
|
const checkpointDirectory = await prepareCheckpointDirectory(values["checkpoint-dir"], agent.key, project.id, selection.kind);
|
|
1285
|
+
const judgeStore = await acquireJudgeCheckpoint(checkpointDirectory, project.id, connection.baseUrl, values.judges === true);
|
|
1270
1286
|
const judging = values.judges
|
|
1271
1287
|
? {
|
|
1272
|
-
store:
|
|
1288
|
+
store: judgeStore,
|
|
1273
1289
|
selectionDigest: digest({
|
|
1274
1290
|
datasetVersionId: pins.datasetVersionId,
|
|
1275
1291
|
scorerVersionIds: [...pins.scorerVersionIds].sort(),
|
|
@@ -1385,7 +1401,7 @@ async function runOnce(values, connection, agents, agent, hue, output, signal) {
|
|
|
1385
1401
|
}, baselineId, output, signal, judging);
|
|
1386
1402
|
}
|
|
1387
1403
|
finally {
|
|
1388
|
-
await
|
|
1404
|
+
await judgeStore.release();
|
|
1389
1405
|
}
|
|
1390
1406
|
}
|
|
1391
1407
|
/**
|
|
@@ -1425,7 +1441,7 @@ async function checkRun(client, values, pins, selection, agent, output) {
|
|
|
1425
1441
|
mode: kind,
|
|
1426
1442
|
worldCases: kind === "direct" ? 0 : worldCases,
|
|
1427
1443
|
directCases: kind === "direct" ? cases : cases - worldCases,
|
|
1428
|
-
agent: { key: agent.key, revision: agent.revision, concurrency: values.concurrency ?? "
|
|
1444
|
+
agent: { key: agent.key, revision: agent.revision, concurrency: values.concurrency ?? "4" },
|
|
1429
1445
|
};
|
|
1430
1446
|
if (values.json)
|
|
1431
1447
|
process.stdout.write(`${JSON.stringify({ check })}\n`);
|
|
@@ -1602,19 +1618,16 @@ async function runWorker(values, connection, agents, agent, hue, output, signal)
|
|
|
1602
1618
|
const client = new ObservedClient(connection);
|
|
1603
1619
|
const environmentClient = createEnvironmentClient(connection);
|
|
1604
1620
|
const wait = integer("wait", values.wait, 300, 0, 86_400);
|
|
1605
|
-
const concurrency = integer("concurrency", values.concurrency,
|
|
1621
|
+
const concurrency = integer("concurrency", values.concurrency, 4, 1, 100);
|
|
1606
1622
|
let maxRuns = values["max-runs"] === undefined
|
|
1607
1623
|
? undefined
|
|
1608
1624
|
: integer("max-runs", values["max-runs"], 1, 1, 1_000_000);
|
|
1609
1625
|
const project = await client.checkConnection();
|
|
1610
1626
|
const checkpointDirectory = await prepareCheckpointDirectory(values["checkpoint-dir"], agent.key, project.id, "worker", slug(agent.revision) || "dev");
|
|
1627
|
+
const judgeStore = await acquireJudgeCheckpoint(checkpointDirectory, project.id, connection.baseUrl, values.judges === true);
|
|
1611
1628
|
const judging = values.judges
|
|
1612
1629
|
? {
|
|
1613
|
-
store:
|
|
1614
|
-
kind: "cli-judge-dispatch",
|
|
1615
|
-
projectId: project.id,
|
|
1616
|
-
baseUrl: connection.baseUrl,
|
|
1617
|
-
}),
|
|
1630
|
+
store: judgeStore,
|
|
1618
1631
|
selectionDigest: digest(agent),
|
|
1619
1632
|
source: {
|
|
1620
1633
|
directory: checkpointDirectory,
|
|
@@ -1787,7 +1800,7 @@ async function runWorker(values, connection, agents, agent, hue, output, signal)
|
|
|
1787
1800
|
return 0;
|
|
1788
1801
|
}
|
|
1789
1802
|
finally {
|
|
1790
|
-
await
|
|
1803
|
+
await judgeStore.release();
|
|
1791
1804
|
}
|
|
1792
1805
|
}
|
|
1793
1806
|
/**
|
|
@@ -1831,8 +1844,11 @@ export async function runEvalCommand(argv) {
|
|
|
1831
1844
|
worker = values.worker;
|
|
1832
1845
|
if (values.content && values["no-content"])
|
|
1833
1846
|
throw new UsageError("Choose either --content or --no-content");
|
|
1847
|
+
if (values.judges && values["no-judges"])
|
|
1848
|
+
throw new UsageError("Choose either --judges or --no-judges");
|
|
1834
1849
|
if (values.judges && values["no-output"])
|
|
1835
1850
|
throw new UsageError("--judges requires stored case outputs; remove --no-output");
|
|
1851
|
+
values.judges = !values["no-judges"] && !values["no-output"];
|
|
1836
1852
|
// A usage error before anything is prepared or created: no run is left to recover from it.
|
|
1837
1853
|
traceNotAcceptedPolicy(values["trace-not-accepted"]);
|
|
1838
1854
|
json = values.json;
|
package/dist/client.d.ts
CHANGED
|
@@ -83,6 +83,18 @@ export declare class HueClient {
|
|
|
83
83
|
* provider and surface as `hue.mcp.provider` / `hue.mcp.surface`.
|
|
84
84
|
*/
|
|
85
85
|
tool<T>(name: string, input: unknown, execute: () => Promise<T> | T, options?: ToolOptions): Promise<T>;
|
|
86
|
+
/**
|
|
87
|
+
* Runs `callback` as a sub-agent: inside an `invoke_agent {name}` span carrying
|
|
88
|
+
* `gen_ai.operation.name` `invoke_agent`, `gen_ai.agent.name` and `hue.agent.role` `subagent`.
|
|
89
|
+
* The span nests under the active span, normally the `tool()` span of the call that started
|
|
90
|
+
* the sub-agent, or under `options.parentContext`, for example `hue.extract(headers)` in
|
|
91
|
+
* another process. The sub-agent's own model and tool spans nest under it, and session, user
|
|
92
|
+
* and workspace identifiers are inherited unless `options` sets them. `options.input` and a
|
|
93
|
+
* defined result are recorded as `input.value` / `output.value` when `captureContent` is true;
|
|
94
|
+
* a callback that calls `setOutput` keeps its own output. Errors are recorded and rethrown as
|
|
95
|
+
* in `withSpan`. A blank or non-string name is recorded as `unknown` and counted.
|
|
96
|
+
*/
|
|
97
|
+
subagent<T>(name: string, callback: (span: HueSpan) => Promise<T> | T, options?: SpanOptions): Promise<T>;
|
|
86
98
|
/**
|
|
87
99
|
* Runs `callback` inside a GenAI client span for one direct provider call, named
|
|
88
100
|
* `{operation} {model}` unless `options.name` is given and carrying `gen_ai.operation.name`,
|
package/dist/client.js
CHANGED
|
@@ -11,6 +11,7 @@ import { defaultResource, resourceFromAttributes } from "@opentelemetry/resource
|
|
|
11
11
|
import { HUE_SCOPE } from "./config.js";
|
|
12
12
|
import { hostedServerAddresses, hostedToolActivity, hostedToolProvider, providerErrorDescription, } from "./provider-tools.js";
|
|
13
13
|
import { toolCatalogSummary } from "./tool-definitions.js";
|
|
14
|
+
import { toolEndpointAttributes } from "./tool-endpoint.js";
|
|
14
15
|
import { contentLimitsOf, encodeBoundedContent, encodeContent, noopSpan, safeSpan, spanSource, } from "./safety.js";
|
|
15
16
|
import { TRUNCATED_KEY, withTruncatedKeys } from "./privacy.js";
|
|
16
17
|
import { countFallback, MAX_BLOB_BYTES, newTally, sizeOf } from "./blobs.js";
|
|
@@ -387,6 +388,15 @@ export class HueClient {
|
|
|
387
388
|
stamp("mcp.server.version", options.mcp?.version);
|
|
388
389
|
stamp("hue.mcp.provider", options.mcp?.provider);
|
|
389
390
|
stamp("hue.mcp.surface", options.mcp?.surface);
|
|
391
|
+
if (this.enabled && !this.closed && this.captureContent) {
|
|
392
|
+
try {
|
|
393
|
+
if (options.endpoint !== undefined)
|
|
394
|
+
Object.assign(attributes, toolEndpointAttributes(options.endpoint));
|
|
395
|
+
}
|
|
396
|
+
catch {
|
|
397
|
+
this.instrumentationFailed();
|
|
398
|
+
}
|
|
399
|
+
}
|
|
390
400
|
return this.withSpan(`execute_tool ${name}`, async ({ span }) => {
|
|
391
401
|
this.setContent(span, "gen_ai.tool.call.arguments", input);
|
|
392
402
|
const result = await execute();
|
|
@@ -398,6 +408,46 @@ export class HueClient {
|
|
|
398
408
|
...(options.parentContext ? { parentContext: options.parentContext } : {}),
|
|
399
409
|
});
|
|
400
410
|
}
|
|
411
|
+
/**
|
|
412
|
+
* Runs `callback` as a sub-agent: inside an `invoke_agent {name}` span carrying
|
|
413
|
+
* `gen_ai.operation.name` `invoke_agent`, `gen_ai.agent.name` and `hue.agent.role` `subagent`.
|
|
414
|
+
* The span nests under the active span, normally the `tool()` span of the call that started
|
|
415
|
+
* the sub-agent, or under `options.parentContext`, for example `hue.extract(headers)` in
|
|
416
|
+
* another process. The sub-agent's own model and tool spans nest under it, and session, user
|
|
417
|
+
* and workspace identifiers are inherited unless `options` sets them. `options.input` and a
|
|
418
|
+
* defined result are recorded as `input.value` / `output.value` when `captureContent` is true;
|
|
419
|
+
* a callback that calls `setOutput` keeps its own output. Errors are recorded and rethrown as
|
|
420
|
+
* in `withSpan`. A blank or non-string name is recorded as `unknown` and counted.
|
|
421
|
+
*/
|
|
422
|
+
async subagent(name, callback, options = {}) {
|
|
423
|
+
let agent = name;
|
|
424
|
+
if (!isLabel(name)) {
|
|
425
|
+
agent = "unknown";
|
|
426
|
+
if (this.enabled && !this.closed)
|
|
427
|
+
this.instrumentationFailed();
|
|
428
|
+
}
|
|
429
|
+
return this.withSpan(`invoke_agent ${agent}`, async (span) => {
|
|
430
|
+
let outputSet = false;
|
|
431
|
+
const result = await callback({
|
|
432
|
+
...span,
|
|
433
|
+
setOutput: (value) => {
|
|
434
|
+
outputSet = true;
|
|
435
|
+
span.setOutput(value);
|
|
436
|
+
},
|
|
437
|
+
});
|
|
438
|
+
if (result !== undefined && !outputSet)
|
|
439
|
+
span.setOutput(result);
|
|
440
|
+
return result;
|
|
441
|
+
}, {
|
|
442
|
+
...options,
|
|
443
|
+
attributes: {
|
|
444
|
+
...options.attributes,
|
|
445
|
+
"gen_ai.operation.name": "invoke_agent",
|
|
446
|
+
"gen_ai.agent.name": agent,
|
|
447
|
+
"hue.agent.role": "subagent",
|
|
448
|
+
},
|
|
449
|
+
});
|
|
450
|
+
}
|
|
401
451
|
/**
|
|
402
452
|
* Runs `callback` inside a GenAI client span for one direct provider call, named
|
|
403
453
|
* `{operation} {model}` unless `options.name` is given and carrying `gen_ai.operation.name`,
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import type { Attributes } from "@opentelemetry/api";
|
|
2
|
+
import type { ToolEndpoint } from "./types.js";
|
|
3
|
+
/** Copy evidence, never send a request or resolve an endpoint from a tool name. */
|
|
4
|
+
export declare function toolEndpointAttributes(endpoint: ToolEndpoint): Attributes;
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import { isCredentialKey } from "./tool-definitions.js";
|
|
2
|
+
/** Copy evidence, never send a request or resolve an endpoint from a tool name. */
|
|
3
|
+
export function toolEndpointAttributes(endpoint) {
|
|
4
|
+
if (!endpoint ||
|
|
5
|
+
typeof endpoint.url !== "string" ||
|
|
6
|
+
endpoint.url.length > 8192 ||
|
|
7
|
+
/[\s\\\p{Cc}]/u.test(endpoint.url) ||
|
|
8
|
+
/%(?![0-9a-f]{2})/i.test(endpoint.url) ||
|
|
9
|
+
!["request", "client_config", "receipt"].includes(endpoint.source))
|
|
10
|
+
throw new Error("Invalid tool endpoint evidence");
|
|
11
|
+
const url = new URL(endpoint.url);
|
|
12
|
+
if (!["http:", "https:"].includes(url.protocol))
|
|
13
|
+
throw new Error("Invalid tool endpoint protocol");
|
|
14
|
+
const names = endpoint.captureQueryParameters ?? [];
|
|
15
|
+
if (!Array.isArray(names) ||
|
|
16
|
+
names.length > 32 ||
|
|
17
|
+
names.some((name) => typeof name !== "string" ||
|
|
18
|
+
!/^[a-zA-Z][a-zA-Z0-9_.-]{0,63}$/.test(name) ||
|
|
19
|
+
isCredentialKey(name)))
|
|
20
|
+
throw new Error("Invalid tool endpoint query allowlist");
|
|
21
|
+
// Keep the raw pairs, including duplicates and empty values. Re-encoding or selecting the
|
|
22
|
+
// first value can turn missing, malformed or ambiguous routing evidence into a usable ID.
|
|
23
|
+
const query = url.search
|
|
24
|
+
.slice(1)
|
|
25
|
+
.split("&")
|
|
26
|
+
.filter((pair) => {
|
|
27
|
+
const key = decodeURIComponent(pair.split("=", 1)[0].replace(/\+/g, " "));
|
|
28
|
+
return names.includes(key);
|
|
29
|
+
})
|
|
30
|
+
.join("&");
|
|
31
|
+
url.username = "";
|
|
32
|
+
url.password = "";
|
|
33
|
+
url.hash = "";
|
|
34
|
+
url.search = query;
|
|
35
|
+
const attributes = {
|
|
36
|
+
"url.full": url.href,
|
|
37
|
+
"hue.tool.endpoint.source": endpoint.source,
|
|
38
|
+
};
|
|
39
|
+
if (endpoint.method !== undefined) {
|
|
40
|
+
if (typeof endpoint.method !== "string" ||
|
|
41
|
+
!/^[!#$%&'*+.^_`|~0-9A-Za-z-]{1,32}$/.test(endpoint.method))
|
|
42
|
+
throw new Error("Invalid tool endpoint method");
|
|
43
|
+
attributes["http.request.method"] = endpoint.method;
|
|
44
|
+
}
|
|
45
|
+
return attributes;
|
|
46
|
+
}
|
package/dist/transport.js
CHANGED
|
@@ -16,7 +16,7 @@ import { MediaDigestMemo, isMediaName } from "./inline-files.js";
|
|
|
16
16
|
import { locateParts } from "./attachment.js";
|
|
17
17
|
import { snapshotLog, snapshotSpan } from "./snapshot.js";
|
|
18
18
|
import { DROPPED_RECORDS_KEY, isContentKey, isTruncatedMarker, redactLog, redactSpan, TRUNCATED_KEY, truncatedMarker, withTruncatedKeys, } from "./privacy.js";
|
|
19
|
-
import { sdkVersion } from "./version.js";
|
|
19
|
+
import { otlpExporterVersion, sdkVersion } from "./version.js";
|
|
20
20
|
/** A record's trace: a span's context, a log record's span identifiers; "" when unreadable. */
|
|
21
21
|
function recordTraceId(record) {
|
|
22
22
|
try {
|
|
@@ -2021,7 +2021,7 @@ class ReportingExporter {
|
|
|
2021
2021
|
await this.forceFlush();
|
|
2022
2022
|
}
|
|
2023
2023
|
}
|
|
2024
|
-
// OpenTelemetry's OTLP/HTTP transport rules
|
|
2024
|
+
// OpenTelemetry's OTLP/HTTP transport rules, restated because that
|
|
2025
2025
|
// transport discards response headers, which Hue uses to announce features.
|
|
2026
2026
|
const MAX_RETRIES = 5;
|
|
2027
2027
|
const INITIAL_BACKOFF_MILLIS = 1000;
|
|
@@ -2042,7 +2042,7 @@ const RETRYABLE_NETWORK_ERRORS = new Set([
|
|
|
2042
2042
|
"EHOSTUNREACH",
|
|
2043
2043
|
]);
|
|
2044
2044
|
/** OpenTelemetry's exporter token, which follows Hue's in the User-Agent as it always has. */
|
|
2045
|
-
const OTEL_USER_AGENT =
|
|
2045
|
+
const OTEL_USER_AGENT = `OTel-OTLP-Exporter-JavaScript/${otlpExporterVersion}`;
|
|
2046
2046
|
function retryAfterMillis(value) {
|
|
2047
2047
|
if (value == null)
|
|
2048
2048
|
return undefined;
|
package/dist/types.d.ts
CHANGED
|
@@ -201,11 +201,21 @@ export interface McpServerInfo {
|
|
|
201
201
|
/** Hue surface such as `google.gmail/mcp`, recorded as `hue.mcp.surface`. */
|
|
202
202
|
surface?: string;
|
|
203
203
|
}
|
|
204
|
-
/**
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
204
|
+
/** Explicit routing evidence for {@link HueClient.tool}; values come from the application. */
|
|
205
|
+
export interface ToolEndpoint {
|
|
206
|
+
/** Actual HTTP(S) URL used by the tool. Never synthesize it from a tool or server label. */
|
|
207
|
+
url: string;
|
|
208
|
+
/** Where the URL was observed. Recorded as `hue.tool.endpoint.source`. */
|
|
209
|
+
source: "request" | "client_config" | "receipt";
|
|
210
|
+
/** Actual HTTP method, when known; the SDK supplies no default. */
|
|
211
|
+
method?: string;
|
|
212
|
+
/** Non-secret query names to retain, including every occurrence. All others are removed. */
|
|
213
|
+
captureQueryParameters?: readonly string[];
|
|
214
|
+
}
|
|
215
|
+
/** Options for {@link HueClient.tool}, including call identity and opt-in routing evidence. */
|
|
208
216
|
export interface ToolOptions extends Pick<SpanOptions, "parentContext"> {
|
|
217
|
+
/** Opt-in routing evidence. Only captured with content enabled, through existing redaction. */
|
|
218
|
+
endpoint?: ToolEndpoint;
|
|
209
219
|
/** Provider-issued identifier of this tool call, recorded as `gen_ai.tool.call.id`. */
|
|
210
220
|
callId?: string;
|
|
211
221
|
/**
|
package/dist/version.d.ts
CHANGED
|
@@ -1,2 +1,4 @@
|
|
|
1
1
|
/** Package version shared by the instrumentation scope and the export User-Agent. */
|
|
2
|
-
export declare const sdkVersion = "0.
|
|
2
|
+
export declare const sdkVersion = "0.21.0";
|
|
3
|
+
/** OTLP dependency version sent in the export User-Agent. */
|
|
4
|
+
export declare const otlpExporterVersion = "0.223.0";
|
package/dist/version.js
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
1
|
// Generated by scripts/write-version.mjs from package.json; do not edit by hand.
|
|
2
2
|
/** Package version shared by the instrumentation scope and the export User-Agent. */
|
|
3
|
-
export const sdkVersion = "0.
|
|
3
|
+
export const sdkVersion = "0.21.0";
|
|
4
|
+
/** OTLP dependency version sent in the export User-Agent. */
|
|
5
|
+
export const otlpExporterVersion = "0.223.0";
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@hue-run/sdk",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.21.0",
|
|
4
4
|
"private": false,
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"publishConfig": {
|
|
@@ -91,22 +91,22 @@
|
|
|
91
91
|
},
|
|
92
92
|
"dependencies": {
|
|
93
93
|
"@babel/parser": "7.29.9",
|
|
94
|
-
"@opentelemetry/api-logs": "0.
|
|
95
|
-
"@opentelemetry/core": "2.
|
|
96
|
-
"@opentelemetry/otlp-exporter-base": "0.
|
|
97
|
-
"@opentelemetry/otlp-transformer": "0.
|
|
98
|
-
"@opentelemetry/resources": "2.
|
|
99
|
-
"@opentelemetry/sdk-logs": "0.
|
|
100
|
-
"@opentelemetry/sdk-trace": "2.
|
|
94
|
+
"@opentelemetry/api-logs": "0.223.0",
|
|
95
|
+
"@opentelemetry/core": "2.12.0",
|
|
96
|
+
"@opentelemetry/otlp-exporter-base": "0.223.0",
|
|
97
|
+
"@opentelemetry/otlp-transformer": "0.223.0",
|
|
98
|
+
"@opentelemetry/resources": "2.12.0",
|
|
99
|
+
"@opentelemetry/sdk-logs": "0.223.0",
|
|
100
|
+
"@opentelemetry/sdk-trace": "2.12.0",
|
|
101
101
|
"zod": "^3.25.76 || ^4.6.5"
|
|
102
102
|
},
|
|
103
103
|
"devDependencies": {
|
|
104
|
-
"@ai-sdk/otel": "1.0.
|
|
104
|
+
"@ai-sdk/otel": "1.0.137",
|
|
105
105
|
"@opentelemetry/api": "1.9.1",
|
|
106
|
-
"@opentelemetry/context-async-hooks": "2.
|
|
106
|
+
"@opentelemetry/context-async-hooks": "2.12.0",
|
|
107
107
|
"@types/bun": "1.4.2",
|
|
108
108
|
"@types/node": "26.6.4",
|
|
109
|
-
"ai": "7.0.
|
|
109
|
+
"ai": "7.0.137",
|
|
110
110
|
"ajv": "8.20.0",
|
|
111
111
|
"protobufjs": "8.8.0",
|
|
112
112
|
"typescript": "7.0.2"
|