@hue-run/sdk 0.20.7 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLI.md +25 -18
- package/EVALUATIONS.md +1 -1
- package/README.md +99 -5
- package/dist/cli/eval.js +31 -15
- package/dist/cli/login.js +72 -12
- package/dist/client.d.ts +12 -0
- package/dist/client.js +55 -3
- package/dist/receipt.d.ts +2 -2
- package/dist/receipt.js +22 -3
- package/dist/tool-endpoint.d.ts +4 -0
- package/dist/tool-endpoint.js +46 -0
- package/dist/transport.js +8 -7
- package/dist/types.d.ts +19 -4
- package/dist/version.d.ts +3 -1
- package/dist/version.js +3 -1
- package/package.json +13 -13
package/CLI.md
CHANGED
|
@@ -42,7 +42,7 @@ project manifest are refused because managers can update ancestor locks; Python
|
|
|
42
42
|
The generated `hue.setup.mjs` or `hue_setup.py` always selects `captureContent: false` /
|
|
43
43
|
`capture_content=False`. For a supported application, setup installs the dependency and adds the
|
|
44
44
|
managed import and middleware registration to the existing entrypoint; an unreferenced helper is
|
|
45
|
-
not a completed integration. TypeScript uses `@hue-run/sdk@0.
|
|
45
|
+
not a completed integration. TypeScript uses `@hue-run/sdk@0.21.0`, `@opentelemetry/api@1.9.1` and
|
|
46
46
|
`@opentelemetry/context-async-hooks@2.11.0`; Python setup uses its separately tested package pin.
|
|
47
47
|
Content capture requires an ordinary account-managed key and a later explicit application decision.
|
|
48
48
|
|
|
@@ -381,8 +381,8 @@ and Hue never receives or stores them. The command is part of the `hue` executab
|
|
|
381
381
|
# LANGFUSE_PUBLIC_KEY=pk-lf-...
|
|
382
382
|
# LANGFUSE_SECRET_KEY=sk-lf-...
|
|
383
383
|
# LANGFUSE_BASE_URL=https://cloud.langfuse.com
|
|
384
|
-
npx -y @hue-run/sdk@0.
|
|
385
|
-
npx -y @hue-run/sdk@0.
|
|
384
|
+
npx -y @hue-run/sdk@0.21.0 import langfuse --env-path .env.langfuse --from 2026-01-01 --dry-run
|
|
385
|
+
npx -y @hue-run/sdk@0.21.0 import langfuse --env-path .env.langfuse --from 2026-01-01 --media
|
|
386
386
|
```
|
|
387
387
|
|
|
388
388
|
`LANGFUSE_PUBLIC_KEY` and `LANGFUSE_SECRET_KEY` come from the environment or `--env-path`
|
|
@@ -698,7 +698,7 @@ node packages/sdk-typescript/scripts/verify-package.mjs --artifacts-dir .artifac
|
|
|
698
698
|
# Set project to an existing supported fixture; use the same directory on resume.
|
|
699
699
|
project=/absolute/path/to/supported-fixture
|
|
700
700
|
node packages/sdk-typescript/scripts/verify-setup-live.mjs \
|
|
701
|
-
--archive .artifacts/typescript/hue-run-sdk-0.
|
|
701
|
+
--archive .artifacts/typescript/hue-run-sdk-0.21.0.tgz \
|
|
702
702
|
--origin https://STAGING_ORIGIN \
|
|
703
703
|
--project "$project" --command setup \
|
|
704
704
|
--evidence .context/setup-staging-before-claim.json
|
|
@@ -709,7 +709,7 @@ the private local handoff and finish the real browser claim, then reconcile the
|
|
|
709
709
|
|
|
710
710
|
```sh
|
|
711
711
|
node packages/sdk-typescript/scripts/verify-setup-live.mjs \
|
|
712
|
-
--archive .artifacts/typescript/hue-run-sdk-0.
|
|
712
|
+
--archive .artifacts/typescript/hue-run-sdk-0.21.0.tgz \
|
|
713
713
|
--origin https://STAGING_ORIGIN \
|
|
714
714
|
--project "$project" --command claim \
|
|
715
715
|
--evidence .context/setup-staging-after-claim.json
|
|
@@ -751,10 +751,10 @@ otherwise. Never hand the agent Hue-native tools (`context.tools`, `context.mcp`
|
|
|
751
751
|
production agent lacks. Pin the SDK, so the agent's repository gains no dependency:
|
|
752
752
|
|
|
753
753
|
```sh
|
|
754
|
-
npx -y @hue-run/sdk@0.
|
|
755
|
-
npx -y @hue-run/sdk@0.
|
|
756
|
-
npx -y @hue-run/sdk@0.
|
|
757
|
-
npx -y @hue-run/sdk@0.
|
|
754
|
+
npx -y @hue-run/sdk@0.21.0 eval --case "Refund an eligible charge" --command "node agent.js"
|
|
755
|
+
npx -y @hue-run/sdk@0.21.0 eval --case https://app.hue.run/projects/demo/scenarios/<id> --command "python agent.py" --timeout 120 --baseline <run id>
|
|
756
|
+
npx -y @hue-run/sdk@0.21.0 eval --set "Billing regressions" --scorer-version <id> --command "node agent.js" --save-version
|
|
757
|
+
npx -y @hue-run/sdk@0.21.0 eval --worker --command "node agent.js" --agent-key support-agent --revision <new-agent-revision>
|
|
758
758
|
```
|
|
759
759
|
|
|
760
760
|
One invocation creates one run holding every selected case; it prints the run URL
|
|
@@ -792,7 +792,7 @@ export default function runMyAgent(inputs: JsonValue, context: SimulationTargetC
|
|
|
792
792
|
```
|
|
793
793
|
|
|
794
794
|
```sh
|
|
795
|
-
npx -y @hue-run/sdk@0.
|
|
795
|
+
npx -y @hue-run/sdk@0.21.0 eval --case "Refund an eligible charge" ./hue-agent.ts --content
|
|
796
796
|
```
|
|
797
797
|
|
|
798
798
|
`--scenario` remains an alias for `--case` for existing scripts. Pass one selection flag.
|
|
@@ -834,21 +834,28 @@ former; a case failed for its telemetry has `state: "error"`,
|
|
|
834
834
|
Hue-owned `world_outcome` checks are graded after the world seals. An experiment always covers
|
|
835
835
|
every case of the saved version; there is no case subset.
|
|
836
836
|
|
|
837
|
-
|
|
838
|
-
`
|
|
839
|
-
|
|
837
|
+
`hue eval` runs up to four cases at a time by default in one-shot and worker modes.
|
|
838
|
+
`--concurrency 1` runs cases serially. Any explicit `--concurrency` value from 1 to 100 takes precedence.
|
|
839
|
+
Each case keeps its own files, environment and checkpoint.
|
|
840
|
+
|
|
841
|
+
The CLI requests the run's pinned `llm_judge` evaluators by default after outputs upload,
|
|
842
|
+
before verdict polling, in direct, simulation and worker modes. `--no-judges` skips these requests;
|
|
843
|
+
`--judges` remains accepted for existing commands. Requests use Hue's hosted judge budget;
|
|
840
844
|
Hue applies its admission controls to new jobs. An accepted request remains replayable if the
|
|
841
|
-
budget is later disabled. Cases without stored output are excluded
|
|
842
|
-
`--judges` with `--no-output` is refused.
|
|
845
|
+
budget is later disabled. Cases without stored output are excluded. `--no-output` skips judge
|
|
846
|
+
requests, and explicitly combining `--judges` with `--no-output` is refused. Without a pinned
|
|
847
|
+
`llm_judge`, the CLI requests no model jobs. Creating a judge alone does not start one.
|
|
848
|
+
`--check` creates no judge jobs.
|
|
843
849
|
|
|
844
|
-
With `--
|
|
850
|
+
With `--wait 0`, the CLI requests the jobs and returns their current verdict state;
|
|
845
851
|
pending jobs continue in Hue. Uploaded outputs are checkpointed before the attempt or worker
|
|
846
852
|
queue is marked completed. If this handoff or dispatch fails, or its acknowledgement is lost,
|
|
847
853
|
rerun the unchanged command with the same checkpoint directory. The CLI resumes judging the uploaded
|
|
848
|
-
outputs without invoking the agent, using the same idempotency keys.
|
|
854
|
+
outputs without invoking the agent, using the same idempotency keys. Changing to `--no-judges`
|
|
855
|
+
or `--no-output` while that handoff is pending stops before a new agent run. After dispatch succeeds,
|
|
849
856
|
the next ordinary invocation starts a new run as usual.
|
|
850
857
|
|
|
851
|
-
`--worker
|
|
858
|
+
`--worker` discards cancelled or released judge handoffs. Once judge requests succeed, a
|
|
852
859
|
verdict-read failure is logged and the worker keeps accepting runs.
|
|
853
860
|
|
|
854
861
|
`--command "<shell command>"` spawns the command once per case with the world's environment
|
package/EVALUATIONS.md
CHANGED
|
@@ -184,7 +184,7 @@ Scorer deferral shipped in TypeScript `0.3.1`: the local runner executes only th
|
|
|
184
184
|
|
|
185
185
|
`world_outcome` pins (`hue.conversion_outcome.v1` and `.v2`, and the `hue.outcome_assertions.v2` and `.v3` case-outcome evaluators, `.v3` with its pinned judge in `config.judge`) run inside Hue and need no local callback or executable source digest. A supporting server owns their execution from saved world evidence. Legacy `local_code` pins still require the exact registered callback; changing the worker cannot convert those immutable pins into hosted ones.
|
|
186
186
|
|
|
187
|
-
Manual results require a human session. Hosted model-judge dispatch is explicit: call `createJudgeJobs(runId,{idempotencyKey,jobs:[{evaluationItemId,scorerVersionId}]})`. Hue checks its judge budget when admitting new requests. `getJudgeBudget()` reports its current state, and on current servers the organization's monthly AI budget as `organizationBudget`, but replay an interrupted request with the same key and body even if that state changed.
|
|
187
|
+
Manual results require a human session. Hosted model-judge dispatch is explicit: call `createJudgeJobs(runId,{idempotencyKey,jobs:[{evaluationItemId,scorerVersionId}]})`. Hue checks its judge budget when admitting new requests. `getJudgeBudget()` reports its current state, and on current servers the organization's monthly AI budget as `organizationBudget`, but replay an interrupted request with the same key and body even if that state changed. [`hue eval`](./CLI.md#evaluate-an-agent-against-a-case) dispatches pinned model judges by default after outputs upload; `--no-judges` skips them. Creating or publishing a judge alone starts no model job. `listJudgeJobs`, `getJudgeJob` and `cancelJudgeJob` expose job progress and cancellation requests. These methods never claim that local execution has hosted provenance. Hosted job endpoints are covered by HTTP contract tests here; live hosted model execution is a separate platform acceptance phase. `listResults` and `getResult` read recorded local or hosted results.
|
|
188
188
|
|
|
189
189
|
When present, the budget's `authentication` reports credential resolution only. An
|
|
190
190
|
`available` status or `configured: true` does not prove that a provider accepted the
|
package/README.md
CHANGED
|
@@ -202,6 +202,22 @@ helpers (`setInput`, `setOutput`, `tool` arguments and results, `recordMessages`
|
|
|
202
202
|
that is not JSON, such as a `Date` or a class instance, is omitted with an instrumentation failure
|
|
203
203
|
while the callback result is returned unchanged.
|
|
204
204
|
|
|
205
|
+
### Tool endpoint evidence (unreleased)
|
|
206
|
+
|
|
207
|
+
This checkout adds optional `ToolOptions.endpoint`: `{ url, source, method?, captureQueryParameters? }`.
|
|
208
|
+
Use the actual HTTP(S) request URL, client config URL or receipt URL; set `source` to `request`,
|
|
209
|
+
`client_config` or `receipt`. Supply `method` only when known. With content capture enabled,
|
|
210
|
+
Hue records `url.full`, `hue.tool.endpoint.source` and optional `http.request.method` through the
|
|
211
|
+
existing redactor (`attributes.url.full` for the URL). URL userinfo and fragments are removed;
|
|
212
|
+
query values are omitted unless their names appear in `captureQueryParameters`. Credential names
|
|
213
|
+
are rejected even if allowlisted. Duplicate and empty allowed query values remain intact.
|
|
214
|
+
|
|
215
|
+
Metadata-only mode omits endpoint evidence. Invalid evidence is omitted and counted without
|
|
216
|
+
changing the tool's result or exception. This helper does not observe HTTP automatically or infer
|
|
217
|
+
account identifiers. Do not supply secret-bearing URL paths, and preserve existing masking of
|
|
218
|
+
account identifiers. See the [setup skill](../../skills/hue/SKILL.md#http-tool-endpoint-evidence)
|
|
219
|
+
for dispatcher integration and source checkout requirements.
|
|
220
|
+
|
|
205
221
|
## Images, documents and audio
|
|
206
222
|
|
|
207
223
|
Hue stores the bytes of images, documents and audio apart from the spans and log records that
|
|
@@ -536,8 +552,8 @@ the Langfuse project's own keys (`LANGFUSE_PUBLIC_KEY`, `LANGFUSE_SECRET_KEY`, a
|
|
|
536
552
|
`HUE_API_KEY`:
|
|
537
553
|
|
|
538
554
|
```sh
|
|
539
|
-
npx -y @hue-run/sdk@0.
|
|
540
|
-
npx -y @hue-run/sdk@0.
|
|
555
|
+
npx -y @hue-run/sdk@0.21.0 import langfuse --env-path .env.langfuse --from 2026-01-01 --dry-run
|
|
556
|
+
npx -y @hue-run/sdk@0.21.0 import langfuse --env-path .env.langfuse --from 2026-01-01 --media
|
|
541
557
|
```
|
|
542
558
|
|
|
543
559
|
- It reads Langfuse Cloud and self-hosted v4 through the Observations API v2, and self-hosted v3
|
|
@@ -588,6 +604,79 @@ npx -y @hue-run/sdk@latest traces import upload ./exports.hue-import --authoriza
|
|
|
588
604
|
See [Import a bundle of existing traces](CLI.md#import-a-bundle-of-existing-traces) for the
|
|
589
605
|
formats, the report, what Hue refuses and the exit codes.
|
|
590
606
|
|
|
607
|
+
## Sub-agents
|
|
608
|
+
|
|
609
|
+
Run a sub-agent with `hue.subagent(name, callback, options)`, inside the tool call that starts it:
|
|
610
|
+
|
|
611
|
+
```ts
|
|
612
|
+
const findings = await hue.tool("research", { topic }, () =>
|
|
613
|
+
hue.subagent(
|
|
614
|
+
"researcher",
|
|
615
|
+
async () => {
|
|
616
|
+
const notes = await hue.model("gpt-5-mini", () => callModel(topic), { provider: "openai" });
|
|
617
|
+
return hue.tool("summarize", notes, () => summarize(notes));
|
|
618
|
+
},
|
|
619
|
+
{ input: topic },
|
|
620
|
+
),
|
|
621
|
+
);
|
|
622
|
+
```
|
|
623
|
+
|
|
624
|
+
The sub-agent runs inside one `invoke_agent researcher` span with `gen_ai.operation.name`
|
|
625
|
+
`invoke_agent`, `gen_ai.agent.name` `researcher` and `hue.agent.role` `subagent`. That span is a
|
|
626
|
+
child of the active span, here the `execute_tool research` span, and the sub-agent's own model and
|
|
627
|
+
tool spans are its children, so Hue scopes them to the sub-agent. Sub-agents nest: a sub-agent's
|
|
628
|
+
tool can start another. The span is `INTERNAL` unless `options.kind` says otherwise; session, user
|
|
629
|
+
and workspace identifiers are inherited unless `options` sets them. `options.input` and the
|
|
630
|
+
callback's defined result are recorded as `input.value` and `output.value` when `captureContent`
|
|
631
|
+
is true (a callback that calls `setOutput` keeps its own output), and errors are recorded and
|
|
632
|
+
rethrown as in `withSpan`.
|
|
633
|
+
|
|
634
|
+
The calling agent's role is not set for you: Hue reads roles only from the spans that state them,
|
|
635
|
+
and the span a sub-agent starts under is usually a tool call, not the agent. To label the calling
|
|
636
|
+
agent, give its own span the role:
|
|
637
|
+
|
|
638
|
+
```ts
|
|
639
|
+
await hue.withSpan("invoke_agent planner", () => runPlanner(), {
|
|
640
|
+
attributes: {
|
|
641
|
+
"gen_ai.operation.name": "invoke_agent",
|
|
642
|
+
"gen_ai.agent.name": "planner",
|
|
643
|
+
"hue.agent.role": "orchestrator",
|
|
644
|
+
},
|
|
645
|
+
});
|
|
646
|
+
```
|
|
647
|
+
|
|
648
|
+
When the sub-agent runs in another process, send the caller's trace context with the request and
|
|
649
|
+
start the sub-agent under it:
|
|
650
|
+
|
|
651
|
+
```ts
|
|
652
|
+
// The calling process, inside the tool call that hands the task over.
|
|
653
|
+
await hue.tool("research", { topic }, async () => {
|
|
654
|
+
const headers: Record<string, string> = { "content-type": "application/json" };
|
|
655
|
+
hue.inject(headers); // adds `traceparent` only
|
|
656
|
+
const response = await fetch(researcherUrl, {
|
|
657
|
+
method: "POST",
|
|
658
|
+
headers,
|
|
659
|
+
body: JSON.stringify({ topic, sessionId }),
|
|
660
|
+
});
|
|
661
|
+
return response.json();
|
|
662
|
+
});
|
|
663
|
+
|
|
664
|
+
// The sub-agent's process, for example an Express handler.
|
|
665
|
+
const { topic, sessionId } = req.body;
|
|
666
|
+
const findings = await hue.subagent("researcher", () => research(topic), {
|
|
667
|
+
parentContext: hue.extract(req.headers),
|
|
668
|
+
sessionId,
|
|
669
|
+
});
|
|
670
|
+
```
|
|
671
|
+
|
|
672
|
+
The `invoke_agent researcher` span then belongs to the caller's trace, with the caller's
|
|
673
|
+
`execute_tool research` span as its parent. `hue.extract()` reads a plain header object such as
|
|
674
|
+
Node's `req.headers`; pass `Object.fromEntries(request.headers)` for a Fetch `Request`. Only
|
|
675
|
+
`traceparent` crosses the boundary, never the API key, baggage or the session, so pass the
|
|
676
|
+
session identifier yourself as above. Accept `traceparent` only from your own services: whoever
|
|
677
|
+
sets it chooses the trace the sub-agent's spans join. On an endpoint that also serves other
|
|
678
|
+
callers, extract it only from authenticated internal requests and start a new trace otherwise.
|
|
679
|
+
|
|
591
680
|
## Vercel AI SDK 6
|
|
592
681
|
|
|
593
682
|
AI SDK 6 accepts a per-call tracer through `experimental_telemetry`. Pass
|
|
@@ -610,7 +699,9 @@ installed `ai` major version once per process and throws a `TypeError` below 7.
|
|
|
610
699
|
|
|
611
700
|
## Vercel AI SDK 7
|
|
612
701
|
|
|
613
|
-
Compatible optional peers are `ai@^7.0.
|
|
702
|
+
Compatible optional peers are `ai@^7.0.0` and `@ai-sdk/otel@^1.0.0` (install `@ai-sdk/otel` with
|
|
703
|
+
the matching patch number, for example `ai@7.0.99` with `@ai-sdk/otel@1.0.99`; each `@ai-sdk/otel`
|
|
704
|
+
release depends on exactly that `ai` release), alongside
|
|
614
705
|
`@opentelemetry/api@1.9.1`. For an app without global AI SDK telemetry integrations,
|
|
615
706
|
configure telemetry on each agent or generation call:
|
|
616
707
|
|
|
@@ -780,7 +871,8 @@ and similar attributes on your own providers.
|
|
|
780
871
|
|
|
781
872
|
For external parent context pass `parentContext` to `withSpan`. Across processes, use
|
|
782
873
|
`hue.inject(carrier)` inside the producing span and `hue.extract(carrier)` in the worker; both
|
|
783
|
-
speak W3C `traceparent` only and never include the API key or baggage.
|
|
874
|
+
speak W3C `traceparent` only and never include the API key or baggage. Extract it only from
|
|
875
|
+
trusted internal callers, as described under [Sub-agents](#sub-agents). Hue registers no global
|
|
784
876
|
propagator, so `propagation.inject()` from `@opentelemetry/api` is a no-op unless your
|
|
785
877
|
application configured one. `getContext()` exposes the helper's current context for APIs taking
|
|
786
878
|
an explicit context. Session, user and workspace identifiers (`sessionId` as
|
|
@@ -1043,7 +1135,9 @@ receipt request. It does not flush, run your application, create a test span, or
|
|
|
1043
1135
|
read captured values. `fields` reports the presence of stored normalized input,
|
|
1044
1136
|
output, model, usage, and session data across the trace; it does not establish
|
|
1045
1137
|
content correctness or that every possible span has arrived. Leave unknown usage
|
|
1046
|
-
and intentionally disabled content out of `requiredFields`.
|
|
1138
|
+
and intentionally disabled content out of `requiredFields`. When Hue reports the trace
|
|
1139
|
+
missing, `project` names the project the key searched: a trace absent there was sent with a
|
|
1140
|
+
different key (for example one still exported in the shell) or is not stored yet.
|
|
1047
1141
|
|
|
1048
1142
|
The default budget is 10 seconds; set `timeoutMillis` up to 60,000. Only a recognized
|
|
1049
1143
|
missing trace, HTTP 429, or HTTP 503 is retried, respecting `Retry-After` and the
|
package/dist/cli/eval.js
CHANGED
|
@@ -92,15 +92,16 @@ Output and limits:
|
|
|
92
92
|
--check One-shot: check the key, selection and agent, print what the
|
|
93
93
|
run would do and exit without creating a run or a world
|
|
94
94
|
--checkpoint-dir <path> Private checkpoint directory (default: .hue/eval/<agent-key>)
|
|
95
|
-
--concurrency <n> Cases in flight, 1-100 (default:
|
|
95
|
+
--concurrency <n> Cases in flight, 1-100 (default: 4)
|
|
96
96
|
--trace-not-accepted <policy> A case whose telemetry Hue did not accept in time: fail_case
|
|
97
97
|
(default) completes it as failed with the evidence omitted;
|
|
98
98
|
pending completes it in its true state with the trace still to
|
|
99
99
|
arrive, which Hue attaches once its export is complete
|
|
100
100
|
--timeout <seconds> Per-case --command timeout (default: 600)
|
|
101
101
|
--wait <seconds> Verdict wait after the run finishes (default: 300)
|
|
102
|
-
--judges
|
|
103
|
-
|
|
102
|
+
--no-judges Skip pinned hosted model judges (requested by default
|
|
103
|
+
after outputs upload; uses the project's judge budget)
|
|
104
|
+
--judges Request pinned judges (compatibility flag)
|
|
104
105
|
-h, --help Show this help
|
|
105
106
|
|
|
106
107
|
HUE_API_KEY must be a "Read and write" project key; it is never printed.
|
|
@@ -154,7 +155,8 @@ function parse(argv) {
|
|
|
154
155
|
content: { type: "boolean" },
|
|
155
156
|
"no-content": { type: "boolean", default: false },
|
|
156
157
|
"no-output": { type: "boolean", default: false },
|
|
157
|
-
judges: { type: "boolean"
|
|
158
|
+
judges: { type: "boolean" },
|
|
159
|
+
"no-judges": { type: "boolean", default: false },
|
|
158
160
|
"trace-not-accepted": { type: "string" },
|
|
159
161
|
"save-version": { type: "boolean", default: false },
|
|
160
162
|
check: { type: "boolean", default: false },
|
|
@@ -1123,6 +1125,19 @@ export async function dispatchPinnedJudges(client, experimentId, subjectIds, sig
|
|
|
1123
1125
|
await dispatch();
|
|
1124
1126
|
return count;
|
|
1125
1127
|
}
|
|
1128
|
+
/** Keep the judge lock through execution so an opt-out cannot race a pending dispatch. */
|
|
1129
|
+
async function acquireJudgeCheckpoint(directory, projectId, baseUrl, enabled) {
|
|
1130
|
+
const store = await CheckpointStore.acquire(checkpointPath(directory, ["judge-dispatch", "judge checkpoint"]), { kind: "cli-judge-dispatch", projectId, baseUrl });
|
|
1131
|
+
try {
|
|
1132
|
+
if (!enabled && (await store.read("pending")))
|
|
1133
|
+
throw new Error("Recover the pending hosted checks with the original flags before using --no-judges or --no-output");
|
|
1134
|
+
return store;
|
|
1135
|
+
}
|
|
1136
|
+
catch (error) {
|
|
1137
|
+
await store.release();
|
|
1138
|
+
throw error;
|
|
1139
|
+
}
|
|
1140
|
+
}
|
|
1126
1141
|
/** Durable handoff before the original attempt marker or queue acknowledgement. */
|
|
1127
1142
|
async function checkpointJudges(judging, run, completion) {
|
|
1128
1143
|
await judging.store.write("pending", {
|
|
@@ -1242,7 +1257,7 @@ function parseBaseline(value) {
|
|
|
1242
1257
|
}
|
|
1243
1258
|
async function runOnce(values, connection, agents, agent, hue, output, signal) {
|
|
1244
1259
|
const client = new EvaluationClient(connection);
|
|
1245
|
-
const concurrency = integer("concurrency", values.concurrency,
|
|
1260
|
+
const concurrency = integer("concurrency", values.concurrency, 4, 1, 100);
|
|
1246
1261
|
const baselineId = parseBaseline(values.baseline);
|
|
1247
1262
|
const mode = parseMode(values.mode);
|
|
1248
1263
|
integer("wait", values.wait, 300, 0, 86_400);
|
|
@@ -1267,9 +1282,10 @@ async function runOnce(values, connection, agents, agent, hue, output, signal) {
|
|
|
1267
1282
|
// Each kind of run has its own checkpoint directory and identity: a mixed run never resumes
|
|
1268
1283
|
// from, or is resumed by, a direct or simulation run's checkpoint.
|
|
1269
1284
|
const checkpointDirectory = await prepareCheckpointDirectory(values["checkpoint-dir"], agent.key, project.id, selection.kind);
|
|
1285
|
+
const judgeStore = await acquireJudgeCheckpoint(checkpointDirectory, project.id, connection.baseUrl, values.judges === true);
|
|
1270
1286
|
const judging = values.judges
|
|
1271
1287
|
? {
|
|
1272
|
-
store:
|
|
1288
|
+
store: judgeStore,
|
|
1273
1289
|
selectionDigest: digest({
|
|
1274
1290
|
datasetVersionId: pins.datasetVersionId,
|
|
1275
1291
|
scorerVersionIds: [...pins.scorerVersionIds].sort(),
|
|
@@ -1385,7 +1401,7 @@ async function runOnce(values, connection, agents, agent, hue, output, signal) {
|
|
|
1385
1401
|
}, baselineId, output, signal, judging);
|
|
1386
1402
|
}
|
|
1387
1403
|
finally {
|
|
1388
|
-
await
|
|
1404
|
+
await judgeStore.release();
|
|
1389
1405
|
}
|
|
1390
1406
|
}
|
|
1391
1407
|
/**
|
|
@@ -1425,7 +1441,7 @@ async function checkRun(client, values, pins, selection, agent, output) {
|
|
|
1425
1441
|
mode: kind,
|
|
1426
1442
|
worldCases: kind === "direct" ? 0 : worldCases,
|
|
1427
1443
|
directCases: kind === "direct" ? cases : cases - worldCases,
|
|
1428
|
-
agent: { key: agent.key, revision: agent.revision, concurrency: values.concurrency ?? "
|
|
1444
|
+
agent: { key: agent.key, revision: agent.revision, concurrency: values.concurrency ?? "4" },
|
|
1429
1445
|
};
|
|
1430
1446
|
if (values.json)
|
|
1431
1447
|
process.stdout.write(`${JSON.stringify({ check })}\n`);
|
|
@@ -1602,19 +1618,16 @@ async function runWorker(values, connection, agents, agent, hue, output, signal)
|
|
|
1602
1618
|
const client = new ObservedClient(connection);
|
|
1603
1619
|
const environmentClient = createEnvironmentClient(connection);
|
|
1604
1620
|
const wait = integer("wait", values.wait, 300, 0, 86_400);
|
|
1605
|
-
const concurrency = integer("concurrency", values.concurrency,
|
|
1621
|
+
const concurrency = integer("concurrency", values.concurrency, 4, 1, 100);
|
|
1606
1622
|
let maxRuns = values["max-runs"] === undefined
|
|
1607
1623
|
? undefined
|
|
1608
1624
|
: integer("max-runs", values["max-runs"], 1, 1, 1_000_000);
|
|
1609
1625
|
const project = await client.checkConnection();
|
|
1610
1626
|
const checkpointDirectory = await prepareCheckpointDirectory(values["checkpoint-dir"], agent.key, project.id, "worker", slug(agent.revision) || "dev");
|
|
1627
|
+
const judgeStore = await acquireJudgeCheckpoint(checkpointDirectory, project.id, connection.baseUrl, values.judges === true);
|
|
1611
1628
|
const judging = values.judges
|
|
1612
1629
|
? {
|
|
1613
|
-
store:
|
|
1614
|
-
kind: "cli-judge-dispatch",
|
|
1615
|
-
projectId: project.id,
|
|
1616
|
-
baseUrl: connection.baseUrl,
|
|
1617
|
-
}),
|
|
1630
|
+
store: judgeStore,
|
|
1618
1631
|
selectionDigest: digest(agent),
|
|
1619
1632
|
source: {
|
|
1620
1633
|
directory: checkpointDirectory,
|
|
@@ -1787,7 +1800,7 @@ async function runWorker(values, connection, agents, agent, hue, output, signal)
|
|
|
1787
1800
|
return 0;
|
|
1788
1801
|
}
|
|
1789
1802
|
finally {
|
|
1790
|
-
await
|
|
1803
|
+
await judgeStore.release();
|
|
1791
1804
|
}
|
|
1792
1805
|
}
|
|
1793
1806
|
/**
|
|
@@ -1831,8 +1844,11 @@ export async function runEvalCommand(argv) {
|
|
|
1831
1844
|
worker = values.worker;
|
|
1832
1845
|
if (values.content && values["no-content"])
|
|
1833
1846
|
throw new UsageError("Choose either --content or --no-content");
|
|
1847
|
+
if (values.judges && values["no-judges"])
|
|
1848
|
+
throw new UsageError("Choose either --judges or --no-judges");
|
|
1834
1849
|
if (values.judges && values["no-output"])
|
|
1835
1850
|
throw new UsageError("--judges requires stored case outputs; remove --no-output");
|
|
1851
|
+
values.judges = !values["no-judges"] && !values["no-output"];
|
|
1836
1852
|
// A usage error before anything is prepared or created: no run is left to recover from it.
|
|
1837
1853
|
traceNotAcceptedPolicy(values["trace-not-accepted"]);
|
|
1838
1854
|
json = values.json;
|
package/dist/cli/login.js
CHANGED
|
@@ -220,11 +220,8 @@ async function readBoundedText(response) {
|
|
|
220
220
|
function isRecord(value) {
|
|
221
221
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
222
222
|
}
|
|
223
|
-
/**
|
|
224
|
-
|
|
225
|
-
* with `GET /api/v1/datasets`. No redirects are followed.
|
|
226
|
-
*/
|
|
227
|
-
async function checkEvaluationsKey(fetchImpl, origin, apiKey) {
|
|
223
|
+
/** Mirrors `checkConnection()` with `GET /api/v1/projects/current`; `detail` is the project name. */
|
|
224
|
+
async function checkProject(fetchImpl, origin, apiKey) {
|
|
228
225
|
let response;
|
|
229
226
|
try {
|
|
230
227
|
response = await fetchImpl(`${origin}/api/v1/projects/current`, {
|
|
@@ -253,16 +250,21 @@ async function checkEvaluationsKey(fetchImpl, origin, apiKey) {
|
|
|
253
250
|
detail: `Hue answered HTTP ${response.status} while checking the evaluations key.`,
|
|
254
251
|
};
|
|
255
252
|
}
|
|
256
|
-
let projectName;
|
|
257
253
|
try {
|
|
258
254
|
const project = JSON.parse(await readBoundedText(response));
|
|
259
255
|
if (!isRecord(project) || typeof project.name !== "string")
|
|
260
256
|
throw new Error("Invalid project");
|
|
261
|
-
|
|
257
|
+
return { ok: true, detail: project.name };
|
|
262
258
|
}
|
|
263
259
|
catch {
|
|
264
260
|
return { ok: false, rejected: false, detail: "Hue returned an unexpected project response." };
|
|
265
261
|
}
|
|
262
|
+
}
|
|
263
|
+
/** {@link checkProject}, then confirms evaluation access with `GET /api/v1/datasets`. */
|
|
264
|
+
async function checkEvaluationsKey(fetchImpl, origin, apiKey) {
|
|
265
|
+
const project = await checkProject(fetchImpl, origin, apiKey);
|
|
266
|
+
if (!project.ok)
|
|
267
|
+
return project;
|
|
266
268
|
// Every valid key reaches the project check; only a key with write access can list eval sets,
|
|
267
269
|
// so a Read or Tracing only key is refused here instead of failing later in `hue eval`.
|
|
268
270
|
let evaluations;
|
|
@@ -290,7 +292,7 @@ async function checkEvaluationsKey(fetchImpl, origin, apiKey) {
|
|
|
290
292
|
rejected: false,
|
|
291
293
|
detail: `Hue answered HTTP ${evaluations.status} while checking evaluation access.`,
|
|
292
294
|
};
|
|
293
|
-
return
|
|
295
|
+
return project;
|
|
294
296
|
}
|
|
295
297
|
/** Reads JSON-RPC messages from a JSON body or a `text/event-stream` body. */
|
|
296
298
|
function parseJsonRpcMessages(text, contentType) {
|
|
@@ -689,6 +691,21 @@ export async function runLoginCommand(argv, io = {}) {
|
|
|
689
691
|
out(kind === "evaluations"
|
|
690
692
|
? `Evaluations access accepted for project "${check.detail}".`
|
|
691
693
|
: `Coding-agent access accepted; the Hue MCP server lists ${check.detail} tools.`);
|
|
694
|
+
// Loaders such as `uv run --env-file` and dotenv keep an exported variable over the file,
|
|
695
|
+
// so a stale shell key silently sends traces to another project. Name projects, never keys.
|
|
696
|
+
if (kind === "evaluations" && env.HUE_API_KEY && env.HUE_API_KEY !== value) {
|
|
697
|
+
// The shell key only goes to the server it was exported for.
|
|
698
|
+
const shellOrigin = env.HUE_BASE_URL ?? DEFAULT_ORIGIN;
|
|
699
|
+
const shell = shellOrigin === origin ? await checkProject(fetchImpl, origin, env.HUE_API_KEY) : null;
|
|
700
|
+
const owner = !shell
|
|
701
|
+
? ` for ${shellOrigin}`
|
|
702
|
+
: shell.ok
|
|
703
|
+
? ` for project "${shell.detail}"`
|
|
704
|
+
: shell.rejected
|
|
705
|
+
? " that Hue does not accept"
|
|
706
|
+
: " whose project could not be checked";
|
|
707
|
+
stderr.write(`Warning: this shell already exports a different HUE_API_KEY${owner}. It takes precedence over ${envDisplay} (project "${check.detail}") in tools that load env files without overriding them; unset it before running the application.\n`);
|
|
708
|
+
}
|
|
692
709
|
}
|
|
693
710
|
if (!failure) {
|
|
694
711
|
const text = mergeEnvText(envFile.text, updates);
|
|
@@ -738,12 +755,55 @@ export async function runLoginCommand(argv, io = {}) {
|
|
|
738
755
|
}
|
|
739
756
|
if (failure)
|
|
740
757
|
return fail(failure.message, failure.code);
|
|
741
|
-
|
|
758
|
+
const steps = [];
|
|
742
759
|
if (stored.includes("coding-agent"))
|
|
743
760
|
// `hue mcp install` defaults to production; a non-default origin needs its own endpoint.
|
|
744
761
|
// Without --auth it configures sign-in unless HUE_MCP_KEY is exported, so name the key stored.
|
|
745
|
-
|
|
746
|
-
if (stored.includes("evaluations"))
|
|
747
|
-
|
|
762
|
+
steps.push(`hue mcp install --client ${(await detectMcpClient(cwd)) ?? "<claude-code|codex|conductor|cursor|vscode|windsurf|gemini>"} --auth key${mcpUrl === mcpUrlForOrigin(DEFAULT_ORIGIN) ? "" : ` --url ${mcpUrl}`}`);
|
|
763
|
+
if (stored.includes("evaluations")) {
|
|
764
|
+
const agent = await detectEvalAgent(cwd);
|
|
765
|
+
if (agent)
|
|
766
|
+
steps.push(`hue eval --case "<name>" ${agent}${envDisplay === DEFAULT_ENV_FILE ? "" : ` --env-file ${envDisplay}`}`);
|
|
767
|
+
}
|
|
768
|
+
if (steps.length > 0) {
|
|
769
|
+
out("Next steps:");
|
|
770
|
+
for (const step of steps)
|
|
771
|
+
out(` ${step}`);
|
|
772
|
+
}
|
|
748
773
|
return 0;
|
|
749
774
|
}
|
|
775
|
+
/** Client configuration present in the repository root; `hue mcp install` writes the first three. */
|
|
776
|
+
const MCP_CLIENT_MARKERS = [
|
|
777
|
+
["claude-code", ".mcp.json"],
|
|
778
|
+
["cursor", ".cursor"],
|
|
779
|
+
["vscode", ".vscode"],
|
|
780
|
+
["claude-code", ".claude"],
|
|
781
|
+
["codex", ".codex"],
|
|
782
|
+
["gemini", ".gemini"],
|
|
783
|
+
["windsurf", ".windsurf"],
|
|
784
|
+
];
|
|
785
|
+
/** The one client whose configuration the repository holds; undefined when none or several do. */
|
|
786
|
+
async function detectMcpClient(cwd) {
|
|
787
|
+
const found = new Set();
|
|
788
|
+
for (const [client, marker] of MCP_CLIENT_MARKERS)
|
|
789
|
+
if (await access(join(cwd, marker)).then(() => true, () => false))
|
|
790
|
+
found.add(client);
|
|
791
|
+
return found.size === 1 ? [...found][0] : undefined;
|
|
792
|
+
}
|
|
793
|
+
/** Manifests in the repository root that name its language; the CLI stays outside `hue setup`. */
|
|
794
|
+
const LANGUAGE_MARKERS = [
|
|
795
|
+
["./hue-agent.ts", "tsconfig.json"],
|
|
796
|
+
['--command "<agent start command>"', "pyproject.toml"],
|
|
797
|
+
['--command "<agent start command>"', "requirements.txt"],
|
|
798
|
+
['--command "<agent start command>"', "uv.lock"],
|
|
799
|
+
['--command "<agent start command>"', "poetry.lock"],
|
|
800
|
+
// Last: a Python repository often carries a package.json for its tooling.
|
|
801
|
+
["./hue-agent.ts", "package.json"],
|
|
802
|
+
];
|
|
803
|
+
/** The `hue eval` agent argument for the repository's language; undefined when it is unknown. */
|
|
804
|
+
async function detectEvalAgent(cwd) {
|
|
805
|
+
for (const [agent, marker] of LANGUAGE_MARKERS)
|
|
806
|
+
if (await access(join(cwd, marker)).then(() => true, () => false))
|
|
807
|
+
return agent;
|
|
808
|
+
return undefined;
|
|
809
|
+
}
|
package/dist/client.d.ts
CHANGED
|
@@ -83,6 +83,18 @@ export declare class HueClient {
|
|
|
83
83
|
* provider and surface as `hue.mcp.provider` / `hue.mcp.surface`.
|
|
84
84
|
*/
|
|
85
85
|
tool<T>(name: string, input: unknown, execute: () => Promise<T> | T, options?: ToolOptions): Promise<T>;
|
|
86
|
+
/**
|
|
87
|
+
* Runs `callback` as a sub-agent: inside an `invoke_agent {name}` span carrying
|
|
88
|
+
* `gen_ai.operation.name` `invoke_agent`, `gen_ai.agent.name` and `hue.agent.role` `subagent`.
|
|
89
|
+
* The span nests under the active span, normally the `tool()` span of the call that started
|
|
90
|
+
* the sub-agent, or under `options.parentContext`, for example `hue.extract(headers)` in
|
|
91
|
+
* another process. The sub-agent's own model and tool spans nest under it, and session, user
|
|
92
|
+
* and workspace identifiers are inherited unless `options` sets them. `options.input` and a
|
|
93
|
+
* defined result are recorded as `input.value` / `output.value` when `captureContent` is true;
|
|
94
|
+
* a callback that calls `setOutput` keeps its own output. Errors are recorded and rethrown as
|
|
95
|
+
* in `withSpan`. A blank or non-string name is recorded as `unknown` and counted.
|
|
96
|
+
*/
|
|
97
|
+
subagent<T>(name: string, callback: (span: HueSpan) => Promise<T> | T, options?: SpanOptions): Promise<T>;
|
|
86
98
|
/**
|
|
87
99
|
* Runs `callback` inside a GenAI client span for one direct provider call, named
|
|
88
100
|
* `{operation} {model}` unless `options.name` is given and carrying `gen_ai.operation.name`,
|
package/dist/client.js
CHANGED
|
@@ -11,6 +11,7 @@ import { defaultResource, resourceFromAttributes } from "@opentelemetry/resource
|
|
|
11
11
|
import { HUE_SCOPE } from "./config.js";
|
|
12
12
|
import { hostedServerAddresses, hostedToolActivity, hostedToolProvider, providerErrorDescription, } from "./provider-tools.js";
|
|
13
13
|
import { toolCatalogSummary } from "./tool-definitions.js";
|
|
14
|
+
import { toolEndpointAttributes } from "./tool-endpoint.js";
|
|
14
15
|
import { contentLimitsOf, encodeBoundedContent, encodeContent, noopSpan, safeSpan, spanSource, } from "./safety.js";
|
|
15
16
|
import { TRUNCATED_KEY, withTruncatedKeys } from "./privacy.js";
|
|
16
17
|
import { countFallback, MAX_BLOB_BYTES, newTally, sizeOf } from "./blobs.js";
|
|
@@ -269,10 +270,12 @@ export class HueClient {
|
|
|
269
270
|
* @throws HueTraceVerificationError for authentication, unsupported endpoint, transport or invalid response failures.
|
|
270
271
|
* @throws TypeError for an invalid trace ID or options.
|
|
271
272
|
*/
|
|
272
|
-
verifyTrace(traceId, options = {}) {
|
|
273
|
+
async verifyTrace(traceId, options = {}) {
|
|
273
274
|
if (!this.enabled)
|
|
274
|
-
|
|
275
|
-
|
|
275
|
+
throw new HueConnectionError("Hue telemetry is disabled");
|
|
276
|
+
// Once Hue reports the trace missing, name the project this key searched so a wrong key reads
|
|
277
|
+
// differently from a trace that is not stored yet.
|
|
278
|
+
return verifyTrace(this.transport.options, traceId, options, () => this.checkConnection().catch(() => undefined));
|
|
276
279
|
}
|
|
277
280
|
/** The helper's current context: the innermost active Hue span, else the OpenTelemetry active context. */
|
|
278
281
|
getContext() {
|
|
@@ -385,6 +388,15 @@ export class HueClient {
|
|
|
385
388
|
stamp("mcp.server.version", options.mcp?.version);
|
|
386
389
|
stamp("hue.mcp.provider", options.mcp?.provider);
|
|
387
390
|
stamp("hue.mcp.surface", options.mcp?.surface);
|
|
391
|
+
if (this.enabled && !this.closed && this.captureContent) {
|
|
392
|
+
try {
|
|
393
|
+
if (options.endpoint !== undefined)
|
|
394
|
+
Object.assign(attributes, toolEndpointAttributes(options.endpoint));
|
|
395
|
+
}
|
|
396
|
+
catch {
|
|
397
|
+
this.instrumentationFailed();
|
|
398
|
+
}
|
|
399
|
+
}
|
|
388
400
|
return this.withSpan(`execute_tool ${name}`, async ({ span }) => {
|
|
389
401
|
this.setContent(span, "gen_ai.tool.call.arguments", input);
|
|
390
402
|
const result = await execute();
|
|
@@ -396,6 +408,46 @@ export class HueClient {
|
|
|
396
408
|
...(options.parentContext ? { parentContext: options.parentContext } : {}),
|
|
397
409
|
});
|
|
398
410
|
}
|
|
411
|
+
/**
|
|
412
|
+
* Runs `callback` as a sub-agent: inside an `invoke_agent {name}` span carrying
|
|
413
|
+
* `gen_ai.operation.name` `invoke_agent`, `gen_ai.agent.name` and `hue.agent.role` `subagent`.
|
|
414
|
+
* The span nests under the active span, normally the `tool()` span of the call that started
|
|
415
|
+
* the sub-agent, or under `options.parentContext`, for example `hue.extract(headers)` in
|
|
416
|
+
* another process. The sub-agent's own model and tool spans nest under it, and session, user
|
|
417
|
+
* and workspace identifiers are inherited unless `options` sets them. `options.input` and a
|
|
418
|
+
* defined result are recorded as `input.value` / `output.value` when `captureContent` is true;
|
|
419
|
+
* a callback that calls `setOutput` keeps its own output. Errors are recorded and rethrown as
|
|
420
|
+
* in `withSpan`. A blank or non-string name is recorded as `unknown` and counted.
|
|
421
|
+
*/
|
|
422
|
+
async subagent(name, callback, options = {}) {
|
|
423
|
+
let agent = name;
|
|
424
|
+
if (!isLabel(name)) {
|
|
425
|
+
agent = "unknown";
|
|
426
|
+
if (this.enabled && !this.closed)
|
|
427
|
+
this.instrumentationFailed();
|
|
428
|
+
}
|
|
429
|
+
return this.withSpan(`invoke_agent ${agent}`, async (span) => {
|
|
430
|
+
let outputSet = false;
|
|
431
|
+
const result = await callback({
|
|
432
|
+
...span,
|
|
433
|
+
setOutput: (value) => {
|
|
434
|
+
outputSet = true;
|
|
435
|
+
span.setOutput(value);
|
|
436
|
+
},
|
|
437
|
+
});
|
|
438
|
+
if (result !== undefined && !outputSet)
|
|
439
|
+
span.setOutput(result);
|
|
440
|
+
return result;
|
|
441
|
+
}, {
|
|
442
|
+
...options,
|
|
443
|
+
attributes: {
|
|
444
|
+
...options.attributes,
|
|
445
|
+
"gen_ai.operation.name": "invoke_agent",
|
|
446
|
+
"gen_ai.agent.name": agent,
|
|
447
|
+
"hue.agent.role": "subagent",
|
|
448
|
+
},
|
|
449
|
+
});
|
|
450
|
+
}
|
|
399
451
|
/**
|
|
400
452
|
* Runs `callback` inside a GenAI client span for one direct provider call, named
|
|
401
453
|
* `{operation} {model}` unless `options.name` is given and carrying `gen_ai.operation.name`,
|
package/dist/receipt.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { HueOptions, TraceVerification, VerifyTraceOptions } from "./types.js";
|
|
1
|
+
import type { ProjectConnection, HueOptions, TraceVerification, VerifyTraceOptions } from "./types.js";
|
|
2
2
|
/**
|
|
3
3
|
* Thrown by {@link HueClient.verifyTrace} when verification cannot proceed: denied key, unsupported
|
|
4
4
|
* or refusing server, unreachable network or an invalid receipt. A timeout is not an error; it
|
|
@@ -18,4 +18,4 @@ export declare class HueTraceVerificationError extends Error {
|
|
|
18
18
|
/** Observe persisted evidence after the application and its exporter have finished. */
|
|
19
19
|
export declare function verifyTrace(connection: Pick<HueOptions, "apiKey"> & {
|
|
20
20
|
baseUrl: string;
|
|
21
|
-
}, traceId: string, options?: VerifyTraceOptions): Promise<TraceVerification>;
|
|
21
|
+
}, traceId: string, options?: VerifyTraceOptions, onMissing?: () => Promise<ProjectConnection | undefined>): Promise<TraceVerification>;
|
package/dist/receipt.js
CHANGED
|
@@ -145,14 +145,16 @@ async function pause(milliseconds, signal) {
|
|
|
145
145
|
});
|
|
146
146
|
}
|
|
147
147
|
/** Observe persisted evidence after the application and its exporter have finished. */
|
|
148
|
-
export async function verifyTrace(connection, traceId, options = {}) {
|
|
149
|
-
return verifyTraceAtPath(connection, traceId, options, false, fetch);
|
|
148
|
+
export async function verifyTrace(connection, traceId, options = {}, onMissing) {
|
|
149
|
+
return verifyTraceAtPath(connection, traceId, options, false, fetch, undefined, onMissing);
|
|
150
150
|
}
|
|
151
151
|
/** @internal Dedicated receipt path for setup credentials; ordinary clients stay unchanged. */
|
|
152
152
|
export async function verifySetupTrace(connection, traceId, options, fetcher, signal) {
|
|
153
153
|
return verifyTraceAtPath(connection, traceId, options, true, fetcher, signal);
|
|
154
154
|
}
|
|
155
|
-
async function verifyTraceAtPath(connection, traceId, options, setup, fetcher, signal
|
|
155
|
+
async function verifyTraceAtPath(connection, traceId, options, setup, fetcher, signal,
|
|
156
|
+
/** Looks up the project the key searched once Hue has reported the trace missing. */
|
|
157
|
+
onMissing) {
|
|
156
158
|
if (!validId(traceId, 32))
|
|
157
159
|
throw new TypeError("traceId must be a nonzero lowercase 32-character OpenTelemetry trace ID");
|
|
158
160
|
if (options === null || typeof options !== "object" || Array.isArray(options))
|
|
@@ -181,6 +183,8 @@ async function verifyTraceAtPath(connection, traceId, options, setup, fetcher, s
|
|
|
181
183
|
const timer = setTimeout(() => controller.abort(), timeout);
|
|
182
184
|
const requestSignal = signal ? AbortSignal.any([signal, controller.signal]) : controller.signal;
|
|
183
185
|
let receipt = null;
|
|
186
|
+
let missing = false;
|
|
187
|
+
let lookup;
|
|
184
188
|
let delay = 250;
|
|
185
189
|
try {
|
|
186
190
|
while (!requestSignal.aborted && performance.now() < deadline) {
|
|
@@ -204,6 +208,7 @@ async function verifyTraceAtPath(connection, traceId, options, setup, fetcher, s
|
|
|
204
208
|
const body = await readJson(response);
|
|
205
209
|
if (!record(body) || body.code !== "TRACE_NOT_FOUND")
|
|
206
210
|
throw new HueTraceVerificationError("http", "This Hue server does not support trace receipts. Check the server version and baseUrl.", 404);
|
|
211
|
+
missing = true;
|
|
207
212
|
}
|
|
208
213
|
else {
|
|
209
214
|
await response.body?.cancel();
|
|
@@ -221,6 +226,12 @@ async function verifyTraceAtPath(connection, traceId, options, setup, fetcher, s
|
|
|
221
226
|
if (remaining <= 0)
|
|
222
227
|
break;
|
|
223
228
|
const wait = Math.max(delay, retryAfter);
|
|
229
|
+
// Once the backoff would outlast the budget, the rest of it names the project the key
|
|
230
|
+
// searched instead of sleeping.
|
|
231
|
+
if (wait >= remaining && missing && onMissing) {
|
|
232
|
+
lookup = onMissing();
|
|
233
|
+
break;
|
|
234
|
+
}
|
|
224
235
|
await pause(Math.min(wait, remaining), requestSignal);
|
|
225
236
|
// A truncated backoff exhausts this call even if a timer wakes just early.
|
|
226
237
|
if (wait >= remaining)
|
|
@@ -240,5 +251,13 @@ async function verifyTraceAtPath(connection, traceId, options, setup, fetcher, s
|
|
|
240
251
|
finally {
|
|
241
252
|
clearTimeout(timer);
|
|
242
253
|
}
|
|
254
|
+
if (lookup && !receipt) {
|
|
255
|
+
const project = await Promise.race([
|
|
256
|
+
lookup,
|
|
257
|
+
pause(Math.max(0, deadline - performance.now()), controller.signal),
|
|
258
|
+
]);
|
|
259
|
+
if (project)
|
|
260
|
+
return { verified: false, receipt, project };
|
|
261
|
+
}
|
|
243
262
|
return { verified: false, receipt };
|
|
244
263
|
}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import type { Attributes } from "@opentelemetry/api";
|
|
2
|
+
import type { ToolEndpoint } from "./types.js";
|
|
3
|
+
/** Copy evidence, never send a request or resolve an endpoint from a tool name. */
|
|
4
|
+
export declare function toolEndpointAttributes(endpoint: ToolEndpoint): Attributes;
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import { isCredentialKey } from "./tool-definitions.js";
|
|
2
|
+
/** Copy evidence, never send a request or resolve an endpoint from a tool name. */
|
|
3
|
+
export function toolEndpointAttributes(endpoint) {
|
|
4
|
+
if (!endpoint ||
|
|
5
|
+
typeof endpoint.url !== "string" ||
|
|
6
|
+
endpoint.url.length > 8192 ||
|
|
7
|
+
/[\s\\\p{Cc}]/u.test(endpoint.url) ||
|
|
8
|
+
/%(?![0-9a-f]{2})/i.test(endpoint.url) ||
|
|
9
|
+
!["request", "client_config", "receipt"].includes(endpoint.source))
|
|
10
|
+
throw new Error("Invalid tool endpoint evidence");
|
|
11
|
+
const url = new URL(endpoint.url);
|
|
12
|
+
if (!["http:", "https:"].includes(url.protocol))
|
|
13
|
+
throw new Error("Invalid tool endpoint protocol");
|
|
14
|
+
const names = endpoint.captureQueryParameters ?? [];
|
|
15
|
+
if (!Array.isArray(names) ||
|
|
16
|
+
names.length > 32 ||
|
|
17
|
+
names.some((name) => typeof name !== "string" ||
|
|
18
|
+
!/^[a-zA-Z][a-zA-Z0-9_.-]{0,63}$/.test(name) ||
|
|
19
|
+
isCredentialKey(name)))
|
|
20
|
+
throw new Error("Invalid tool endpoint query allowlist");
|
|
21
|
+
// Keep the raw pairs, including duplicates and empty values. Re-encoding or selecting the
|
|
22
|
+
// first value can turn missing, malformed or ambiguous routing evidence into a usable ID.
|
|
23
|
+
const query = url.search
|
|
24
|
+
.slice(1)
|
|
25
|
+
.split("&")
|
|
26
|
+
.filter((pair) => {
|
|
27
|
+
const key = decodeURIComponent(pair.split("=", 1)[0].replace(/\+/g, " "));
|
|
28
|
+
return names.includes(key);
|
|
29
|
+
})
|
|
30
|
+
.join("&");
|
|
31
|
+
url.username = "";
|
|
32
|
+
url.password = "";
|
|
33
|
+
url.hash = "";
|
|
34
|
+
url.search = query;
|
|
35
|
+
const attributes = {
|
|
36
|
+
"url.full": url.href,
|
|
37
|
+
"hue.tool.endpoint.source": endpoint.source,
|
|
38
|
+
};
|
|
39
|
+
if (endpoint.method !== undefined) {
|
|
40
|
+
if (typeof endpoint.method !== "string" ||
|
|
41
|
+
!/^[!#$%&'*+.^_`|~0-9A-Za-z-]{1,32}$/.test(endpoint.method))
|
|
42
|
+
throw new Error("Invalid tool endpoint method");
|
|
43
|
+
attributes["http.request.method"] = endpoint.method;
|
|
44
|
+
}
|
|
45
|
+
return attributes;
|
|
46
|
+
}
|
package/dist/transport.js
CHANGED
|
@@ -16,7 +16,7 @@ import { MediaDigestMemo, isMediaName } from "./inline-files.js";
|
|
|
16
16
|
import { locateParts } from "./attachment.js";
|
|
17
17
|
import { snapshotLog, snapshotSpan } from "./snapshot.js";
|
|
18
18
|
import { DROPPED_RECORDS_KEY, isContentKey, isTruncatedMarker, redactLog, redactSpan, TRUNCATED_KEY, truncatedMarker, withTruncatedKeys, } from "./privacy.js";
|
|
19
|
-
import { sdkVersion } from "./version.js";
|
|
19
|
+
import { otlpExporterVersion, sdkVersion } from "./version.js";
|
|
20
20
|
/** A record's trace: a span's context, a log record's span identifiers; "" when unreadable. */
|
|
21
21
|
function recordTraceId(record) {
|
|
22
22
|
try {
|
|
@@ -2021,7 +2021,7 @@ class ReportingExporter {
|
|
|
2021
2021
|
await this.forceFlush();
|
|
2022
2022
|
}
|
|
2023
2023
|
}
|
|
2024
|
-
// OpenTelemetry's OTLP/HTTP transport rules
|
|
2024
|
+
// OpenTelemetry's OTLP/HTTP transport rules, restated because that
|
|
2025
2025
|
// transport discards response headers, which Hue uses to announce features.
|
|
2026
2026
|
const MAX_RETRIES = 5;
|
|
2027
2027
|
const INITIAL_BACKOFF_MILLIS = 1000;
|
|
@@ -2042,7 +2042,7 @@ const RETRYABLE_NETWORK_ERRORS = new Set([
|
|
|
2042
2042
|
"EHOSTUNREACH",
|
|
2043
2043
|
]);
|
|
2044
2044
|
/** OpenTelemetry's exporter token, which follows Hue's in the User-Agent as it always has. */
|
|
2045
|
-
const OTEL_USER_AGENT =
|
|
2045
|
+
const OTEL_USER_AGENT = `OTel-OTLP-Exporter-JavaScript/${otlpExporterVersion}`;
|
|
2046
2046
|
function retryAfterMillis(value) {
|
|
2047
2047
|
if (value == null)
|
|
2048
2048
|
return undefined;
|
|
@@ -2141,12 +2141,13 @@ class OtlpHttpTransport {
|
|
|
2141
2141
|
if (this.aborted)
|
|
2142
2142
|
throw new Error("Hue export deadline exceeded");
|
|
2143
2143
|
const url = new URL(this.url);
|
|
2144
|
-
const protocol = url.protocol;
|
|
2145
2144
|
// Loaded on first use, as OpenTelemetry's exporter does, so importing Hue never loads http
|
|
2146
|
-
// before the application's http instrumentation can patch it.
|
|
2147
|
-
//
|
|
2145
|
+
// before the application's http instrumentation can patch it. Each specifier is a literal:
|
|
2146
|
+
// bundlers (Next.js Turbopack, webpack) cannot resolve a computed one and replace the import
|
|
2147
|
+
// with a throw, so every export would fail. A request measured before it was sent is not
|
|
2148
|
+
// compressed again.
|
|
2148
2149
|
const [{ Agent: ConnectionAgent, request }, body] = await Promise.all([
|
|
2149
|
-
|
|
2150
|
+
url.protocol === "https:" ? import("node:https") : import("node:http"),
|
|
2150
2151
|
this.body && data === this.body.data ? this.body.compressed : compress(data),
|
|
2151
2152
|
]);
|
|
2152
2153
|
if (this.aborted)
|
package/dist/types.d.ts
CHANGED
|
@@ -201,11 +201,21 @@ export interface McpServerInfo {
|
|
|
201
201
|
/** Hue surface such as `google.gmail/mcp`, recorded as `hue.mcp.surface`. */
|
|
202
202
|
surface?: string;
|
|
203
203
|
}
|
|
204
|
-
/**
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
204
|
+
/** Explicit routing evidence for {@link HueClient.tool}; values come from the application. */
|
|
205
|
+
export interface ToolEndpoint {
|
|
206
|
+
/** Actual HTTP(S) URL used by the tool. Never synthesize it from a tool or server label. */
|
|
207
|
+
url: string;
|
|
208
|
+
/** Where the URL was observed. Recorded as `hue.tool.endpoint.source`. */
|
|
209
|
+
source: "request" | "client_config" | "receipt";
|
|
210
|
+
/** Actual HTTP method, when known; the SDK supplies no default. */
|
|
211
|
+
method?: string;
|
|
212
|
+
/** Non-secret query names to retain, including every occurrence. All others are removed. */
|
|
213
|
+
captureQueryParameters?: readonly string[];
|
|
214
|
+
}
|
|
215
|
+
/** Options for {@link HueClient.tool}, including call identity and opt-in routing evidence. */
|
|
208
216
|
export interface ToolOptions extends Pick<SpanOptions, "parentContext"> {
|
|
217
|
+
/** Opt-in routing evidence. Only captured with content enabled, through existing redaction. */
|
|
218
|
+
endpoint?: ToolEndpoint;
|
|
209
219
|
/** Provider-issued identifier of this tool call, recorded as `gen_ai.tool.call.id`. */
|
|
210
220
|
callId?: string;
|
|
211
221
|
/**
|
|
@@ -390,4 +400,9 @@ export interface TraceVerification {
|
|
|
390
400
|
verified: boolean;
|
|
391
401
|
/** Latest receipt observed, or `null` when the trace was never found. */
|
|
392
402
|
receipt: TraceReceipt | null;
|
|
403
|
+
/**
|
|
404
|
+
* The project the key searched, looked up when no receipt arrived: a trace missing there was
|
|
405
|
+
* sent with a different key or is not stored yet. Absent when that lookup failed.
|
|
406
|
+
*/
|
|
407
|
+
project?: ProjectConnection;
|
|
393
408
|
}
|
package/dist/version.d.ts
CHANGED
|
@@ -1,2 +1,4 @@
|
|
|
1
1
|
/** Package version shared by the instrumentation scope and the export User-Agent. */
|
|
2
|
-
export declare const sdkVersion = "0.
|
|
2
|
+
export declare const sdkVersion = "0.21.0";
|
|
3
|
+
/** OTLP dependency version sent in the export User-Agent. */
|
|
4
|
+
export declare const otlpExporterVersion = "0.223.0";
|
package/dist/version.js
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
1
|
// Generated by scripts/write-version.mjs from package.json; do not edit by hand.
|
|
2
2
|
/** Package version shared by the instrumentation scope and the export User-Agent. */
|
|
3
|
-
export const sdkVersion = "0.
|
|
3
|
+
export const sdkVersion = "0.21.0";
|
|
4
|
+
/** OTLP dependency version sent in the export User-Agent. */
|
|
5
|
+
export const otlpExporterVersion = "0.223.0";
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@hue-run/sdk",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.21.0",
|
|
4
4
|
"private": false,
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"publishConfig": {
|
|
@@ -91,30 +91,30 @@
|
|
|
91
91
|
},
|
|
92
92
|
"dependencies": {
|
|
93
93
|
"@babel/parser": "7.29.9",
|
|
94
|
-
"@opentelemetry/api-logs": "0.
|
|
95
|
-
"@opentelemetry/core": "2.
|
|
96
|
-
"@opentelemetry/otlp-exporter-base": "0.
|
|
97
|
-
"@opentelemetry/otlp-transformer": "0.
|
|
98
|
-
"@opentelemetry/resources": "2.
|
|
99
|
-
"@opentelemetry/sdk-logs": "0.
|
|
100
|
-
"@opentelemetry/sdk-trace": "2.
|
|
94
|
+
"@opentelemetry/api-logs": "0.223.0",
|
|
95
|
+
"@opentelemetry/core": "2.12.0",
|
|
96
|
+
"@opentelemetry/otlp-exporter-base": "0.223.0",
|
|
97
|
+
"@opentelemetry/otlp-transformer": "0.223.0",
|
|
98
|
+
"@opentelemetry/resources": "2.12.0",
|
|
99
|
+
"@opentelemetry/sdk-logs": "0.223.0",
|
|
100
|
+
"@opentelemetry/sdk-trace": "2.12.0",
|
|
101
101
|
"zod": "^3.25.76 || ^4.6.5"
|
|
102
102
|
},
|
|
103
103
|
"devDependencies": {
|
|
104
|
-
"@ai-sdk/otel": "1.0.
|
|
104
|
+
"@ai-sdk/otel": "1.0.137",
|
|
105
105
|
"@opentelemetry/api": "1.9.1",
|
|
106
|
-
"@opentelemetry/context-async-hooks": "2.
|
|
106
|
+
"@opentelemetry/context-async-hooks": "2.12.0",
|
|
107
107
|
"@types/bun": "1.4.2",
|
|
108
108
|
"@types/node": "26.6.4",
|
|
109
|
-
"ai": "7.0.
|
|
109
|
+
"ai": "7.0.137",
|
|
110
110
|
"ajv": "8.20.0",
|
|
111
111
|
"protobufjs": "8.8.0",
|
|
112
112
|
"typescript": "7.0.2"
|
|
113
113
|
},
|
|
114
114
|
"peerDependencies": {
|
|
115
|
-
"@ai-sdk/otel": "^1.0.
|
|
115
|
+
"@ai-sdk/otel": "^1.0.0",
|
|
116
116
|
"@opentelemetry/api": "^1.9.1",
|
|
117
|
-
"ai": "^6.0.0 || ^7.0.
|
|
117
|
+
"ai": "^6.0.0 || ^7.0.0",
|
|
118
118
|
"ajv": "^8.17.0"
|
|
119
119
|
},
|
|
120
120
|
"peerDependenciesMeta": {
|