@hue-run/sdk 0.1.5 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ENVIRONMENTS.md +182 -0
- package/EVALUATIONS.md +12 -0
- package/README.md +192 -18
- package/dist/ai-sdk.d.ts +9 -1
- package/dist/ai-sdk.js +34 -8
- package/dist/client.d.ts +121 -6
- package/dist/client.js +329 -56
- package/dist/config.d.ts +11 -2
- package/dist/config.js +36 -7
- package/dist/environment/client.d.ts +73 -0
- package/dist/environment/client.js +209 -0
- package/dist/environment/tools.d.ts +30 -0
- package/dist/environment/tools.js +24 -0
- package/dist/environment/types.d.ts +429 -0
- package/dist/environment/types.js +1 -0
- package/dist/environment.d.ts +5 -0
- package/dist/environment.js +2 -0
- package/dist/evals/attempt.d.ts +454 -0
- package/dist/evals/attempt.js +687 -0
- package/dist/evals/client.d.ts +99 -5
- package/dist/evals/client.js +136 -7
- package/dist/evals/environment-evidence.d.ts +6 -0
- package/dist/evals/environment-evidence.js +123 -0
- package/dist/evals/environment-json.d.ts +3 -0
- package/dist/evals/environment-json.js +76 -0
- package/dist/evals/json.d.ts +9 -1
- package/dist/evals/json.js +14 -6
- package/dist/evals/runner.d.ts +61 -2
- package/dist/evals/runner.js +71 -9
- package/dist/evals/scorer-publication.d.ts +2 -0
- package/dist/evals/scorer-publication.js +84 -0
- package/dist/evals/scorers.d.ts +11 -0
- package/dist/evals/scorers.js +56 -5
- package/dist/evals/simulation.d.ts +184 -0
- package/dist/evals/simulation.js +603 -0
- package/dist/evals/types.d.ts +304 -0
- package/dist/evals.d.ts +5 -1
- package/dist/evals.js +3 -1
- package/dist/experimental-telemetry.d.ts +8 -0
- package/dist/experimental-telemetry.js +13 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.js +2 -0
- package/dist/managed.d.ts +51 -1
- package/dist/managed.js +11 -1
- package/dist/privacy.d.ts +2 -0
- package/dist/privacy.js +16 -1
- package/dist/receipt.d.ts +12 -1
- package/dist/receipt.js +10 -1
- package/dist/safety.d.ts +1 -2
- package/dist/snapshot.js +4 -0
- package/dist/transport.d.ts +41 -9
- package/dist/transport.js +80 -22
- package/dist/types.d.ts +144 -8
- package/dist/version.d.ts +2 -0
- package/dist/version.js +3 -0
- package/package.json +51 -15
package/ENVIRONMENTS.md
ADDED
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
# Simulated environments
|
|
2
|
+
|
|
3
|
+
Install the optional evaluation runtime-contract peer with the SDK:
|
|
4
|
+
|
|
5
|
+
```bash
|
|
6
|
+
npm install @hue-run/sdk zod
|
|
7
|
+
```
|
|
8
|
+
|
|
9
|
+
Your agent runs in your process while a disposable simulated world runs in Hue. The world is
|
|
10
|
+
authoritative and records an ordered journal; Hue does not execute your agent code or provider
|
|
11
|
+
credentials.
|
|
12
|
+
|
|
13
|
+
## Run a scenario like a test
|
|
14
|
+
|
|
15
|
+
`runSimulation` is the one-shot developer path. It calls your existing callback directly, so
|
|
16
|
+
IDE breakpoints and cooperative cancellation work. It does not register a worker, poll for jobs,
|
|
17
|
+
host your laptop or require an inbound tunnel.
|
|
18
|
+
|
|
19
|
+
```ts
|
|
20
|
+
import { createHue } from "@hue-run/sdk";
|
|
21
|
+
import { createEnvironmentClient } from "@hue-run/sdk/environment";
|
|
22
|
+
import { createEvaluationClient, runSimulation } from "@hue-run/sdk/evals";
|
|
23
|
+
|
|
24
|
+
const connection = { apiKey: process.env.HUE_API_KEY! };
|
|
25
|
+
const hue = createHue({ ...connection, serviceName: "agent-test", captureContent: true });
|
|
26
|
+
|
|
27
|
+
try {
|
|
28
|
+
const report = await runSimulation({
|
|
29
|
+
client: createEvaluationClient(connection),
|
|
30
|
+
environmentClient: createEnvironmentClient(connection),
|
|
31
|
+
hue,
|
|
32
|
+
checkpointDirectory: ".hue-checkpoints/refund-scenario",
|
|
33
|
+
scenario: { kind: "experiment", experimentId: process.env.HUE_EXPERIMENT_ID! },
|
|
34
|
+
persistResultContent: true,
|
|
35
|
+
traceEvidence: { mode: "required" },
|
|
36
|
+
target: (inputs, { tools, mcp, config, signal }) =>
|
|
37
|
+
runMyExistingAgent({ inputs, config, tools, mcp, signal }),
|
|
38
|
+
onProgress(event) {
|
|
39
|
+
if (event.type === "run_created") console.log(`Inspect this run: ${event.runUrl}`);
|
|
40
|
+
},
|
|
41
|
+
});
|
|
42
|
+
console.log(report.runUrl);
|
|
43
|
+
} finally {
|
|
44
|
+
await hue.shutdownSafe();
|
|
45
|
+
}
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
The referenced app-authored experiment is a template. Each completed invocation clones its
|
|
49
|
+
exact frozen dataset, configuration and scorer-version pins into a fresh experiment and creates
|
|
50
|
+
one isolated world per case. An interrupted invocation resumes through the private checkpoint
|
|
51
|
+
directory. If the agent may have run without a saved outcome, resume fails explicitly and never
|
|
52
|
+
calls it again. If Hue cannot confirm whether the world sealed, the execution likewise stays
|
|
53
|
+
uncertain and a resume refuses to re-invoke the agent.
|
|
54
|
+
|
|
55
|
+
`tools` contains framework-neutral local callables. `mcp` is a short-lived bearer for the same
|
|
56
|
+
run's closed catalog when a model provider executes MCP remotely. It is scoped to one execution
|
|
57
|
+
and world and is not the Hue project key. Configuration alone does not redirect real provider
|
|
58
|
+
calls; give one of these connections to the agent's actual tool boundary. `environmentRunId`
|
|
59
|
+
identifies the same world for adapter control operations such as
|
|
60
|
+
`environmentClient.recordCoverageGap`; it is not a credential. The MCP token is delivered only
|
|
61
|
+
to the callback and is never written to checkpoints.
|
|
62
|
+
|
|
63
|
+
### Pinned provider-profile preflight
|
|
64
|
+
|
|
65
|
+
An experiment with an immutable `attemptBaselineV2` can require the local process to describe
|
|
66
|
+
the agent configuration it is actually about to run. Supply `actualAgentManifest`, the exact
|
|
67
|
+
ordered `requestedProviders`, and an `mcpSurface` selected from that request. Hue compares the
|
|
68
|
+
agent, prompt, model, tools, approvals, orchestration, MCP catalogs and native-helper
|
|
69
|
+
configuration before the callback or model runs. Missing evidence stays explicitly `missing`; it
|
|
70
|
+
is never treated as a match.
|
|
71
|
+
|
|
72
|
+
On a ready decision, `context.connectionBundle` contains the selected V2 provider surfaces and
|
|
73
|
+
`context.mcp` remains the backwards-compatible projection of the selected MCP surface. Endpoints,
|
|
74
|
+
bearers, expiry and credential generation stay in callback memory: the runner does not write
|
|
75
|
+
them to checkpoints or progress events, and it never mutates global `process.env`. A durable
|
|
76
|
+
`environment_incomplete` decision skips both the callback and scoring. If a ready response or
|
|
77
|
+
world seal cannot be confirmed, the checkpoint remains uncertain and resume neither reacquires
|
|
78
|
+
credentials nor invokes the callback again.
|
|
79
|
+
|
|
80
|
+
This is currently a control-plane contract. Hue can issue provider endpoints under
|
|
81
|
+
`/api/v1/provider-facades/{bindingId}/{grantId}`, but a provider data-plane facade call has not
|
|
82
|
+
yet been proven by the released integration. The existing generic Hue MCP capability remains the
|
|
83
|
+
runnable hosted-tool path; do not interpret preparation or local-tool tests as evidence of a
|
|
84
|
+
hosted Gmail or Slack MCP call.
|
|
85
|
+
|
|
86
|
+
## Repository-authored scenarios
|
|
87
|
+
|
|
88
|
+
Repository scenarios publish through the same validated environment, dataset, scorer and
|
|
89
|
+
experiment APIs as Hue-authored scenarios. Stable slugs reuse matching immutable content
|
|
90
|
+
digests; changed definitions, tasks or scorers publish new versions. Hue does not synchronize
|
|
91
|
+
files back from its UI, and the helper refuses an unrelated mutable dataset draft instead of
|
|
92
|
+
overwriting it.
|
|
93
|
+
|
|
94
|
+
Repository publication supports the public `ScorerDefinition` union: exact match, includes,
|
|
95
|
+
JSON Schema, local code, manual and model-judge definitions. `runSimulation` applies the same
|
|
96
|
+
identity-affecting defaults as Hue before resolving versions and rejects unknown or server-only
|
|
97
|
+
kinds. In particular, `document_verifier` is not part of this SDK contract and is rejected rather
|
|
98
|
+
than published with a guessed digest.
|
|
99
|
+
|
|
100
|
+
```ts
|
|
101
|
+
const scenario = {
|
|
102
|
+
kind: "repository" as const,
|
|
103
|
+
name: "Refund an eligible charge",
|
|
104
|
+
slug: "refund-eligible-charge",
|
|
105
|
+
environment: {
|
|
106
|
+
name: "Refund fixture",
|
|
107
|
+
slug: "refund-fixture",
|
|
108
|
+
definition: refundWorld,
|
|
109
|
+
},
|
|
110
|
+
cases: [
|
|
111
|
+
{
|
|
112
|
+
externalKey: "eligible-charge",
|
|
113
|
+
inputs: { request: "Refund charge ch_2" },
|
|
114
|
+
metadata: { suite: "billing" },
|
|
115
|
+
},
|
|
116
|
+
],
|
|
117
|
+
scorers: [{ name: "Refund saved", slug: "refund-saved", scorer: refundScorer }],
|
|
118
|
+
config: { agentMode: "support" },
|
|
119
|
+
};
|
|
120
|
+
|
|
121
|
+
await runSimulation({
|
|
122
|
+
client,
|
|
123
|
+
environmentClient,
|
|
124
|
+
hue,
|
|
125
|
+
checkpointDirectory: ".hue-checkpoints/refund-scenario",
|
|
126
|
+
scenario,
|
|
127
|
+
persistResultContent: false,
|
|
128
|
+
traceEvidence: { mode: "required" },
|
|
129
|
+
target: (inputs, context) => runMyExistingAgent({ inputs, ...context }),
|
|
130
|
+
});
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
Repeat the command after an edit for a fresh attempt and world. The run URL joins task, trace,
|
|
134
|
+
world effects, final state, target outcome and scorer results. Target failures and cancellations
|
|
135
|
+
seal the world as `abandoned`; scorer errors remain separate. Cancellation is cooperative, so
|
|
136
|
+
pass `context.signal` into the provider or agent call.
|
|
137
|
+
|
|
138
|
+
## Direct environment tools
|
|
139
|
+
|
|
140
|
+
For lower-level use, create a run and bind its generated catalog:
|
|
141
|
+
|
|
142
|
+
```ts
|
|
143
|
+
import { randomUUID } from "node:crypto";
|
|
144
|
+
import { bindEnvironmentTools, createEnvironmentClient } from "@hue-run/sdk/environment";
|
|
145
|
+
|
|
146
|
+
const client = createEnvironmentClient(connection);
|
|
147
|
+
const run = await client.createRun({
|
|
148
|
+
idempotencyKey: randomUUID(),
|
|
149
|
+
environmentVersionId,
|
|
150
|
+
});
|
|
151
|
+
const tools = bindEnvironmentTools({ hue, client, run });
|
|
152
|
+
await tools.refund_charge!.execute({ charge_id: "ch_2" });
|
|
153
|
+
await client.finishRun(run.id, { idempotencyKey: randomUUID(), status: "completed" });
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
An observation with `status: "error"` is a recorded world answer, not a transport exception.
|
|
157
|
+
Run mutations retry with stable invocation/idempotency identities. Registry writes do not retry
|
|
158
|
+
automatically because identity creation and publication have no request key.
|
|
159
|
+
|
|
160
|
+
## Coverage gaps
|
|
161
|
+
|
|
162
|
+
A provider adapter can record a known valid provider request that the environment cannot
|
|
163
|
+
implement with `client.recordCoverageGap(run.id, { idempotencyKey, provider, operation, code,
|
|
164
|
+
args, description })`. Use a durable UUID idempotency key and repeat the identical request to
|
|
165
|
+
recover a lost acknowledgement. This is a runner/adapter control operation, not an agent tool.
|
|
166
|
+
Arguments must be a JSON object of at most 16,000 encoded bytes.
|
|
167
|
+
|
|
168
|
+
Hue preserves the first report, marks `validity: "environment_incomplete"`, and refuses new
|
|
169
|
+
actions while still replaying already-recorded invocation receipts. `coverageGap` retains the
|
|
170
|
+
request and reporting provenance. An absent gap means `not_assessed`; it does not establish
|
|
171
|
+
provider parity.
|
|
172
|
+
|
|
173
|
+
Local scoring and historical rescoring skip incomplete evidence before calling a scorer, even
|
|
174
|
+
when the target returned no output or failed. Unsupported caller syntax and real provider errors
|
|
175
|
+
are not automatically coverage gaps; the adapter must identify a known missing provider
|
|
176
|
+
behavior. `runSimulation` checks the authoritative world when its callback throws: a durably
|
|
177
|
+
recorded gap finishes as environment-incomplete rather than `TargetError`, while a gap or seal
|
|
178
|
+
that cannot be confirmed stays uncertain and never causes the agent to be replayed.
|
|
179
|
+
|
|
180
|
+
The hosted MCP connection exposes Hue's bounded native actions; it is not general Gmail or
|
|
181
|
+
Slack HTTP parity and does not proxy arbitrary provider traffic. Forking, in-place reset and
|
|
182
|
+
arbitrary-step diffs are outside this interface.
|
package/EVALUATIONS.md
CHANGED
|
@@ -1,5 +1,12 @@
|
|
|
1
1
|
# Local evaluations
|
|
2
2
|
|
|
3
|
+
Install the optional runtime-contract peer with the SDK before importing
|
|
4
|
+
`@hue-run/sdk/evals`:
|
|
5
|
+
|
|
6
|
+
```bash
|
|
7
|
+
npm install @hue-run/sdk zod
|
|
8
|
+
```
|
|
9
|
+
|
|
3
10
|
The SDK executes targets and scorers on your machine. Hue stores pinned definitions, experiment progress and results. It does not execute uploaded source code. Follow the [installation guide](https://docs.hue.run/installation) to add `@hue-run/sdk` to your application.
|
|
4
11
|
|
|
5
12
|
```ts
|
|
@@ -63,6 +70,11 @@ try {
|
|
|
63
70
|
|
|
64
71
|
Create another experiment with the same frozen version and different `config` to compare configurations. The runner reads the exact experiment case/version and scorer definitions; it never resolves a mutable latest version. `rescore` accepts an existing evaluation-run ID and has no target callback. Subject IDs refer to immutable saved outputs and trace evidence.
|
|
65
72
|
|
|
73
|
+
For the shorter agent-against-a-hosted-world workflow, use `runSimulation`. It owns immutable
|
|
74
|
+
resolution, a fresh linked world per case, local and hosted MCP tools, finalization, sealed
|
|
75
|
+
evidence and scoring while retaining this runner's checkpoint guarantees. See
|
|
76
|
+
[Simulated environments](ENVIRONMENTS.md#run-a-scenario-like-a-test).
|
|
77
|
+
|
|
66
78
|
## Content and result states
|
|
67
79
|
|
|
68
80
|
Both choices are required and independent:
|
package/README.md
CHANGED
|
@@ -1,9 +1,17 @@
|
|
|
1
|
+
<p align="center">
|
|
2
|
+
<img alt="Hue" src="https://raw.githubusercontent.com/hue-run/hue-sdk/df0443f98c6096ff331fd0400715e4f3a1936607/.github/assets/hue-ascii-neutral.png" width="720">
|
|
3
|
+
</p>
|
|
4
|
+
|
|
1
5
|
# Hue TypeScript SDK
|
|
2
6
|
|
|
3
|
-
|
|
7
|
+
[](https://www.npmjs.com/package/@hue-run/sdk) 
|
|
8
|
+
|
|
9
|
+
A client for Hue's standard OTLP HTTP endpoints on Node.js 22 or 24 and Bun 1.4.2. It uses the
|
|
4
10
|
OpenTelemetry JavaScript SDK and official OTLP protobuf exporter components for
|
|
5
11
|
traces and correlated logs. The package is named `@hue-run/sdk`.
|
|
6
12
|
|
|
13
|
+
[Documentation](https://docs.hue.run) · [Sign in](https://app.hue.run)
|
|
14
|
+
|
|
7
15
|
## Install
|
|
8
16
|
|
|
9
17
|
```bash
|
|
@@ -16,7 +24,8 @@ Or with Bun:
|
|
|
16
24
|
bun add @hue-run/sdk
|
|
17
25
|
```
|
|
18
26
|
|
|
19
|
-
Run the command in your application's server package.
|
|
27
|
+
Run the command in your application's server package. The package is ESM; CommonJS applications on
|
|
28
|
+
Node.js 22.12 or later load it with `require("@hue-run/sdk")`. See the [compatibility guide](https://docs.hue.run/sdks/compatibility) before adding Hue to an application with existing OpenTelemetry or AI SDK dependencies.
|
|
20
29
|
|
|
21
30
|
## Start
|
|
22
31
|
|
|
@@ -52,9 +61,90 @@ await hue.shutdown(); // flushes and releases providers owned by this client
|
|
|
52
61
|
The default destination is `https://app.hue.run`. Data goes to
|
|
53
62
|
`/api/v1/otlp/v1/traces` and `/api/v1/otlp/v1/logs` with a Bearer project key.
|
|
54
63
|
Set `baseUrl` only for another Hue deployment. It must be an origin without an API path; a trailing slash is accepted.
|
|
55
|
-
HTTPS is required except for loopback HTTP
|
|
56
|
-
|
|
57
|
-
|
|
64
|
+
HTTPS is required except for loopback HTTP or the explicit
|
|
65
|
+
[`allowInsecureHttp`](#local-development-without-a-hue-account) opt-in. Redirects are refused for both
|
|
66
|
+
project checks and exports.
|
|
67
|
+
|
|
68
|
+
`checkConnection()` rejects with `HueConnectionError`: its fixed message is safe to log, `status`
|
|
69
|
+
carries the HTTP status when Hue answered, and `cause` carries the underlying network, timeout or
|
|
70
|
+
parsing error. `serviceVersion` and `resourceAttributes` (for example
|
|
71
|
+
`{ "deployment.environment.name": "production", "service.namespace": "agents" }`) describe the
|
|
72
|
+
deployment; a client that owns its providers merges them into its resource, with `serviceName`
|
|
73
|
+
and `serviceVersion` taking precedence over same-named keys.
|
|
74
|
+
|
|
75
|
+
## Model spans without a framework adapter
|
|
76
|
+
|
|
77
|
+
When you call a provider SDK directly, `hue.model()` creates the GenAI client span for the call.
|
|
78
|
+
Inside it, `setInput` and `setOutput` record `gen_ai.input.messages` / `gen_ai.output.messages`
|
|
79
|
+
when `captureContent` is true. Those attributes carry the OpenTelemetry GenAI message shape
|
|
80
|
+
(`{ role, parts: [{ type: "text", content }] }`, with `finish_reason` on output messages) defined
|
|
81
|
+
by the semantic conventions'
|
|
82
|
+
[input messages](https://github.com/open-telemetry/semantic-conventions/blob/v1.41.0/docs/gen-ai/gen-ai-input-messages.json)
|
|
83
|
+
and
|
|
84
|
+
[output messages](https://github.com/open-telemetry/semantic-conventions/blob/v1.41.0/docs/gen-ai/gen-ai-output-messages.json)
|
|
85
|
+
JSON schemas, so any semantic-convention-aware backend can read them. Convert provider-native
|
|
86
|
+
messages before recording them:
|
|
87
|
+
|
|
88
|
+
```ts
|
|
89
|
+
await hue.model(
|
|
90
|
+
"gpt-5-mini",
|
|
91
|
+
async (span) => {
|
|
92
|
+
span.setInput(
|
|
93
|
+
messages.map((message) => ({
|
|
94
|
+
role: message.role,
|
|
95
|
+
parts: [{ type: "text", content: message.content }],
|
|
96
|
+
})),
|
|
97
|
+
);
|
|
98
|
+
const response = await openai.chat.completions.create({ model: "gpt-5-mini", messages });
|
|
99
|
+
span.setOutput(
|
|
100
|
+
response.choices.map((choice) => ({
|
|
101
|
+
role: choice.message.role,
|
|
102
|
+
parts: [{ type: "text", content: choice.message.content ?? "" }],
|
|
103
|
+
finish_reason: choice.finish_reason,
|
|
104
|
+
})),
|
|
105
|
+
);
|
|
106
|
+
span.setUsage({
|
|
107
|
+
inputTokens: response.usage?.prompt_tokens,
|
|
108
|
+
outputTokens: response.usage?.completion_tokens,
|
|
109
|
+
});
|
|
110
|
+
return response;
|
|
111
|
+
},
|
|
112
|
+
{ provider: "openai" },
|
|
113
|
+
);
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
The span is named `{operation} {model}` (`operation` defaults to `chat`) with
|
|
117
|
+
`gen_ai.operation.name`, `gen_ai.request.model` and `gen_ai.provider.name`. Like `withSpan`, the
|
|
118
|
+
options come after the callback and also accept `name`, `sessionId`, `userId`, `input` (recorded as
|
|
119
|
+
`gen_ai.input.messages`) and `parentContext`. `setUsage` records
|
|
120
|
+
nonnegative integer `gen_ai.usage.input_tokens` / `output_tokens`; other values are omitted and
|
|
121
|
+
counted as instrumentation failures. Unknown usage stays absent. `hue.tool(name, input, execute)`
|
|
122
|
+
creates an `execute_tool {name}` span with `gen_ai.tool.name`, arguments and result; an optional
|
|
123
|
+
fourth argument `{ callId }` records the provider's tool call id as `gen_ai.tool.call.id`. Content
|
|
124
|
+
helpers (`setInput`, `setOutput`, `tool` arguments and results, `recordMessages`,
|
|
125
|
+
`SpanOptions.input`) accept any value and encode plain JSON data (`JsonValue`) at runtime; a value
|
|
126
|
+
that is not JSON, such as a `Date` or a class instance, is omitted with an instrumentation failure
|
|
127
|
+
while the callback result is returned unchanged.
|
|
128
|
+
|
|
129
|
+
## Vercel AI SDK 6
|
|
130
|
+
|
|
131
|
+
AI SDK 6 accepts a per-call tracer through `experimental_telemetry`. Pass
|
|
132
|
+
`hueExperimentalTelemetry(hue)` from the core entry point; the generated spans parent under
|
|
133
|
+
`withSpan`, inherit session/user identifiers, and record prompts and responses only when
|
|
134
|
+
`captureContent` is true:
|
|
135
|
+
|
|
136
|
+
```ts
|
|
137
|
+
import { hueExperimentalTelemetry } from "@hue-run/sdk";
|
|
138
|
+
|
|
139
|
+
const result = await generateText({
|
|
140
|
+
model,
|
|
141
|
+
prompt,
|
|
142
|
+
experimental_telemetry: hueExperimentalTelemetry(hue),
|
|
143
|
+
});
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
This requires no `@ai-sdk/otel` peer. `hueTelemetry` remains AI SDK 7 only: it reads the
|
|
147
|
+
installed `ai` major version once per process and throws a `TypeError` below 7.
|
|
58
148
|
|
|
59
149
|
## Vercel AI SDK 7
|
|
60
150
|
|
|
@@ -110,7 +200,8 @@ normal OTel setup; Hue does not silently replace it.
|
|
|
110
200
|
`captureContent: false` disables manual input/output/messages/tool content and
|
|
111
201
|
removes recognized GenAI, Vercel, OpenInference and OpenLLMetry content attributes,
|
|
112
202
|
legacy GenAI content events, log bodies, status messages and exception text before
|
|
113
|
-
export.
|
|
203
|
+
export. The exported `contentPrefixes` array lists the attribute keys (and their dotted
|
|
204
|
+
children) that are removed. Model/provider/token metadata remains available. Generic custom attribute
|
|
114
205
|
names cannot be classified automatically; use them deliberately.
|
|
115
206
|
|
|
116
207
|
`captureContent: true` captures supplied content. Accepted content is stored by Hue;
|
|
@@ -121,15 +212,26 @@ Invalid/oversized helper content is omitted with an instrumentation failure; the
|
|
|
121
212
|
export batch. Do not put user content or secrets in span names or scope names.
|
|
122
213
|
|
|
123
214
|
Manual helpers encode JSON values without converting null into absence. Unknown
|
|
124
|
-
outputs and usage remain absent. This SDK does not estimate tokens or cost.
|
|
125
|
-
|
|
126
|
-
|
|
215
|
+
outputs and usage remain absent. This SDK does not estimate tokens or cost. A thrown
|
|
216
|
+
application error marks the span with `error.type` (the error's `name`), an ERROR status and an
|
|
217
|
+
`exception` event carrying only the type; exception messages and stack traces are never recorded by
|
|
218
|
+
the helpers, whatever `captureContent` is, and the error is rethrown unchanged. `withSpan` ends its
|
|
219
|
+
span in `finally`.
|
|
220
|
+
|
|
221
|
+
`recordMessages` emits the `gen_ai.client.inference.operation.details` log record correlated with
|
|
222
|
+
the active span, with the messages in its body. The record also carries `gen_ai.operation.name`,
|
|
223
|
+
`gen_ai.provider.name` and `gen_ai.request.model` as attributes, copied from the enclosing
|
|
224
|
+
`hue.model()` span or passed as `operation`, `provider` and `model`, and `gen_ai.conversation.id`
|
|
225
|
+
from the active session, so a collector fan-out to another GenAI-aware backend keeps the request
|
|
226
|
+
context.
|
|
127
227
|
|
|
128
228
|
## Existing OpenTelemetry providers
|
|
129
229
|
|
|
130
230
|
Attach processors while constructing your providers. Hue uses local async context
|
|
131
231
|
for its own helpers and never registers/replaces the global tracer, logger, or
|
|
132
|
-
context manager.
|
|
232
|
+
context manager. When your application has registered a context manager, Hue helpers also make
|
|
233
|
+
their span the active OpenTelemetry span for the duration of the callback, so spans from other
|
|
234
|
+
instrumentations (HTTP clients, provider SDKs) that use the global API parent under it.
|
|
133
235
|
|
|
134
236
|
```ts
|
|
135
237
|
import { TracerProvider } from "@opentelemetry/sdk-trace";
|
|
@@ -149,10 +251,60 @@ await hue.shutdown(); // flushes; does not shut down these externally owned prov
|
|
|
149
251
|
// During application shutdown, shut down your providers, then await transport.shutdown().
|
|
150
252
|
```
|
|
151
253
|
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
254
|
+
The application's providers own the resource in this mode, so `resourceAttributes` on the
|
|
255
|
+
transport options is ignored and reported as a `warning` issue; set `deployment.environment.name`
|
|
256
|
+
and similar attributes on your own providers.
|
|
257
|
+
|
|
258
|
+
For external parent context pass `parentContext` to `withSpan`. Across processes, use
|
|
259
|
+
`hue.inject(carrier)` inside the producing span and `hue.extract(carrier)` in the worker; both
|
|
260
|
+
speak W3C `traceparent` only and never include the API key or baggage. Hue registers no global
|
|
261
|
+
propagator, so `propagation.inject()` from `@opentelemetry/api` is a no-op unless your
|
|
262
|
+
application configured one. `getContext()` exposes the helper's current context for APIs taking
|
|
263
|
+
an explicit context. Session/user identifiers are inherited within a client callback and are
|
|
264
|
+
stamped only on spans created through Hue's tracer (helpers and the AI SDK adapters); spans from
|
|
265
|
+
other instrumentations on a shared provider carry them only if that instrumentation sets them.
|
|
266
|
+
Separate requests require separate callbacks.
|
|
267
|
+
|
|
268
|
+
In attach mode Hue's span processor exports every span that ends on that provider, the same
|
|
269
|
+
default as other OpenTelemetry exporters. To send only part of a provider's spans, wrap the
|
|
270
|
+
processor:
|
|
271
|
+
|
|
272
|
+
```ts
|
|
273
|
+
const aiSpansOnly = {
|
|
274
|
+
onStart: () => {},
|
|
275
|
+
onEnd: (span) => {
|
|
276
|
+
if ("gen_ai.operation.name" in span.attributes || span.name.startsWith("ai."))
|
|
277
|
+
transport.spanProcessor.onEnd(span);
|
|
278
|
+
},
|
|
279
|
+
forceFlush: () => transport.spanProcessor.forceFlush(),
|
|
280
|
+
shutdown: () => transport.spanProcessor.shutdown(),
|
|
281
|
+
};
|
|
282
|
+
```
|
|
283
|
+
|
|
284
|
+
## Local development without a Hue account
|
|
285
|
+
|
|
286
|
+
Hue speaks standard OTLP, so any local collector works. Point `baseUrl` at a loopback receiver
|
|
287
|
+
that accepts `/api/v1/otlp/v1/traces` and `/api/v1/otlp/v1/logs` (for example an OpenTelemetry
|
|
288
|
+
Collector `otlp` receiver with `http.traces_url_path` and `logs_url_path` set to those paths,
|
|
289
|
+
forwarding to Jaeger or the debug exporter) and pass any placeholder `apiKey`; HTTP is allowed
|
|
290
|
+
for loopback origins. `checkConnection()` and `verifyTrace()` are Hue-only diagnostics and are
|
|
291
|
+
not available against a generic collector.
|
|
292
|
+
|
|
293
|
+
A collector on a private network is not loopback: a docker-compose sibling such as
|
|
294
|
+
`http://otel-collector:4318` or an in-cluster service requires the explicit opt-in
|
|
295
|
+
`allowInsecureHttp: true`. The client then records a one-time `warning` issue because the key and
|
|
296
|
+
telemetry travel unencrypted. Use a placeholder key with such a collector, and never enable the
|
|
297
|
+
option for a real project key on a network you do not control.
|
|
298
|
+
|
|
299
|
+
```ts
|
|
300
|
+
const hue = createHue({
|
|
301
|
+
apiKey: "local-placeholder",
|
|
302
|
+
serviceName: "my-agent",
|
|
303
|
+
captureContent: true,
|
|
304
|
+
baseUrl: "http://otel-collector:4318",
|
|
305
|
+
allowInsecureHttp: true,
|
|
306
|
+
});
|
|
307
|
+
```
|
|
156
308
|
|
|
157
309
|
## Delivery behavior
|
|
158
310
|
|
|
@@ -209,9 +361,25 @@ HTTP `status`. Missing expected spans and required fields remain explicit; a 200
|
|
|
209
361
|
response alone is not success. A successful result verifies those requested
|
|
210
362
|
conditions only. Use Hue's UI to inspect captured values and redaction.
|
|
211
363
|
|
|
364
|
+
## Dependencies
|
|
365
|
+
|
|
366
|
+
The tracing core depends only on official `@opentelemetry/*` packages. The optional
|
|
367
|
+
`@hue-run/sdk/evals` entry point uses `zod` for its bounded runtime contracts; install that peer
|
|
368
|
+
when you use evaluations or simulations. JSON Schema scoring also uses `ajv`, an optional peer
|
|
369
|
+
loaded inside a worker only when `builtins.jsonSchema` scores a case; without it that scorer
|
|
370
|
+
reports `SchemaValidatorUnavailable`. Install it when you use that scorer:
|
|
371
|
+
|
|
372
|
+
```bash
|
|
373
|
+
npm install zod
|
|
374
|
+
# Add ajv too when using builtins.jsonSchema.
|
|
375
|
+
npm install ajv
|
|
376
|
+
```
|
|
377
|
+
|
|
378
|
+
See [THIRD_PARTY_NOTICES.md](https://github.com/hue-run/hue-sdk/blob/main/THIRD_PARTY_NOTICES.md) for licenses.
|
|
379
|
+
|
|
212
380
|
## Package verification
|
|
213
381
|
|
|
214
|
-
From the repository root with Node 24 and Bun 1.
|
|
382
|
+
From the repository root with Node 24 and Bun 1.4.2 on PATH:
|
|
215
383
|
|
|
216
384
|
```sh
|
|
217
385
|
node packages/sdk-typescript/scripts/verify-package.mjs
|
|
@@ -223,7 +391,7 @@ HTTP exporter suite against that installed package, and installs/builds the
|
|
|
223
391
|
standalone reference chatbot. It prints the artifact paths. No package is
|
|
224
392
|
published. The chatbot README describes running that external installation.
|
|
225
393
|
|
|
226
|
-
|
|
394
|
+
## Local evaluation workflows
|
|
227
395
|
|
|
228
396
|
The optional `@hue-run/sdk/evals` entry point supports dataset/scorer registration, frozen-version experiments, local built-in/custom scoring, upload resume, and historical rescoring. See the [evaluation guide](https://docs.hue.run/evaluations/first-evaluation) for the complete journey, content policy and checkpoint recovery contract.
|
|
229
397
|
|
|
@@ -240,10 +408,16 @@ export const POST = createManagedTargetHandler({
|
|
|
240
408
|
// Application-owned functions: keep your current provider, tools and tracing.
|
|
241
409
|
target: async ({ input, config, inputFiles, signal }) =>
|
|
242
410
|
runAgentForEvaluation({ input, config, inputFiles, signal }),
|
|
411
|
+
tracer: hue.tracer, // Required with a Hue-owned client: Hue never registers a global tracer.
|
|
243
412
|
flushTelemetry: () => hue.flush(), // Existing Hue client; flush traces and logs.
|
|
244
413
|
});
|
|
245
414
|
```
|
|
246
415
|
|
|
416
|
+
Without `tracer`, the handler falls back to the global OpenTelemetry tracer, its span is not
|
|
417
|
+
recorded, and every invocation returns `uncertain`. An application that only uses `createHue()`
|
|
418
|
+
also needs an OpenTelemetry context manager installed for the handler's span to propagate; see
|
|
419
|
+
the [managed-run guide](https://docs.hue.run/evaluations/managed-runs).
|
|
420
|
+
|
|
247
421
|
`runAgentForEvaluation` adapts your application result to `{ output, files? }`.
|
|
248
422
|
Files contain `filename`, `contentType`, actual `Uint8Array` data and an optional
|
|
249
423
|
`primary` flag. The helper verifies input bytes, claims the invocation, saves the
|
|
@@ -252,12 +426,12 @@ Use a 120-second host request limit for the default 90-second execution and
|
|
|
252
426
|
30-second finalization budget; your target must honor `signal`.
|
|
253
427
|
|
|
254
428
|
See the [managed-run guide](https://docs.hue.run/evaluations/managed-runs) and the
|
|
255
|
-
[full adapter contract](MANAGED_TARGETS.md) for registration, file handling,
|
|
429
|
+
[full adapter contract](https://github.com/hue-run/hue-sdk/blob/main/packages/sdk-typescript/MANAGED_TARGETS.md) for registration, file handling,
|
|
256
430
|
existing-provider flush callbacks and recovery. Local/CI runners remain available.
|
|
257
431
|
|
|
258
432
|
## Serving safely
|
|
259
433
|
|
|
260
|
-
Use `createHueSafe(options)` for best-effort startup. Invalid initialization returns a disabled client
|
|
434
|
+
Use `createHueSafe(options)` for best-effort startup. Invalid initialization returns a disabled client that keeps your `onExportIssue` hook and records the reason as an instrumentation failure. Pass `enabled: false` to disable Hue without a key or a `captureContent` choice; disabled helpers still execute the application callback, and `inject()` keeps propagating the application's own trace context. `flushSafe({ timeoutMillis: 1000 })` and `shutdownSafe({ timeoutMillis: 1000 })` return `{ ok, timedOut, report }` without rejecting. Strict initialization, connection checks and `flush()` remain available for diagnostics; do not gate application readiness or responses on them.
|
|
261
435
|
|
|
262
436
|
Capture/serialization/redaction/provider failures omit unsafe telemetry, record failures, and preserve the original business result/error. Async diagnostic rejections are contained; diagnostics are rate-limited. The default `maxQueueBytes` is 8 MiB across traces/logs including in-flight work, alongside the existing record cap. `pendingBytes` is a current queue gauge; `droppedSpans`, `droppedLogs` and `instrumentationFailures` are cumulative failure counters. This is a telemetry budget, not a total process memory ceiling. A timeout bounds the caller and does not cancel a borrowed provider. Never retry the business operation to recover telemetry. See [production safety](https://docs.hue.run/guides/production-safety).
|
|
263
437
|
|
package/dist/ai-sdk.d.ts
CHANGED
|
@@ -1,4 +1,12 @@
|
|
|
1
1
|
import type { TelemetryOptions } from "ai";
|
|
2
2
|
import type { HueClient } from "./client.js";
|
|
3
|
-
/**
|
|
3
|
+
/**
|
|
4
|
+
* Per-call telemetry for AI SDK 7: pass as an agent's or generation call's `telemetry` option.
|
|
5
|
+
* Spans come from Hue's tracer, so they parent under `withSpan` and inherit session/user
|
|
6
|
+
* identifiers; prompt and response recording follow `captureContent`. It does not change global
|
|
7
|
+
* AI SDK integrations.
|
|
8
|
+
*
|
|
9
|
+
* @throws TypeError when the client is enabled and the installed `ai` major version is below 7;
|
|
10
|
+
* AI SDK 6 applications use `hueExperimentalTelemetry` from `@hue-run/sdk` instead.
|
|
11
|
+
*/
|
|
4
12
|
export declare function hueTelemetry(hue: HueClient): TelemetryOptions;
|
package/dist/ai-sdk.js
CHANGED
|
@@ -1,13 +1,40 @@
|
|
|
1
|
+
import { createRequire } from "node:module";
|
|
1
2
|
import { OpenTelemetry } from "@ai-sdk/otel";
|
|
2
|
-
|
|
3
|
+
let installedAiMajor;
|
|
4
|
+
/**
|
|
5
|
+
* The installed `ai` major version, read once per process. An unreadable or unparsable version
|
|
6
|
+
* is left to the peer dependency range rather than rejected here.
|
|
7
|
+
*/
|
|
8
|
+
function aiMajor() {
|
|
9
|
+
if (installedAiMajor === undefined) {
|
|
10
|
+
let major = Number.NaN;
|
|
11
|
+
try {
|
|
12
|
+
const manifest = createRequire(import.meta.url)("ai/package.json");
|
|
13
|
+
major = Number(String(manifest.version).split(".")[0]);
|
|
14
|
+
}
|
|
15
|
+
catch {
|
|
16
|
+
/* The peer range decides. */
|
|
17
|
+
}
|
|
18
|
+
installedAiMajor = Number.isInteger(major) ? major : Number.NaN;
|
|
19
|
+
}
|
|
20
|
+
return Number.isNaN(installedAiMajor) ? undefined : installedAiMajor;
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Per-call telemetry for AI SDK 7: pass as an agent's or generation call's `telemetry` option.
|
|
24
|
+
* Spans come from Hue's tracer, so they parent under `withSpan` and inherit session/user
|
|
25
|
+
* identifiers; prompt and response recording follow `captureContent`. It does not change global
|
|
26
|
+
* AI SDK integrations.
|
|
27
|
+
*
|
|
28
|
+
* @throws TypeError when the client is enabled and the installed `ai` major version is below 7;
|
|
29
|
+
* AI SDK 6 applications use `hueExperimentalTelemetry` from `@hue-run/sdk` instead.
|
|
30
|
+
*/
|
|
3
31
|
export function hueTelemetry(hue) {
|
|
4
|
-
// Core tracing also installs beside AI SDK 6. This adapter
|
|
5
|
-
//
|
|
32
|
+
// Core tracing also installs beside AI SDK 6. This adapter needs the v7 per-call
|
|
33
|
+
// integration API, so fail explicitly during configuration instead of silently on v6.
|
|
6
34
|
if (hue.enabled) {
|
|
7
|
-
const
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
throw new TypeError("hueTelemetry requires ai@^7.0.99; AI SDK 6 applications can use Hue core tracing with their existing OpenTelemetry integration");
|
|
35
|
+
const major = aiMajor();
|
|
36
|
+
if (major !== undefined && major < 7)
|
|
37
|
+
throw new TypeError("hueTelemetry requires ai@7 or later; AI SDK 6 applications pass hueExperimentalTelemetry(hue) from @hue-run/sdk as experimental_telemetry");
|
|
11
38
|
}
|
|
12
39
|
return {
|
|
13
40
|
isEnabled: hue.enabled,
|
|
@@ -16,4 +43,3 @@ export function hueTelemetry(hue) {
|
|
|
16
43
|
integrations: [new OpenTelemetry({ tracer: hue.tracer, usage: true })],
|
|
17
44
|
};
|
|
18
45
|
}
|
|
19
|
-
import { createRequire } from "node:module";
|