@hue-run/sdk 0.7.0 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLI.md +27 -14
- package/ENVIRONMENTS.md +63 -1
- package/README.md +121 -5
- package/dist/ai-sdk.d.ts +3 -3
- package/dist/ai-sdk.js +3 -3
- package/dist/cli/env-file.d.ts +22 -0
- package/dist/cli/env-file.js +21 -0
- package/dist/cli/eval.js +100 -30
- package/dist/cli/login.d.ts +1 -1
- package/dist/cli/login.js +9 -4
- package/dist/client.d.ts +31 -3
- package/dist/client.js +199 -7
- package/dist/config.d.ts +2 -0
- package/dist/config.js +2 -0
- package/dist/environment/client.d.ts +16 -2
- package/dist/environment/client.js +46 -3
- package/dist/environment/tools.d.ts +2 -2
- package/dist/environment/tools.js +2 -2
- package/dist/environment/types.d.ts +134 -4
- package/dist/environment/world.d.ts +50 -0
- package/dist/environment/world.js +105 -0
- package/dist/environment.d.ts +2 -0
- package/dist/environment.js +1 -0
- package/dist/evals/environment-target.d.ts +53 -2
- package/dist/evals/environment-target.js +114 -10
- package/dist/evals/local-worker.d.ts +12 -5
- package/dist/evals/local-worker.js +13 -5
- package/dist/evals/runner.d.ts +1 -1
- package/dist/evals/runner.js +2 -2
- package/dist/evals/simulation.d.ts +14 -5
- package/dist/evals/simulation.js +26 -8
- package/dist/experimental-telemetry.d.ts +3 -2
- package/dist/experimental-telemetry.js +3 -2
- package/dist/inline-files.d.ts +10 -0
- package/dist/inline-files.js +86 -0
- package/dist/privacy.js +54 -5
- package/dist/provider-tools.d.ts +39 -0
- package/dist/provider-tools.js +222 -0
- package/dist/tool-definitions.d.ts +20 -0
- package/dist/tool-definitions.js +274 -0
- package/dist/transport.js +4 -2
- package/dist/types.d.ts +76 -3
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +1 -1
|
@@ -31,11 +31,16 @@ function localAgentTargetContext(context) {
|
|
|
31
31
|
executionId: context.executionId,
|
|
32
32
|
environmentRunId: context.environmentRunId,
|
|
33
33
|
trace: { traceId: context.trace.traceId, spanId: context.trace.spanId },
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
34
|
+
...(context.world ? { world: structuredClone(context.world) } : {}),
|
|
35
|
+
...(context.mcp
|
|
36
|
+
? {
|
|
37
|
+
mcp: {
|
|
38
|
+
url: context.mcp.url,
|
|
39
|
+
token: context.mcp.token,
|
|
40
|
+
expiresAt: context.mcp.expiresAt,
|
|
41
|
+
},
|
|
42
|
+
}
|
|
43
|
+
: {}),
|
|
39
44
|
...(context.connectionBundle
|
|
40
45
|
? { connectionBundle: structuredClone(context.connectionBundle) }
|
|
41
46
|
: {}),
|
|
@@ -178,6 +183,9 @@ export async function runLocalAgent(options) {
|
|
|
178
183
|
inputs,
|
|
179
184
|
context,
|
|
180
185
|
requested,
|
|
186
|
+
// The registered revision is the agent revision under test.
|
|
187
|
+
agentRevision: options.agent.revision,
|
|
188
|
+
deprecationWarnings: options.deprecationWarnings,
|
|
181
189
|
signal: options.signal,
|
|
182
190
|
target: (targetInputs, targetContext) => target(structuredClone(targetInputs), targetContext.tools, localAgentTargetContext(targetContext)),
|
|
183
191
|
});
|
package/dist/evals/runner.d.ts
CHANGED
|
@@ -50,7 +50,7 @@ interface RunnerOptions {
|
|
|
50
50
|
* (for example a Hue-operated grading worker that owns the evaluator source) instead of
|
|
51
51
|
* refusing the run. Their IDs are reported in `deferredScorerVersionIds`. */
|
|
52
52
|
deferUnboundLocalScorers?: boolean;
|
|
53
|
-
/** Cases in flight at once, 1–
|
|
53
|
+
/** Cases in flight at once, 1–64. Default 1. */
|
|
54
54
|
concurrency?: number;
|
|
55
55
|
/** Deadline for JSON Schema scoring in its worker, 100–60000 ms. Default 2000. */
|
|
56
56
|
schemaTimeoutMillis?: number;
|
package/dist/evals/runner.js
CHANGED
|
@@ -80,8 +80,8 @@ function settings(options) {
|
|
|
80
80
|
if (typeof options.persistResultContent !== "boolean")
|
|
81
81
|
throw new TypeError("Choose persistResultContent explicitly: true or false");
|
|
82
82
|
const concurrency = options.concurrency ?? 1;
|
|
83
|
-
if (!Number.isInteger(concurrency) || concurrency < 1 || concurrency >
|
|
84
|
-
throw new RangeError("concurrency must be 1–
|
|
83
|
+
if (!Number.isInteger(concurrency) || concurrency < 1 || concurrency > 64)
|
|
84
|
+
throw new RangeError("concurrency must be 1–64");
|
|
85
85
|
const timeout = options.schemaTimeoutMillis ?? 2000;
|
|
86
86
|
if (!Number.isInteger(timeout) || timeout < 100 || timeout > 60_000)
|
|
87
87
|
throw new RangeError("schemaTimeoutMillis must be 100–60000");
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { HueClient } from "../client.js";
|
|
2
2
|
import { type EnvironmentClient } from "../environment/client.js";
|
|
3
3
|
import type { EnvironmentTool } from "../environment/tools.js";
|
|
4
|
-
import type { EnvironmentIdentity, PublishableEnvironmentDefinition } from "../environment/types.js";
|
|
4
|
+
import type { EnvironmentIdentity, PublishableEnvironmentDefinition, WorldHandoff } from "../environment/types.js";
|
|
5
5
|
import { type EvaluationClient } from "./client.js";
|
|
6
6
|
import type { ActualAgentManifestInputV2, AttemptConnectionBundleV2, RequestedAttemptProviderV2 } from "./attempt.js";
|
|
7
7
|
import { type RunnerReport } from "./runner.js";
|
|
@@ -109,10 +109,15 @@ export interface SimulationTargetContext {
|
|
|
109
109
|
executionId: string;
|
|
110
110
|
/** Stable world identity for adapter control operations such as coverage reporting. */
|
|
111
111
|
environmentRunId: string;
|
|
112
|
-
/** Framework-neutral local callables backed by this attempt's isolated world
|
|
112
|
+
/** Framework-neutral local callables backed by this attempt's isolated world; empty for a
|
|
113
|
+
* gateway world, whose calls go to the provider mirrors in `world`. */
|
|
113
114
|
tools: Record<string, EnvironmentTool>;
|
|
114
|
-
/**
|
|
115
|
-
|
|
115
|
+
/** The mirror URLs, world token, environment carriers and MCP configuration of a gateway
|
|
116
|
+
* world; absent for a world created while the gateway was off. */
|
|
117
|
+
world?: WorldHandoff;
|
|
118
|
+
/** One MCP endpoint and bearer: the gateway world's first MCP mirror with the world token, or
|
|
119
|
+
* the deprecated execution-scoped `hue_sim_` capability of a legacy world. */
|
|
120
|
+
mcp?: SimulationMcpCapability;
|
|
116
121
|
/** Credential-bearing provider connections for this callback only. Hue never
|
|
117
122
|
* checkpoints, logs or adds this response to parity digests. */
|
|
118
123
|
connectionBundle?: AttemptConnectionBundleV2;
|
|
@@ -149,7 +154,7 @@ export interface RunSimulationOptions {
|
|
|
149
154
|
};
|
|
150
155
|
/** Local scorer callbacks bound by their declared source digests. */
|
|
151
156
|
localScorers?: LocalScorer[];
|
|
152
|
-
/** Cases in flight, 1–
|
|
157
|
+
/** Cases in flight, 1–64; defaults to 1. */
|
|
153
158
|
concurrency?: number;
|
|
154
159
|
/** JSON Schema worker deadline in milliseconds. */
|
|
155
160
|
schemaTimeoutMillis?: number;
|
|
@@ -157,6 +162,10 @@ export interface RunSimulationOptions {
|
|
|
157
162
|
maxSteps?: number;
|
|
158
163
|
/** Per-world lease in seconds, 1–86400. */
|
|
159
164
|
ttlSeconds?: number;
|
|
165
|
+
/** The agent revision under test, sent on world create for the world's fingerprint. */
|
|
166
|
+
agentRevision?: string;
|
|
167
|
+
/** Emit a one-time `DeprecationWarning` when the deployment serves a legacy world; on by default. */
|
|
168
|
+
deprecationWarnings?: boolean;
|
|
160
169
|
/** Cooperative caller cancellation signal. */
|
|
161
170
|
signal?: AbortSignal;
|
|
162
171
|
/** Optional display name for the fresh experiment. */
|
package/dist/evals/simulation.js
CHANGED
|
@@ -378,10 +378,21 @@ export async function runSimulation(options) {
|
|
|
378
378
|
baseUrl: options.client.baseUrl,
|
|
379
379
|
});
|
|
380
380
|
try {
|
|
381
|
-
|
|
381
|
+
// The agent revision is part of the attempt: a resume after it changed would otherwise
|
|
382
|
+
// finish the remaining cases under a different revision than the completed ones. A
|
|
383
|
+
// checkpoint written before revisions were tracked carries the definition digest alone and
|
|
384
|
+
// still matches its definition.
|
|
385
|
+
const scenarioDigest = digest({
|
|
386
|
+
definition: definitionIdentity(definition),
|
|
387
|
+
agentRevision: options.agentRevision ?? null,
|
|
388
|
+
});
|
|
382
389
|
let attempt = await store.read("active-attempt");
|
|
383
|
-
|
|
384
|
-
|
|
390
|
+
const previousDigest = digest(definitionIdentity(definition));
|
|
391
|
+
if (attempt &&
|
|
392
|
+
attempt.stage !== "completed" &&
|
|
393
|
+
attempt.scenarioDigest !== scenarioDigest &&
|
|
394
|
+
attempt.scenarioDigest !== previousDigest)
|
|
395
|
+
throw new Error("Recover the unfinished simulation before running a changed scenario or agent revision");
|
|
385
396
|
if (!attempt || attempt.stage === "completed") {
|
|
386
397
|
attempt = {
|
|
387
398
|
scenarioDigest,
|
|
@@ -432,6 +443,8 @@ export async function runSimulation(options) {
|
|
|
432
443
|
requested,
|
|
433
444
|
maxSteps: options.maxSteps,
|
|
434
445
|
ttlSeconds: options.ttlSeconds,
|
|
446
|
+
agentRevision: options.agentRevision,
|
|
447
|
+
deprecationWarnings: options.deprecationWarnings,
|
|
435
448
|
signal: options.signal,
|
|
436
449
|
onProgress: (event) => options.onProgress?.({
|
|
437
450
|
...event,
|
|
@@ -448,11 +461,16 @@ export async function runSimulation(options) {
|
|
|
448
461
|
executionId: targetContext.executionId,
|
|
449
462
|
environmentRunId: targetContext.environmentRunId,
|
|
450
463
|
tools: targetContext.tools,
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
464
|
+
...(targetContext.world ? { world: structuredClone(targetContext.world) } : {}),
|
|
465
|
+
...(targetContext.mcp
|
|
466
|
+
? {
|
|
467
|
+
mcp: {
|
|
468
|
+
url: targetContext.mcp.url,
|
|
469
|
+
token: targetContext.mcp.token,
|
|
470
|
+
expiresAt: targetContext.mcp.expiresAt,
|
|
471
|
+
},
|
|
472
|
+
}
|
|
473
|
+
: {}),
|
|
456
474
|
...(targetContext.connectionBundle
|
|
457
475
|
? { connectionBundle: structuredClone(targetContext.connectionBundle) }
|
|
458
476
|
: {}),
|
|
@@ -2,7 +2,8 @@ import type { HueClient } from "./client.js";
|
|
|
2
2
|
import type { ExperimentalTelemetrySettings } from "./types.js";
|
|
3
3
|
/**
|
|
4
4
|
* Per-call telemetry for AI SDK 6: pass as `experimental_telemetry`. Spans are created with Hue's
|
|
5
|
-
* tracer, so they parent under `withSpan` and inherit session
|
|
6
|
-
* recording follows `captureContent`. AI SDK 7 applications use `hueTelemetry`
|
|
5
|
+
* tracer, so they parent under `withSpan` and inherit session, user and workspace identifiers, and
|
|
6
|
+
* prompt/response recording follows `captureContent`. AI SDK 7 applications use `hueTelemetry`
|
|
7
|
+
* from `@hue-run/sdk/ai-sdk`.
|
|
7
8
|
*/
|
|
8
9
|
export declare function hueExperimentalTelemetry(hue: HueClient): ExperimentalTelemetrySettings;
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Per-call telemetry for AI SDK 6: pass as `experimental_telemetry`. Spans are created with Hue's
|
|
3
|
-
* tracer, so they parent under `withSpan` and inherit session
|
|
4
|
-
* recording follows `captureContent`. AI SDK 7 applications use `hueTelemetry`
|
|
3
|
+
* tracer, so they parent under `withSpan` and inherit session, user and workspace identifiers, and
|
|
4
|
+
* prompt/response recording follows `captureContent`. AI SDK 7 applications use `hueTelemetry`
|
|
5
|
+
* from `@hue-run/sdk/ai-sdk`.
|
|
5
6
|
*/
|
|
6
7
|
export function hueExperimentalTelemetry(hue) {
|
|
7
8
|
return {
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/** Inline file content larger than this many UTF-8 bytes is exported as its digest instead. */
|
|
2
|
+
export declare const INLINE_FILE_LIMIT: number;
|
|
3
|
+
/**
|
|
4
|
+
* Replaces inline files longer than {@link INLINE_FILE_LIMIT} in a recorded message attribute
|
|
5
|
+
* with their SHA-256 and byte size, so a span that inlines a large file exports the file's
|
|
6
|
+
* identity instead of being rejected for its size. The part keeps its other fields (`type`,
|
|
7
|
+
* `mime_type`, `modality`, `mediaType`, …). Other attributes, shorter messages and values that
|
|
8
|
+
* are not JSON are returned unchanged.
|
|
9
|
+
*/
|
|
10
|
+
export declare function hashInlineFiles(key: string, value: unknown): unknown;
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { MAX_BODY_BYTES } from "./config.js";
|
|
3
|
+
/** Inline file content larger than this many UTF-8 bytes is exported as its digest instead. */
|
|
4
|
+
export const INLINE_FILE_LIMIT = 64 * 1024;
|
|
5
|
+
const MAX_INLINE_FILE_TEXT = 8 * MAX_BODY_BYTES;
|
|
6
|
+
/** Message attributes whose JSON can inline files: GenAI blob parts and AI SDK 6 file parts. */
|
|
7
|
+
const messageKeys = new Set([
|
|
8
|
+
"gen_ai.input.messages",
|
|
9
|
+
"gen_ai.output.messages",
|
|
10
|
+
"ai.prompt.messages",
|
|
11
|
+
]);
|
|
12
|
+
/** Strict base64: alphabet characters only, padded to a multiple of four. */
|
|
13
|
+
const base64 = /^[A-Za-z0-9+/]*={0,2}$/;
|
|
14
|
+
const base64DataUrl = /^data:[^,]*;base64,/;
|
|
15
|
+
function binaryMimeType(value) {
|
|
16
|
+
return (typeof value === "string" &&
|
|
17
|
+
!/^text\//i.test(value) &&
|
|
18
|
+
value.toLowerCase() !== "application/json");
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* The bytes an inline file part carries: base64 (plain or as a `data:` URL) is decoded, and
|
|
22
|
+
* anything else, such as a text file's content, is taken as UTF-8.
|
|
23
|
+
*/
|
|
24
|
+
function fileBytes(content, mimeType) {
|
|
25
|
+
const prefix = base64DataUrl.exec(content)?.[0].length ?? 0;
|
|
26
|
+
const payload = content.slice(prefix);
|
|
27
|
+
return (prefix > 0 || binaryMimeType(mimeType)) &&
|
|
28
|
+
payload.length % 4 === 0 &&
|
|
29
|
+
base64.test(payload)
|
|
30
|
+
? Buffer.from(payload, "base64")
|
|
31
|
+
: Buffer.from(content, "utf8");
|
|
32
|
+
}
|
|
33
|
+
/** The key holding a part's inline content: GenAI `blob` parts and AI SDK 6 `file` parts. */
|
|
34
|
+
function contentKey(part) {
|
|
35
|
+
return part.type === "blob" ? "content" : part.type === "file" ? "data" : undefined;
|
|
36
|
+
}
|
|
37
|
+
function hashNode(value, state, depth) {
|
|
38
|
+
if (depth > 256)
|
|
39
|
+
throw new Error("Message exceeds the supported nesting limit");
|
|
40
|
+
if (Array.isArray(value))
|
|
41
|
+
return value.map((item) => hashNode(item, state, depth + 1));
|
|
42
|
+
if (value === null || typeof value !== "object")
|
|
43
|
+
return value;
|
|
44
|
+
const part = value;
|
|
45
|
+
const key = contentKey(part);
|
|
46
|
+
const inline = key === undefined ? undefined : part[key];
|
|
47
|
+
const mimeType = part.mime_type ?? part.mediaType;
|
|
48
|
+
if (key !== undefined && typeof inline === "string") {
|
|
49
|
+
const bytes = fileBytes(inline, mimeType);
|
|
50
|
+
if (bytes.byteLength > INLINE_FILE_LIMIT) {
|
|
51
|
+
const { [key]: _omitted, ...rest } = part;
|
|
52
|
+
state.changed = true;
|
|
53
|
+
return {
|
|
54
|
+
...rest,
|
|
55
|
+
sha256: createHash("sha256").update(bytes).digest("hex"),
|
|
56
|
+
size: bytes.byteLength,
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
return Object.fromEntries(Object.entries(part).map(([name, item]) => [name, hashNode(item, state, depth + 1)]));
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Replaces inline files longer than {@link INLINE_FILE_LIMIT} in a recorded message attribute
|
|
64
|
+
* with their SHA-256 and byte size, so a span that inlines a large file exports the file's
|
|
65
|
+
* identity instead of being rejected for its size. The part keeps its other fields (`type`,
|
|
66
|
+
* `mime_type`, `modality`, `mediaType`, …). Other attributes, shorter messages and values that
|
|
67
|
+
* are not JSON are returned unchanged.
|
|
68
|
+
*/
|
|
69
|
+
export function hashInlineFiles(key, value) {
|
|
70
|
+
if (!messageKeys.has(key) ||
|
|
71
|
+
typeof value !== "string" ||
|
|
72
|
+
Buffer.byteLength(value, "utf8") <= INLINE_FILE_LIMIT ||
|
|
73
|
+
Buffer.byteLength(value, "utf8") > MAX_INLINE_FILE_TEXT ||
|
|
74
|
+
!(value.includes('"blob"') || value.includes('"file"')))
|
|
75
|
+
return value;
|
|
76
|
+
try {
|
|
77
|
+
const parsed = JSON.parse(value);
|
|
78
|
+
const state = { changed: false };
|
|
79
|
+
const hashed = hashNode(parsed, state, 0);
|
|
80
|
+
return state.changed ? JSON.stringify(hashed) : value;
|
|
81
|
+
}
|
|
82
|
+
catch {
|
|
83
|
+
// Not JSON, or nested too deeply to inspect: export decides the value's fate as before.
|
|
84
|
+
return value;
|
|
85
|
+
}
|
|
86
|
+
}
|
package/dist/privacy.js
CHANGED
|
@@ -1,5 +1,8 @@
|
|
|
1
|
+
import { SpanStatusCode } from "@opentelemetry/api";
|
|
1
2
|
import { resourceFromAttributes } from "@opentelemetry/resources";
|
|
2
3
|
import { MAX_BODY_BYTES, MAX_CONTENT_BYTES } from "./config.js";
|
|
4
|
+
import { scrubToolCredentials, withToolCatalogSummary } from "./tool-definitions.js";
|
|
5
|
+
import { hashInlineFiles } from "./inline-files.js";
|
|
3
6
|
/** Attribute keys (and their dotted children) removed in metadata-only mode. */
|
|
4
7
|
export const contentPrefixes = [
|
|
5
8
|
"gen_ai.input.messages",
|
|
@@ -95,9 +98,16 @@ function redactValue(value, path, options, budget, depth = 0) {
|
|
|
95
98
|
return value;
|
|
96
99
|
}
|
|
97
100
|
function attributes(source, options, path, budget) {
|
|
98
|
-
|
|
101
|
+
// Metadata-only export summarizes the tool definitions it removes by name and digest.
|
|
102
|
+
const summarized = options.captureContent ? source : withToolCatalogSummary(source);
|
|
103
|
+
return Object.fromEntries(Object.entries(summarized).flatMap(([key, value]) => !options.captureContent && isContentKey(key)
|
|
99
104
|
? []
|
|
100
|
-
: [
|
|
105
|
+
: [
|
|
106
|
+
[
|
|
107
|
+
key,
|
|
108
|
+
redactValue(scrubToolCredentials(key, hashInlineFiles(key, value)), `${path}.${key}`, options, budget),
|
|
109
|
+
],
|
|
110
|
+
]));
|
|
101
111
|
}
|
|
102
112
|
function redactResource(resource, options, cache, budget) {
|
|
103
113
|
let result = cache.get(resource);
|
|
@@ -107,8 +117,43 @@ function redactResource(resource, options, cache, budget) {
|
|
|
107
117
|
}
|
|
108
118
|
return result;
|
|
109
119
|
}
|
|
120
|
+
/**
|
|
121
|
+
* Server identity and failure of an AI SDK 7 provider-executed (`extension`) tool span, read from
|
|
122
|
+
* its recorded result before content is stripped. OpenAI hosted MCP results carry
|
|
123
|
+
* `{ type: "call", serverLabel, name, arguments, output?, error? }`; nothing else names the server.
|
|
124
|
+
*/
|
|
125
|
+
function hostedMcpCall(attributes) {
|
|
126
|
+
const result = attributes["gen_ai.tool.call.result"];
|
|
127
|
+
if (attributes["gen_ai.tool.type"] !== "extension" || typeof result !== "string")
|
|
128
|
+
return { failed: false };
|
|
129
|
+
let parsed;
|
|
130
|
+
try {
|
|
131
|
+
parsed = JSON.parse(result);
|
|
132
|
+
}
|
|
133
|
+
catch {
|
|
134
|
+
return { failed: false };
|
|
135
|
+
}
|
|
136
|
+
if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed))
|
|
137
|
+
return { failed: false };
|
|
138
|
+
const { type, serverLabel, error } = parsed;
|
|
139
|
+
if (type !== "call")
|
|
140
|
+
return { failed: false };
|
|
141
|
+
const valid = typeof serverLabel === "string" &&
|
|
142
|
+
serverLabel.trim() !== "" &&
|
|
143
|
+
serverLabel.length <= 256 &&
|
|
144
|
+
!serverLabel.includes("\u0000") &&
|
|
145
|
+
serverLabel.isWellFormed();
|
|
146
|
+
return { ...(valid ? { serverName: serverLabel } : {}), failed: error != null };
|
|
147
|
+
}
|
|
110
148
|
export function redactSpan(span, options, cache) {
|
|
111
149
|
const budget = { bytes: 0, nodes: 0 };
|
|
150
|
+
// Derived before metadata-only stripping so the identity survives without the result itself.
|
|
151
|
+
const hosted = hostedMcpCall(span.attributes);
|
|
152
|
+
const source = {
|
|
153
|
+
...(hosted.serverName === undefined ? {} : { "mcp.server.name": hosted.serverName }),
|
|
154
|
+
...(hosted.failed ? { "error.type": "mcp_error" } : {}),
|
|
155
|
+
...span.attributes,
|
|
156
|
+
};
|
|
112
157
|
return {
|
|
113
158
|
name: span.name,
|
|
114
159
|
kind: span.kind,
|
|
@@ -119,12 +164,16 @@ export function redactSpan(span, options, cache) {
|
|
|
119
164
|
duration: span.duration,
|
|
120
165
|
ended: span.ended,
|
|
121
166
|
status: {
|
|
122
|
-
code: span.status.code
|
|
123
|
-
|
|
167
|
+
code: hosted.failed && span.status.code === SpanStatusCode.UNSET
|
|
168
|
+
? SpanStatusCode.ERROR
|
|
169
|
+
: span.status.code,
|
|
170
|
+
...(options.captureContent &&
|
|
171
|
+
span.status.message !== undefined &&
|
|
172
|
+
!(hosted.failed && span.status.code === SpanStatusCode.UNSET)
|
|
124
173
|
? { message: String(redactValue(span.status.message, "status.message", options, budget)) }
|
|
125
174
|
: {}),
|
|
126
175
|
},
|
|
127
|
-
attributes: attributes(
|
|
176
|
+
attributes: attributes(source, options, "attributes", budget),
|
|
128
177
|
events: span.events
|
|
129
178
|
.filter((event) => options.captureContent ||
|
|
130
179
|
!/^gen_ai\.(?:system|user|assistant|tool|choice)/.test(event.name))
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import type { HostedToolProvider } from "./types.js";
|
|
2
|
+
/** One provider-executed tool call. `arguments` and `result` are content. */
|
|
3
|
+
export interface HostedToolCall {
|
|
4
|
+
name: string;
|
|
5
|
+
callId?: string;
|
|
6
|
+
/** The provider's label (OpenAI `server_label`) or name (Anthropic `server_name`) for the MCP server. */
|
|
7
|
+
server?: string;
|
|
8
|
+
arguments?: unknown;
|
|
9
|
+
result?: unknown;
|
|
10
|
+
/** Low-cardinality failure marker for `error.type`; absent when the call succeeded. */
|
|
11
|
+
errorType?: string;
|
|
12
|
+
}
|
|
13
|
+
/** One `mcp_list_tools` result: the tools a hosted MCP server offered. */
|
|
14
|
+
export interface HostedToolListing {
|
|
15
|
+
server: string;
|
|
16
|
+
/** Tool definitions in the OpenTelemetry GenAI shape (`type`, `name`, `description`, `parameters`). */
|
|
17
|
+
definitions: Record<string, unknown>[];
|
|
18
|
+
errorType?: string;
|
|
19
|
+
}
|
|
20
|
+
export interface HostedToolActivity {
|
|
21
|
+
calls: HostedToolCall[];
|
|
22
|
+
listings: HostedToolListing[];
|
|
23
|
+
/** Items that looked like hosted calls but could not be read. */
|
|
24
|
+
skipped: number;
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Reads the hosted tool calls out of a provider response: the `output` items of an OpenAI
|
|
28
|
+
* Responses API response, or the `content` blocks of an Anthropic Messages API response. An array
|
|
29
|
+
* is taken as those items directly. Anything else yields no calls.
|
|
30
|
+
*/
|
|
31
|
+
export declare function hostedToolActivity(provider: HostedToolProvider, response: unknown): HostedToolActivity;
|
|
32
|
+
/**
|
|
33
|
+
* The host of each hosted MCP server's URL, by label, read from the request that produced the
|
|
34
|
+
* response: OpenAI `tools[].server_url` by `server_label`, Anthropic `mcp_servers[].url` by `name`.
|
|
35
|
+
* Nothing else in the request is read.
|
|
36
|
+
*/
|
|
37
|
+
export declare function hostedServerAddresses(provider: HostedToolProvider, request: unknown): Map<string, string>;
|
|
38
|
+
/** The provider a `model()` call named, when this module can read its responses. */
|
|
39
|
+
export declare function hostedToolProvider(value: unknown): HostedToolProvider | undefined;
|
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
import { MAX_CONTENT_BYTES } from "./config.js";
|
|
2
|
+
const MAX_PROVIDER_ITEMS = 128;
|
|
3
|
+
const MAX_PROVIDER_DEFINITIONS = 512;
|
|
4
|
+
const MAX_PROVIDER_SERVERS = 512;
|
|
5
|
+
function isItem(value) {
|
|
6
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
7
|
+
}
|
|
8
|
+
function text(value) {
|
|
9
|
+
return typeof value === "string" &&
|
|
10
|
+
value.trim() !== "" &&
|
|
11
|
+
value.length <= 256 &&
|
|
12
|
+
!value.includes("\u0000") &&
|
|
13
|
+
value.isWellFormed()
|
|
14
|
+
? value
|
|
15
|
+
: undefined;
|
|
16
|
+
}
|
|
17
|
+
/** MCP arguments arrive as a JSON string; record the structure when it parses, else the text. */
|
|
18
|
+
function jsonArguments(value) {
|
|
19
|
+
if (typeof value !== "string")
|
|
20
|
+
return value;
|
|
21
|
+
if (Buffer.byteLength(value, "utf8") > MAX_CONTENT_BYTES)
|
|
22
|
+
return undefined;
|
|
23
|
+
try {
|
|
24
|
+
return JSON.parse(value);
|
|
25
|
+
}
|
|
26
|
+
catch {
|
|
27
|
+
return value;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
/** OpenAI Responses `output` items. Built-in tools are named by their kind; MCP calls by tool. */
|
|
31
|
+
function openaiCalls(items, activity) {
|
|
32
|
+
const count = Math.min(items.length, MAX_PROVIDER_ITEMS);
|
|
33
|
+
activity.skipped += items.length - count;
|
|
34
|
+
for (let index = 0; index < count; index++) {
|
|
35
|
+
const item = items[index];
|
|
36
|
+
if (!isItem(item))
|
|
37
|
+
continue;
|
|
38
|
+
const callId = text(item.id);
|
|
39
|
+
switch (item.type) {
|
|
40
|
+
case "mcp_call": {
|
|
41
|
+
const name = text(item.name);
|
|
42
|
+
if (!name) {
|
|
43
|
+
activity.skipped++;
|
|
44
|
+
break;
|
|
45
|
+
}
|
|
46
|
+
activity.calls.push({
|
|
47
|
+
name,
|
|
48
|
+
callId,
|
|
49
|
+
server: text(item.server_label),
|
|
50
|
+
arguments: jsonArguments(item.arguments),
|
|
51
|
+
...(item.output !== undefined && item.output !== null ? { result: item.output } : {}),
|
|
52
|
+
...(item.error !== undefined && item.error !== null ? { errorType: "mcp_error" } : {}),
|
|
53
|
+
});
|
|
54
|
+
break;
|
|
55
|
+
}
|
|
56
|
+
case "mcp_list_tools": {
|
|
57
|
+
const server = text(item.server_label);
|
|
58
|
+
if (!server || !Array.isArray(item.tools)) {
|
|
59
|
+
activity.skipped++;
|
|
60
|
+
break;
|
|
61
|
+
}
|
|
62
|
+
const definitions = [];
|
|
63
|
+
const definitionCount = Math.min(item.tools.length, MAX_PROVIDER_DEFINITIONS);
|
|
64
|
+
activity.skipped += item.tools.length - definitionCount;
|
|
65
|
+
for (let index = 0; index < definitionCount; index++) {
|
|
66
|
+
const tool = item.tools[index];
|
|
67
|
+
if (!isItem(tool))
|
|
68
|
+
continue;
|
|
69
|
+
definitions.push({
|
|
70
|
+
type: "function",
|
|
71
|
+
...(tool.name !== undefined ? { name: tool.name } : {}),
|
|
72
|
+
...(tool.description !== undefined ? { description: tool.description } : {}),
|
|
73
|
+
...(tool.input_schema !== undefined ? { parameters: tool.input_schema } : {}),
|
|
74
|
+
...(tool.annotations !== undefined ? { annotations: tool.annotations } : {}),
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
activity.listings.push({
|
|
78
|
+
server,
|
|
79
|
+
definitions,
|
|
80
|
+
...(item.error !== undefined && item.error !== null ? { errorType: "mcp_error" } : {}),
|
|
81
|
+
});
|
|
82
|
+
break;
|
|
83
|
+
}
|
|
84
|
+
case "web_search_call":
|
|
85
|
+
case "file_search_call":
|
|
86
|
+
case "code_interpreter_call": {
|
|
87
|
+
const name = item.type.slice(0, -"_call".length);
|
|
88
|
+
const failed = item.status === "failed" ? { errorType: "failed" } : {};
|
|
89
|
+
if (item.type === "web_search_call")
|
|
90
|
+
activity.calls.push({ name, callId, arguments: item.action, ...failed });
|
|
91
|
+
else if (item.type === "file_search_call")
|
|
92
|
+
activity.calls.push({
|
|
93
|
+
name,
|
|
94
|
+
callId,
|
|
95
|
+
arguments: { queries: item.queries },
|
|
96
|
+
...(item.results !== undefined && item.results !== null
|
|
97
|
+
? { result: item.results }
|
|
98
|
+
: {}),
|
|
99
|
+
...failed,
|
|
100
|
+
});
|
|
101
|
+
else
|
|
102
|
+
activity.calls.push({
|
|
103
|
+
name,
|
|
104
|
+
callId,
|
|
105
|
+
arguments: { code: item.code, container_id: item.container_id },
|
|
106
|
+
...(item.outputs !== undefined && item.outputs !== null
|
|
107
|
+
? { result: item.outputs }
|
|
108
|
+
: {}),
|
|
109
|
+
...failed,
|
|
110
|
+
});
|
|
111
|
+
break;
|
|
112
|
+
}
|
|
113
|
+
default:
|
|
114
|
+
// Messages, reasoning, approval requests and other items are not executed tools.
|
|
115
|
+
break;
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
/** Anthropic Messages `content` blocks: a use block paired with the result block that names it. */
|
|
120
|
+
function anthropicCalls(blocks, activity) {
|
|
121
|
+
const results = new Map();
|
|
122
|
+
const count = Math.min(blocks.length, MAX_PROVIDER_ITEMS);
|
|
123
|
+
const truncated = blocks.length > MAX_PROVIDER_ITEMS;
|
|
124
|
+
activity.skipped += blocks.length - count;
|
|
125
|
+
for (let index = 0; index < count; index++) {
|
|
126
|
+
const block = blocks[index];
|
|
127
|
+
if (isItem(block) &&
|
|
128
|
+
typeof block.type === "string" &&
|
|
129
|
+
block.type.endsWith("_tool_result") &&
|
|
130
|
+
typeof block.tool_use_id === "string")
|
|
131
|
+
results.set(block.tool_use_id, block);
|
|
132
|
+
}
|
|
133
|
+
for (let index = 0; index < count; index++) {
|
|
134
|
+
const block = blocks[index];
|
|
135
|
+
if (!isItem(block) || (block.type !== "mcp_tool_use" && block.type !== "server_tool_use"))
|
|
136
|
+
continue;
|
|
137
|
+
const name = text(block.name);
|
|
138
|
+
const callId = text(block.id);
|
|
139
|
+
if (!name) {
|
|
140
|
+
activity.skipped++;
|
|
141
|
+
continue;
|
|
142
|
+
}
|
|
143
|
+
const result = callId === undefined ? undefined : results.get(callId);
|
|
144
|
+
// When the response was truncated, an unmatched use block may have its result outside the
|
|
145
|
+
// bounded prefix. Do not export it as a successful call with a missing result.
|
|
146
|
+
if (truncated && result === undefined) {
|
|
147
|
+
activity.skipped++;
|
|
148
|
+
continue;
|
|
149
|
+
}
|
|
150
|
+
const content = result?.content;
|
|
151
|
+
let errorType;
|
|
152
|
+
if (result?.is_error === true)
|
|
153
|
+
errorType = "mcp_error";
|
|
154
|
+
else if (isItem(content) && typeof content.type === "string" && content.type.endsWith("_error"))
|
|
155
|
+
errorType = text(content.error_code) ?? "error";
|
|
156
|
+
activity.calls.push({
|
|
157
|
+
name,
|
|
158
|
+
callId,
|
|
159
|
+
...(block.type === "mcp_tool_use" ? { server: text(block.server_name) } : {}),
|
|
160
|
+
arguments: block.input,
|
|
161
|
+
...(content !== undefined ? { result: content } : {}),
|
|
162
|
+
...(errorType ? { errorType } : {}),
|
|
163
|
+
});
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
/**
|
|
167
|
+
* Reads the hosted tool calls out of a provider response: the `output` items of an OpenAI
|
|
168
|
+
* Responses API response, or the `content` blocks of an Anthropic Messages API response. An array
|
|
169
|
+
* is taken as those items directly. Anything else yields no calls.
|
|
170
|
+
*/
|
|
171
|
+
export function hostedToolActivity(provider, response) {
|
|
172
|
+
const activity = { calls: [], listings: [], skipped: 0 };
|
|
173
|
+
const items = Array.isArray(response)
|
|
174
|
+
? response
|
|
175
|
+
: isItem(response)
|
|
176
|
+
? response[provider === "openai" ? "output" : "content"]
|
|
177
|
+
: undefined;
|
|
178
|
+
if (!Array.isArray(items))
|
|
179
|
+
return activity;
|
|
180
|
+
if (provider === "openai")
|
|
181
|
+
openaiCalls(items, activity);
|
|
182
|
+
else
|
|
183
|
+
anthropicCalls(items, activity);
|
|
184
|
+
return activity;
|
|
185
|
+
}
|
|
186
|
+
/**
|
|
187
|
+
* The host of each hosted MCP server's URL, by label, read from the request that produced the
|
|
188
|
+
* response: OpenAI `tools[].server_url` by `server_label`, Anthropic `mcp_servers[].url` by `name`.
|
|
189
|
+
* Nothing else in the request is read.
|
|
190
|
+
*/
|
|
191
|
+
export function hostedServerAddresses(provider, request) {
|
|
192
|
+
const addresses = new Map();
|
|
193
|
+
if (!isItem(request))
|
|
194
|
+
return addresses;
|
|
195
|
+
const entries = request[provider === "openai" ? "tools" : "mcp_servers"];
|
|
196
|
+
if (!Array.isArray(entries))
|
|
197
|
+
return addresses;
|
|
198
|
+
const count = Math.min(entries.length, MAX_PROVIDER_SERVERS);
|
|
199
|
+
for (let index = 0; index < count; index++) {
|
|
200
|
+
const entry = entries[index];
|
|
201
|
+
if (!isItem(entry))
|
|
202
|
+
continue;
|
|
203
|
+
const label = text(provider === "openai" ? entry.server_label : entry.name);
|
|
204
|
+
const url = provider === "openai" ? entry.server_url : entry.url;
|
|
205
|
+
if (!label || typeof url !== "string")
|
|
206
|
+
continue;
|
|
207
|
+
try {
|
|
208
|
+
const { hostname } = new URL(url);
|
|
209
|
+
if (hostname)
|
|
210
|
+
addresses.set(label, hostname);
|
|
211
|
+
}
|
|
212
|
+
catch {
|
|
213
|
+
// Not a URL; there is no address to record.
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
return addresses;
|
|
217
|
+
}
|
|
218
|
+
/** The provider a `model()` call named, when this module can read its responses. */
|
|
219
|
+
export function hostedToolProvider(value) {
|
|
220
|
+
const provider = typeof value === "string" ? value.toLowerCase() : undefined;
|
|
221
|
+
return provider === "openai" || provider === "anthropic" ? provider : undefined;
|
|
222
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Removes hosted-tool credentials from an exported attribute value, before the caller's `redact`.
|
|
3
|
+
* Tool definitions come from OpenTelemetry GenAI (`gen_ai.tool.definitions`), AI SDK 6
|
|
4
|
+
* (`ai.prompt.tools`, one JSON string per tool) and OpenInference (`llm.tools.{i}.tool.json_schema`,
|
|
5
|
+
* plus the raw request in `input.value`). Other attributes, and values that are not JSON, are
|
|
6
|
+
* returned unchanged. Metadata-only export removes all of these attributes anyway.
|
|
7
|
+
*
|
|
8
|
+
* @throws Error when a tool definition is nested too deeply to inspect; the record is then
|
|
9
|
+
* rejected rather than exported with credentials.
|
|
10
|
+
*/
|
|
11
|
+
export declare function scrubToolCredentials(key: string, value: unknown): unknown;
|
|
12
|
+
/**
|
|
13
|
+
* Metadata-only summary of the tool definitions a record carries, which export then removes:
|
|
14
|
+
* `hue.tool.names` lists each definition's name in order, and `hue.tool.definitions.sha256` is
|
|
15
|
+
* the lowercase hex SHA-256 of the RFC 8785 canonical JSON of the credential-scrubbed definition
|
|
16
|
+
* list, so the same catalog has the same digest in both SDKs and across credential rotation.
|
|
17
|
+
* Returns the source unchanged when it has no parseable definitions; attributes the source
|
|
18
|
+
* already sets are kept.
|
|
19
|
+
*/
|
|
20
|
+
export declare function withToolCatalogSummary<T extends Record<string, unknown>>(source: T): T;
|