@hue-run/sdk 0.8.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLI.md +11 -6
- package/ENVIRONMENTS.md +36 -5
- package/README.md +105 -4
- package/dist/ai-sdk.d.ts +3 -3
- package/dist/ai-sdk.js +3 -3
- package/dist/cli/env-file.d.ts +22 -0
- package/dist/cli/env-file.js +21 -0
- package/dist/cli/eval.js +21 -6
- package/dist/cli/login.d.ts +1 -1
- package/dist/cli/login.js +9 -4
- package/dist/client.d.ts +31 -3
- package/dist/client.js +188 -6
- package/dist/config.d.ts +2 -0
- package/dist/config.js +2 -0
- package/dist/environment/client.d.ts +10 -2
- package/dist/environment/client.js +43 -12
- package/dist/environment/tools.d.ts +2 -2
- package/dist/environment/tools.js +2 -2
- package/dist/environment/types.d.ts +33 -12
- package/dist/environment.d.ts +1 -1
- package/dist/environment.js +1 -1
- package/dist/evals/environment-target.d.ts +5 -0
- package/dist/evals/environment-target.js +56 -5
- package/dist/evals/scenarios.d.ts +1 -1
- package/dist/evals/scenarios.js +4 -1
- package/dist/evals/types.d.ts +6 -1
- package/dist/experimental-telemetry.d.ts +3 -2
- package/dist/experimental-telemetry.js +3 -2
- package/dist/inline-files.d.ts +10 -0
- package/dist/inline-files.js +86 -0
- package/dist/privacy.js +54 -5
- package/dist/provider-tools.d.ts +39 -0
- package/dist/provider-tools.js +249 -0
- package/dist/tool-definitions.d.ts +20 -0
- package/dist/tool-definitions.js +274 -0
- package/dist/transport.js +4 -2
- package/dist/types.d.ts +76 -3
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +1 -1
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
|
-
import { HueEnvironmentError } from "../environment/client.js";
|
|
2
|
+
import { HueEnvironmentError, isTransientEnvironmentError, } from "../environment/client.js";
|
|
3
3
|
import { bindEnvironmentTools } from "../environment/tools.js";
|
|
4
4
|
import { legacyMcpCapability, worldHandoff } from "../environment/world.js";
|
|
5
5
|
import { actualAgentManifestV2, attemptBaselineV2, projectMcpConnectionV2, requestedAttemptProvidersV2, validateAttemptConnectionBundleV2, } from "./attempt.js";
|
|
@@ -42,22 +42,73 @@ export function pinRequestedAttemptV2(requested, config) {
|
|
|
42
42
|
}
|
|
43
43
|
return { ...requested, expectedAgentManifestDigest: baseline.data.expectedAgentManifestDigest };
|
|
44
44
|
}
|
|
45
|
-
/**
|
|
45
|
+
/** The completion grace is five seconds; cap an unexpectedly distant timestamp and let reads
|
|
46
|
+
* force the seal after the grace. */
|
|
47
|
+
export const MAX_GRACE_WAIT_MS = 10_000;
|
|
48
|
+
export const SEAL_POLL_MS = 250;
|
|
49
|
+
export const SEAL_WAIT_MS = 30_000;
|
|
50
|
+
/** Finish, then wait for the authoritative run to leave open. */
|
|
46
51
|
async function seal(client, runId, executionId, status) {
|
|
52
|
+
let completingUntil;
|
|
47
53
|
try {
|
|
48
|
-
await client.finishRun(runId, {
|
|
54
|
+
const finished = await client.finishRun(runId, {
|
|
49
55
|
idempotencyKey: `execution:${executionId}:${status}`,
|
|
50
56
|
status,
|
|
51
57
|
});
|
|
58
|
+
if (finished.lifecycle === "completing")
|
|
59
|
+
completingUntil = finished.completingUntil ?? null;
|
|
52
60
|
}
|
|
53
61
|
catch (error) {
|
|
54
|
-
// A gateway world answers 409 once it is completing, sealed or expired
|
|
55
|
-
// outcome the caller wanted or the one it can no longer change.
|
|
62
|
+
// A gateway world answers 409 once it is completing, sealed or expired.
|
|
56
63
|
const recovered = await client.getRun(runId).catch(() => undefined);
|
|
57
64
|
if (recovered?.status !== status &&
|
|
58
65
|
recovered?.status !== "expired" &&
|
|
59
66
|
!(recovered?.status === "open" && recovered.lifecycle === "completing"))
|
|
60
67
|
throw new TargetOutcomeUncertainError(executionId, { cause: error });
|
|
68
|
+
if (recovered?.status === "open")
|
|
69
|
+
completingUntil = recovered.completingUntil ?? null;
|
|
70
|
+
}
|
|
71
|
+
if (completingUntil !== undefined) {
|
|
72
|
+
try {
|
|
73
|
+
await awaitSeal(client, runId, completingUntil);
|
|
74
|
+
}
|
|
75
|
+
catch (error) {
|
|
76
|
+
throw new TargetOutcomeUncertainError(executionId, { cause: error });
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
/** Wait out a completing world's grace, then read until the server seals it. */
|
|
81
|
+
async function awaitSeal(client, runId, completingUntil) {
|
|
82
|
+
const graceEnd = Date.parse(completingUntil ?? "");
|
|
83
|
+
let wait = Number.isFinite(graceEnd)
|
|
84
|
+
? Math.min(Math.max(0, graceEnd - Date.now()), MAX_GRACE_WAIT_MS)
|
|
85
|
+
: 0;
|
|
86
|
+
const deadline = performance.now() + wait + SEAL_WAIT_MS;
|
|
87
|
+
for (;;) {
|
|
88
|
+
await new Promise((resolve) => setTimeout(resolve, wait));
|
|
89
|
+
const remaining = deadline - performance.now();
|
|
90
|
+
if (remaining <= 0)
|
|
91
|
+
throw new Error(`World ${runId} was not sealed after its completion grace`);
|
|
92
|
+
const controller = new AbortController();
|
|
93
|
+
const timer = setTimeout(() => controller.abort(), remaining);
|
|
94
|
+
try {
|
|
95
|
+
if ((await client.getRun(runId, { signal: controller.signal })).status !== "open")
|
|
96
|
+
return;
|
|
97
|
+
}
|
|
98
|
+
catch (error) {
|
|
99
|
+
if (!isTransientEnvironmentError(error))
|
|
100
|
+
throw error;
|
|
101
|
+
if (performance.now() >= deadline)
|
|
102
|
+
throw new Error(`World ${runId} was not sealed after its completion grace`, {
|
|
103
|
+
cause: error,
|
|
104
|
+
});
|
|
105
|
+
}
|
|
106
|
+
finally {
|
|
107
|
+
clearTimeout(timer);
|
|
108
|
+
}
|
|
109
|
+
if (performance.now() >= deadline)
|
|
110
|
+
throw new Error(`World ${runId} was not sealed after its completion grace`);
|
|
111
|
+
wait = SEAL_POLL_MS;
|
|
61
112
|
}
|
|
62
113
|
}
|
|
63
114
|
const deprecations = new Set();
|
|
@@ -10,7 +10,7 @@ export interface ScenarioPins {
|
|
|
10
10
|
datasetId: string;
|
|
11
11
|
/** Pinned dataset version; frozen only when `saved` is true. */
|
|
12
12
|
datasetVersionId: string;
|
|
13
|
-
/** Pinned scorer versions; a Scenario
|
|
13
|
+
/** Pinned scorer versions; a Scenario's publication lists them, its outcome scorer first. */
|
|
14
14
|
scorerVersionIds: string[];
|
|
15
15
|
/** Pinned simulated-world version, or `null` when the selection does not pin one. */
|
|
16
16
|
environmentVersionId: string | null;
|
package/dist/evals/scenarios.js
CHANGED
|
@@ -74,7 +74,10 @@ async function pinsFromScenario(client, scenario, dataset) {
|
|
|
74
74
|
name: dataset.name,
|
|
75
75
|
datasetId: scenario.publication.datasetId,
|
|
76
76
|
datasetVersionId: version.id,
|
|
77
|
-
|
|
77
|
+
// Publications list every pin, the outcome scorer first; older ones carry the single field.
|
|
78
|
+
scorerVersionIds: scenario.publication.scorerVersionIds?.length
|
|
79
|
+
? [...new Set(scenario.publication.scorerVersionIds)]
|
|
80
|
+
: [scenario.publication.scorerVersionId],
|
|
78
81
|
environmentVersionId: scenario.publication.environmentVersionId,
|
|
79
82
|
saved: version.frozenAt !== null,
|
|
80
83
|
revision: version.revision,
|
package/dist/evals/types.d.ts
CHANGED
|
@@ -879,8 +879,13 @@ export interface CaseConversionPublication {
|
|
|
879
879
|
environmentVersionId: string;
|
|
880
880
|
/** Scorer identity of the published outcome checks. */
|
|
881
881
|
scorerId: string;
|
|
882
|
-
/** Immutable scorer version pinned by the Scenario. */
|
|
882
|
+
/** Immutable scorer version pinned by the Scenario for its outcome checks. */
|
|
883
883
|
scorerVersionId: string;
|
|
884
|
+
/**
|
|
885
|
+
* Every immutable scorer version the Scenario pins, `scorerVersionId` first. Absent on
|
|
886
|
+
* Scenarios published before publications listed their pins.
|
|
887
|
+
*/
|
|
888
|
+
scorerVersionIds?: string[];
|
|
884
889
|
}
|
|
885
890
|
/** A Scenario read by {@link EvaluationClient.getCaseConversion}. Extra server fields are ignored. */
|
|
886
891
|
export interface CaseConversion extends Partial<Omit<CaseConversionSummary, "id" | "status">> {
|
|
@@ -2,7 +2,8 @@ import type { HueClient } from "./client.js";
|
|
|
2
2
|
import type { ExperimentalTelemetrySettings } from "./types.js";
|
|
3
3
|
/**
|
|
4
4
|
* Per-call telemetry for AI SDK 6: pass as `experimental_telemetry`. Spans are created with Hue's
|
|
5
|
-
* tracer, so they parent under `withSpan` and inherit session
|
|
6
|
-
* recording follows `captureContent`. AI SDK 7 applications use `hueTelemetry`
|
|
5
|
+
* tracer, so they parent under `withSpan` and inherit session, user and workspace identifiers, and
|
|
6
|
+
* prompt/response recording follows `captureContent`. AI SDK 7 applications use `hueTelemetry`
|
|
7
|
+
* from `@hue-run/sdk/ai-sdk`.
|
|
7
8
|
*/
|
|
8
9
|
export declare function hueExperimentalTelemetry(hue: HueClient): ExperimentalTelemetrySettings;
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Per-call telemetry for AI SDK 6: pass as `experimental_telemetry`. Spans are created with Hue's
|
|
3
|
-
* tracer, so they parent under `withSpan` and inherit session
|
|
4
|
-
* recording follows `captureContent`. AI SDK 7 applications use `hueTelemetry`
|
|
3
|
+
* tracer, so they parent under `withSpan` and inherit session, user and workspace identifiers, and
|
|
4
|
+
* prompt/response recording follows `captureContent`. AI SDK 7 applications use `hueTelemetry`
|
|
5
|
+
* from `@hue-run/sdk/ai-sdk`.
|
|
5
6
|
*/
|
|
6
7
|
export function hueExperimentalTelemetry(hue) {
|
|
7
8
|
return {
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/** Inline file content larger than this many UTF-8 bytes is exported as its digest instead. */
|
|
2
|
+
export declare const INLINE_FILE_LIMIT: number;
|
|
3
|
+
/**
|
|
4
|
+
* Replaces inline files longer than {@link INLINE_FILE_LIMIT} in a recorded message attribute
|
|
5
|
+
* with their SHA-256 and byte size, so a span that inlines a large file exports the file's
|
|
6
|
+
* identity instead of being rejected for its size. The part keeps its other fields (`type`,
|
|
7
|
+
* `mime_type`, `modality`, `mediaType`, …). Other attributes, shorter messages and values that
|
|
8
|
+
* are not JSON are returned unchanged.
|
|
9
|
+
*/
|
|
10
|
+
export declare function hashInlineFiles(key: string, value: unknown): unknown;
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { MAX_BODY_BYTES } from "./config.js";
|
|
3
|
+
/** Inline file content larger than this many UTF-8 bytes is exported as its digest instead. */
|
|
4
|
+
export const INLINE_FILE_LIMIT = 64 * 1024;
|
|
5
|
+
const MAX_INLINE_FILE_TEXT = 8 * MAX_BODY_BYTES;
|
|
6
|
+
/** Message attributes whose JSON can inline files: GenAI blob parts and AI SDK 6 file parts. */
|
|
7
|
+
const messageKeys = new Set([
|
|
8
|
+
"gen_ai.input.messages",
|
|
9
|
+
"gen_ai.output.messages",
|
|
10
|
+
"ai.prompt.messages",
|
|
11
|
+
]);
|
|
12
|
+
/** Strict base64: alphabet characters only, padded to a multiple of four. */
|
|
13
|
+
const base64 = /^[A-Za-z0-9+/]*={0,2}$/;
|
|
14
|
+
const base64DataUrl = /^data:[^,]*;base64,/;
|
|
15
|
+
function binaryMimeType(value) {
|
|
16
|
+
return (typeof value === "string" &&
|
|
17
|
+
!/^text\//i.test(value) &&
|
|
18
|
+
value.toLowerCase() !== "application/json");
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* The bytes an inline file part carries: base64 (plain or as a `data:` URL) is decoded, and
|
|
22
|
+
* anything else, such as a text file's content, is taken as UTF-8.
|
|
23
|
+
*/
|
|
24
|
+
function fileBytes(content, mimeType) {
|
|
25
|
+
const prefix = base64DataUrl.exec(content)?.[0].length ?? 0;
|
|
26
|
+
const payload = content.slice(prefix);
|
|
27
|
+
return (prefix > 0 || binaryMimeType(mimeType)) &&
|
|
28
|
+
payload.length % 4 === 0 &&
|
|
29
|
+
base64.test(payload)
|
|
30
|
+
? Buffer.from(payload, "base64")
|
|
31
|
+
: Buffer.from(content, "utf8");
|
|
32
|
+
}
|
|
33
|
+
/** The key holding a part's inline content: GenAI `blob` parts and AI SDK 6 `file` parts. */
|
|
34
|
+
function contentKey(part) {
|
|
35
|
+
return part.type === "blob" ? "content" : part.type === "file" ? "data" : undefined;
|
|
36
|
+
}
|
|
37
|
+
function hashNode(value, state, depth) {
|
|
38
|
+
if (depth > 256)
|
|
39
|
+
throw new Error("Message exceeds the supported nesting limit");
|
|
40
|
+
if (Array.isArray(value))
|
|
41
|
+
return value.map((item) => hashNode(item, state, depth + 1));
|
|
42
|
+
if (value === null || typeof value !== "object")
|
|
43
|
+
return value;
|
|
44
|
+
const part = value;
|
|
45
|
+
const key = contentKey(part);
|
|
46
|
+
const inline = key === undefined ? undefined : part[key];
|
|
47
|
+
const mimeType = part.mime_type ?? part.mediaType;
|
|
48
|
+
if (key !== undefined && typeof inline === "string") {
|
|
49
|
+
const bytes = fileBytes(inline, mimeType);
|
|
50
|
+
if (bytes.byteLength > INLINE_FILE_LIMIT) {
|
|
51
|
+
const { [key]: _omitted, ...rest } = part;
|
|
52
|
+
state.changed = true;
|
|
53
|
+
return {
|
|
54
|
+
...rest,
|
|
55
|
+
sha256: createHash("sha256").update(bytes).digest("hex"),
|
|
56
|
+
size: bytes.byteLength,
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
return Object.fromEntries(Object.entries(part).map(([name, item]) => [name, hashNode(item, state, depth + 1)]));
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Replaces inline files longer than {@link INLINE_FILE_LIMIT} in a recorded message attribute
|
|
64
|
+
* with their SHA-256 and byte size, so a span that inlines a large file exports the file's
|
|
65
|
+
* identity instead of being rejected for its size. The part keeps its other fields (`type`,
|
|
66
|
+
* `mime_type`, `modality`, `mediaType`, …). Other attributes, shorter messages and values that
|
|
67
|
+
* are not JSON are returned unchanged.
|
|
68
|
+
*/
|
|
69
|
+
export function hashInlineFiles(key, value) {
|
|
70
|
+
if (!messageKeys.has(key) ||
|
|
71
|
+
typeof value !== "string" ||
|
|
72
|
+
Buffer.byteLength(value, "utf8") <= INLINE_FILE_LIMIT ||
|
|
73
|
+
Buffer.byteLength(value, "utf8") > MAX_INLINE_FILE_TEXT ||
|
|
74
|
+
!(value.includes('"blob"') || value.includes('"file"')))
|
|
75
|
+
return value;
|
|
76
|
+
try {
|
|
77
|
+
const parsed = JSON.parse(value);
|
|
78
|
+
const state = { changed: false };
|
|
79
|
+
const hashed = hashNode(parsed, state, 0);
|
|
80
|
+
return state.changed ? JSON.stringify(hashed) : value;
|
|
81
|
+
}
|
|
82
|
+
catch {
|
|
83
|
+
// Not JSON, or nested too deeply to inspect: export decides the value's fate as before.
|
|
84
|
+
return value;
|
|
85
|
+
}
|
|
86
|
+
}
|
package/dist/privacy.js
CHANGED
|
@@ -1,5 +1,8 @@
|
|
|
1
|
+
import { SpanStatusCode } from "@opentelemetry/api";
|
|
1
2
|
import { resourceFromAttributes } from "@opentelemetry/resources";
|
|
2
3
|
import { MAX_BODY_BYTES, MAX_CONTENT_BYTES } from "./config.js";
|
|
4
|
+
import { scrubToolCredentials, withToolCatalogSummary } from "./tool-definitions.js";
|
|
5
|
+
import { hashInlineFiles } from "./inline-files.js";
|
|
3
6
|
/** Attribute keys (and their dotted children) removed in metadata-only mode. */
|
|
4
7
|
export const contentPrefixes = [
|
|
5
8
|
"gen_ai.input.messages",
|
|
@@ -95,9 +98,16 @@ function redactValue(value, path, options, budget, depth = 0) {
|
|
|
95
98
|
return value;
|
|
96
99
|
}
|
|
97
100
|
function attributes(source, options, path, budget) {
|
|
98
|
-
|
|
101
|
+
// Metadata-only export summarizes the tool definitions it removes by name and digest.
|
|
102
|
+
const summarized = options.captureContent ? source : withToolCatalogSummary(source);
|
|
103
|
+
return Object.fromEntries(Object.entries(summarized).flatMap(([key, value]) => !options.captureContent && isContentKey(key)
|
|
99
104
|
? []
|
|
100
|
-
: [
|
|
105
|
+
: [
|
|
106
|
+
[
|
|
107
|
+
key,
|
|
108
|
+
redactValue(scrubToolCredentials(key, hashInlineFiles(key, value)), `${path}.${key}`, options, budget),
|
|
109
|
+
],
|
|
110
|
+
]));
|
|
101
111
|
}
|
|
102
112
|
function redactResource(resource, options, cache, budget) {
|
|
103
113
|
let result = cache.get(resource);
|
|
@@ -107,8 +117,43 @@ function redactResource(resource, options, cache, budget) {
|
|
|
107
117
|
}
|
|
108
118
|
return result;
|
|
109
119
|
}
|
|
120
|
+
/**
|
|
121
|
+
* Server identity and failure of an AI SDK 7 provider-executed (`extension`) tool span, read from
|
|
122
|
+
* its recorded result before content is stripped. OpenAI hosted MCP results carry
|
|
123
|
+
* `{ type: "call", serverLabel, name, arguments, output?, error? }`; nothing else names the server.
|
|
124
|
+
*/
|
|
125
|
+
function hostedMcpCall(attributes) {
|
|
126
|
+
const result = attributes["gen_ai.tool.call.result"];
|
|
127
|
+
if (attributes["gen_ai.tool.type"] !== "extension" || typeof result !== "string")
|
|
128
|
+
return { failed: false };
|
|
129
|
+
let parsed;
|
|
130
|
+
try {
|
|
131
|
+
parsed = JSON.parse(result);
|
|
132
|
+
}
|
|
133
|
+
catch {
|
|
134
|
+
return { failed: false };
|
|
135
|
+
}
|
|
136
|
+
if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed))
|
|
137
|
+
return { failed: false };
|
|
138
|
+
const { type, serverLabel, error } = parsed;
|
|
139
|
+
if (type !== "call")
|
|
140
|
+
return { failed: false };
|
|
141
|
+
const valid = typeof serverLabel === "string" &&
|
|
142
|
+
serverLabel.trim() !== "" &&
|
|
143
|
+
serverLabel.length <= 256 &&
|
|
144
|
+
!serverLabel.includes("\u0000") &&
|
|
145
|
+
serverLabel.isWellFormed();
|
|
146
|
+
return { ...(valid ? { serverName: serverLabel } : {}), failed: error != null };
|
|
147
|
+
}
|
|
110
148
|
export function redactSpan(span, options, cache) {
|
|
111
149
|
const budget = { bytes: 0, nodes: 0 };
|
|
150
|
+
// Derived before metadata-only stripping so the identity survives without the result itself.
|
|
151
|
+
const hosted = hostedMcpCall(span.attributes);
|
|
152
|
+
const source = {
|
|
153
|
+
...(hosted.serverName === undefined ? {} : { "mcp.server.name": hosted.serverName }),
|
|
154
|
+
...(hosted.failed ? { "error.type": "mcp_error" } : {}),
|
|
155
|
+
...span.attributes,
|
|
156
|
+
};
|
|
112
157
|
return {
|
|
113
158
|
name: span.name,
|
|
114
159
|
kind: span.kind,
|
|
@@ -119,12 +164,16 @@ export function redactSpan(span, options, cache) {
|
|
|
119
164
|
duration: span.duration,
|
|
120
165
|
ended: span.ended,
|
|
121
166
|
status: {
|
|
122
|
-
code: span.status.code
|
|
123
|
-
|
|
167
|
+
code: hosted.failed && span.status.code === SpanStatusCode.UNSET
|
|
168
|
+
? SpanStatusCode.ERROR
|
|
169
|
+
: span.status.code,
|
|
170
|
+
...(options.captureContent &&
|
|
171
|
+
span.status.message !== undefined &&
|
|
172
|
+
!(hosted.failed && span.status.code === SpanStatusCode.UNSET)
|
|
124
173
|
? { message: String(redactValue(span.status.message, "status.message", options, budget)) }
|
|
125
174
|
: {}),
|
|
126
175
|
},
|
|
127
|
-
attributes: attributes(
|
|
176
|
+
attributes: attributes(source, options, "attributes", budget),
|
|
128
177
|
events: span.events
|
|
129
178
|
.filter((event) => options.captureContent ||
|
|
130
179
|
!/^gen_ai\.(?:system|user|assistant|tool|choice)/.test(event.name))
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import type { HostedToolProvider } from "./types.js";
|
|
2
|
+
/** One provider-executed tool call. `arguments` and `result` are content. */
|
|
3
|
+
export interface HostedToolCall {
|
|
4
|
+
name: string;
|
|
5
|
+
callId?: string;
|
|
6
|
+
/** The provider's label (OpenAI `server_label`) or name (Anthropic `server_name`) for the MCP server. */
|
|
7
|
+
server?: string;
|
|
8
|
+
arguments?: unknown;
|
|
9
|
+
result?: unknown;
|
|
10
|
+
/** Low-cardinality failure marker for `error.type`; absent when the call succeeded. */
|
|
11
|
+
errorType?: string;
|
|
12
|
+
}
|
|
13
|
+
/** One `mcp_list_tools` result: the tools a hosted MCP server offered. */
|
|
14
|
+
export interface HostedToolListing {
|
|
15
|
+
server: string;
|
|
16
|
+
/** Tool definitions in the OpenTelemetry GenAI shape (`type`, `name`, `description`, `parameters`). */
|
|
17
|
+
definitions: Record<string, unknown>[];
|
|
18
|
+
errorType?: string;
|
|
19
|
+
}
|
|
20
|
+
export interface HostedToolActivity {
|
|
21
|
+
calls: HostedToolCall[];
|
|
22
|
+
listings: HostedToolListing[];
|
|
23
|
+
/** Items that looked like hosted calls but could not be read. */
|
|
24
|
+
skipped: number;
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Reads the hosted tool calls out of a provider response: the `output` items of an OpenAI
|
|
28
|
+
* Responses API response, or the `content` blocks of an Anthropic Messages API response. An array
|
|
29
|
+
* is taken as those items directly. Anything else yields no calls.
|
|
30
|
+
*/
|
|
31
|
+
export declare function hostedToolActivity(provider: HostedToolProvider, response: unknown): HostedToolActivity;
|
|
32
|
+
/**
|
|
33
|
+
* The host of each hosted MCP server's URL, by label, read from the request that produced the
|
|
34
|
+
* response: OpenAI `tools[].server_url` by `server_label`, Anthropic `mcp_servers[].url` by `name`.
|
|
35
|
+
* Nothing else in the request is read.
|
|
36
|
+
*/
|
|
37
|
+
export declare function hostedServerAddresses(provider: HostedToolProvider, request: unknown): Map<string, string>;
|
|
38
|
+
/** The provider a `model()` call named, when this module can read its responses. */
|
|
39
|
+
export declare function hostedToolProvider(value: unknown): HostedToolProvider | undefined;
|
|
@@ -0,0 +1,249 @@
|
|
|
1
|
+
import { MAX_CONTENT_BYTES } from "./config.js";
|
|
2
|
+
const MAX_PROVIDER_ITEMS = 128;
|
|
3
|
+
const MAX_PROVIDER_DEFINITIONS = 512;
|
|
4
|
+
const MAX_PROVIDER_SERVERS = 512;
|
|
5
|
+
function isItem(value) {
|
|
6
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
7
|
+
}
|
|
8
|
+
/** Only provider-executed tool items contribute to invalid-item diagnostics when a response is
|
|
9
|
+
* truncated; messages and reasoning are harmless response content. */
|
|
10
|
+
function isProviderToolItem(provider, value) {
|
|
11
|
+
if (!isItem(value) || typeof value.type !== "string")
|
|
12
|
+
return false;
|
|
13
|
+
if (provider === "openai")
|
|
14
|
+
return (value.type === "mcp_call" ||
|
|
15
|
+
value.type === "mcp_list_tools" ||
|
|
16
|
+
value.type === "web_search_call" ||
|
|
17
|
+
value.type === "file_search_call" ||
|
|
18
|
+
value.type === "code_interpreter_call");
|
|
19
|
+
return value.type === "mcp_tool_use" || value.type === "server_tool_use";
|
|
20
|
+
}
|
|
21
|
+
function countProviderToolItems(provider, items, start = 0) {
|
|
22
|
+
let count = 0;
|
|
23
|
+
for (const key of Object.keys(items)) {
|
|
24
|
+
const index = Number(key);
|
|
25
|
+
if (!Number.isInteger(index) || index < start || index >= items.length || String(index) !== key)
|
|
26
|
+
continue;
|
|
27
|
+
if (isProviderToolItem(provider, items[index]))
|
|
28
|
+
count++;
|
|
29
|
+
}
|
|
30
|
+
return count;
|
|
31
|
+
}
|
|
32
|
+
function text(value) {
|
|
33
|
+
return typeof value === "string" &&
|
|
34
|
+
value.trim() !== "" &&
|
|
35
|
+
value.length <= 256 &&
|
|
36
|
+
!value.includes("\u0000") &&
|
|
37
|
+
value.isWellFormed()
|
|
38
|
+
? value
|
|
39
|
+
: undefined;
|
|
40
|
+
}
|
|
41
|
+
function errorCode(value) {
|
|
42
|
+
return typeof value === "string" && /^[a-z0-9_]{1,64}$/.test(value) ? value : "error";
|
|
43
|
+
}
|
|
44
|
+
/** MCP arguments arrive as a JSON string; record the structure when it parses, else the text. */
|
|
45
|
+
function jsonArguments(value) {
|
|
46
|
+
if (typeof value !== "string")
|
|
47
|
+
return value;
|
|
48
|
+
if (Buffer.byteLength(value, "utf8") > MAX_CONTENT_BYTES)
|
|
49
|
+
return undefined;
|
|
50
|
+
try {
|
|
51
|
+
return JSON.parse(value);
|
|
52
|
+
}
|
|
53
|
+
catch {
|
|
54
|
+
return value;
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
/** OpenAI Responses `output` items. Built-in tools are named by their kind; MCP calls by tool. */
|
|
58
|
+
function openaiCalls(items, activity) {
|
|
59
|
+
const count = Math.min(items.length, MAX_PROVIDER_ITEMS);
|
|
60
|
+
activity.skipped += countProviderToolItems("openai", items, count);
|
|
61
|
+
for (let index = 0; index < count; index++) {
|
|
62
|
+
const item = items[index];
|
|
63
|
+
if (!isItem(item))
|
|
64
|
+
continue;
|
|
65
|
+
const callId = text(item.id);
|
|
66
|
+
switch (item.type) {
|
|
67
|
+
case "mcp_call": {
|
|
68
|
+
const name = text(item.name);
|
|
69
|
+
if (!name) {
|
|
70
|
+
activity.skipped++;
|
|
71
|
+
break;
|
|
72
|
+
}
|
|
73
|
+
activity.calls.push({
|
|
74
|
+
name,
|
|
75
|
+
callId,
|
|
76
|
+
server: text(item.server_label),
|
|
77
|
+
arguments: jsonArguments(item.arguments),
|
|
78
|
+
...(item.output !== undefined && item.output !== null ? { result: item.output } : {}),
|
|
79
|
+
...(item.error !== undefined && item.error !== null ? { errorType: "mcp_error" } : {}),
|
|
80
|
+
});
|
|
81
|
+
break;
|
|
82
|
+
}
|
|
83
|
+
case "mcp_list_tools": {
|
|
84
|
+
const server = text(item.server_label);
|
|
85
|
+
if (!server || !Array.isArray(item.tools)) {
|
|
86
|
+
activity.skipped++;
|
|
87
|
+
break;
|
|
88
|
+
}
|
|
89
|
+
const definitions = [];
|
|
90
|
+
const definitionCount = Math.min(item.tools.length, MAX_PROVIDER_DEFINITIONS);
|
|
91
|
+
activity.skipped += item.tools.length - definitionCount;
|
|
92
|
+
for (let index = 0; index < definitionCount; index++) {
|
|
93
|
+
const tool = item.tools[index];
|
|
94
|
+
if (!isItem(tool))
|
|
95
|
+
continue;
|
|
96
|
+
definitions.push({
|
|
97
|
+
type: "function",
|
|
98
|
+
...(tool.name !== undefined ? { name: tool.name } : {}),
|
|
99
|
+
...(tool.description !== undefined ? { description: tool.description } : {}),
|
|
100
|
+
...(tool.input_schema !== undefined ? { parameters: tool.input_schema } : {}),
|
|
101
|
+
...(tool.annotations !== undefined ? { annotations: tool.annotations } : {}),
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
activity.listings.push({
|
|
105
|
+
server,
|
|
106
|
+
definitions,
|
|
107
|
+
...(item.error !== undefined && item.error !== null ? { errorType: "mcp_error" } : {}),
|
|
108
|
+
});
|
|
109
|
+
break;
|
|
110
|
+
}
|
|
111
|
+
case "web_search_call":
|
|
112
|
+
case "file_search_call":
|
|
113
|
+
case "code_interpreter_call": {
|
|
114
|
+
const name = item.type.slice(0, -"_call".length);
|
|
115
|
+
const failed = item.status === "failed" ? { errorType: "failed" } : {};
|
|
116
|
+
if (item.type === "web_search_call")
|
|
117
|
+
activity.calls.push({ name, callId, arguments: item.action, ...failed });
|
|
118
|
+
else if (item.type === "file_search_call")
|
|
119
|
+
activity.calls.push({
|
|
120
|
+
name,
|
|
121
|
+
callId,
|
|
122
|
+
arguments: { queries: item.queries },
|
|
123
|
+
...(item.results !== undefined && item.results !== null
|
|
124
|
+
? { result: item.results }
|
|
125
|
+
: {}),
|
|
126
|
+
...failed,
|
|
127
|
+
});
|
|
128
|
+
else
|
|
129
|
+
activity.calls.push({
|
|
130
|
+
name,
|
|
131
|
+
callId,
|
|
132
|
+
arguments: { code: item.code, container_id: item.container_id },
|
|
133
|
+
...(item.outputs !== undefined && item.outputs !== null
|
|
134
|
+
? { result: item.outputs }
|
|
135
|
+
: {}),
|
|
136
|
+
...failed,
|
|
137
|
+
});
|
|
138
|
+
break;
|
|
139
|
+
}
|
|
140
|
+
default:
|
|
141
|
+
// Messages, reasoning, approval requests and other items are not executed tools.
|
|
142
|
+
break;
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
/** Anthropic Messages `content` blocks: a use block paired with the result block that names it. */
|
|
147
|
+
function anthropicCalls(blocks, activity) {
|
|
148
|
+
const results = new Map();
|
|
149
|
+
const count = Math.min(blocks.length, MAX_PROVIDER_ITEMS);
|
|
150
|
+
const truncated = blocks.length > MAX_PROVIDER_ITEMS;
|
|
151
|
+
activity.skipped += countProviderToolItems("anthropic", blocks, count);
|
|
152
|
+
for (let index = 0; index < count; index++) {
|
|
153
|
+
const block = blocks[index];
|
|
154
|
+
if (isItem(block) &&
|
|
155
|
+
typeof block.type === "string" &&
|
|
156
|
+
block.type.endsWith("_tool_result") &&
|
|
157
|
+
typeof block.tool_use_id === "string")
|
|
158
|
+
results.set(block.tool_use_id, block);
|
|
159
|
+
}
|
|
160
|
+
for (let index = 0; index < count; index++) {
|
|
161
|
+
const block = blocks[index];
|
|
162
|
+
if (!isItem(block) || (block.type !== "mcp_tool_use" && block.type !== "server_tool_use"))
|
|
163
|
+
continue;
|
|
164
|
+
const name = text(block.name);
|
|
165
|
+
const callId = text(block.id);
|
|
166
|
+
if (!name) {
|
|
167
|
+
activity.skipped++;
|
|
168
|
+
continue;
|
|
169
|
+
}
|
|
170
|
+
const result = callId === undefined ? undefined : results.get(callId);
|
|
171
|
+
// When the response was truncated, an unmatched use block may have its result outside the
|
|
172
|
+
// bounded prefix. Do not export it as a successful call with a missing result.
|
|
173
|
+
if (truncated && result === undefined) {
|
|
174
|
+
activity.skipped++;
|
|
175
|
+
continue;
|
|
176
|
+
}
|
|
177
|
+
const content = result?.content;
|
|
178
|
+
let errorType;
|
|
179
|
+
if (result?.is_error === true)
|
|
180
|
+
errorType = "mcp_error";
|
|
181
|
+
else if (isItem(content) && typeof content.type === "string" && content.type.endsWith("_error"))
|
|
182
|
+
errorType = errorCode(content.error_code);
|
|
183
|
+
activity.calls.push({
|
|
184
|
+
name,
|
|
185
|
+
callId,
|
|
186
|
+
...(block.type === "mcp_tool_use" ? { server: text(block.server_name) } : {}),
|
|
187
|
+
arguments: block.input,
|
|
188
|
+
...(content !== undefined ? { result: content } : {}),
|
|
189
|
+
...(errorType ? { errorType } : {}),
|
|
190
|
+
});
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
/**
|
|
194
|
+
* Reads the hosted tool calls out of a provider response: the `output` items of an OpenAI
|
|
195
|
+
* Responses API response, or the `content` blocks of an Anthropic Messages API response. An array
|
|
196
|
+
* is taken as those items directly. Anything else yields no calls.
|
|
197
|
+
*/
|
|
198
|
+
export function hostedToolActivity(provider, response) {
|
|
199
|
+
const activity = { calls: [], listings: [], skipped: 0 };
|
|
200
|
+
const items = Array.isArray(response)
|
|
201
|
+
? response
|
|
202
|
+
: isItem(response)
|
|
203
|
+
? response[provider === "openai" ? "output" : "content"]
|
|
204
|
+
: undefined;
|
|
205
|
+
if (!Array.isArray(items))
|
|
206
|
+
return activity;
|
|
207
|
+
if (provider === "openai")
|
|
208
|
+
openaiCalls(items, activity);
|
|
209
|
+
else
|
|
210
|
+
anthropicCalls(items, activity);
|
|
211
|
+
return activity;
|
|
212
|
+
}
|
|
213
|
+
/**
|
|
214
|
+
* The host of each hosted MCP server's URL, by label, read from the request that produced the
|
|
215
|
+
* response: OpenAI `tools[].server_url` by `server_label`, Anthropic `mcp_servers[].url` by `name`.
|
|
216
|
+
* Nothing else in the request is read.
|
|
217
|
+
*/
|
|
218
|
+
export function hostedServerAddresses(provider, request) {
|
|
219
|
+
const addresses = new Map();
|
|
220
|
+
if (!isItem(request))
|
|
221
|
+
return addresses;
|
|
222
|
+
const entries = request[provider === "openai" ? "tools" : "mcp_servers"];
|
|
223
|
+
if (!Array.isArray(entries))
|
|
224
|
+
return addresses;
|
|
225
|
+
const count = Math.min(entries.length, MAX_PROVIDER_SERVERS);
|
|
226
|
+
for (let index = 0; index < count; index++) {
|
|
227
|
+
const entry = entries[index];
|
|
228
|
+
if (!isItem(entry))
|
|
229
|
+
continue;
|
|
230
|
+
const label = text(provider === "openai" ? entry.server_label : entry.name);
|
|
231
|
+
const url = provider === "openai" ? entry.server_url : entry.url;
|
|
232
|
+
if (!label || typeof url !== "string")
|
|
233
|
+
continue;
|
|
234
|
+
try {
|
|
235
|
+
const { hostname } = new URL(url);
|
|
236
|
+
if (hostname)
|
|
237
|
+
addresses.set(label, hostname);
|
|
238
|
+
}
|
|
239
|
+
catch {
|
|
240
|
+
// Not a URL; there is no address to record.
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
return addresses;
|
|
244
|
+
}
|
|
245
|
+
/** The provider a `model()` call named, when this module can read its responses. */
|
|
246
|
+
export function hostedToolProvider(value) {
|
|
247
|
+
const provider = typeof value === "string" ? value.toLowerCase() : undefined;
|
|
248
|
+
return provider === "openai" || provider === "anthropic" ? provider : undefined;
|
|
249
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Removes hosted-tool credentials from an exported attribute value, before the caller's `redact`.
|
|
3
|
+
* Tool definitions come from OpenTelemetry GenAI (`gen_ai.tool.definitions`), AI SDK 6
|
|
4
|
+
* (`ai.prompt.tools`, one JSON string per tool) and OpenInference (`llm.tools.{i}.tool.json_schema`,
|
|
5
|
+
* plus the raw request in `input.value`). Other attributes, and values that are not JSON, are
|
|
6
|
+
* returned unchanged. Metadata-only export removes all of these attributes anyway.
|
|
7
|
+
*
|
|
8
|
+
* @throws Error when a tool definition is nested too deeply to inspect; the record is then
|
|
9
|
+
* rejected rather than exported with credentials.
|
|
10
|
+
*/
|
|
11
|
+
export declare function scrubToolCredentials(key: string, value: unknown): unknown;
|
|
12
|
+
/**
|
|
13
|
+
* Metadata-only summary of the tool definitions a record carries, which export then removes:
|
|
14
|
+
* `hue.tool.names` lists each definition's name in order, and `hue.tool.definitions.sha256` is
|
|
15
|
+
* the lowercase hex SHA-256 of the RFC 8785 canonical JSON of the credential-scrubbed definition
|
|
16
|
+
* list, so the same catalog has the same digest in both SDKs and across credential rotation.
|
|
17
|
+
* Returns the source unchanged when it has no parseable definitions; attributes the source
|
|
18
|
+
* already sets are kept.
|
|
19
|
+
*/
|
|
20
|
+
export declare function withToolCatalogSummary<T extends Record<string, unknown>>(source: T): T;
|