@hue-run/sdk 0.10.0 → 0.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLI.md +154 -28
- package/ENVIRONMENTS.md +5 -1
- package/EVALUATIONS.md +2 -2
- package/README.md +19 -8
- package/dist/cli/eval.js +43 -17
- package/dist/cli/listen.d.ts +76 -0
- package/dist/cli/listen.js +816 -0
- package/dist/cli/mcp.d.ts +66 -7
- package/dist/cli/mcp.js +273 -74
- package/dist/client.d.ts +7 -4
- package/dist/client.js +36 -10
- package/dist/evals/client.d.ts +5 -1
- package/dist/evals/client.js +17 -11
- package/dist/evals/files.d.ts +10 -2
- package/dist/evals/files.js +68 -15
- package/dist/evals/json.js +116 -9
- package/dist/evals/runner.js +35 -8
- package/dist/evals/scorer-publication.js +58 -3
- package/dist/evals/types.d.ts +44 -0
- package/dist/evals/verdicts.d.ts +11 -0
- package/dist/evals/verdicts.js +33 -4
- package/dist/inline-files.js +11 -3
- package/dist/provider-tools.d.ts +12 -1
- package/dist/provider-tools.js +192 -106
- package/dist/safety.d.ts +7 -1
- package/dist/safety.js +9 -7
- package/dist/setup/cli.js +31 -8
- package/dist/setup/peers.d.ts +13 -0
- package/dist/setup/peers.js +44 -0
- package/dist/tool-definitions.d.ts +18 -0
- package/dist/tool-definitions.js +232 -0
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +1 -1
package/dist/evals/types.d.ts
CHANGED
|
@@ -202,6 +202,33 @@ export type ScorerDefinition = {
|
|
|
202
202
|
entry: "hue.conversion_outcome.v1";
|
|
203
203
|
/** The seven fixed boolean metrics defined by the entry. */
|
|
204
204
|
metrics: MetricDefinition[];
|
|
205
|
+
} | {
|
|
206
|
+
/** Scored inside Hue using immutable world evidence; the local runner defers it. */
|
|
207
|
+
kind: "world_outcome";
|
|
208
|
+
/** v1's rubric with the destination reported as `recipient`, `thread` and `subject`. */
|
|
209
|
+
entry: "hue.conversion_outcome.v2";
|
|
210
|
+
/** The ten fixed boolean metrics defined by the entry. */
|
|
211
|
+
metrics: MetricDefinition[];
|
|
212
|
+
} | {
|
|
213
|
+
/** A case's reviewed outcome assertions, scored inside Hue; the local runner defers it. */
|
|
214
|
+
kind: "world_outcome";
|
|
215
|
+
/** Pinned Hue-executed assertions evaluator. */
|
|
216
|
+
entry: "hue.outcome_assertions.v2";
|
|
217
|
+
/** `task_success` and the fixed assertion counts defined by the entry. */
|
|
218
|
+
metrics: MetricDefinition[];
|
|
219
|
+
} | {
|
|
220
|
+
/** The v2 assertions plus the case's calibrated judges, scored inside Hue; the local runner
|
|
221
|
+
* defers it. */
|
|
222
|
+
kind: "world_outcome";
|
|
223
|
+
/** Pinned Hue-executed assertions-and-judges evaluator. */
|
|
224
|
+
entry: "hue.outcome_assertions.v3";
|
|
225
|
+
/** `task_success`, the assertion counts and the judge counts defined by the entry. */
|
|
226
|
+
metrics: MetricDefinition[];
|
|
227
|
+
/** The judge pin: model, template and sampling policy belong to the version. */
|
|
228
|
+
config: {
|
|
229
|
+
/** The judge every case scored by this version uses. */
|
|
230
|
+
judge: OutcomeJudgeConfig;
|
|
231
|
+
};
|
|
205
232
|
} | {
|
|
206
233
|
/** Scored by a person in Hue; the local runner defers it. */
|
|
207
234
|
kind: "manual";
|
|
@@ -215,6 +242,23 @@ export type ScorerDefinition = {
|
|
|
215
242
|
/** Metrics the judge reports. */
|
|
216
243
|
metrics: MetricDefinition[];
|
|
217
244
|
};
|
|
245
|
+
/** The judge a `hue.outcome_assertions.v3` version pins. */
|
|
246
|
+
export interface OutcomeJudgeConfig {
|
|
247
|
+
/** A provider-prefixed model id, such as `anthropic/claude-fable-5.1`. */
|
|
248
|
+
model: string;
|
|
249
|
+
/** The model's provider. */
|
|
250
|
+
provider: string;
|
|
251
|
+
/** SHA-256 hex digest of the judge template: its fixed instructions and evidence rendering. */
|
|
252
|
+
template: string;
|
|
253
|
+
/** Answers sampled per rubric item, 1–9; a majority decides. */
|
|
254
|
+
samples: number;
|
|
255
|
+
/** Sampling temperature, 0–2. */
|
|
256
|
+
temperature: number;
|
|
257
|
+
/** Output token limit per answer. */
|
|
258
|
+
maxOutputTokens: number;
|
|
259
|
+
/** Per-answer timeout in milliseconds. */
|
|
260
|
+
timeoutMs: number;
|
|
261
|
+
}
|
|
218
262
|
/** Configuration of a hosted judge scorer. */
|
|
219
263
|
export interface JudgeConfig {
|
|
220
264
|
/** Judge model identifier. */
|
package/dist/evals/verdicts.d.ts
CHANGED
|
@@ -16,6 +16,10 @@ export interface WaitForResultsOptions {
|
|
|
16
16
|
pollIntervalMillis?: number;
|
|
17
17
|
/** Stops waiting early; the partial state is returned with `complete: false`. */
|
|
18
18
|
signal?: AbortSignal;
|
|
19
|
+
/** Pinned versions of Hue's own judges (definition kind `world_judge`), whose results Hue can
|
|
20
|
+
* mark advisory. Any evaluator can write the same evidence, so no other result is advisory.
|
|
21
|
+
* {@link collectExperimentVerdicts} reads them from the experiment. */
|
|
22
|
+
judgeScorerVersionIds?: string[];
|
|
19
23
|
}
|
|
20
24
|
/** One stored result joined to its evaluation item. */
|
|
21
25
|
export interface VerdictResult {
|
|
@@ -40,6 +44,10 @@ export interface VerdictResult {
|
|
|
40
44
|
notApplicable?: boolean;
|
|
41
45
|
/** What a not-applicable evaluator needs that the case lacks, such as `outcome_criteria`. */
|
|
42
46
|
requires?: string;
|
|
47
|
+
/** Whether Hue recorded the result as advisory, as it does for every judge today: a result of a
|
|
48
|
+
* pinned Hue judge whose evidence says `advisory: true`, not an error, and whose metrics carry
|
|
49
|
+
* no `passed`. It is shown with its verdict but never decides a case. Absent means false. */
|
|
50
|
+
advisory?: boolean;
|
|
43
51
|
}
|
|
44
52
|
/** Outcome of {@link waitForResults}. */
|
|
45
53
|
export interface VerdictResults {
|
|
@@ -79,6 +87,9 @@ export interface CaseVerdict {
|
|
|
79
87
|
/** Pinned scorer versions Hue recorded as not applicable to the case; always set by
|
|
80
88
|
* {@link summarizeVerdicts}. */
|
|
81
89
|
notApplicable?: string[];
|
|
90
|
+
/** Pinned scorer versions whose results for the case are advisory: their metrics are listed
|
|
91
|
+
* but never decide the case; always set by {@link summarizeVerdicts}. */
|
|
92
|
+
advisory?: string[];
|
|
82
93
|
/** Reported metrics across pinned scorers, in result order. */
|
|
83
94
|
metrics: (Metric & {
|
|
84
95
|
/** Scorer version that reported the metric. */
|
package/dist/evals/verdicts.js
CHANGED
|
@@ -42,6 +42,7 @@ export async function waitForResults(client, options) {
|
|
|
42
42
|
throw new RangeError("timeoutMillis must be ≥ 0");
|
|
43
43
|
if (!Number.isInteger(interval) || interval < 250 || interval > 60_000)
|
|
44
44
|
throw new RangeError("pollIntervalMillis must be 250–60000");
|
|
45
|
+
const judges = new Set(options.judgeScorerVersionIds ?? []);
|
|
45
46
|
const wanted = options.subjectIds ? new Set(options.subjectIds) : undefined;
|
|
46
47
|
const pins = [...new Set(options.scorerVersionIds)];
|
|
47
48
|
const deadline = Date.now() + timeout;
|
|
@@ -70,6 +71,12 @@ export async function waitForResults(client, options) {
|
|
|
70
71
|
// never does, and a scored or errored result keeps its verdict whatever it carries.
|
|
71
72
|
const notApplicable = stored.state === "skipped" && stored.notApplicable === true;
|
|
72
73
|
const evidence = stored.evidence;
|
|
74
|
+
// Any evaluator can write `advisory: true` in its evidence; only Hue's own judges'
|
|
75
|
+
// results are advisory, never as an error or with a metric that says it passed.
|
|
76
|
+
const advisory = judges.has(stored.scorerVersionId) &&
|
|
77
|
+
stored.state !== "error" &&
|
|
78
|
+
evidence?.advisory === true &&
|
|
79
|
+
!(stored.state === "scored" && stored.metrics.some((metric) => "passed" in metric));
|
|
73
80
|
results.push({
|
|
74
81
|
id: stored.id,
|
|
75
82
|
itemId: stored.itemId,
|
|
@@ -80,6 +87,7 @@ export async function waitForResults(client, options) {
|
|
|
80
87
|
explanation: stored.explanation ?? null,
|
|
81
88
|
error: stored.error ?? null,
|
|
82
89
|
notApplicable,
|
|
90
|
+
advisory,
|
|
83
91
|
...(notApplicable && typeof evidence?.requires === "string"
|
|
84
92
|
? { requires: evidence.requires }
|
|
85
93
|
: {}),
|
|
@@ -115,14 +123,21 @@ export function summarizeVerdicts(results, options) {
|
|
|
115
123
|
const metrics = own.flatMap((result) => result.metrics.map((metric) => ({ ...metric, scorerVersionId: result.scorerVersionId })));
|
|
116
124
|
// An evaluator that does not apply to the case neither passes nor fails it.
|
|
117
125
|
const skippedAsInapplicable = (result) => result.state === "skipped" && result.notApplicable === true;
|
|
126
|
+
const isAdvisory = (result) => result.advisory === true &&
|
|
127
|
+
result.state !== "error" &&
|
|
128
|
+
result.metrics.every((metric) => metric.passed === undefined);
|
|
118
129
|
const inapplicable = own.filter(skippedAsInapplicable);
|
|
119
130
|
const applicable = own.filter((result) => !skippedAsInapplicable(result));
|
|
120
|
-
|
|
121
|
-
const
|
|
131
|
+
// An advisory result, such as a judge's, is shown but never decides the case, as in Hue.
|
|
132
|
+
const advisory = applicable.filter(isAdvisory);
|
|
133
|
+
const deciding = applicable.filter((result) => !isAdvisory(result));
|
|
134
|
+
const errors = deciding.filter((result) => result.state === "error");
|
|
135
|
+
const scored = deciding.filter((result) => result.state === "scored");
|
|
122
136
|
const failing = scored.filter((result) => !result.metrics.every(metricPassed));
|
|
123
137
|
const missing = [...pins].some((pin) => !own.some((result) => result.scorerVersionId === pin));
|
|
124
138
|
const noneApplies = !missing && own.length > 0 && !applicable.length;
|
|
125
|
-
const
|
|
139
|
+
const noneDecides = !missing && applicable.length > 0 && !deciding.length;
|
|
140
|
+
const state = errors.length || noneApplies || noneDecides
|
|
126
141
|
? "error"
|
|
127
142
|
: failing.length
|
|
128
143
|
? "failed"
|
|
@@ -141,6 +156,7 @@ export function summarizeVerdicts(results, options) {
|
|
|
141
156
|
state,
|
|
142
157
|
passed: state === "passed",
|
|
143
158
|
notApplicable: inapplicable.map((result) => result.scorerVersionId),
|
|
159
|
+
advisory: advisory.map((result) => result.scorerVersionId),
|
|
144
160
|
metrics,
|
|
145
161
|
explanations: [
|
|
146
162
|
...(noneApplies
|
|
@@ -148,7 +164,15 @@ export function summarizeVerdicts(results, options) {
|
|
|
148
164
|
`No pinned evaluator applies to this case${requires.length ? ` (they need ${requires.join(" or ")})` : ""}; pin one that grades it`,
|
|
149
165
|
]
|
|
150
166
|
: []),
|
|
151
|
-
...
|
|
167
|
+
...(noneDecides
|
|
168
|
+
? [
|
|
169
|
+
"Only advisory evaluators ran for this case, and they never decide it; pin one that grades it",
|
|
170
|
+
...advisory
|
|
171
|
+
.filter((result) => result.state === "skipped")
|
|
172
|
+
.map((result) => `An advisory evaluator skipped the case: ${result.explanation ?? "no reason given"}`),
|
|
173
|
+
]
|
|
174
|
+
: []),
|
|
175
|
+
...deciding
|
|
152
176
|
.filter((result) => result.state !== "scored" || failing.includes(result))
|
|
153
177
|
.map((result) => result.explanation)
|
|
154
178
|
.filter((explanation) => !!explanation),
|
|
@@ -203,10 +227,15 @@ export async function collectExperimentVerdicts(client, options) {
|
|
|
203
227
|
const experiment = await client.getExperiment(options.experimentId);
|
|
204
228
|
const scorerVersionIds = experiment.evaluation.scorerVersions.map((version) => version.id);
|
|
205
229
|
const { experimentId: _experimentId, ...wait } = options;
|
|
230
|
+
// Hue's own judges, the only evaluators whose results can be advisory.
|
|
231
|
+
const judgeScorerVersionIds = experiment.evaluation.scorerVersions
|
|
232
|
+
.filter((version) => version.definition?.kind === "world_judge")
|
|
233
|
+
.map((version) => version.id);
|
|
206
234
|
const results = await waitForResults(client, {
|
|
207
235
|
...wait,
|
|
208
236
|
runId: experiment.evaluation.id,
|
|
209
237
|
scorerVersionIds,
|
|
238
|
+
judgeScorerVersionIds,
|
|
210
239
|
});
|
|
211
240
|
const experimentItems = await allPages((after) => client.listExperimentItems(experiment.id, { after }));
|
|
212
241
|
const linked = new Set(experimentItems.map((item) => item.execution?.subjectId).filter((id) => !!id));
|
package/dist/inline-files.js
CHANGED
|
@@ -17,8 +17,16 @@ export function isMessageKey(key) {
|
|
|
17
17
|
}
|
|
18
18
|
/** Strict base64: alphabet characters only, padded to a multiple of four. */
|
|
19
19
|
const base64 = /^[A-Za-z0-9+/]*={0,2}$/;
|
|
20
|
-
/** An RFC 2397 `data:` URL's
|
|
21
|
-
|
|
20
|
+
/** An RFC 2397 `data:` URL's header, up to the comma before its data. Its `;` parameters are read
|
|
21
|
+
* in code: a pattern repeated per parameter fails on a pathological count (Node's stack overflows,
|
|
22
|
+
* Bun stops matching). */
|
|
23
|
+
const dataUrl = /^data:([^,]*),/;
|
|
24
|
+
/** Whether a `data:` URL header's parameters, after its media type, include `base64`. */
|
|
25
|
+
function base64Header(header) {
|
|
26
|
+
const semicolon = header.indexOf(";");
|
|
27
|
+
// One `;`-separated parameter is exactly `base64`, without splitting them all.
|
|
28
|
+
return semicolon !== -1 && `${header.slice(semicolon)};`.includes(";base64;");
|
|
29
|
+
}
|
|
22
30
|
const hexPair = /^[0-9A-Fa-f]{2}$/;
|
|
23
31
|
/** RFC 2397 data without `;base64`: percent-escaped octets, other characters as UTF-8. Undefined
|
|
24
32
|
* when a `%` does not start an escape. */
|
|
@@ -47,7 +55,7 @@ function percentDecoded(payload) {
|
|
|
47
55
|
function fileBytes(content) {
|
|
48
56
|
const url = dataUrl.exec(content);
|
|
49
57
|
const payload = url ? content.slice(url[0].length) : content;
|
|
50
|
-
const decoded = url && !url[
|
|
58
|
+
const decoded = url && !base64Header(url[1])
|
|
51
59
|
? percentDecoded(payload)
|
|
52
60
|
: payload.length % 4 === 0 && base64.test(payload)
|
|
53
61
|
? Buffer.from(payload, "base64")
|
package/dist/provider-tools.d.ts
CHANGED
|
@@ -1,14 +1,19 @@
|
|
|
1
1
|
import type { HostedToolProvider } from "./types.js";
|
|
2
|
-
/** One provider-executed tool call. `arguments` and `
|
|
2
|
+
/** One provider-executed tool call. `arguments`, `result` and `errorText` are content. */
|
|
3
3
|
export interface HostedToolCall {
|
|
4
4
|
name: string;
|
|
5
5
|
callId?: string;
|
|
6
|
+
/** 0-based position of the call's item in the response: the OpenAI `output` index or the
|
|
7
|
+
* Anthropic `content` block index. */
|
|
8
|
+
position: number;
|
|
6
9
|
/** The provider's label (OpenAI `server_label`) or name (Anthropic `server_name`) for the MCP server. */
|
|
7
10
|
server?: string;
|
|
8
11
|
arguments?: unknown;
|
|
9
12
|
result?: unknown;
|
|
10
13
|
/** Low-cardinality failure marker for `error.type`; absent when the call succeeded. */
|
|
11
14
|
errorType?: string;
|
|
15
|
+
/** The provider's own error text for a failed OpenAI MCP call, unscrubbed and unbounded. */
|
|
16
|
+
errorText?: string;
|
|
12
17
|
}
|
|
13
18
|
/** One `mcp_list_tools` result: the tools a hosted MCP server offered. */
|
|
14
19
|
export interface HostedToolListing {
|
|
@@ -23,6 +28,12 @@ export interface HostedToolActivity {
|
|
|
23
28
|
/** Items that looked like hosted calls but could not be read. */
|
|
24
29
|
skipped: number;
|
|
25
30
|
}
|
|
31
|
+
/**
|
|
32
|
+
* A provider's error text as a failed call exports it under content capture: credentials
|
|
33
|
+
* scrubbed first, then at most 1,024 code points with `…` marking a cut, lone surrogates
|
|
34
|
+
* replaced and NUL removed. The Python SDK produces the same text.
|
|
35
|
+
*/
|
|
36
|
+
export declare function providerErrorDescription(value: string): string;
|
|
26
37
|
/**
|
|
27
38
|
* Reads the hosted tool calls out of a provider response: the `output` items of an OpenAI
|
|
28
39
|
* Responses API response, or the `content` blocks of an Anthropic Messages API response. An array
|
package/dist/provider-tools.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { MAX_CONTENT_BYTES } from "./config.js";
|
|
2
|
+
import { scrubCredentialText } from "./tool-definitions.js";
|
|
2
3
|
const MAX_PROVIDER_ITEMS = 128;
|
|
3
4
|
const MAX_PROVIDER_DEFINITIONS = 512;
|
|
4
5
|
const MAX_PROVIDER_SERVERS = 512;
|
|
@@ -19,13 +20,28 @@ function isProviderToolItem(provider, value) {
|
|
|
19
20
|
return value.type === "mcp_tool_use" || value.type === "server_tool_use";
|
|
20
21
|
}
|
|
21
22
|
function countProviderToolItems(provider, items, start = 0) {
|
|
23
|
+
let keys;
|
|
24
|
+
try {
|
|
25
|
+
keys = Object.keys(items);
|
|
26
|
+
}
|
|
27
|
+
catch {
|
|
28
|
+
// A tail that cannot be listed still holds at least one item nothing could read.
|
|
29
|
+
return 1;
|
|
30
|
+
}
|
|
22
31
|
let count = 0;
|
|
23
|
-
for (const key of
|
|
32
|
+
for (const key of keys) {
|
|
24
33
|
const index = Number(key);
|
|
25
34
|
if (!Number.isInteger(index) || index < start || index >= items.length || String(index) !== key)
|
|
26
35
|
continue;
|
|
27
|
-
|
|
36
|
+
try {
|
|
37
|
+
if (isProviderToolItem(provider, items[index]))
|
|
38
|
+
count++;
|
|
39
|
+
}
|
|
40
|
+
catch {
|
|
41
|
+
// A tail item whose `type` cannot be read counts as skipped; the bounded prefix is still
|
|
42
|
+
// recorded, as the Python SDK does.
|
|
28
43
|
count++;
|
|
44
|
+
}
|
|
29
45
|
}
|
|
30
46
|
return count;
|
|
31
47
|
}
|
|
@@ -41,6 +57,42 @@ function text(value) {
|
|
|
41
57
|
function errorCode(value) {
|
|
42
58
|
return typeof value === "string" && /^[a-z0-9_]{1,64}$/.test(value) ? value : "error";
|
|
43
59
|
}
|
|
60
|
+
/** The most of a provider's error text scrubbed, and the most exported, in code points. */
|
|
61
|
+
const MAX_ERROR_SCAN = 16_384;
|
|
62
|
+
const MAX_ERROR_TEXT = 1_024;
|
|
63
|
+
/** The first `count` code points of `text`. */
|
|
64
|
+
function codePoints(text, count) {
|
|
65
|
+
let kept = "";
|
|
66
|
+
let seen = 0;
|
|
67
|
+
for (const character of text) {
|
|
68
|
+
if (seen++ === count)
|
|
69
|
+
return { text: kept, cut: true };
|
|
70
|
+
kept += character;
|
|
71
|
+
}
|
|
72
|
+
return { text: kept, cut: false };
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* A provider's error text as a failed call exports it under content capture: credentials
|
|
76
|
+
* scrubbed first, then at most 1,024 code points with `…` marking a cut, lone surrogates
|
|
77
|
+
* replaced and NUL removed. The Python SDK produces the same text.
|
|
78
|
+
*/
|
|
79
|
+
export function providerErrorDescription(value) {
|
|
80
|
+
const scanned = codePoints(value.toWellFormed().replaceAll("\u0000", ""), MAX_ERROR_SCAN);
|
|
81
|
+
const bounded = codePoints(scrubCredentialText(scanned.text), MAX_ERROR_TEXT);
|
|
82
|
+
return bounded.cut || scanned.cut ? `${bounded.text}…` : bounded.text;
|
|
83
|
+
}
|
|
84
|
+
/** An OpenAI MCP call's `error`: its text, or an error object's string `message`. */
|
|
85
|
+
function errorText(value) {
|
|
86
|
+
let message;
|
|
87
|
+
try {
|
|
88
|
+
message = isItem(value) ? value.message : value;
|
|
89
|
+
}
|
|
90
|
+
catch {
|
|
91
|
+
// An error whose message cannot be read still fails its call; only the text is lost.
|
|
92
|
+
return undefined;
|
|
93
|
+
}
|
|
94
|
+
return typeof message === "string" && message.trim() !== "" ? message : undefined;
|
|
95
|
+
}
|
|
44
96
|
/** MCP arguments arrive as a JSON string; record the structure when it parses, else the text. */
|
|
45
97
|
function jsonArguments(value) {
|
|
46
98
|
if (typeof value !== "string")
|
|
@@ -59,88 +111,105 @@ function openaiCalls(items, activity) {
|
|
|
59
111
|
const count = Math.min(items.length, MAX_PROVIDER_ITEMS);
|
|
60
112
|
activity.skipped += countProviderToolItems("openai", items, count);
|
|
61
113
|
for (let index = 0; index < count; index++) {
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
});
|
|
114
|
+
try {
|
|
115
|
+
openaiItem(items[index], index, activity);
|
|
116
|
+
}
|
|
117
|
+
catch {
|
|
118
|
+
// An item that cannot be read is skipped; the others are still recorded.
|
|
119
|
+
activity.skipped++;
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
function openaiItem(item, position, activity) {
|
|
124
|
+
if (!isItem(item))
|
|
125
|
+
return;
|
|
126
|
+
const callId = text(item.id);
|
|
127
|
+
switch (item.type) {
|
|
128
|
+
case "mcp_call": {
|
|
129
|
+
const name = text(item.name);
|
|
130
|
+
if (!name) {
|
|
131
|
+
activity.skipped++;
|
|
81
132
|
break;
|
|
82
133
|
}
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
134
|
+
const failed = item.error !== undefined && item.error !== null;
|
|
135
|
+
const detail = failed ? errorText(item.error) : undefined;
|
|
136
|
+
activity.calls.push({
|
|
137
|
+
name,
|
|
138
|
+
callId,
|
|
139
|
+
position,
|
|
140
|
+
server: text(item.server_label),
|
|
141
|
+
arguments: jsonArguments(item.arguments),
|
|
142
|
+
...(item.output !== undefined && item.output !== null ? { result: item.output } : {}),
|
|
143
|
+
...(failed ? { errorType: "mcp_error" } : {}),
|
|
144
|
+
...(detail !== undefined ? { errorText: detail } : {}),
|
|
145
|
+
});
|
|
146
|
+
break;
|
|
147
|
+
}
|
|
148
|
+
case "mcp_list_tools": {
|
|
149
|
+
const server = text(item.server_label);
|
|
150
|
+
if (!server || !Array.isArray(item.tools)) {
|
|
151
|
+
activity.skipped++;
|
|
152
|
+
break;
|
|
153
|
+
}
|
|
154
|
+
const definitions = [];
|
|
155
|
+
const definitionCount = Math.min(item.tools.length, MAX_PROVIDER_DEFINITIONS);
|
|
156
|
+
activity.skipped += item.tools.length - definitionCount;
|
|
157
|
+
for (let index = 0; index < definitionCount; index++) {
|
|
158
|
+
try {
|
|
93
159
|
const tool = item.tools[index];
|
|
94
160
|
if (!isItem(tool))
|
|
95
161
|
continue;
|
|
162
|
+
// A null field is left out, as absent, so both SDKs digest the same definition.
|
|
96
163
|
definitions.push({
|
|
97
164
|
type: "function",
|
|
98
|
-
...(tool.name
|
|
99
|
-
...(tool.description
|
|
100
|
-
...(tool.input_schema
|
|
101
|
-
...(tool.annotations
|
|
165
|
+
...(tool.name != null ? { name: tool.name } : {}),
|
|
166
|
+
...(tool.description != null ? { description: tool.description } : {}),
|
|
167
|
+
...(tool.input_schema != null ? { parameters: tool.input_schema } : {}),
|
|
168
|
+
...(tool.annotations != null ? { annotations: tool.annotations } : {}),
|
|
102
169
|
});
|
|
103
170
|
}
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
});
|
|
109
|
-
break;
|
|
110
|
-
}
|
|
111
|
-
case "web_search_call":
|
|
112
|
-
case "file_search_call":
|
|
113
|
-
case "code_interpreter_call": {
|
|
114
|
-
const name = item.type.slice(0, -"_call".length);
|
|
115
|
-
const failed = item.status === "failed" ? { errorType: "failed" } : {};
|
|
116
|
-
if (item.type === "web_search_call")
|
|
117
|
-
activity.calls.push({ name, callId, arguments: item.action, ...failed });
|
|
118
|
-
else if (item.type === "file_search_call")
|
|
119
|
-
activity.calls.push({
|
|
120
|
-
name,
|
|
121
|
-
callId,
|
|
122
|
-
arguments: { queries: item.queries },
|
|
123
|
-
...(item.results !== undefined && item.results !== null
|
|
124
|
-
? { result: item.results }
|
|
125
|
-
: {}),
|
|
126
|
-
...failed,
|
|
127
|
-
});
|
|
128
|
-
else
|
|
129
|
-
activity.calls.push({
|
|
130
|
-
name,
|
|
131
|
-
callId,
|
|
132
|
-
arguments: { code: item.code, container_id: item.container_id },
|
|
133
|
-
...(item.outputs !== undefined && item.outputs !== null
|
|
134
|
-
? { result: item.outputs }
|
|
135
|
-
: {}),
|
|
136
|
-
...failed,
|
|
137
|
-
});
|
|
138
|
-
break;
|
|
171
|
+
catch {
|
|
172
|
+
// A definition that cannot be read is skipped; the rest of the listing is kept.
|
|
173
|
+
activity.skipped++;
|
|
174
|
+
}
|
|
139
175
|
}
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
176
|
+
activity.listings.push({
|
|
177
|
+
server,
|
|
178
|
+
definitions,
|
|
179
|
+
...(item.error !== undefined && item.error !== null ? { errorType: "mcp_error" } : {}),
|
|
180
|
+
});
|
|
181
|
+
break;
|
|
143
182
|
}
|
|
183
|
+
case "web_search_call":
|
|
184
|
+
case "file_search_call":
|
|
185
|
+
case "code_interpreter_call": {
|
|
186
|
+
const name = item.type.slice(0, -"_call".length);
|
|
187
|
+
const failed = item.status === "failed" ? { errorType: "failed" } : {};
|
|
188
|
+
if (item.type === "web_search_call")
|
|
189
|
+
activity.calls.push({ name, callId, position, arguments: item.action, ...failed });
|
|
190
|
+
else if (item.type === "file_search_call")
|
|
191
|
+
activity.calls.push({
|
|
192
|
+
name,
|
|
193
|
+
callId,
|
|
194
|
+
position,
|
|
195
|
+
arguments: { queries: item.queries },
|
|
196
|
+
...(item.results !== undefined && item.results !== null ? { result: item.results } : {}),
|
|
197
|
+
...failed,
|
|
198
|
+
});
|
|
199
|
+
else
|
|
200
|
+
activity.calls.push({
|
|
201
|
+
name,
|
|
202
|
+
callId,
|
|
203
|
+
position,
|
|
204
|
+
arguments: { code: item.code, container_id: item.container_id },
|
|
205
|
+
...(item.outputs !== undefined && item.outputs !== null ? { result: item.outputs } : {}),
|
|
206
|
+
...failed,
|
|
207
|
+
});
|
|
208
|
+
break;
|
|
209
|
+
}
|
|
210
|
+
default:
|
|
211
|
+
// Messages, reasoning, approval requests and other items are not executed tools.
|
|
212
|
+
break;
|
|
144
213
|
}
|
|
145
214
|
}
|
|
146
215
|
/** Anthropic Messages `content` blocks: a use block paired with the result block that names it. */
|
|
@@ -149,47 +218,64 @@ function anthropicCalls(blocks, activity) {
|
|
|
149
218
|
const count = Math.min(blocks.length, MAX_PROVIDER_ITEMS);
|
|
150
219
|
const truncated = blocks.length > MAX_PROVIDER_ITEMS;
|
|
151
220
|
activity.skipped += countProviderToolItems("anthropic", blocks, count);
|
|
221
|
+
// Each block is read once; one that cannot be read is skipped and the rest still recorded.
|
|
222
|
+
const readable = [];
|
|
152
223
|
for (let index = 0; index < count; index++) {
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
224
|
+
try {
|
|
225
|
+
const block = blocks[index];
|
|
226
|
+
if (isItem(block) &&
|
|
227
|
+
typeof block.type === "string" &&
|
|
228
|
+
block.type.endsWith("_tool_result") &&
|
|
229
|
+
typeof block.tool_use_id === "string")
|
|
230
|
+
results.set(block.tool_use_id, block);
|
|
231
|
+
readable.push(block);
|
|
232
|
+
}
|
|
233
|
+
catch {
|
|
234
|
+
activity.skipped++;
|
|
235
|
+
readable.push(undefined);
|
|
236
|
+
}
|
|
159
237
|
}
|
|
160
238
|
for (let index = 0; index < count; index++) {
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
continue;
|
|
164
|
-
const name = text(block.name);
|
|
165
|
-
const callId = text(block.id);
|
|
166
|
-
if (!name) {
|
|
167
|
-
activity.skipped++;
|
|
168
|
-
continue;
|
|
239
|
+
try {
|
|
240
|
+
anthropicBlock(readable[index], index, results, truncated, activity);
|
|
169
241
|
}
|
|
170
|
-
|
|
171
|
-
// When the response was truncated, an unmatched use block may have its result outside the
|
|
172
|
-
// bounded prefix. Do not export it as a successful call with a missing result.
|
|
173
|
-
if (truncated && result === undefined) {
|
|
242
|
+
catch {
|
|
174
243
|
activity.skipped++;
|
|
175
|
-
continue;
|
|
176
244
|
}
|
|
177
|
-
const content = result?.content;
|
|
178
|
-
let errorType;
|
|
179
|
-
if (result?.is_error === true)
|
|
180
|
-
errorType = "mcp_error";
|
|
181
|
-
else if (isItem(content) && typeof content.type === "string" && content.type.endsWith("_error"))
|
|
182
|
-
errorType = errorCode(content.error_code);
|
|
183
|
-
activity.calls.push({
|
|
184
|
-
name,
|
|
185
|
-
callId,
|
|
186
|
-
...(block.type === "mcp_tool_use" ? { server: text(block.server_name) } : {}),
|
|
187
|
-
arguments: block.input,
|
|
188
|
-
...(content !== undefined ? { result: content } : {}),
|
|
189
|
-
...(errorType ? { errorType } : {}),
|
|
190
|
-
});
|
|
191
245
|
}
|
|
192
246
|
}
|
|
247
|
+
function anthropicBlock(block, position, results, truncated, activity) {
|
|
248
|
+
if (!isItem(block) || (block.type !== "mcp_tool_use" && block.type !== "server_tool_use"))
|
|
249
|
+
return;
|
|
250
|
+
const name = text(block.name);
|
|
251
|
+
const callId = text(block.id);
|
|
252
|
+
if (!name) {
|
|
253
|
+
activity.skipped++;
|
|
254
|
+
return;
|
|
255
|
+
}
|
|
256
|
+
const result = callId === undefined ? undefined : results.get(callId);
|
|
257
|
+
// When the response was truncated, an unmatched use block may have its result outside the
|
|
258
|
+
// bounded prefix. Do not export it as a successful call with a missing result.
|
|
259
|
+
if (truncated && result === undefined) {
|
|
260
|
+
activity.skipped++;
|
|
261
|
+
return;
|
|
262
|
+
}
|
|
263
|
+
const content = result?.content;
|
|
264
|
+
let errorType;
|
|
265
|
+
if (result?.is_error === true)
|
|
266
|
+
errorType = "mcp_error";
|
|
267
|
+
else if (isItem(content) && typeof content.type === "string" && content.type.endsWith("_error"))
|
|
268
|
+
errorType = errorCode(content.error_code);
|
|
269
|
+
activity.calls.push({
|
|
270
|
+
name,
|
|
271
|
+
callId,
|
|
272
|
+
position,
|
|
273
|
+
...(block.type === "mcp_tool_use" ? { server: text(block.server_name) } : {}),
|
|
274
|
+
arguments: block.input,
|
|
275
|
+
...(content !== undefined ? { result: content } : {}),
|
|
276
|
+
...(errorType ? { errorType } : {}),
|
|
277
|
+
});
|
|
278
|
+
}
|
|
193
279
|
/**
|
|
194
280
|
* Reads the hosted tool calls out of a provider response: the `output` items of an OpenAI
|
|
195
281
|
* Responses API response, or the `content` blocks of an Anthropic Messages API response. An array
|
package/dist/safety.d.ts
CHANGED
|
@@ -1,7 +1,13 @@
|
|
|
1
1
|
import { type Span } from "@opentelemetry/api";
|
|
2
2
|
export declare function noopSpan(): Span;
|
|
3
3
|
export declare function safeSpan(source: Span, failed: () => void): Span;
|
|
4
|
+
/** The bounds `encodeContent` enforces while it copies a value. */
|
|
5
|
+
export interface EncodeLimits {
|
|
6
|
+
bytes: number;
|
|
7
|
+
nodes: number;
|
|
8
|
+
depth: number;
|
|
9
|
+
}
|
|
4
10
|
/** Validate a bounded data tree without invoking toJSON or property getters. */
|
|
5
|
-
export declare function encodeContent(value: unknown): string;
|
|
11
|
+
export declare function encodeContent(value: unknown, limits?: EncodeLimits): string;
|
|
6
12
|
/** Bounded conservative accounting for the record data retained by our queue, not total process RSS. */
|
|
7
13
|
export declare function estimateRecordBytes(value: unknown, limit: number): number;
|