@hue-run/sdk 0.12.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLI.md +14 -7
- package/ENVIRONMENTS.md +18 -4
- package/EVALUATIONS.md +3 -1
- package/README.md +29 -13
- package/dist/cli/eval.js +31 -8
- package/dist/client.d.ts +8 -0
- package/dist/client.js +23 -2
- package/dist/environment/types.d.ts +8 -1
- package/dist/environment/world.d.ts +8 -0
- package/dist/environment/world.js +17 -0
- package/dist/environment.d.ts +1 -1
- package/dist/environment.js +1 -1
- package/dist/evals/failure.d.ts +35 -0
- package/dist/evals/failure.js +252 -0
- package/dist/evals/local-worker.d.ts +1 -1
- package/dist/evals/runner.d.ts +21 -2
- package/dist/evals/runner.js +73 -13
- package/dist/evals/simulation.d.ts +1 -1
- package/dist/evals/types.d.ts +18 -4
- package/dist/evals.d.ts +1 -1
- package/dist/privacy.d.ts +18 -0
- package/dist/privacy.js +147 -57
- package/dist/safety.d.ts +18 -0
- package/dist/safety.js +76 -23
- package/dist/transport.d.ts +5 -0
- package/dist/transport.js +277 -57
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,252 @@
|
|
|
1
|
+
import { HueConnectionError } from "../client.js";
|
|
2
|
+
import { HueEnvironmentError } from "../environment/client.js";
|
|
3
|
+
import { HueApiError } from "./client.js";
|
|
4
|
+
const TIMEOUT_NAMES = new Set(["TimeoutError", "APIConnectionTimeoutError", "APITimeoutError"]);
|
|
5
|
+
/** Node's and Undici's codes for a call that timed out, connecting or waiting on the answer. */
|
|
6
|
+
const TIMEOUT_CODES = new Set([
|
|
7
|
+
"ETIMEDOUT",
|
|
8
|
+
"UND_ERR_CONNECT_TIMEOUT",
|
|
9
|
+
"UND_ERR_HEADERS_TIMEOUT",
|
|
10
|
+
"UND_ERR_BODY_TIMEOUT",
|
|
11
|
+
]);
|
|
12
|
+
const NETWORK_CODES = new Set([
|
|
13
|
+
"ECONNRESET",
|
|
14
|
+
"ECONNREFUSED",
|
|
15
|
+
"ENOTFOUND",
|
|
16
|
+
"EAI_AGAIN",
|
|
17
|
+
"EPIPE",
|
|
18
|
+
"ENETUNREACH",
|
|
19
|
+
"EHOSTUNREACH",
|
|
20
|
+
"UND_ERR_SOCKET",
|
|
21
|
+
]);
|
|
22
|
+
/** Links followed from the thrown error, and errors read at most. */
|
|
23
|
+
const MAX_DEPTH = 6;
|
|
24
|
+
const MAX_ERRORS = 64;
|
|
25
|
+
/** The errors an error links to: its `cause`, and the `errors` and `lastError` of an
|
|
26
|
+
* `AggregateError` or a retry error. A property that throws when read links nothing. */
|
|
27
|
+
function links(error) {
|
|
28
|
+
try {
|
|
29
|
+
const { cause, lastError, errors } = error;
|
|
30
|
+
return [cause, lastError, ...(Array.isArray(errors) ? errors.slice(0, MAX_ERRORS) : [])];
|
|
31
|
+
}
|
|
32
|
+
catch {
|
|
33
|
+
return [];
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
/** The thrown error and every error within `MAX_DEPTH` links of it, each once, so a cycle ends
|
|
37
|
+
* the walk. */
|
|
38
|
+
function linked(error) {
|
|
39
|
+
const found = [];
|
|
40
|
+
const seen = new Set();
|
|
41
|
+
let level = [error];
|
|
42
|
+
for (let depth = 0; depth <= MAX_DEPTH && level.length; depth++) {
|
|
43
|
+
const next = [];
|
|
44
|
+
for (const item of level) {
|
|
45
|
+
if (typeof item !== "object" || item === null || seen.has(item))
|
|
46
|
+
continue;
|
|
47
|
+
if (found.length === MAX_ERRORS)
|
|
48
|
+
return found;
|
|
49
|
+
seen.add(item);
|
|
50
|
+
found.push(item);
|
|
51
|
+
next.push(...links(item));
|
|
52
|
+
}
|
|
53
|
+
level = next;
|
|
54
|
+
}
|
|
55
|
+
return found;
|
|
56
|
+
}
|
|
57
|
+
/** The error's `name` and its class names, since some SDKs never set `name`. */
|
|
58
|
+
function names(error) {
|
|
59
|
+
const found = [];
|
|
60
|
+
const name = error.name;
|
|
61
|
+
if (typeof name === "string")
|
|
62
|
+
found.push(name);
|
|
63
|
+
let prototype = Object.getPrototypeOf(error);
|
|
64
|
+
for (let depth = 0; depth < MAX_DEPTH && prototype; depth++) {
|
|
65
|
+
const constructor = prototype.constructor;
|
|
66
|
+
if (typeof constructor?.name === "string")
|
|
67
|
+
found.push(constructor.name);
|
|
68
|
+
prototype = Object.getPrototypeOf(prototype);
|
|
69
|
+
}
|
|
70
|
+
return found;
|
|
71
|
+
}
|
|
72
|
+
/** The MCP TypeScript SDK's transport errors, which carry the HTTP status as `code` and no
|
|
73
|
+
* response: a POST the server answered with a failing status (`StreamableHTTPError`) or an SSE
|
|
74
|
+
* stream it refused (`SseError`). Recognized by class name, never imported. */
|
|
75
|
+
const MCP_TRANSPORT_NAMES = new Set(["StreamableHTTPError", "SseError"]);
|
|
76
|
+
/** The status an MCP transport error carries, or undefined for any other error. */
|
|
77
|
+
function transportStatus(error) {
|
|
78
|
+
const { code } = error;
|
|
79
|
+
return typeof code === "number" && names(error).some((name) => MCP_TRANSPORT_NAMES.has(name))
|
|
80
|
+
? code
|
|
81
|
+
: undefined;
|
|
82
|
+
}
|
|
83
|
+
const retryable = (status) => typeof status === "number" &&
|
|
84
|
+
(status === 408 || status === 429 || (status >= 500 && status < 600));
|
|
85
|
+
function timedOut(error) {
|
|
86
|
+
const { code } = error;
|
|
87
|
+
return ((typeof code === "string" && TIMEOUT_CODES.has(code)) ||
|
|
88
|
+
names(error).some((name) => TIMEOUT_NAMES.has(name)));
|
|
89
|
+
}
|
|
90
|
+
/** An error one of Hue's own clients threw, which carries the status Hue answered or none. */
|
|
91
|
+
function hueClientError(error) {
|
|
92
|
+
return (error instanceof HueConnectionError ||
|
|
93
|
+
error instanceof HueEnvironmentError ||
|
|
94
|
+
error instanceof HueApiError);
|
|
95
|
+
}
|
|
96
|
+
function disconnected(error) {
|
|
97
|
+
// Hue's clients report a request that got no usable response without a status.
|
|
98
|
+
if (hueClientError(error) && error.status === undefined)
|
|
99
|
+
return true;
|
|
100
|
+
const { code, message } = error;
|
|
101
|
+
const classes = names(error);
|
|
102
|
+
return ((typeof code === "string" && NETWORK_CODES.has(code)) ||
|
|
103
|
+
classes.includes("APIConnectionError") ||
|
|
104
|
+
(classes.includes("TypeError") && message === "fetch failed"));
|
|
105
|
+
}
|
|
106
|
+
/** A retryable status a service answered with: on Hue's own clients, or on an error that also
|
|
107
|
+
* carries the exchange it came from (the OpenAI and Anthropic SDKs' `headers`, the AI SDK's
|
|
108
|
+
* `url` or `responseHeaders`, a fetch `response`), so an agent's own error that only names a
|
|
109
|
+
* status stays the agent's. */
|
|
110
|
+
function refused(error) {
|
|
111
|
+
if (hueClientError(error))
|
|
112
|
+
return retryable(error.status);
|
|
113
|
+
// An MCP transport error is the exchange itself: the server answered the status it carries.
|
|
114
|
+
if (retryable(transportStatus(error)))
|
|
115
|
+
return true;
|
|
116
|
+
const { status, statusCode, headers, response, url, responseHeaders } = error;
|
|
117
|
+
const exchanged = [headers, response, url, responseHeaders].some((item) => item !== undefined && item !== null);
|
|
118
|
+
// A fetch `Response` carries the status itself.
|
|
119
|
+
const answered = response !== null && typeof response === "object"
|
|
120
|
+
? response
|
|
121
|
+
: {};
|
|
122
|
+
return (exchanged &&
|
|
123
|
+
(retryable(status) ||
|
|
124
|
+
retryable(statusCode) ||
|
|
125
|
+
retryable(answered.status) ||
|
|
126
|
+
retryable(answered.statusCode)));
|
|
127
|
+
}
|
|
128
|
+
/** A status Hue's own client answered with that is neither a success nor retryable, a 3xx or a
|
|
129
|
+
* 4xx: Hue refused the caller's key, project, environment version or world, so the case could
|
|
130
|
+
* not run as configured. The clients fetch with `redirect: "error"`, so a redirect reaches here
|
|
131
|
+
* only as a status some caller constructed; a real one surfaces without a status, as a failed
|
|
132
|
+
* connection. Only Hue's clients show it; another service's 4xx stays the agent's. */
|
|
133
|
+
function rejected(error) {
|
|
134
|
+
const { status } = error;
|
|
135
|
+
return (hueClientError(error) &&
|
|
136
|
+
typeof status === "number" &&
|
|
137
|
+
status >= 300 &&
|
|
138
|
+
status < 500 &&
|
|
139
|
+
!retryable(status));
|
|
140
|
+
}
|
|
141
|
+
/** Whether any error shows the signal; one whose properties throw when read shows none. */
|
|
142
|
+
const any = (errors, signal) => errors.some((error) => {
|
|
143
|
+
try {
|
|
144
|
+
return signal(error);
|
|
145
|
+
}
|
|
146
|
+
catch {
|
|
147
|
+
return false;
|
|
148
|
+
}
|
|
149
|
+
});
|
|
150
|
+
/** The service failure a target's throw shows, by the errors linked from it, in precedence
|
|
151
|
+
* order: a timeout, a failed connection, a retryable refusal, a status Hue's own client was
|
|
152
|
+
* refused with; null for the agent's own error. */
|
|
153
|
+
export function serviceFailureType(error) {
|
|
154
|
+
const errors = linked(error);
|
|
155
|
+
if (any(errors, timedOut))
|
|
156
|
+
return "TimedOut";
|
|
157
|
+
if (any(errors, disconnected))
|
|
158
|
+
return "ConnectionFailed";
|
|
159
|
+
if (any(errors, refused))
|
|
160
|
+
return "ServiceRefused";
|
|
161
|
+
if (any(errors, rejected))
|
|
162
|
+
return "ConfigurationRejected";
|
|
163
|
+
return null;
|
|
164
|
+
}
|
|
165
|
+
/** The header Hue's gateway names its diagnostic in, beside the JSON body's `diagnostic`. */
|
|
166
|
+
const DIAGNOSTIC_HEADER = "x-hue-diagnostic";
|
|
167
|
+
/** A diagnostic's spelling: a short lower-case token (`authorization_unavailable`). */
|
|
168
|
+
const DIAGNOSTIC = /^[a-z][a-z0-9_]{0,63}$/;
|
|
169
|
+
/** The most of a body or message read for a diagnostic. */
|
|
170
|
+
const MAX_BODY = 8192;
|
|
171
|
+
/** A header's value from a `Headers`, a record of names to values (case-insensitive), or a
|
|
172
|
+
* record of names to arrays, as the HTTP clients spell them; undefined otherwise. */
|
|
173
|
+
function headerValue(headers, name) {
|
|
174
|
+
if (headers === null || typeof headers !== "object")
|
|
175
|
+
return undefined;
|
|
176
|
+
const get = headers.get;
|
|
177
|
+
if (typeof get === "function") {
|
|
178
|
+
const value = get.call(headers, name);
|
|
179
|
+
return typeof value === "string" ? value : undefined;
|
|
180
|
+
}
|
|
181
|
+
for (const [key, value] of Object.entries(headers)) {
|
|
182
|
+
if (key.toLowerCase() !== name)
|
|
183
|
+
continue;
|
|
184
|
+
const first = Array.isArray(value) ? value[0] : value;
|
|
185
|
+
return typeof first === "string" ? first : undefined;
|
|
186
|
+
}
|
|
187
|
+
return undefined;
|
|
188
|
+
}
|
|
189
|
+
/** The `diagnostic` a JSON body names: an object's, at the top or in a JSON-RPC error's `data`,
|
|
190
|
+
* or the object a string carries from its first brace (an MCP transport error's message quotes
|
|
191
|
+
* the body it was answered). */
|
|
192
|
+
function bodyDiagnostic(body) {
|
|
193
|
+
if (typeof body === "string") {
|
|
194
|
+
const start = body.indexOf("{");
|
|
195
|
+
if (start < 0 || body.length - start > MAX_BODY)
|
|
196
|
+
return undefined;
|
|
197
|
+
try {
|
|
198
|
+
return bodyDiagnostic(JSON.parse(body.slice(start)));
|
|
199
|
+
}
|
|
200
|
+
catch {
|
|
201
|
+
return undefined;
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
if (body === null || typeof body !== "object")
|
|
205
|
+
return undefined;
|
|
206
|
+
// Hue's REST refusal names it at the top; its MCP refusal, a JSON-RPC error, in `error.data`.
|
|
207
|
+
const { diagnostic, error } = body;
|
|
208
|
+
if (typeof diagnostic === "string")
|
|
209
|
+
return diagnostic;
|
|
210
|
+
const data = error !== null && typeof error === "object" ? error.data : undefined;
|
|
211
|
+
const named = data !== null && typeof data === "object"
|
|
212
|
+
? data.diagnostic
|
|
213
|
+
: undefined;
|
|
214
|
+
return typeof named === "string" ? named : undefined;
|
|
215
|
+
}
|
|
216
|
+
/** The diagnostic one error carries: the one it names itself (Hue's own clients read the header
|
|
217
|
+
* into `diagnostic`), else the header on its `headers`, `responseHeaders` or `response.headers`,
|
|
218
|
+
* else the body on its `responseBody`, `body` or, for an MCP transport error, its message. */
|
|
219
|
+
function diagnosticOf(error) {
|
|
220
|
+
const { diagnostic, headers, responseHeaders, response, responseBody, body, message } = error;
|
|
221
|
+
const answered = response !== null && typeof response === "object"
|
|
222
|
+
? response.headers
|
|
223
|
+
: undefined;
|
|
224
|
+
return ((typeof diagnostic === "string" ? diagnostic : undefined) ??
|
|
225
|
+
headerValue(headers, DIAGNOSTIC_HEADER) ??
|
|
226
|
+
headerValue(responseHeaders, DIAGNOSTIC_HEADER) ??
|
|
227
|
+
headerValue(answered, DIAGNOSTIC_HEADER) ??
|
|
228
|
+
bodyDiagnostic(responseBody) ??
|
|
229
|
+
bodyDiagnostic(body) ??
|
|
230
|
+
(transportStatus(error) !== undefined ? bodyDiagnostic(message) : undefined));
|
|
231
|
+
}
|
|
232
|
+
/**
|
|
233
|
+
* The service's own word for what failed, when the answer a target's throw carries names one:
|
|
234
|
+
* Hue's gateway diagnostic (`authorization_unavailable`, `gateway_failure`), read from the
|
|
235
|
+
* `x-hue-diagnostic` header or the JSON body's `diagnostic` of any error linked from the throw,
|
|
236
|
+
* nearest first. Undefined when no answer names one, or the name is not a diagnostic's token.
|
|
237
|
+
* Read for a service failure (`serviceFailureType`), where the runner records it as the error's
|
|
238
|
+
* `cause`.
|
|
239
|
+
*/
|
|
240
|
+
export function serviceFailureCause(error) {
|
|
241
|
+
for (const item of linked(error)) {
|
|
242
|
+
try {
|
|
243
|
+
const diagnostic = diagnosticOf(item);
|
|
244
|
+
if (diagnostic !== undefined && DIAGNOSTIC.test(diagnostic))
|
|
245
|
+
return diagnostic;
|
|
246
|
+
}
|
|
247
|
+
catch {
|
|
248
|
+
// A property that throws when read names nothing.
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
return undefined;
|
|
252
|
+
}
|
|
@@ -104,7 +104,7 @@ export interface RunLocalAgentOptions {
|
|
|
104
104
|
concurrency?: number;
|
|
105
105
|
/** What a case does when its telemetry is not accepted; see
|
|
106
106
|
* `traceNotAccepted` of {@link runExperiment}. Defaults to `"stop"`. */
|
|
107
|
-
traceNotAccepted?: "stop" | "fail_case";
|
|
107
|
+
traceNotAccepted?: "stop" | "fail_case" | "pending";
|
|
108
108
|
/** Called as each case is failed for its telemetry; see `onTelemetryNotAccepted` of
|
|
109
109
|
* {@link runExperiment}. */
|
|
110
110
|
onTelemetryNotAccepted?(entry: TelemetryNotAccepted): void | Promise<void>;
|
package/dist/evals/runner.d.ts
CHANGED
|
@@ -123,9 +123,13 @@ export interface RunExperimentOptions extends RunnerOptions {
|
|
|
123
123
|
* rejects. `"fail_case"` completes the execution as failed instead, with the evidence omitted
|
|
124
124
|
* under the reason `telemetry_not_accepted` and without its output or generated files, so no
|
|
125
125
|
* scorer can pass it; reports it in {@link RunnerReport.telemetryNotAccepted} and goes on with
|
|
126
|
-
* the other cases.
|
|
126
|
+
* the other cases. `"pending"` completes the execution in its true state with
|
|
127
|
+
* `traceEvidence: "pending"` and the count of spans the case ended: Hue scores the sealed world
|
|
128
|
+
* at once and attaches the trace once it holds every span, or records the evidence omitted a
|
|
129
|
+
* day later; reports it in {@link RunnerReport.tracePending} and goes on. A saved outcome whose
|
|
130
|
+
* export the receipt does not accept on resume completes the same way.
|
|
127
131
|
*/
|
|
128
|
-
traceNotAccepted?: "stop" | "fail_case";
|
|
132
|
+
traceNotAccepted?: "stop" | "fail_case" | "pending";
|
|
129
133
|
/** Called when this call completes a case as failed under `traceNotAccepted: "fail_case"`,
|
|
130
134
|
* before the run goes on, so the failure is known even if a later case stops the run. A call
|
|
131
135
|
* that resumes an already completed case lists it in the report without calling again. */
|
|
@@ -152,6 +156,21 @@ export interface RunnerReport {
|
|
|
152
156
|
/** Cases completed as failed because their telemetry was not accepted, under
|
|
153
157
|
* `traceNotAccepted: "fail_case"`; absent when there were none. */
|
|
154
158
|
telemetryNotAccepted?: TelemetryNotAccepted[];
|
|
159
|
+
/** Cases completed with their trace evidence pending, under `traceNotAccepted: "pending"`;
|
|
160
|
+
* absent when there were none. */
|
|
161
|
+
tracePending?: TracePending[];
|
|
162
|
+
}
|
|
163
|
+
/** A case completed with its trace still to arrive. */
|
|
164
|
+
export interface TracePending {
|
|
165
|
+
/** Frozen case ID. */
|
|
166
|
+
caseId: string;
|
|
167
|
+
/** The case's external key. */
|
|
168
|
+
caseKey: string;
|
|
169
|
+
/** Execution completed pending. */
|
|
170
|
+
executionId: string;
|
|
171
|
+
/** How many spans the case ended, which Hue holds the arriving trace to; absent when the
|
|
172
|
+
* checkpoint recorded no count. */
|
|
173
|
+
spanCount?: number;
|
|
155
174
|
}
|
|
156
175
|
/** Export issue counts, never content: which signal, what happened, the HTTP status when Hue
|
|
157
176
|
* answered, and how many records. */
|
package/dist/evals/runner.js
CHANGED
|
@@ -6,6 +6,7 @@ import { ROOT_CONTEXT } from "@opentelemetry/api";
|
|
|
6
6
|
import { HueExportError } from "../transport.js";
|
|
7
7
|
import { HueApiError } from "./client.js";
|
|
8
8
|
import { loadEnvironmentEvidence } from "./environment-evidence.js";
|
|
9
|
+
import { serviceFailureCause, serviceFailureType } from "./failure.js";
|
|
9
10
|
import { CheckpointStore } from "./checkpoint.js";
|
|
10
11
|
import { onForcedExit } from "./exit-cleanup.js";
|
|
11
12
|
import { assertSafeFileNames, downloadCaseFiles, downloadCaseInputs, localOutputFiles, verifyCaseFiles, OutputFileError, stageOutputFiles, targetFileRoles, uploadOutputFiles, } from "./files.js";
|
|
@@ -252,6 +253,20 @@ async function pool(items, concurrency, execute) {
|
|
|
252
253
|
/** Whether Hue holds the case's telemetry, read from the trace receipt: its root span and at
|
|
253
254
|
* least as many spans as the case ended; false when they are not there by the wait's end or the
|
|
254
255
|
* receipt cannot be read, and for a checkpoint an older runner saved without the span. */
|
|
256
|
+
/** Turn a prepared completion into a pending one: the trace is declared but still to arrive,
|
|
257
|
+
* and the span count the checkpoint recorded at the flush goes with it, so Hue freezes the trace
|
|
258
|
+
* only once it holds every span the case ended, never a root span alone. */
|
|
259
|
+
function pendingCompletion(prepared) {
|
|
260
|
+
prepared.complete.traceEvidence = "pending";
|
|
261
|
+
delete prepared.complete.omissionReason;
|
|
262
|
+
if (prepared.trace?.spans !== undefined)
|
|
263
|
+
prepared.complete.traceSpanCount = prepared.trace.spans;
|
|
264
|
+
}
|
|
265
|
+
/** The reverse, once the receipt accepted the trace after all: required evidence, no count. */
|
|
266
|
+
function acceptedCompletion(prepared) {
|
|
267
|
+
prepared.complete.traceEvidence = "required";
|
|
268
|
+
delete prepared.complete.traceSpanCount;
|
|
269
|
+
}
|
|
255
270
|
async function traceLanded(hue, trace) {
|
|
256
271
|
// Without the count of spans the case ended, the receipt cannot tell a lost child span from a
|
|
257
272
|
// landed trace, and does not accept.
|
|
@@ -417,17 +432,28 @@ export async function runExperiment(options) {
|
|
|
417
432
|
const filesRoot = resolve(options.filesDirectory ?? join(options.checkpointDirectory, "files"));
|
|
418
433
|
const sanitize = (message) => message.slice(0, 4000).toWellFormed().replaceAll("\u0000", "");
|
|
419
434
|
// A target stopped at its world's deadline is a `TargetTimeout`, told apart from an error the
|
|
420
|
-
// target raised itself: the agent did not finish, rather than finishing wrongly.
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
435
|
+
// target raised itself: the agent did not finish, rather than finishing wrongly. A service the
|
|
436
|
+
// agent called that refused, dropped the connection or timed out is named apart from the
|
|
437
|
+
// agent's own error (`failure.ts`), so Hue counts it as infrastructure, not as the agent, with
|
|
438
|
+
// the service's own word for what failed as the error's `cause` when its answer names one
|
|
439
|
+
// (Hue's gateway diagnostic).
|
|
440
|
+
const errorPayload = (error) => {
|
|
441
|
+
const service = error instanceof OutputTooLargeError || error instanceof TargetTimeoutError
|
|
442
|
+
? null
|
|
443
|
+
: serviceFailureType(error);
|
|
444
|
+
const cause = service ? serviceFailureCause(error) : undefined;
|
|
445
|
+
return {
|
|
446
|
+
type: error instanceof OutputTooLargeError
|
|
447
|
+
? "OutputTooLarge"
|
|
448
|
+
: error instanceof TargetTimeoutError
|
|
449
|
+
? "TargetTimeout"
|
|
450
|
+
: (service ?? "TargetError"),
|
|
451
|
+
...(cause === undefined ? {} : { cause }),
|
|
452
|
+
...(options.persistResultContent && error instanceof Error
|
|
453
|
+
? { message: sanitize(error.message) }
|
|
454
|
+
: {}),
|
|
455
|
+
};
|
|
456
|
+
};
|
|
431
457
|
/** Publish staged files, score with every verified file on disk and save the completion. */
|
|
432
458
|
async function prepare(file, saved, frozenCase, caseDirectory,
|
|
433
459
|
/** The target's output, in memory when result content is not persisted. */
|
|
@@ -707,7 +733,15 @@ export async function runExperiment(options) {
|
|
|
707
733
|
if (options.traceEvidence.mode !== "omit")
|
|
708
734
|
throw error;
|
|
709
735
|
}
|
|
710
|
-
if (exportError !== undefined &&
|
|
736
|
+
if (exportError !== undefined &&
|
|
737
|
+
options.traceEvidence.mode !== "omit" &&
|
|
738
|
+
options.traceNotAccepted === "pending") {
|
|
739
|
+
// The outcome stands as the target gave it; Hue attaches the trace once it holds
|
|
740
|
+
// every span the case ended, which the checkpoint's count tells it.
|
|
741
|
+
pendingCompletion(checkpoint);
|
|
742
|
+
await store.write(file, checkpoint);
|
|
743
|
+
}
|
|
744
|
+
else if (exportError !== undefined && options.traceEvidence.mode !== "omit") {
|
|
711
745
|
if (options.traceNotAccepted !== "fail_case")
|
|
712
746
|
throw exportError;
|
|
713
747
|
// Required evidence that Hue did not accept fails the case rather than leaving its
|
|
@@ -746,9 +780,26 @@ export async function runExperiment(options) {
|
|
|
746
780
|
prepared.complete.traceEvidence !== "omit" &&
|
|
747
781
|
(await traceLanded(options.hue, prepared.trace))) {
|
|
748
782
|
prepared.exportState = "accepted";
|
|
783
|
+
// A completion made pending by an earlier receipt read is required again: the trace
|
|
784
|
+
// landed, so Hue freezes it at completion as for any accepted case.
|
|
785
|
+
if (prepared.complete.traceEvidence === "pending")
|
|
786
|
+
acceptedCompletion(prepared);
|
|
749
787
|
await store.write(file, prepared);
|
|
750
788
|
}
|
|
751
|
-
|
|
789
|
+
// Under the pending policy a saved outcome the receipt does not accept, whether no flush
|
|
790
|
+
// decided it or an export failure named its trace, completes with the trace pending rather
|
|
791
|
+
// than being refused: Hue attaches the trace when it arrives in full.
|
|
792
|
+
if (options.traceNotAccepted === "pending" &&
|
|
793
|
+
prepared.exportState !== "accepted" &&
|
|
794
|
+
prepared.exportState !== "not_accepted" &&
|
|
795
|
+
prepared.complete.traceEvidence !== "omit" &&
|
|
796
|
+
prepared.complete.traceEvidence !== "pending") {
|
|
797
|
+
pendingCompletion(prepared);
|
|
798
|
+
await store.write(file, prepared);
|
|
799
|
+
}
|
|
800
|
+
if (prepared.exportState !== "accepted" &&
|
|
801
|
+
prepared.complete.traceEvidence !== "omit" &&
|
|
802
|
+
prepared.complete.traceEvidence !== "pending")
|
|
752
803
|
throw new TraceExportUnacknowledgedError(prepared.executionId);
|
|
753
804
|
const save = () => store.write(file, prepared);
|
|
754
805
|
const notAccepted = prepared.exportState === "not_accepted"
|
|
@@ -770,6 +821,15 @@ export async function runExperiment(options) {
|
|
|
770
821
|
}
|
|
771
822
|
if (notAccepted)
|
|
772
823
|
(report.telemetryNotAccepted ??= []).push(notAccepted);
|
|
824
|
+
if (prepared.complete.traceEvidence === "pending")
|
|
825
|
+
(report.tracePending ??= []).push({
|
|
826
|
+
caseId: item.id,
|
|
827
|
+
caseKey: item.externalKey,
|
|
828
|
+
executionId: prepared.executionId,
|
|
829
|
+
...(prepared.complete.traceSpanCount !== undefined
|
|
830
|
+
? { spanCount: prepared.complete.traceSpanCount }
|
|
831
|
+
: {}),
|
|
832
|
+
});
|
|
773
833
|
const results = await uploadScores(options, experiment.evaluation.id, prepared.scores, save, versions);
|
|
774
834
|
report.subjectIds.push(prepared.completion.subjectId);
|
|
775
835
|
report.resultIds.push(...results);
|
|
@@ -160,7 +160,7 @@ export interface RunSimulationOptions {
|
|
|
160
160
|
};
|
|
161
161
|
/** What a case does when required telemetry is not accepted; see
|
|
162
162
|
* `traceNotAccepted` of {@link runExperiment}. Defaults to `"stop"`. */
|
|
163
|
-
traceNotAccepted?: "stop" | "fail_case";
|
|
163
|
+
traceNotAccepted?: "stop" | "fail_case" | "pending";
|
|
164
164
|
/** Called as each case is failed for its telemetry; see `onTelemetryNotAccepted` of
|
|
165
165
|
* {@link runExperiment}. */
|
|
166
166
|
onTelemetryNotAccepted?(entry: TelemetryNotAccepted): void | Promise<void>;
|
package/dist/evals/types.d.ts
CHANGED
|
@@ -542,6 +542,16 @@ export interface TypedError {
|
|
|
542
542
|
type: string;
|
|
543
543
|
/** Optional message; stored only when result content is persisted. */
|
|
544
544
|
message?: string;
|
|
545
|
+
/** For a runner of your own that writes a `type` of its own: the class Hue files it under
|
|
546
|
+
* (`agent`, `infrastructure`, `configuration`, `telemetry` or `unsupported`), which Hue reads
|
|
547
|
+
* only for a name it does not know. The SDK's own types need none. */
|
|
548
|
+
class?: "agent" | "infrastructure" | "configuration" | "telemetry" | "unsupported";
|
|
549
|
+
/** For a service failure (`ServiceRefused`, `ConnectionFailed`, `TimedOut`,
|
|
550
|
+
* `ConfigurationRejected`): the service's own word for what failed, when its answer names one,
|
|
551
|
+
* such as Hue's gateway diagnostic (`authorization_unavailable`) from the answer's
|
|
552
|
+
* `x-hue-diagnostic` header or its JSON body's `diagnostic`. A short lower-case token; absent
|
|
553
|
+
* when the answer names none. */
|
|
554
|
+
cause?: string;
|
|
545
555
|
}
|
|
546
556
|
/** Input for {@link EvaluationClient.startExecution}. */
|
|
547
557
|
export interface StartExecution {
|
|
@@ -570,10 +580,13 @@ export interface CompleteExecution {
|
|
|
570
580
|
error?: TypedError;
|
|
571
581
|
/** Trace revision the stored snapshot must have reached. */
|
|
572
582
|
expectedTraceRevision?: number;
|
|
573
|
-
/** Whether stored trace evidence is required
|
|
574
|
-
|
|
583
|
+
/** Whether stored trace evidence is required, explicitly omitted, or pending: declared and
|
|
584
|
+
* still to arrive, which Hue attaches once the export is complete. */
|
|
585
|
+
traceEvidence?: "required" | "omit" | "pending";
|
|
575
586
|
/** Why trace evidence was omitted. */
|
|
576
587
|
omissionReason?: string;
|
|
588
|
+
/** With `pending`: how many spans the case ended, which Hue holds the arriving trace to. */
|
|
589
|
+
traceSpanCount?: number;
|
|
577
590
|
}
|
|
578
591
|
/** Result of {@link EvaluationClient.completeExecution}. */
|
|
579
592
|
export interface Completion {
|
|
@@ -633,8 +646,9 @@ export interface Subject {
|
|
|
633
646
|
experimentId: string;
|
|
634
647
|
/** Attempt number of the execution. */
|
|
635
648
|
attempt: number;
|
|
636
|
-
/** Whether trace evidence was captured, omitted with a reason
|
|
637
|
-
|
|
649
|
+
/** Whether trace evidence was captured, omitted with a reason, not requested, or is pending
|
|
650
|
+
* (declared and still to arrive; Hue attaches it once the export is complete). */
|
|
651
|
+
traceEvidence: "captured" | "omitted" | "not_requested" | "pending";
|
|
638
652
|
/** Declared OpenTelemetry trace ID, or `null`. */
|
|
639
653
|
traceExternalId: string | null;
|
|
640
654
|
/** Reason trace evidence was omitted, or `null`. */
|
package/dist/evals.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
export { ArtifactSizeError, createEvaluationClient, EvaluationClient, HueApiError, isTransientApiError, } from "./evals/client.js";
|
|
2
2
|
export type { EvaluationClientOptions } from "./evals/client.js";
|
|
3
3
|
export { runExperiment, rescore, UncertainExecutionError, OutcomeSerializationError, TargetCancelledError, TargetOutcomeUncertainError, TargetTimeoutError, TraceExportUnacknowledgedError, } from "./evals/runner.js";
|
|
4
|
-
export type { RunExperimentOptions, RunExperimentTargetContext, RescoreOptions, RunnerReport, TelemetryIssueCount, TelemetryNotAccepted, } from "./evals/runner.js";
|
|
4
|
+
export type { RunExperimentOptions, RunExperimentTargetContext, RescoreOptions, RunnerReport, TelemetryIssueCount, TelemetryNotAccepted, TracePending, } from "./evals/runner.js";
|
|
5
5
|
export { TargetResult, withFiles } from "./evals/types.js";
|
|
6
6
|
export { CaseFileError, outputContentTypes, outputFileLimits, OutputFileError, safeFilename, targetFileRoles, } from "./evals/files.js";
|
|
7
7
|
export type { CaseFileErrorCode } from "./evals/files.js";
|
package/dist/privacy.d.ts
CHANGED
|
@@ -2,8 +2,26 @@ import type { ReadableSpan } from "@opentelemetry/sdk-trace";
|
|
|
2
2
|
import type { ReadableLogRecord } from "@opentelemetry/sdk-logs";
|
|
3
3
|
import { type Resource } from "@opentelemetry/resources";
|
|
4
4
|
import type { HueOptions } from "./types.js";
|
|
5
|
+
import { truncateUtf8 } from "./safety.js";
|
|
6
|
+
export { truncateUtf8 };
|
|
5
7
|
/** Attribute keys (and their dotted children) removed in metadata-only mode. */
|
|
6
8
|
export declare const contentPrefixes: string[];
|
|
9
|
+
export declare function isContentKey(key: string): boolean;
|
|
10
|
+
/** The record attribute listing the keys whose values were cut to fit Hue's 256 KiB value cap:
|
|
11
|
+
* Hue's receiver writes it for the values it cuts, and the SDK writes it for the values it cuts
|
|
12
|
+
* before export, under the same name, so a reader learns of a cut value from one place. */
|
|
13
|
+
export declare const TRUNCATED_KEY = "hue.truncated";
|
|
14
|
+
/** The size of a structured value the marker replaced. */
|
|
15
|
+
export declare const TRUNCATED_BYTES_KEY = "hue.truncated_bytes";
|
|
16
|
+
/** The root span attribute counting this trace's records the SDK could not export at all (a
|
|
17
|
+
* record over the request limit even with every content value shed, or one its queue had no
|
|
18
|
+
* room for): what Hue stores of the trace is incomplete by that many records. */
|
|
19
|
+
export declare const DROPPED_RECORDS_KEY = "hue.sdk.dropped_records";
|
|
20
|
+
/** Hue's receiver's own marker for a structured value it replaced: nothing of the value, its
|
|
21
|
+
* size, and the flag a reader tests. */
|
|
22
|
+
export declare function truncatedMarker(bytes: number): Record<string, unknown>;
|
|
23
|
+
/** `listed` with `keys` added, each once, in order. */
|
|
24
|
+
export declare function withTruncatedKeys(listed: unknown, keys: readonly string[]): string[];
|
|
7
25
|
export type ResourceCache = WeakMap<Resource, Resource>;
|
|
8
26
|
export declare function redactSpan(span: ReadableSpan, options: HueOptions, cache: ResourceCache): ReadableSpan;
|
|
9
27
|
export declare function redactLog(log: ReadableLogRecord, options: HueOptions, cache: ResourceCache): ReadableLogRecord;
|