@hue-run/sdk 0.8.1 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLI.md +100 -29
- package/ENVIRONMENTS.md +73 -8
- package/EVALUATIONS.md +39 -19
- package/README.md +51 -22
- package/dist/cli/eval-direct.d.ts +12 -1
- package/dist/cli/eval-direct.js +145 -33
- package/dist/cli/eval.d.ts +28 -1
- package/dist/cli/eval.js +469 -72
- package/dist/client.js +10 -20
- package/dist/environment/client.d.ts +10 -2
- package/dist/environment/client.js +43 -12
- package/dist/environment/types.d.ts +30 -10
- package/dist/environment.d.ts +1 -1
- package/dist/environment.js +1 -1
- package/dist/evals/checkpoint.d.ts +15 -1
- package/dist/evals/checkpoint.js +34 -3
- package/dist/evals/client.d.ts +19 -3
- package/dist/evals/client.js +82 -34
- package/dist/evals/environment-target.d.ts +23 -3
- package/dist/evals/environment-target.js +73 -10
- package/dist/evals/exit-cleanup.d.ts +4 -0
- package/dist/evals/exit-cleanup.js +26 -0
- package/dist/evals/files.d.ts +59 -1
- package/dist/evals/files.js +187 -27
- package/dist/evals/local-worker.d.ts +21 -3
- package/dist/evals/local-worker.js +14 -0
- package/dist/evals/runner.d.ts +45 -0
- package/dist/evals/runner.js +164 -13
- package/dist/evals/scenarios.d.ts +8 -6
- package/dist/evals/scenarios.js +75 -15
- package/dist/evals/simulation.d.ts +17 -4
- package/dist/evals/simulation.js +8 -0
- package/dist/evals/types.d.ts +10 -1
- package/dist/evals/verdicts.d.ts +15 -2
- package/dist/evals/verdicts.js +43 -9
- package/dist/evals.d.ts +4 -3
- package/dist/evals.js +2 -2
- package/dist/inline-files.d.ts +4 -0
- package/dist/inline-files.js +40 -17
- package/dist/provider-tools.js +30 -3
- package/dist/snapshot.d.ts +2 -2
- package/dist/snapshot.js +47 -12
- package/dist/transport.js +2 -2
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +1 -1
package/dist/client.js
CHANGED
|
@@ -40,7 +40,7 @@ function identifier(value) {
|
|
|
40
40
|
throw new TypeError("Session/user/workspace identifiers must contain 1–4096 valid characters");
|
|
41
41
|
return value;
|
|
42
42
|
}
|
|
43
|
-
/** A usable metadata label:
|
|
43
|
+
/** A usable metadata label: non-blank, at most 256 UTF-16 code units, with no NUL or unpaired surrogate. */
|
|
44
44
|
function isLabel(value) {
|
|
45
45
|
return (typeof value === "string" &&
|
|
46
46
|
value.trim() !== "" &&
|
|
@@ -48,18 +48,6 @@ function isLabel(value) {
|
|
|
48
48
|
!value.includes("\u0000") &&
|
|
49
49
|
value.isWellFormed());
|
|
50
50
|
}
|
|
51
|
-
/** A source label uses the stricter wire-safe validation without changing existing labels. */
|
|
52
|
-
function isSourceLabel(value) {
|
|
53
|
-
return (typeof value === "string" &&
|
|
54
|
-
value.trim() !== "" &&
|
|
55
|
-
value.length <= 256 &&
|
|
56
|
-
!value.includes("\u0000") &&
|
|
57
|
-
value.isWellFormed());
|
|
58
|
-
}
|
|
59
|
-
/** A label that is also free of NUL and unpaired surrogates, which export would reject. */
|
|
60
|
-
function isTextLabel(value) {
|
|
61
|
-
return isLabel(value) && !value.includes("\u0000") && value.isWellFormed();
|
|
62
|
-
}
|
|
63
51
|
/**
|
|
64
52
|
* Runs `work` exactly once with `active` as OpenTelemetry's current context, so instrumentations
|
|
65
53
|
* that use the global API parent under the Hue span. Without a registered context manager
|
|
@@ -330,11 +318,11 @@ export class HueClient {
|
|
|
330
318
|
"gen_ai.operation.name": "execute_tool",
|
|
331
319
|
"gen_ai.tool.name": name,
|
|
332
320
|
};
|
|
333
|
-
const stamp = (key, value
|
|
321
|
+
const stamp = (key, value) => {
|
|
334
322
|
if (value === undefined)
|
|
335
323
|
return;
|
|
336
324
|
// A blank or non-string label is omitted and counted; the tool call itself still runs.
|
|
337
|
-
if (
|
|
325
|
+
if (isLabel(value))
|
|
338
326
|
attributes[key] = value;
|
|
339
327
|
else if (this.enabled && !this.closed)
|
|
340
328
|
this.transport.instrumentationFailure();
|
|
@@ -342,8 +330,8 @@ export class HueClient {
|
|
|
342
330
|
stamp("gen_ai.tool.call.id", options.callId);
|
|
343
331
|
stamp("mcp.server.name", options.mcp?.name);
|
|
344
332
|
stamp("mcp.server.version", options.mcp?.version);
|
|
345
|
-
stamp("hue.mcp.provider", options.mcp?.provider
|
|
346
|
-
stamp("hue.mcp.surface", options.mcp?.surface
|
|
333
|
+
stamp("hue.mcp.provider", options.mcp?.provider);
|
|
334
|
+
stamp("hue.mcp.surface", options.mcp?.surface);
|
|
347
335
|
return this.withSpan(`execute_tool ${name}`, async ({ span }) => {
|
|
348
336
|
this.setContent(span, "gen_ai.tool.call.arguments", input);
|
|
349
337
|
const result = await execute();
|
|
@@ -555,7 +543,9 @@ export class HueClient {
|
|
|
555
543
|
const attributes = {};
|
|
556
544
|
if (label === undefined)
|
|
557
545
|
return attributes;
|
|
558
|
-
const info = options.servers
|
|
546
|
+
const info = options.servers && Object.hasOwn(options.servers, label)
|
|
547
|
+
? options.servers[label]
|
|
548
|
+
: undefined;
|
|
559
549
|
for (const [key, value] of [
|
|
560
550
|
["mcp.server.name", info?.name ?? label],
|
|
561
551
|
["mcp.server.version", info?.version],
|
|
@@ -630,7 +620,7 @@ export class HueClient {
|
|
|
630
620
|
const { role, mediaType, data, name } = file;
|
|
631
621
|
if (role !== "input" && role !== "attachment" && role !== "output")
|
|
632
622
|
throw new TypeError("Invalid file role");
|
|
633
|
-
if (!
|
|
623
|
+
if (!isLabel(mediaType))
|
|
634
624
|
throw new TypeError("Invalid media type");
|
|
635
625
|
let sha256 = typeof file.sha256 === "string" ? file.sha256.toLowerCase() : file.sha256;
|
|
636
626
|
let byteSize = file.byteSize;
|
|
@@ -673,7 +663,7 @@ export class HueClient {
|
|
|
673
663
|
if (byteSize !== undefined)
|
|
674
664
|
attributes["hue.file.size"] = byteSize;
|
|
675
665
|
if (this.captureContent && name !== undefined) {
|
|
676
|
-
if (
|
|
666
|
+
if (isLabel(name))
|
|
677
667
|
attributes["hue.file.name"] = name;
|
|
678
668
|
else
|
|
679
669
|
this.transport.instrumentationFailure();
|
|
@@ -16,12 +16,18 @@ export declare class HueEnvironmentError extends Error {
|
|
|
16
16
|
readonly status?: number | undefined;
|
|
17
17
|
/** Hue's `Retry-After` in milliseconds, bounded, when a 429 or 503 carried one. */
|
|
18
18
|
readonly retryAfterMs?: number | undefined;
|
|
19
|
+
/** The server's `X-Hue-Diagnostic` code, when it is a short snake-case value. */
|
|
20
|
+
readonly diagnostic?: string | undefined;
|
|
19
21
|
constructor(
|
|
20
22
|
/** HTTP status when Hue answered; absent for transport, timeout or parse failure. */
|
|
21
23
|
status?: number | undefined,
|
|
22
24
|
/** Hue's `Retry-After` in milliseconds, bounded, when a 429 or 503 carried one. */
|
|
23
|
-
retryAfterMs?: number | undefined
|
|
25
|
+
retryAfterMs?: number | undefined,
|
|
26
|
+
/** The server's `X-Hue-Diagnostic` code, when it is a short snake-case value. */
|
|
27
|
+
diagnostic?: string | undefined);
|
|
24
28
|
}
|
|
29
|
+
/** A connection failure or a status the client retries; seal polling continues through these. */
|
|
30
|
+
export declare function isTransientEnvironmentError(error: unknown): boolean;
|
|
25
31
|
/** Typed client for authored environments, isolated runs and immutable journals. */
|
|
26
32
|
export declare class EnvironmentClient {
|
|
27
33
|
/** Validated Hue origin. */
|
|
@@ -47,7 +53,9 @@ export declare class EnvironmentClient {
|
|
|
47
53
|
/** Creates or recovers one fresh isolated world using a stable idempotency key. */
|
|
48
54
|
createRun(input: CreateRunInput): Promise<EnvironmentRun>;
|
|
49
55
|
/** Reads authoritative current or sealed world state. */
|
|
50
|
-
getRun(runId: string
|
|
56
|
+
getRun(runId: string, options?: {
|
|
57
|
+
signal?: AbortSignal;
|
|
58
|
+
}): Promise<{
|
|
51
59
|
id: string;
|
|
52
60
|
environmentVersionId: string;
|
|
53
61
|
executionId: string | null;
|
|
@@ -4,20 +4,35 @@ import { aggregateBounds, json, uuid, valueBounds } from "../evals/json.js";
|
|
|
4
4
|
export class HueEnvironmentError extends Error {
|
|
5
5
|
status;
|
|
6
6
|
retryAfterMs;
|
|
7
|
+
diagnostic;
|
|
7
8
|
constructor(
|
|
8
9
|
/** HTTP status when Hue answered; absent for transport, timeout or parse failure. */
|
|
9
10
|
status,
|
|
10
11
|
/** Hue's `Retry-After` in milliseconds, bounded, when a 429 or 503 carried one. */
|
|
11
|
-
retryAfterMs
|
|
12
|
+
retryAfterMs,
|
|
13
|
+
/** The server's `X-Hue-Diagnostic` code, when it is a short snake-case value. */
|
|
14
|
+
diagnostic) {
|
|
12
15
|
super(status
|
|
13
|
-
? `Hue environment request failed (HTTP ${status})`
|
|
16
|
+
? `Hue environment request failed (HTTP ${status}${diagnostic ? `, ${diagnostic}` : ""})`
|
|
14
17
|
: "Hue environment connection or response failed");
|
|
15
18
|
this.status = status;
|
|
16
19
|
this.retryAfterMs = retryAfterMs;
|
|
20
|
+
this.diagnostic = diagnostic;
|
|
17
21
|
this.name = "HueEnvironmentError";
|
|
18
22
|
}
|
|
19
23
|
}
|
|
24
|
+
const DIAGNOSTIC = /^[a-z_]{1,64}$/;
|
|
25
|
+
/** The response's diagnostic code, or undefined when absent or invalid. */
|
|
26
|
+
function diagnosticOf(response) {
|
|
27
|
+
const value = response.headers.get("x-hue-diagnostic");
|
|
28
|
+
return value !== null && DIAGNOSTIC.test(value) ? value : undefined;
|
|
29
|
+
}
|
|
20
30
|
const RETRYABLE = new Set([408, 429, 500, 502, 503, 504]);
|
|
31
|
+
/** A connection failure or a status the client retries; seal polling continues through these. */
|
|
32
|
+
export function isTransientEnvironmentError(error) {
|
|
33
|
+
return (error instanceof HueEnvironmentError &&
|
|
34
|
+
(error.status === undefined || RETRYABLE.has(error.status)));
|
|
35
|
+
}
|
|
21
36
|
/** A `Retry-After` longer than this waits this long: Hue asks for a second, never minutes. */
|
|
22
37
|
const MAX_RETRY_AFTER_MS = 10_000;
|
|
23
38
|
const TRACEPARENT = /^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$/;
|
|
@@ -64,9 +79,10 @@ export class EnvironmentClient {
|
|
|
64
79
|
throw new RangeError("maxAttempts must be 1–10");
|
|
65
80
|
this.maxAttempts = attempts;
|
|
66
81
|
}
|
|
67
|
-
async send(method, path, payload) {
|
|
82
|
+
async send(method, path, payload, options = {}) {
|
|
68
83
|
let response;
|
|
69
84
|
try {
|
|
85
|
+
const timeout = AbortSignal.timeout(this.timeoutMillis);
|
|
70
86
|
response = await fetch(`${this.baseUrl}/api/v1${path}`, {
|
|
71
87
|
method,
|
|
72
88
|
headers: {
|
|
@@ -75,7 +91,7 @@ export class EnvironmentClient {
|
|
|
75
91
|
},
|
|
76
92
|
body: payload,
|
|
77
93
|
redirect: "error",
|
|
78
|
-
signal: AbortSignal.
|
|
94
|
+
signal: options.signal ? AbortSignal.any([options.signal, timeout]) : timeout,
|
|
79
95
|
});
|
|
80
96
|
}
|
|
81
97
|
catch {
|
|
@@ -83,7 +99,7 @@ export class EnvironmentClient {
|
|
|
83
99
|
}
|
|
84
100
|
if (!response.ok) {
|
|
85
101
|
await response.body?.cancel();
|
|
86
|
-
throw new HueEnvironmentError(response.status, response.status === 429 || response.status === 503 ? retryAfterMillis(response) : undefined);
|
|
102
|
+
throw new HueEnvironmentError(response.status, response.status === 429 || response.status === 503 ? retryAfterMillis(response) : undefined, diagnosticOf(response));
|
|
87
103
|
}
|
|
88
104
|
try {
|
|
89
105
|
const reader = response.body?.getReader();
|
|
@@ -111,25 +127,40 @@ export class EnvironmentClient {
|
|
|
111
127
|
throw new HueEnvironmentError();
|
|
112
128
|
}
|
|
113
129
|
}
|
|
114
|
-
async request(method, path, body) {
|
|
130
|
+
async request(method, path, body, options = {}) {
|
|
115
131
|
// Serialize once: a body this client cannot encode is a caller error that no retry fixes.
|
|
116
132
|
const payload = body === undefined
|
|
117
133
|
? undefined
|
|
118
134
|
: JSON.stringify(json(Object.fromEntries(Object.entries(body).filter(([, value]) => value !== undefined)), REQUEST_BOUNDS));
|
|
119
135
|
for (let attempt = 1;; attempt++) {
|
|
120
136
|
try {
|
|
121
|
-
return await this.send(method, path, payload);
|
|
137
|
+
return await this.send(method, path, payload, options);
|
|
122
138
|
}
|
|
123
139
|
catch (error) {
|
|
124
140
|
if (!(error instanceof HueEnvironmentError))
|
|
125
141
|
throw error;
|
|
126
|
-
|
|
127
|
-
|
|
142
|
+
if (options.signal?.aborted)
|
|
143
|
+
throw error;
|
|
144
|
+
if (!isTransientEnvironmentError(error) || attempt >= this.maxAttempts)
|
|
128
145
|
throw error;
|
|
129
146
|
// Hue's admission refusals say how long to wait; anything else backs off.
|
|
130
147
|
const backoff = Math.min(100 * 2 ** (attempt - 1), 2000);
|
|
131
148
|
const wait = error.retryAfterMs ?? backoff + Math.random() * backoff;
|
|
132
|
-
await new Promise((resolve) =>
|
|
149
|
+
await new Promise((resolve, reject) => {
|
|
150
|
+
const timer = setTimeout(() => {
|
|
151
|
+
options.signal?.removeEventListener("abort", abort);
|
|
152
|
+
resolve();
|
|
153
|
+
}, wait);
|
|
154
|
+
const abort = () => {
|
|
155
|
+
clearTimeout(timer);
|
|
156
|
+
options.signal?.removeEventListener("abort", abort);
|
|
157
|
+
reject(new HueEnvironmentError());
|
|
158
|
+
};
|
|
159
|
+
if (options.signal?.aborted)
|
|
160
|
+
abort();
|
|
161
|
+
else
|
|
162
|
+
options.signal?.addEventListener("abort", abort, { once: true });
|
|
163
|
+
});
|
|
133
164
|
}
|
|
134
165
|
}
|
|
135
166
|
}
|
|
@@ -194,8 +225,8 @@ export class EnvironmentClient {
|
|
|
194
225
|
});
|
|
195
226
|
}
|
|
196
227
|
/** Reads authoritative current or sealed world state. */
|
|
197
|
-
async getRun(runId) {
|
|
198
|
-
const run = await this.request("GET", `/environment-runs/${uuid(runId)}
|
|
228
|
+
async getRun(runId, options = {}) {
|
|
229
|
+
const run = await this.request("GET", `/environment-runs/${uuid(runId)}`, undefined, options);
|
|
199
230
|
return { validity: "not_assessed", coverageGap: null, ...run };
|
|
200
231
|
}
|
|
201
232
|
/** Record a known coverage gap with durable identity; retries reuse the exact request. */
|
|
@@ -173,6 +173,35 @@ export interface EnvironmentDefinition {
|
|
|
173
173
|
/** The extendable legacy name remains V1. Publication and runs select their
|
|
174
174
|
* explicit version; provider context is validated by the authoritative server. */
|
|
175
175
|
export type EnvironmentDefinitionV1 = EnvironmentDefinition;
|
|
176
|
+
/** Gmail mailbox carrier holding messages and drafts in the simplified shape Hue's
|
|
177
|
+
* `hue.gmail.*` actions read. */
|
|
178
|
+
export interface GmailMailboxConfigurationV1 {
|
|
179
|
+
/** Gmail mailbox configuration discriminator. */
|
|
180
|
+
kind: "gmail_mailbox/v1";
|
|
181
|
+
/** Collection containing synthetic messages. */
|
|
182
|
+
messagesCollection: string;
|
|
183
|
+
/** Collection containing synthetic drafts. */
|
|
184
|
+
draftsCollection: string;
|
|
185
|
+
/** Synthetic mailbox address. */
|
|
186
|
+
mailboxAddress: string;
|
|
187
|
+
}
|
|
188
|
+
/** Gmail mailbox carrier served by Hue's simulation gateway: messages, drafts and labels in the
|
|
189
|
+
* entity shapes the Gmail mirrors serve, with threads derived from each message's `threadId`. A
|
|
190
|
+
* definition whose provider instances all use it may publish with no actions. */
|
|
191
|
+
export interface GmailMailboxConfigurationV2 {
|
|
192
|
+
/** Gmail mailbox configuration discriminator. */
|
|
193
|
+
kind: "gmail_mailbox/v2";
|
|
194
|
+
/** Collection containing synthetic messages. */
|
|
195
|
+
messagesCollection: string;
|
|
196
|
+
/** Collection containing synthetic drafts, each naming a `DRAFT`-labelled message. */
|
|
197
|
+
draftsCollection: string;
|
|
198
|
+
/** Collection containing synthetic labels. */
|
|
199
|
+
labelsCollection: string;
|
|
200
|
+
/** Synthetic mailbox address; Hue accepts only `owner@example.test`, in any letter case. */
|
|
201
|
+
mailboxAddress: string;
|
|
202
|
+
}
|
|
203
|
+
/** Either Gmail mailbox carrier, discriminated by `kind`; Hue validates both at publication. */
|
|
204
|
+
export type GmailMailboxConfiguration = GmailMailboxConfigurationV1 | GmailMailboxConfigurationV2;
|
|
176
205
|
/** One synthetic Gmail principal and its world-state collection bindings. */
|
|
177
206
|
export interface GmailProviderInstance {
|
|
178
207
|
/** Stable instance key referenced by attempt provider selection. */
|
|
@@ -182,16 +211,7 @@ export interface GmailProviderInstance {
|
|
|
182
211
|
/** Synthetic principal UUID, canonicalized to lowercase by Hue. */
|
|
183
212
|
syntheticPrincipalId: string;
|
|
184
213
|
/** Versioned mapping from Gmail concepts to authored-world collections. */
|
|
185
|
-
configuration:
|
|
186
|
-
/** Gmail mailbox configuration discriminator. */
|
|
187
|
-
kind: "gmail_mailbox/v1";
|
|
188
|
-
/** Collection containing synthetic messages. */
|
|
189
|
-
messagesCollection: string;
|
|
190
|
-
/** Collection containing synthetic drafts. */
|
|
191
|
-
draftsCollection: string;
|
|
192
|
-
/** Synthetic mailbox address. */
|
|
193
|
-
mailboxAddress: string;
|
|
194
|
-
};
|
|
214
|
+
configuration: GmailMailboxConfiguration;
|
|
195
215
|
}
|
|
196
216
|
/** V2 authored world with immutable provider-instance bindings. */
|
|
197
217
|
export interface EnvironmentDefinitionV2 extends Omit<EnvironmentDefinition, "schemaVersion"> {
|
package/dist/environment.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export { createEnvironmentClient, EnvironmentClient, HueEnvironmentError, } from "./environment/client.js";
|
|
1
|
+
export { createEnvironmentClient, EnvironmentClient, HueEnvironmentError, isTransientEnvironmentError, } from "./environment/client.js";
|
|
2
2
|
export type { EnvironmentClientOptions } from "./environment/client.js";
|
|
3
3
|
export { bindEnvironmentTools } from "./environment/tools.js";
|
|
4
4
|
export type { BindEnvironmentToolsOptions, EnvironmentTool } from "./environment/tools.js";
|
package/dist/environment.js
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
export { createEnvironmentClient, EnvironmentClient, HueEnvironmentError, } from "./environment/client.js";
|
|
1
|
+
export { createEnvironmentClient, EnvironmentClient, HueEnvironmentError, isTransientEnvironmentError, } from "./environment/client.js";
|
|
2
2
|
export { bindEnvironmentTools } from "./environment/tools.js";
|
|
3
3
|
export { agentEnvironment, HUE_CONTROL_PLANE_VARIABLES, isHueControlPlaneCredential, legacyMcpCapability, stripHueControlPlaneCredentials, worldHandoff, writeMcpConfig, } from "./environment/world.js";
|
|
@@ -1,6 +1,20 @@
|
|
|
1
|
-
/**
|
|
1
|
+
/** The checkpoint belongs to a run started with a different identity. When only its content
|
|
2
|
+
* policy differs, `startedWith` holds the policy the unfinished run was started with. */
|
|
3
|
+
export declare class CheckpointIdentityError extends Error {
|
|
4
|
+
readonly startedWith?: {
|
|
5
|
+
persistResultContent?: unknown;
|
|
6
|
+
captureContent?: unknown;
|
|
7
|
+
} | undefined;
|
|
8
|
+
constructor(startedWith?: {
|
|
9
|
+
persistResultContent?: unknown;
|
|
10
|
+
captureContent?: unknown;
|
|
11
|
+
} | undefined);
|
|
12
|
+
}
|
|
13
|
+
/** One owner per directory. A crash leaves .lock for explicit operator recovery; a forced exit of
|
|
14
|
+
* `hue eval`, which stops its agents first, releases it. */
|
|
2
15
|
export declare class CheckpointStore {
|
|
3
16
|
readonly directory: string;
|
|
17
|
+
private untrack;
|
|
4
18
|
private constructor();
|
|
5
19
|
static acquire(directory: string, identity: unknown): Promise<CheckpointStore>;
|
|
6
20
|
read<T>(key: string): Promise<T | undefined>;
|
package/dist/evals/checkpoint.js
CHANGED
|
@@ -1,11 +1,40 @@
|
|
|
1
|
-
import { constants } from "node:fs";
|
|
1
|
+
import { constants, rmSync } from "node:fs";
|
|
2
2
|
import { lstat, mkdir, open, rename, rm } from "node:fs/promises";
|
|
3
3
|
import { join, resolve } from "node:path";
|
|
4
4
|
import { randomUUID } from "node:crypto";
|
|
5
|
+
import { onForcedExit } from "./exit-cleanup.js";
|
|
5
6
|
import { digest } from "./json.js";
|
|
6
|
-
|
|
7
|
+
const CONTENT_POLICY = ["persistResultContent", "captureContent"];
|
|
8
|
+
/** The checkpoint belongs to a run started with a different identity. When only its content
|
|
9
|
+
* policy differs, `startedWith` holds the policy the unfinished run was started with. */
|
|
10
|
+
export class CheckpointIdentityError extends Error {
|
|
11
|
+
startedWith;
|
|
12
|
+
constructor(startedWith) {
|
|
13
|
+
super(startedWith
|
|
14
|
+
? "Checkpoint content policy differs from the one this unfinished run was started with"
|
|
15
|
+
: "Checkpoint identity differs from this project, run, pins or content policy");
|
|
16
|
+
this.startedWith = startedWith;
|
|
17
|
+
this.name = "CheckpointIdentityError";
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
/** The prior policy when two identities differ only in their content policy. */
|
|
21
|
+
function contentPolicyOnly(prior, identity) {
|
|
22
|
+
if (!prior || !identity || typeof prior !== "object" || typeof identity !== "object")
|
|
23
|
+
return undefined;
|
|
24
|
+
const before = prior;
|
|
25
|
+
const after = identity;
|
|
26
|
+
const keys = new Set([...Object.keys(before), ...Object.keys(after)]);
|
|
27
|
+
for (const key of keys)
|
|
28
|
+
if (!CONTENT_POLICY.includes(key) &&
|
|
29
|
+
JSON.stringify(before[key]) !== JSON.stringify(after[key]))
|
|
30
|
+
return undefined;
|
|
31
|
+
return Object.fromEntries(CONTENT_POLICY.map((key) => [key, before[key]]));
|
|
32
|
+
}
|
|
33
|
+
/** One owner per directory. A crash leaves .lock for explicit operator recovery; a forced exit of
|
|
34
|
+
* `hue eval`, which stops its agents first, releases it. */
|
|
7
35
|
export class CheckpointStore {
|
|
8
36
|
directory;
|
|
37
|
+
untrack = () => { };
|
|
9
38
|
constructor(directory) {
|
|
10
39
|
this.directory = directory;
|
|
11
40
|
}
|
|
@@ -18,6 +47,7 @@ export class CheckpointStore {
|
|
|
18
47
|
const store = new CheckpointStore(root);
|
|
19
48
|
try {
|
|
20
49
|
await mkdir(join(root, ".lock"), { mode: 0o700 });
|
|
50
|
+
store.untrack = onForcedExit(() => rmSync(join(root, ".lock"), { recursive: true, force: true }));
|
|
21
51
|
}
|
|
22
52
|
catch {
|
|
23
53
|
throw new Error("Checkpoint directory is locked; confirm its owner stopped before explicitly removing .lock");
|
|
@@ -27,7 +57,7 @@ export class CheckpointStore {
|
|
|
27
57
|
const expected = { format: 1, identity, digest: digest(identity) };
|
|
28
58
|
const prior = await store.read("manifest");
|
|
29
59
|
if (prior && (prior.format !== 1 || prior.digest !== expected.digest))
|
|
30
|
-
throw new
|
|
60
|
+
throw new CheckpointIdentityError(prior.format === 1 ? contentPolicyOnly(prior.identity, identity) : undefined);
|
|
31
61
|
if (!prior)
|
|
32
62
|
await store.write("manifest", expected);
|
|
33
63
|
return store;
|
|
@@ -85,5 +115,6 @@ export class CheckpointStore {
|
|
|
85
115
|
}
|
|
86
116
|
async release() {
|
|
87
117
|
await rm(join(this.directory, ".lock"), { recursive: true });
|
|
118
|
+
this.untrack();
|
|
88
119
|
}
|
|
89
120
|
}
|
package/dist/evals/client.d.ts
CHANGED
|
@@ -18,10 +18,20 @@ export declare class HueApiError extends Error {
|
|
|
18
18
|
/** HTTP status when Hue answered; absent for network, timeout and parsing failures. */
|
|
19
19
|
status?: number | undefined);
|
|
20
20
|
}
|
|
21
|
+
/** An artifact download that ran past the size its caller expected; the rest is not read. */
|
|
22
|
+
export declare class ArtifactSizeError extends Error {
|
|
23
|
+
/** The size the caller expected, in bytes. */
|
|
24
|
+
readonly maxBytes: number;
|
|
25
|
+
constructor(
|
|
26
|
+
/** The size the caller expected, in bytes. */
|
|
27
|
+
maxBytes: number);
|
|
28
|
+
}
|
|
21
29
|
/**
|
|
22
30
|
* Typed client for Hue's evaluation REST API: datasets, scorers, experiments, executions, runs,
|
|
23
31
|
* results and hosted judge jobs. No implicit mutation retry: callers retain stable idempotency keys
|
|
24
|
-
* for experiments and results.
|
|
32
|
+
* for experiments and results. The one exception is a request Hue refused before acting on it with
|
|
33
|
+
* a short `Retry-After` (HTTP 429 or 503, at most 5 seconds): it is sent again after that wait, up
|
|
34
|
+
* to four times. Responses are bounded to 4 MiB.
|
|
25
35
|
*/
|
|
26
36
|
export declare class EvaluationClient {
|
|
27
37
|
/** Validated Hue origin. */
|
|
@@ -29,9 +39,15 @@ export declare class EvaluationClient {
|
|
|
29
39
|
private readonly apiKey;
|
|
30
40
|
private readonly timeoutMillis;
|
|
31
41
|
constructor(options: EvaluationClientOptions);
|
|
42
|
+
/** Sends one request to Hue, again after the wait while Hue refuses it with a short
|
|
43
|
+
* `Retry-After`. `init` runs for every attempt, so each has its own timeout. */
|
|
44
|
+
private send;
|
|
32
45
|
private request;
|
|
33
|
-
/** Verified bytes of one ready artifact in this project, bounded to the 25 MiB pilot file size.
|
|
34
|
-
|
|
46
|
+
/** Verified bytes of one ready artifact in this project, bounded to the 25 MiB pilot file size.
|
|
47
|
+
* With `maxBytes`, the download stops one byte past it and throws `ArtifactSizeError`. */
|
|
48
|
+
downloadArtifact(id: string, options?: {
|
|
49
|
+
maxBytes?: number;
|
|
50
|
+
}): Promise<Uint8Array>;
|
|
35
51
|
/** Stage bytes at the storage capability Hue issued. The Hue key is never sent to storage. */
|
|
36
52
|
uploadArtifactBytes(upload: ArtifactUpload, bytes: Uint8Array, contentType: string): Promise<void>;
|
|
37
53
|
private page;
|
package/dist/evals/client.js
CHANGED
|
@@ -12,6 +12,17 @@ export class HueApiError extends Error {
|
|
|
12
12
|
this.name = "HueApiError";
|
|
13
13
|
}
|
|
14
14
|
}
|
|
15
|
+
/** An artifact download that ran past the size its caller expected; the rest is not read. */
|
|
16
|
+
export class ArtifactSizeError extends Error {
|
|
17
|
+
maxBytes;
|
|
18
|
+
constructor(
|
|
19
|
+
/** The size the caller expected, in bytes. */
|
|
20
|
+
maxBytes) {
|
|
21
|
+
super(`Artifact download exceeded the expected ${maxBytes} bytes`);
|
|
22
|
+
this.maxBytes = maxBytes;
|
|
23
|
+
this.name = "ArtifactSizeError";
|
|
24
|
+
}
|
|
25
|
+
}
|
|
15
26
|
const registryFieldAliases = [
|
|
16
27
|
["datasetId", "evalSetId"],
|
|
17
28
|
["datasetVersionId", "evalSetVersionId"],
|
|
@@ -91,10 +102,30 @@ function productRunFields(value, kind) {
|
|
|
91
102
|
result.scoring = result.evaluation;
|
|
92
103
|
return result;
|
|
93
104
|
}
|
|
105
|
+
/** Times a refused request is sent again before its refusal is the caller's error. */
|
|
106
|
+
const REFUSAL_RETRIES = 4;
|
|
107
|
+
/** A refusal asking for a longer wait fails at once rather than holding the caller. */
|
|
108
|
+
const MAX_REFUSAL_RETRY_AFTER_SECONDS = 5;
|
|
109
|
+
/**
|
|
110
|
+
* The whole-second `Retry-After` of a 429 or 503 asking for at most 5 seconds. Hue sends one only
|
|
111
|
+
* when it refused the request before acting on it, such as a busy key check, so sending any method
|
|
112
|
+
* again is safe. A date, a longer wait or any other failure, a timeout included, is not retried.
|
|
113
|
+
*/
|
|
114
|
+
function refusalRetryAfter(response) {
|
|
115
|
+
if (response.status !== 429 && response.status !== 503)
|
|
116
|
+
return undefined;
|
|
117
|
+
const header = response.headers.get("retry-after")?.trim() ?? "";
|
|
118
|
+
if (!/^\d{1,6}$/.test(header))
|
|
119
|
+
return undefined;
|
|
120
|
+
const seconds = Number(header);
|
|
121
|
+
return seconds <= MAX_REFUSAL_RETRY_AFTER_SECONDS ? seconds : undefined;
|
|
122
|
+
}
|
|
94
123
|
/**
|
|
95
124
|
* Typed client for Hue's evaluation REST API: datasets, scorers, experiments, executions, runs,
|
|
96
125
|
* results and hosted judge jobs. No implicit mutation retry: callers retain stable idempotency keys
|
|
97
|
-
* for experiments and results.
|
|
126
|
+
* for experiments and results. The one exception is a request Hue refused before acting on it with
|
|
127
|
+
* a short `Retry-After` (HTTP 429 or 503, at most 5 seconds): it is sent again after that wait, up
|
|
128
|
+
* to four times. Responses are bounded to 4 MiB.
|
|
98
129
|
*/
|
|
99
130
|
export class EvaluationClient {
|
|
100
131
|
/** Validated Hue origin. */
|
|
@@ -111,27 +142,40 @@ export class EvaluationClient {
|
|
|
111
142
|
this.apiKey = validated.apiKey;
|
|
112
143
|
this.timeoutMillis = validated.timeoutMillis;
|
|
113
144
|
}
|
|
145
|
+
/** Sends one request to Hue, again after the wait while Hue refuses it with a short
|
|
146
|
+
* `Retry-After`. `init` runs for every attempt, so each has its own timeout. */
|
|
147
|
+
async send(url, init) {
|
|
148
|
+
for (let attempt = 0;; attempt++) {
|
|
149
|
+
let response;
|
|
150
|
+
try {
|
|
151
|
+
response = await fetch(url, init());
|
|
152
|
+
}
|
|
153
|
+
catch {
|
|
154
|
+
throw new HueApiError();
|
|
155
|
+
}
|
|
156
|
+
const seconds = attempt < REFUSAL_RETRIES ? refusalRetryAfter(response) : undefined;
|
|
157
|
+
if (seconds === undefined)
|
|
158
|
+
return response;
|
|
159
|
+
await response.body?.cancel().catch(() => undefined);
|
|
160
|
+
// Never sooner than asked; the jitter spreads out parallel requests refused together.
|
|
161
|
+
await new Promise((resolve) => setTimeout(resolve, seconds * 1000 * (1 + Math.random() * 0.5)));
|
|
162
|
+
}
|
|
163
|
+
}
|
|
114
164
|
async request(method, path, body, bounds = { ...valueBounds, bytes: 1024 * 1024 }) {
|
|
115
165
|
// Optional top-level fields are omitted intentionally; nested undefined remains invalid.
|
|
116
166
|
const payload = body === undefined
|
|
117
167
|
? undefined
|
|
118
168
|
: JSON.stringify(json(Object.fromEntries(Object.entries(body).filter(([, value]) => value !== undefined)), bounds));
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
signal: AbortSignal.timeout(this.timeoutMillis),
|
|
130
|
-
});
|
|
131
|
-
}
|
|
132
|
-
catch {
|
|
133
|
-
throw new HueApiError();
|
|
134
|
-
}
|
|
169
|
+
const response = await this.send(`${this.baseUrl}/api/v1${path}`, () => ({
|
|
170
|
+
method,
|
|
171
|
+
headers: {
|
|
172
|
+
Authorization: `Bearer ${this.apiKey}`,
|
|
173
|
+
...(payload ? { "Content-Type": "application/json" } : {}),
|
|
174
|
+
},
|
|
175
|
+
body: payload,
|
|
176
|
+
redirect: "error",
|
|
177
|
+
signal: AbortSignal.timeout(this.timeoutMillis),
|
|
178
|
+
}));
|
|
135
179
|
if (!response.ok) {
|
|
136
180
|
await response.body?.cancel();
|
|
137
181
|
throw new HueApiError(response.status);
|
|
@@ -162,20 +206,18 @@ export class EvaluationClient {
|
|
|
162
206
|
throw new HueApiError();
|
|
163
207
|
}
|
|
164
208
|
}
|
|
165
|
-
/** Verified bytes of one ready artifact in this project, bounded to the 25 MiB pilot file size.
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
throw new HueApiError();
|
|
178
|
-
}
|
|
209
|
+
/** Verified bytes of one ready artifact in this project, bounded to the 25 MiB pilot file size.
|
|
210
|
+
* With `maxBytes`, the download stops one byte past it and throws `ArtifactSizeError`. */
|
|
211
|
+
async downloadArtifact(id, options = {}) {
|
|
212
|
+
const expected = options.maxBytes;
|
|
213
|
+
if (expected !== undefined && (!Number.isSafeInteger(expected) || expected < 0))
|
|
214
|
+
throw new RangeError("maxBytes must be a non-negative integer");
|
|
215
|
+
const response = await this.send(`${this.baseUrl}/api/v1/artifacts/${uuid(id)}/download`, () => ({
|
|
216
|
+
method: "GET",
|
|
217
|
+
headers: { Authorization: `Bearer ${this.apiKey}` },
|
|
218
|
+
redirect: "error",
|
|
219
|
+
signal: AbortSignal.timeout(Math.max(this.timeoutMillis, 120_000)),
|
|
220
|
+
}));
|
|
179
221
|
if (!response.ok) {
|
|
180
222
|
await response.body?.cancel();
|
|
181
223
|
throw new HueApiError(response.status);
|
|
@@ -192,6 +234,8 @@ export class EvaluationClient {
|
|
|
192
234
|
if (done)
|
|
193
235
|
break;
|
|
194
236
|
size += value.byteLength;
|
|
237
|
+
if (expected !== undefined && size > expected)
|
|
238
|
+
throw new ArtifactSizeError(expected);
|
|
195
239
|
if (size > 25 * 1024 * 1024)
|
|
196
240
|
throw new Error("Oversized artifact");
|
|
197
241
|
chunks.push(value);
|
|
@@ -202,7 +246,9 @@ export class EvaluationClient {
|
|
|
202
246
|
}
|
|
203
247
|
return new Uint8Array(Buffer.concat(chunks));
|
|
204
248
|
}
|
|
205
|
-
catch {
|
|
249
|
+
catch (error) {
|
|
250
|
+
if (error instanceof ArtifactSizeError)
|
|
251
|
+
throw error;
|
|
206
252
|
throw new HueApiError();
|
|
207
253
|
}
|
|
208
254
|
}
|
|
@@ -592,8 +638,10 @@ export class EvaluationClient {
|
|
|
592
638
|
return this.request("GET", "/judge-budget");
|
|
593
639
|
}
|
|
594
640
|
/** Register or refresh the fixed local agent key and revision. */
|
|
595
|
-
registerLocalAgent(input) {
|
|
596
|
-
|
|
641
|
+
async registerLocalAgent(input) {
|
|
642
|
+
const agent = await this.request("POST", "/local-agent-worker/register", input);
|
|
643
|
+
// Hue's response names the key `agentKey`; read either name.
|
|
644
|
+
return { ...agent, key: agent.key ?? agent.agentKey ?? input.key };
|
|
597
645
|
}
|
|
598
646
|
/** Claim a queued run for this agent and durable worker identity. */
|
|
599
647
|
claimLocalAgentRun(input) {
|