@hue-run/sdk 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLI.md +100 -29
- package/ENVIRONMENTS.md +44 -9
- package/EVALUATIONS.md +39 -19
- package/README.md +51 -22
- package/dist/cli/eval-direct.d.ts +12 -1
- package/dist/cli/eval-direct.js +145 -33
- package/dist/cli/eval.d.ts +28 -1
- package/dist/cli/eval.js +469 -72
- package/dist/evals/checkpoint.d.ts +15 -1
- package/dist/evals/checkpoint.js +34 -3
- package/dist/evals/client.d.ts +19 -3
- package/dist/evals/client.js +82 -34
- package/dist/evals/environment-target.d.ts +18 -3
- package/dist/evals/environment-target.js +22 -10
- package/dist/evals/exit-cleanup.d.ts +4 -0
- package/dist/evals/exit-cleanup.js +26 -0
- package/dist/evals/files.d.ts +59 -1
- package/dist/evals/files.js +187 -27
- package/dist/evals/local-worker.d.ts +21 -3
- package/dist/evals/local-worker.js +14 -0
- package/dist/evals/runner.d.ts +45 -0
- package/dist/evals/runner.js +164 -13
- package/dist/evals/scenarios.d.ts +7 -5
- package/dist/evals/scenarios.js +71 -14
- package/dist/evals/simulation.d.ts +17 -4
- package/dist/evals/simulation.js +8 -0
- package/dist/evals/types.d.ts +4 -0
- package/dist/evals/verdicts.d.ts +15 -2
- package/dist/evals/verdicts.js +43 -9
- package/dist/evals.d.ts +4 -3
- package/dist/evals.js +2 -2
- package/dist/inline-files.d.ts +4 -0
- package/dist/inline-files.js +40 -17
- package/dist/snapshot.d.ts +2 -2
- package/dist/snapshot.js +47 -12
- package/dist/transport.js +2 -2
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +1 -1
|
@@ -1,6 +1,20 @@
|
|
|
1
|
-
/**
|
|
1
|
+
/** The checkpoint belongs to a run started with a different identity. When only its content
|
|
2
|
+
* policy differs, `startedWith` holds the policy the unfinished run was started with. */
|
|
3
|
+
export declare class CheckpointIdentityError extends Error {
|
|
4
|
+
readonly startedWith?: {
|
|
5
|
+
persistResultContent?: unknown;
|
|
6
|
+
captureContent?: unknown;
|
|
7
|
+
} | undefined;
|
|
8
|
+
constructor(startedWith?: {
|
|
9
|
+
persistResultContent?: unknown;
|
|
10
|
+
captureContent?: unknown;
|
|
11
|
+
} | undefined);
|
|
12
|
+
}
|
|
13
|
+
/** One owner per directory. A crash leaves .lock for explicit operator recovery; a forced exit of
|
|
14
|
+
* `hue eval`, which stops its agents first, releases it. */
|
|
2
15
|
export declare class CheckpointStore {
|
|
3
16
|
readonly directory: string;
|
|
17
|
+
private untrack;
|
|
4
18
|
private constructor();
|
|
5
19
|
static acquire(directory: string, identity: unknown): Promise<CheckpointStore>;
|
|
6
20
|
read<T>(key: string): Promise<T | undefined>;
|
package/dist/evals/checkpoint.js
CHANGED
|
@@ -1,11 +1,40 @@
|
|
|
1
|
-
import { constants } from "node:fs";
|
|
1
|
+
import { constants, rmSync } from "node:fs";
|
|
2
2
|
import { lstat, mkdir, open, rename, rm } from "node:fs/promises";
|
|
3
3
|
import { join, resolve } from "node:path";
|
|
4
4
|
import { randomUUID } from "node:crypto";
|
|
5
|
+
import { onForcedExit } from "./exit-cleanup.js";
|
|
5
6
|
import { digest } from "./json.js";
|
|
6
|
-
|
|
7
|
+
const CONTENT_POLICY = ["persistResultContent", "captureContent"];
|
|
8
|
+
/** The checkpoint belongs to a run started with a different identity. When only its content
|
|
9
|
+
* policy differs, `startedWith` holds the policy the unfinished run was started with. */
|
|
10
|
+
export class CheckpointIdentityError extends Error {
|
|
11
|
+
startedWith;
|
|
12
|
+
constructor(startedWith) {
|
|
13
|
+
super(startedWith
|
|
14
|
+
? "Checkpoint content policy differs from the one this unfinished run was started with"
|
|
15
|
+
: "Checkpoint identity differs from this project, run, pins or content policy");
|
|
16
|
+
this.startedWith = startedWith;
|
|
17
|
+
this.name = "CheckpointIdentityError";
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
/** The prior policy when two identities differ only in their content policy. */
|
|
21
|
+
function contentPolicyOnly(prior, identity) {
|
|
22
|
+
if (!prior || !identity || typeof prior !== "object" || typeof identity !== "object")
|
|
23
|
+
return undefined;
|
|
24
|
+
const before = prior;
|
|
25
|
+
const after = identity;
|
|
26
|
+
const keys = new Set([...Object.keys(before), ...Object.keys(after)]);
|
|
27
|
+
for (const key of keys)
|
|
28
|
+
if (!CONTENT_POLICY.includes(key) &&
|
|
29
|
+
JSON.stringify(before[key]) !== JSON.stringify(after[key]))
|
|
30
|
+
return undefined;
|
|
31
|
+
return Object.fromEntries(CONTENT_POLICY.map((key) => [key, before[key]]));
|
|
32
|
+
}
|
|
33
|
+
/** One owner per directory. A crash leaves .lock for explicit operator recovery; a forced exit of
|
|
34
|
+
* `hue eval`, which stops its agents first, releases it. */
|
|
7
35
|
export class CheckpointStore {
|
|
8
36
|
directory;
|
|
37
|
+
untrack = () => { };
|
|
9
38
|
constructor(directory) {
|
|
10
39
|
this.directory = directory;
|
|
11
40
|
}
|
|
@@ -18,6 +47,7 @@ export class CheckpointStore {
|
|
|
18
47
|
const store = new CheckpointStore(root);
|
|
19
48
|
try {
|
|
20
49
|
await mkdir(join(root, ".lock"), { mode: 0o700 });
|
|
50
|
+
store.untrack = onForcedExit(() => rmSync(join(root, ".lock"), { recursive: true, force: true }));
|
|
21
51
|
}
|
|
22
52
|
catch {
|
|
23
53
|
throw new Error("Checkpoint directory is locked; confirm its owner stopped before explicitly removing .lock");
|
|
@@ -27,7 +57,7 @@ export class CheckpointStore {
|
|
|
27
57
|
const expected = { format: 1, identity, digest: digest(identity) };
|
|
28
58
|
const prior = await store.read("manifest");
|
|
29
59
|
if (prior && (prior.format !== 1 || prior.digest !== expected.digest))
|
|
30
|
-
throw new
|
|
60
|
+
throw new CheckpointIdentityError(prior.format === 1 ? contentPolicyOnly(prior.identity, identity) : undefined);
|
|
31
61
|
if (!prior)
|
|
32
62
|
await store.write("manifest", expected);
|
|
33
63
|
return store;
|
|
@@ -85,5 +115,6 @@ export class CheckpointStore {
|
|
|
85
115
|
}
|
|
86
116
|
async release() {
|
|
87
117
|
await rm(join(this.directory, ".lock"), { recursive: true });
|
|
118
|
+
this.untrack();
|
|
88
119
|
}
|
|
89
120
|
}
|
package/dist/evals/client.d.ts
CHANGED
|
@@ -18,10 +18,20 @@ export declare class HueApiError extends Error {
|
|
|
18
18
|
/** HTTP status when Hue answered; absent for network, timeout and parsing failures. */
|
|
19
19
|
status?: number | undefined);
|
|
20
20
|
}
|
|
21
|
+
/** An artifact download that ran past the size its caller expected; the rest is not read. */
|
|
22
|
+
export declare class ArtifactSizeError extends Error {
|
|
23
|
+
/** The size the caller expected, in bytes. */
|
|
24
|
+
readonly maxBytes: number;
|
|
25
|
+
constructor(
|
|
26
|
+
/** The size the caller expected, in bytes. */
|
|
27
|
+
maxBytes: number);
|
|
28
|
+
}
|
|
21
29
|
/**
|
|
22
30
|
* Typed client for Hue's evaluation REST API: datasets, scorers, experiments, executions, runs,
|
|
23
31
|
* results and hosted judge jobs. No implicit mutation retry: callers retain stable idempotency keys
|
|
24
|
-
* for experiments and results.
|
|
32
|
+
* for experiments and results. The one exception is a request Hue refused before acting on it with
|
|
33
|
+
* a short `Retry-After` (HTTP 429 or 503, at most 5 seconds): it is sent again after that wait, up
|
|
34
|
+
* to four times. Responses are bounded to 4 MiB.
|
|
25
35
|
*/
|
|
26
36
|
export declare class EvaluationClient {
|
|
27
37
|
/** Validated Hue origin. */
|
|
@@ -29,9 +39,15 @@ export declare class EvaluationClient {
|
|
|
29
39
|
private readonly apiKey;
|
|
30
40
|
private readonly timeoutMillis;
|
|
31
41
|
constructor(options: EvaluationClientOptions);
|
|
42
|
+
/** Sends one request to Hue, again after the wait while Hue refuses it with a short
|
|
43
|
+
* `Retry-After`. `init` runs for every attempt, so each has its own timeout. */
|
|
44
|
+
private send;
|
|
32
45
|
private request;
|
|
33
|
-
/** Verified bytes of one ready artifact in this project, bounded to the 25 MiB pilot file size.
|
|
34
|
-
|
|
46
|
+
/** Verified bytes of one ready artifact in this project, bounded to the 25 MiB pilot file size.
|
|
47
|
+
* With `maxBytes`, the download stops one byte past it and throws `ArtifactSizeError`. */
|
|
48
|
+
downloadArtifact(id: string, options?: {
|
|
49
|
+
maxBytes?: number;
|
|
50
|
+
}): Promise<Uint8Array>;
|
|
35
51
|
/** Stage bytes at the storage capability Hue issued. The Hue key is never sent to storage. */
|
|
36
52
|
uploadArtifactBytes(upload: ArtifactUpload, bytes: Uint8Array, contentType: string): Promise<void>;
|
|
37
53
|
private page;
|
package/dist/evals/client.js
CHANGED
|
@@ -12,6 +12,17 @@ export class HueApiError extends Error {
|
|
|
12
12
|
this.name = "HueApiError";
|
|
13
13
|
}
|
|
14
14
|
}
|
|
15
|
+
/** An artifact download that ran past the size its caller expected; the rest is not read. */
|
|
16
|
+
export class ArtifactSizeError extends Error {
|
|
17
|
+
maxBytes;
|
|
18
|
+
constructor(
|
|
19
|
+
/** The size the caller expected, in bytes. */
|
|
20
|
+
maxBytes) {
|
|
21
|
+
super(`Artifact download exceeded the expected ${maxBytes} bytes`);
|
|
22
|
+
this.maxBytes = maxBytes;
|
|
23
|
+
this.name = "ArtifactSizeError";
|
|
24
|
+
}
|
|
25
|
+
}
|
|
15
26
|
const registryFieldAliases = [
|
|
16
27
|
["datasetId", "evalSetId"],
|
|
17
28
|
["datasetVersionId", "evalSetVersionId"],
|
|
@@ -91,10 +102,30 @@ function productRunFields(value, kind) {
|
|
|
91
102
|
result.scoring = result.evaluation;
|
|
92
103
|
return result;
|
|
93
104
|
}
|
|
105
|
+
/** Times a refused request is sent again before its refusal is the caller's error. */
|
|
106
|
+
const REFUSAL_RETRIES = 4;
|
|
107
|
+
/** A refusal asking for a longer wait fails at once rather than holding the caller. */
|
|
108
|
+
const MAX_REFUSAL_RETRY_AFTER_SECONDS = 5;
|
|
109
|
+
/**
|
|
110
|
+
* The whole-second `Retry-After` of a 429 or 503 asking for at most 5 seconds. Hue sends one only
|
|
111
|
+
* when it refused the request before acting on it, such as a busy key check, so sending any method
|
|
112
|
+
* again is safe. A date, a longer wait or any other failure, a timeout included, is not retried.
|
|
113
|
+
*/
|
|
114
|
+
function refusalRetryAfter(response) {
|
|
115
|
+
if (response.status !== 429 && response.status !== 503)
|
|
116
|
+
return undefined;
|
|
117
|
+
const header = response.headers.get("retry-after")?.trim() ?? "";
|
|
118
|
+
if (!/^\d{1,6}$/.test(header))
|
|
119
|
+
return undefined;
|
|
120
|
+
const seconds = Number(header);
|
|
121
|
+
return seconds <= MAX_REFUSAL_RETRY_AFTER_SECONDS ? seconds : undefined;
|
|
122
|
+
}
|
|
94
123
|
/**
|
|
95
124
|
* Typed client for Hue's evaluation REST API: datasets, scorers, experiments, executions, runs,
|
|
96
125
|
* results and hosted judge jobs. No implicit mutation retry: callers retain stable idempotency keys
|
|
97
|
-
* for experiments and results.
|
|
126
|
+
* for experiments and results. The one exception is a request Hue refused before acting on it with
|
|
127
|
+
* a short `Retry-After` (HTTP 429 or 503, at most 5 seconds): it is sent again after that wait, up
|
|
128
|
+
* to four times. Responses are bounded to 4 MiB.
|
|
98
129
|
*/
|
|
99
130
|
export class EvaluationClient {
|
|
100
131
|
/** Validated Hue origin. */
|
|
@@ -111,27 +142,40 @@ export class EvaluationClient {
|
|
|
111
142
|
this.apiKey = validated.apiKey;
|
|
112
143
|
this.timeoutMillis = validated.timeoutMillis;
|
|
113
144
|
}
|
|
145
|
+
/** Sends one request to Hue, again after the wait while Hue refuses it with a short
|
|
146
|
+
* `Retry-After`. `init` runs for every attempt, so each has its own timeout. */
|
|
147
|
+
async send(url, init) {
|
|
148
|
+
for (let attempt = 0;; attempt++) {
|
|
149
|
+
let response;
|
|
150
|
+
try {
|
|
151
|
+
response = await fetch(url, init());
|
|
152
|
+
}
|
|
153
|
+
catch {
|
|
154
|
+
throw new HueApiError();
|
|
155
|
+
}
|
|
156
|
+
const seconds = attempt < REFUSAL_RETRIES ? refusalRetryAfter(response) : undefined;
|
|
157
|
+
if (seconds === undefined)
|
|
158
|
+
return response;
|
|
159
|
+
await response.body?.cancel().catch(() => undefined);
|
|
160
|
+
// Never sooner than asked; the jitter spreads out parallel requests refused together.
|
|
161
|
+
await new Promise((resolve) => setTimeout(resolve, seconds * 1000 * (1 + Math.random() * 0.5)));
|
|
162
|
+
}
|
|
163
|
+
}
|
|
114
164
|
async request(method, path, body, bounds = { ...valueBounds, bytes: 1024 * 1024 }) {
|
|
115
165
|
// Optional top-level fields are omitted intentionally; nested undefined remains invalid.
|
|
116
166
|
const payload = body === undefined
|
|
117
167
|
? undefined
|
|
118
168
|
: JSON.stringify(json(Object.fromEntries(Object.entries(body).filter(([, value]) => value !== undefined)), bounds));
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
signal: AbortSignal.timeout(this.timeoutMillis),
|
|
130
|
-
});
|
|
131
|
-
}
|
|
132
|
-
catch {
|
|
133
|
-
throw new HueApiError();
|
|
134
|
-
}
|
|
169
|
+
const response = await this.send(`${this.baseUrl}/api/v1${path}`, () => ({
|
|
170
|
+
method,
|
|
171
|
+
headers: {
|
|
172
|
+
Authorization: `Bearer ${this.apiKey}`,
|
|
173
|
+
...(payload ? { "Content-Type": "application/json" } : {}),
|
|
174
|
+
},
|
|
175
|
+
body: payload,
|
|
176
|
+
redirect: "error",
|
|
177
|
+
signal: AbortSignal.timeout(this.timeoutMillis),
|
|
178
|
+
}));
|
|
135
179
|
if (!response.ok) {
|
|
136
180
|
await response.body?.cancel();
|
|
137
181
|
throw new HueApiError(response.status);
|
|
@@ -162,20 +206,18 @@ export class EvaluationClient {
|
|
|
162
206
|
throw new HueApiError();
|
|
163
207
|
}
|
|
164
208
|
}
|
|
165
|
-
/** Verified bytes of one ready artifact in this project, bounded to the 25 MiB pilot file size.
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
throw new HueApiError();
|
|
178
|
-
}
|
|
209
|
+
/** Verified bytes of one ready artifact in this project, bounded to the 25 MiB pilot file size.
|
|
210
|
+
* With `maxBytes`, the download stops one byte past it and throws `ArtifactSizeError`. */
|
|
211
|
+
async downloadArtifact(id, options = {}) {
|
|
212
|
+
const expected = options.maxBytes;
|
|
213
|
+
if (expected !== undefined && (!Number.isSafeInteger(expected) || expected < 0))
|
|
214
|
+
throw new RangeError("maxBytes must be a non-negative integer");
|
|
215
|
+
const response = await this.send(`${this.baseUrl}/api/v1/artifacts/${uuid(id)}/download`, () => ({
|
|
216
|
+
method: "GET",
|
|
217
|
+
headers: { Authorization: `Bearer ${this.apiKey}` },
|
|
218
|
+
redirect: "error",
|
|
219
|
+
signal: AbortSignal.timeout(Math.max(this.timeoutMillis, 120_000)),
|
|
220
|
+
}));
|
|
179
221
|
if (!response.ok) {
|
|
180
222
|
await response.body?.cancel();
|
|
181
223
|
throw new HueApiError(response.status);
|
|
@@ -192,6 +234,8 @@ export class EvaluationClient {
|
|
|
192
234
|
if (done)
|
|
193
235
|
break;
|
|
194
236
|
size += value.byteLength;
|
|
237
|
+
if (expected !== undefined && size > expected)
|
|
238
|
+
throw new ArtifactSizeError(expected);
|
|
195
239
|
if (size > 25 * 1024 * 1024)
|
|
196
240
|
throw new Error("Oversized artifact");
|
|
197
241
|
chunks.push(value);
|
|
@@ -202,7 +246,9 @@ export class EvaluationClient {
|
|
|
202
246
|
}
|
|
203
247
|
return new Uint8Array(Buffer.concat(chunks));
|
|
204
248
|
}
|
|
205
|
-
catch {
|
|
249
|
+
catch (error) {
|
|
250
|
+
if (error instanceof ArtifactSizeError)
|
|
251
|
+
throw error;
|
|
206
252
|
throw new HueApiError();
|
|
207
253
|
}
|
|
208
254
|
}
|
|
@@ -592,8 +638,10 @@ export class EvaluationClient {
|
|
|
592
638
|
return this.request("GET", "/judge-budget");
|
|
593
639
|
}
|
|
594
640
|
/** Register or refresh the fixed local agent key and revision. */
|
|
595
|
-
registerLocalAgent(input) {
|
|
596
|
-
|
|
641
|
+
async registerLocalAgent(input) {
|
|
642
|
+
const agent = await this.request("POST", "/local-agent-worker/register", input);
|
|
643
|
+
// Hue's response names the key `agentKey`; read either name.
|
|
644
|
+
return { ...agent, key: agent.key ?? agent.agentKey ?? input.key };
|
|
597
645
|
}
|
|
598
646
|
/** Claim a queued run for this agent and durable worker identity. */
|
|
599
647
|
claimLocalAgentRun(input) {
|
|
@@ -5,7 +5,7 @@ import type { EnvironmentRun, WorldHandoff } from "../environment/types.js";
|
|
|
5
5
|
import type { HueSpan } from "../types.js";
|
|
6
6
|
import { type ActualAgentManifestInputV2, type AttemptBaselineV2, type AttemptConnectionBundleV2, type RequestedAttemptProviderV2, type SurfaceBindingV2 } from "./attempt.js";
|
|
7
7
|
import type { EvaluationClient } from "./client.js";
|
|
8
|
-
import type { ExperimentCase, JsonValue, SimulationMcpCapability } from "./types.js";
|
|
8
|
+
import type { ExperimentCase, JsonValue, LocalFile, SimulationMcpCapability, TargetResult } from "./types.js";
|
|
9
9
|
export type McpSurfaceKeyV2 = Extract<SurfaceBindingV2["surfaceKey"], `${string}/mcp`>;
|
|
10
10
|
export type ActualAgentManifestResolverV2 = ActualAgentManifestInputV2 | ((context: {
|
|
11
11
|
config: JsonValue;
|
|
@@ -64,6 +64,11 @@ export interface EnvironmentTargetContext {
|
|
|
64
64
|
* gateway world has no MCP surface. */
|
|
65
65
|
mcp?: SimulationMcpCapability;
|
|
66
66
|
connectionBundle?: AttemptConnectionBundleV2;
|
|
67
|
+
/** Verified copies of the case's agent-visible input files; evaluator-only files are never
|
|
68
|
+
* among them. Empty when the case has none. */
|
|
69
|
+
files: LocalFile[];
|
|
70
|
+
/** Private directory for this case; return generated files with `withFiles`. */
|
|
71
|
+
outputDirectory: string;
|
|
67
72
|
signal?: AbortSignal;
|
|
68
73
|
}
|
|
69
74
|
interface RunnerTargetContext {
|
|
@@ -71,6 +76,8 @@ interface RunnerTargetContext {
|
|
|
71
76
|
item: ExperimentCase;
|
|
72
77
|
executionId: string;
|
|
73
78
|
span: HueSpan;
|
|
79
|
+
files: LocalFile[];
|
|
80
|
+
outputDirectory: string;
|
|
74
81
|
}
|
|
75
82
|
export interface RunEnvironmentTargetOptions {
|
|
76
83
|
client: EvaluationClient;
|
|
@@ -88,14 +95,22 @@ export interface RunEnvironmentTargetOptions {
|
|
|
88
95
|
* the deployment serves, such as `hue eval`, turns it off. */
|
|
89
96
|
deprecationWarnings?: boolean;
|
|
90
97
|
signal?: AbortSignal;
|
|
98
|
+
/** Test-only: shortens the seal wait. */
|
|
99
|
+
sealTiming?: Partial<SealTiming>;
|
|
91
100
|
onProgress?(event: EnvironmentTargetProgress): void | Promise<void>;
|
|
92
|
-
target(inputs: JsonValue, context: EnvironmentTargetContext): JsonValue | undefined | Promise<JsonValue | undefined>;
|
|
101
|
+
target(inputs: JsonValue, context: EnvironmentTargetContext): JsonValue | TargetResult | undefined | Promise<JsonValue | TargetResult | undefined>;
|
|
93
102
|
}
|
|
94
103
|
/** The completion grace is five seconds; cap an unexpectedly distant timestamp and let reads
|
|
95
104
|
* force the seal after the grace. */
|
|
96
105
|
export declare const MAX_GRACE_WAIT_MS = 10000;
|
|
97
106
|
export declare const SEAL_POLL_MS = 250;
|
|
98
107
|
export declare const SEAL_WAIT_MS = 30000;
|
|
108
|
+
/** The seal wait's bounds. Only tests shorten them; the helpers always use the defaults. */
|
|
109
|
+
export interface SealTiming {
|
|
110
|
+
maxGraceWaitMs: number;
|
|
111
|
+
pollMs: number;
|
|
112
|
+
waitMs: number;
|
|
113
|
+
}
|
|
99
114
|
/** The W3C context of the case span, sent on create so the world span parents on it. The
|
|
100
115
|
* flags are the span's own: an unsampled case span is not exported, and the World API must not
|
|
101
116
|
* be told otherwise. */
|
|
@@ -136,5 +151,5 @@ export declare function createWorldForExecution(client: Pick<EnvironmentClient,
|
|
|
136
151
|
* outbound local workers. Credential-bearing connections stay in this call frame and
|
|
137
152
|
* are never returned to either runner's checkpoint state.
|
|
138
153
|
*/
|
|
139
|
-
export declare function runEnvironmentTarget(options: RunEnvironmentTargetOptions): Promise<JsonValue | undefined>;
|
|
154
|
+
export declare function runEnvironmentTarget(options: RunEnvironmentTargetOptions): Promise<JsonValue | TargetResult | undefined>;
|
|
140
155
|
export {};
|
|
@@ -47,8 +47,13 @@ export function pinRequestedAttemptV2(requested, config) {
|
|
|
47
47
|
export const MAX_GRACE_WAIT_MS = 10_000;
|
|
48
48
|
export const SEAL_POLL_MS = 250;
|
|
49
49
|
export const SEAL_WAIT_MS = 30_000;
|
|
50
|
+
const SEAL_TIMING = {
|
|
51
|
+
maxGraceWaitMs: MAX_GRACE_WAIT_MS,
|
|
52
|
+
pollMs: SEAL_POLL_MS,
|
|
53
|
+
waitMs: SEAL_WAIT_MS,
|
|
54
|
+
};
|
|
50
55
|
/** Finish, then wait for the authoritative run to leave open. */
|
|
51
|
-
async function seal(client, runId, executionId, status) {
|
|
56
|
+
async function seal(client, runId, executionId, status, timing) {
|
|
52
57
|
let completingUntil;
|
|
53
58
|
try {
|
|
54
59
|
const finished = await client.finishRun(runId, {
|
|
@@ -70,7 +75,7 @@ async function seal(client, runId, executionId, status) {
|
|
|
70
75
|
}
|
|
71
76
|
if (completingUntil !== undefined) {
|
|
72
77
|
try {
|
|
73
|
-
await awaitSeal(client, runId, completingUntil);
|
|
78
|
+
await awaitSeal(client, runId, completingUntil, timing);
|
|
74
79
|
}
|
|
75
80
|
catch (error) {
|
|
76
81
|
throw new TargetOutcomeUncertainError(executionId, { cause: error });
|
|
@@ -78,19 +83,21 @@ async function seal(client, runId, executionId, status) {
|
|
|
78
83
|
}
|
|
79
84
|
}
|
|
80
85
|
/** Wait out a completing world's grace, then read until the server seals it. */
|
|
81
|
-
async function awaitSeal(client, runId, completingUntil) {
|
|
86
|
+
async function awaitSeal(client, runId, completingUntil, timing) {
|
|
82
87
|
const graceEnd = Date.parse(completingUntil ?? "");
|
|
83
88
|
let wait = Number.isFinite(graceEnd)
|
|
84
|
-
? Math.min(Math.max(0, graceEnd - Date.now()),
|
|
89
|
+
? Math.min(Math.max(0, graceEnd - Date.now()), timing.maxGraceWaitMs)
|
|
85
90
|
: 0;
|
|
86
|
-
const deadline = performance.now() + wait +
|
|
91
|
+
const deadline = performance.now() + wait + timing.waitMs;
|
|
87
92
|
for (;;) {
|
|
88
93
|
await new Promise((resolve) => setTimeout(resolve, wait));
|
|
89
94
|
const remaining = deadline - performance.now();
|
|
90
95
|
if (remaining <= 0)
|
|
91
96
|
throw new Error(`World ${runId} was not sealed after its completion grace`);
|
|
97
|
+
// The deadline ends the read and the client's retries inside it.
|
|
92
98
|
const controller = new AbortController();
|
|
93
99
|
const timer = setTimeout(() => controller.abort(), remaining);
|
|
100
|
+
wait = timing.pollMs;
|
|
94
101
|
try {
|
|
95
102
|
if ((await client.getRun(runId, { signal: controller.signal })).status !== "open")
|
|
96
103
|
return;
|
|
@@ -102,13 +109,15 @@ async function awaitSeal(client, runId, completingUntil) {
|
|
|
102
109
|
throw new Error(`World ${runId} was not sealed after its completion grace`, {
|
|
103
110
|
cause: error,
|
|
104
111
|
});
|
|
112
|
+
// A 429 or 503 that outlasted the client's retries still says how long to wait.
|
|
113
|
+
wait = Math.max(wait, error.retryAfterMs ?? 0);
|
|
105
114
|
}
|
|
106
115
|
finally {
|
|
107
116
|
clearTimeout(timer);
|
|
108
117
|
}
|
|
109
118
|
if (performance.now() >= deadline)
|
|
110
119
|
throw new Error(`World ${runId} was not sealed after its completion grace`);
|
|
111
|
-
wait =
|
|
120
|
+
wait = Math.min(wait, deadline - performance.now());
|
|
112
121
|
}
|
|
113
122
|
}
|
|
114
123
|
const deprecations = new Set();
|
|
@@ -208,6 +217,7 @@ export async function runEnvironmentTarget(options) {
|
|
|
208
217
|
agentRevision: options.agentRevision,
|
|
209
218
|
});
|
|
210
219
|
const world = worldHandoff(run);
|
|
220
|
+
const timing = { ...SEAL_TIMING, ...options.sealTiming };
|
|
211
221
|
const progress = (event) => options.onProgress?.(event);
|
|
212
222
|
let finalized = false;
|
|
213
223
|
try {
|
|
@@ -268,7 +278,7 @@ export async function runEnvironmentTarget(options) {
|
|
|
268
278
|
: {}),
|
|
269
279
|
});
|
|
270
280
|
if (prepared.status === "environment_incomplete") {
|
|
271
|
-
await seal(options.environmentClient, run.id, context.executionId, "completed");
|
|
281
|
+
await seal(options.environmentClient, run.id, context.executionId, "completed", timing);
|
|
272
282
|
finalized = true;
|
|
273
283
|
await Promise.resolve(progress({ type: "world_sealed", environmentRunId: run.id })).catch(() => undefined);
|
|
274
284
|
return undefined;
|
|
@@ -307,9 +317,11 @@ export async function runEnvironmentTarget(options) {
|
|
|
307
317
|
...(world ? { world } : {}),
|
|
308
318
|
...(mcp ? { mcp } : {}),
|
|
309
319
|
...(connectionBundle ? { connectionBundle } : {}),
|
|
320
|
+
files: structuredClone(context.files),
|
|
321
|
+
outputDirectory: context.outputDirectory,
|
|
310
322
|
signal: options.signal,
|
|
311
323
|
});
|
|
312
|
-
await seal(options.environmentClient, run.id, context.executionId, "completed");
|
|
324
|
+
await seal(options.environmentClient, run.id, context.executionId, "completed", timing);
|
|
313
325
|
finalized = true;
|
|
314
326
|
await Promise.resolve(progress({ type: "world_sealed", environmentRunId: run.id })).catch(() => undefined);
|
|
315
327
|
return output;
|
|
@@ -329,7 +341,7 @@ export async function runEnvironmentTarget(options) {
|
|
|
329
341
|
}
|
|
330
342
|
if (environmentIncomplete) {
|
|
331
343
|
try {
|
|
332
|
-
await seal(options.environmentClient, run.id, context.executionId, "completed");
|
|
344
|
+
await seal(options.environmentClient, run.id, context.executionId, "completed", timing);
|
|
333
345
|
}
|
|
334
346
|
catch (finalizationError) {
|
|
335
347
|
throw new TargetOutcomeUncertainError(context.executionId, {
|
|
@@ -341,7 +353,7 @@ export async function runEnvironmentTarget(options) {
|
|
|
341
353
|
return undefined;
|
|
342
354
|
}
|
|
343
355
|
try {
|
|
344
|
-
await seal(options.environmentClient, run.id, context.executionId, "abandoned");
|
|
356
|
+
await seal(options.environmentClient, run.id, context.executionId, "abandoned", timing);
|
|
345
357
|
}
|
|
346
358
|
catch (finalizationError) {
|
|
347
359
|
throw new TargetOutcomeUncertainError(context.executionId, {
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
/** Register a synchronous removal; the returned function unregisters it. */
|
|
2
|
+
export declare function onForcedExit(cleanup: () => void): () => void;
|
|
3
|
+
/** Run every registered removal, newest first: files before the lock that guards them. */
|
|
4
|
+
export declare function runForcedExitCleanups(): void;
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What this process holds on disk while it runs — a world case's files, a checkpoint lock, a
|
|
3
|
+
* token-bearing MCP configuration — each with a synchronous removal. Normal paths remove them
|
|
4
|
+
* themselves and unregister; a forced exit (a second Ctrl+C in `hue eval`) cannot wait for them,
|
|
5
|
+
* so it runs whatever is still registered, newest first, before `process.exit`.
|
|
6
|
+
*/
|
|
7
|
+
const cleanups = new Set();
|
|
8
|
+
/** Register a synchronous removal; the returned function unregisters it. */
|
|
9
|
+
export function onForcedExit(cleanup) {
|
|
10
|
+
cleanups.add(cleanup);
|
|
11
|
+
return () => {
|
|
12
|
+
cleanups.delete(cleanup);
|
|
13
|
+
};
|
|
14
|
+
}
|
|
15
|
+
/** Run every registered removal, newest first: files before the lock that guards them. */
|
|
16
|
+
export function runForcedExitCleanups() {
|
|
17
|
+
for (const cleanup of [...cleanups].reverse()) {
|
|
18
|
+
try {
|
|
19
|
+
cleanup();
|
|
20
|
+
}
|
|
21
|
+
catch {
|
|
22
|
+
// Best effort on the way out; the rest still run.
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
cleanups.clear();
|
|
26
|
+
}
|
package/dist/evals/files.d.ts
CHANGED
|
@@ -32,13 +32,71 @@ export interface StagedOutputFile {
|
|
|
32
32
|
export declare class OutputFileError extends Error {
|
|
33
33
|
constructor(message: string);
|
|
34
34
|
}
|
|
35
|
-
/**
|
|
35
|
+
/** Stable codes of a pinned file the runner refuses to hand over. */
|
|
36
|
+
export type CaseFileErrorCode = "case_file_mismatch" | "case_file_name_refused";
|
|
37
|
+
/**
|
|
38
|
+
* Thrown when a pinned file cannot be used: its downloaded bytes differ from the manifest's size
|
|
39
|
+
* or SHA-256 (`case_file_mismatch`), or a file meant for an agent working in a world is not
|
|
40
|
+
* named by one safe file name (`case_file_name_refused`). Raised before the case's execution
|
|
41
|
+
* starts, so no execution, world or target call is spent on it.
|
|
42
|
+
*/
|
|
43
|
+
export declare class CaseFileError extends Error {
|
|
44
|
+
/** Stable diagnostic code. */
|
|
45
|
+
readonly code: CaseFileErrorCode;
|
|
46
|
+
/** Hue artifact identity of the refused file. */
|
|
47
|
+
readonly artifactId: string;
|
|
48
|
+
constructor(
|
|
49
|
+
/** Stable diagnostic code. */
|
|
50
|
+
code: CaseFileErrorCode,
|
|
51
|
+
/** Hue artifact identity of the refused file. */
|
|
52
|
+
artifactId: string, message: string);
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Whether `name` can be used unchanged as one file name: not empty, `.` or `..`; no `/`, `\`,
|
|
56
|
+
* C0 or C1 control character, or character Windows reserves (`: < > " | ? *`); no Windows device
|
|
57
|
+
* name (`CON`, `NUL`, `COM1`, … with any extension); no trailing dot or space, which Windows
|
|
58
|
+
* drops; at most 200 UTF-8 bytes. An absolute path or a path with a parent step always contains
|
|
59
|
+
* a separator, so it is refused too.
|
|
60
|
+
*/
|
|
61
|
+
export declare function isSafeFileName(name: string): boolean;
|
|
62
|
+
/** Refuse a pinned file whose name is not one safe file name; see {@link isSafeFileName}. */
|
|
63
|
+
export declare function assertSafeFileNames(files: readonly Pick<CaseFile, "artifactId" | "filename">[]): void;
|
|
64
|
+
/**
|
|
65
|
+
* Reduce a declared file name to a single printable path segment, at most 200 UTF-8 bytes. A
|
|
66
|
+
* longer name keeps its extension, so `.pdf` stays `.pdf`, and its stem is shortened and marked
|
|
67
|
+
* with a short hash of the whole name, so two long names never shorten to the same one.
|
|
68
|
+
*/
|
|
36
69
|
export declare function safeFilename(name: string): string;
|
|
70
|
+
/** The device and inode a listing saw for a file, compared with the file actually opened. */
|
|
71
|
+
export interface FileIdentity {
|
|
72
|
+
dev: number;
|
|
73
|
+
ino: number;
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Read a file an agent wrote without trusting its path: the last component must not be a
|
|
77
|
+
* symlink and the open file must be a regular file of at most `maxBytes`, checked before any byte
|
|
78
|
+
* is read, and, when `expected` is given, the very file a listing saw. The bytes come from that
|
|
79
|
+
* open file. Returns undefined for a missing file only when nothing was expected there; every
|
|
80
|
+
* other refusal is an `OutputFileError` naming `label`.
|
|
81
|
+
*/
|
|
82
|
+
export declare function readAgentFile(path: string, label: string, maxBytes: number, expected?: FileIdentity): Promise<Uint8Array | undefined>;
|
|
83
|
+
/** Check pinned files against the manifest without keeping them: nothing is written to disk. */
|
|
84
|
+
export declare function verifyCaseFiles(client: EvaluationClient, files: readonly SubjectFile[]): Promise<void>;
|
|
37
85
|
/**
|
|
38
86
|
* Save verified copies of a case's pinned input files under `directory`. A file already
|
|
39
87
|
* present with the pinned identity is reused, so a resumed run does not download again.
|
|
40
88
|
*/
|
|
41
89
|
export declare function downloadCaseFiles(client: EvaluationClient, files: SubjectFile[], directory: string, primaryArtifactId?: string | null): Promise<LocalFile[]>;
|
|
90
|
+
/**
|
|
91
|
+
* Download a case's needed inputs under `caseDirectory`: the agent-visible roles into `inputs/`
|
|
92
|
+
* and evaluator-only files into `evaluator-inputs/`, so the directory the target's files live in
|
|
93
|
+
* never holds an evaluator's file. `all` keeps the manifest order for scorers; `target` is the
|
|
94
|
+
* agent-visible subset.
|
|
95
|
+
*/
|
|
96
|
+
export declare function downloadCaseInputs(client: EvaluationClient, files: CaseFile[], caseDirectory: string): Promise<{
|
|
97
|
+
all: LocalFile[];
|
|
98
|
+
target: LocalFile[];
|
|
99
|
+
}>;
|
|
42
100
|
/** Copy the target's generated files next to the checkpoint and record their identities. */
|
|
43
101
|
export declare function stageOutputFiles(files: OutputFile[], directory: string): Promise<StagedOutputFile[]>;
|
|
44
102
|
/**
|