@hue-run/sdk 0.9.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLI.md +240 -49
- package/ENVIRONMENTS.md +48 -9
- package/EVALUATIONS.md +41 -21
- package/README.md +70 -30
- package/dist/cli/eval-direct.d.ts +12 -1
- package/dist/cli/eval-direct.js +145 -33
- package/dist/cli/eval.d.ts +28 -1
- package/dist/cli/eval.js +504 -81
- package/dist/cli/listen.d.ts +76 -0
- package/dist/cli/listen.js +816 -0
- package/dist/cli/mcp.d.ts +65 -7
- package/dist/cli/mcp.js +271 -73
- package/dist/client.d.ts +7 -4
- package/dist/client.js +36 -10
- package/dist/evals/checkpoint.d.ts +15 -1
- package/dist/evals/checkpoint.js +34 -3
- package/dist/evals/client.d.ts +24 -4
- package/dist/evals/client.js +92 -38
- package/dist/evals/environment-target.d.ts +18 -3
- package/dist/evals/environment-target.js +22 -10
- package/dist/evals/exit-cleanup.d.ts +4 -0
- package/dist/evals/exit-cleanup.js +26 -0
- package/dist/evals/files.d.ts +69 -3
- package/dist/evals/files.js +255 -42
- package/dist/evals/json.js +116 -9
- package/dist/evals/local-worker.d.ts +21 -3
- package/dist/evals/local-worker.js +14 -0
- package/dist/evals/runner.d.ts +45 -0
- package/dist/evals/runner.js +199 -21
- package/dist/evals/scenarios.d.ts +7 -5
- package/dist/evals/scenarios.js +71 -14
- package/dist/evals/scorer-publication.js +58 -3
- package/dist/evals/simulation.d.ts +17 -4
- package/dist/evals/simulation.js +8 -0
- package/dist/evals/types.d.ts +48 -0
- package/dist/evals/verdicts.d.ts +26 -2
- package/dist/evals/verdicts.js +72 -9
- package/dist/evals.d.ts +4 -3
- package/dist/evals.js +2 -2
- package/dist/inline-files.d.ts +4 -0
- package/dist/inline-files.js +48 -17
- package/dist/provider-tools.d.ts +12 -1
- package/dist/provider-tools.js +192 -106
- package/dist/safety.d.ts +7 -1
- package/dist/safety.js +9 -7
- package/dist/setup/cli.js +31 -8
- package/dist/setup/peers.d.ts +13 -0
- package/dist/setup/peers.js +44 -0
- package/dist/snapshot.d.ts +2 -2
- package/dist/snapshot.js +47 -12
- package/dist/tool-definitions.d.ts +18 -0
- package/dist/tool-definitions.js +232 -0
- package/dist/transport.js +2 -2
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +1 -1
package/dist/evals/files.d.ts
CHANGED
|
@@ -32,19 +32,85 @@ export interface StagedOutputFile {
|
|
|
32
32
|
export declare class OutputFileError extends Error {
|
|
33
33
|
constructor(message: string);
|
|
34
34
|
}
|
|
35
|
-
/**
|
|
35
|
+
/** Stable codes of a pinned file the runner refuses to hand over. */
|
|
36
|
+
export type CaseFileErrorCode = "case_file_mismatch" | "case_file_name_refused";
|
|
37
|
+
/**
|
|
38
|
+
* Thrown when a pinned file cannot be used: its downloaded bytes differ from the manifest's size
|
|
39
|
+
* or SHA-256 (`case_file_mismatch`), or a file meant for an agent working in a world is not
|
|
40
|
+
* named by one safe file name (`case_file_name_refused`). Raised before the case's execution
|
|
41
|
+
* starts, so no execution, world or target call is spent on it.
|
|
42
|
+
*/
|
|
43
|
+
export declare class CaseFileError extends Error {
|
|
44
|
+
/** Stable diagnostic code. */
|
|
45
|
+
readonly code: CaseFileErrorCode;
|
|
46
|
+
/** Hue artifact identity of the refused file. */
|
|
47
|
+
readonly artifactId: string;
|
|
48
|
+
constructor(
|
|
49
|
+
/** Stable diagnostic code. */
|
|
50
|
+
code: CaseFileErrorCode,
|
|
51
|
+
/** Hue artifact identity of the refused file. */
|
|
52
|
+
artifactId: string, message: string);
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Whether `name` can be used unchanged as one file name: not empty, `.` or `..`; no `/`, `\`,
|
|
56
|
+
* C0 or C1 control character, or character Windows reserves (`: < > " | ? *`); no Windows device
|
|
57
|
+
* name (`CON`, `NUL`, `COM1`, … with any extension); no trailing dot or space, which Windows
|
|
58
|
+
* drops; at most 200 UTF-8 bytes. An absolute path or a path with a parent step always contains
|
|
59
|
+
* a separator, so it is refused too.
|
|
60
|
+
*/
|
|
61
|
+
export declare function isSafeFileName(name: string): boolean;
|
|
62
|
+
/** Refuse a pinned file whose name is not one safe file name; see {@link isSafeFileName}. */
|
|
63
|
+
export declare function assertSafeFileNames(files: readonly Pick<CaseFile, "artifactId" | "filename">[]): void;
|
|
64
|
+
/**
|
|
65
|
+
* Reduce a declared file name to a single printable path segment, at most 200 UTF-8 bytes. A
|
|
66
|
+
* longer name keeps its extension, so `.pdf` stays `.pdf`, and its stem is shortened and marked
|
|
67
|
+
* with a short hash of the whole name, so two long names never shorten to the same one.
|
|
68
|
+
*/
|
|
36
69
|
export declare function safeFilename(name: string): string;
|
|
70
|
+
/** The device and inode a listing saw for a file, compared with the file actually opened. */
|
|
71
|
+
export interface FileIdentity {
|
|
72
|
+
dev: number;
|
|
73
|
+
ino: number;
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Read a file an agent wrote without trusting its path: the last component must not be a
|
|
77
|
+
* symlink and the open file must be a regular file of at most `maxBytes`, checked before any byte
|
|
78
|
+
* is read, and, when `expected` is given, the very file a listing saw. The bytes come from that
|
|
79
|
+
* open file. Returns undefined for a missing file only when nothing was expected there; every
|
|
80
|
+
* other refusal is an `OutputFileError` naming `label`.
|
|
81
|
+
*/
|
|
82
|
+
export declare function readAgentFile(path: string, label: string, maxBytes: number, expected?: FileIdentity): Promise<Uint8Array | undefined>;
|
|
83
|
+
/** Check pinned files against the manifest without keeping them: nothing is written to disk. */
|
|
84
|
+
export declare function verifyCaseFiles(client: EvaluationClient, files: readonly SubjectFile[]): Promise<void>;
|
|
37
85
|
/**
|
|
38
86
|
* Save verified copies of a case's pinned input files under `directory`. A file already
|
|
39
87
|
* present with the pinned identity is reused, so a resumed run does not download again.
|
|
40
88
|
*/
|
|
41
89
|
export declare function downloadCaseFiles(client: EvaluationClient, files: SubjectFile[], directory: string, primaryArtifactId?: string | null): Promise<LocalFile[]>;
|
|
90
|
+
/**
|
|
91
|
+
* Download a case's needed inputs under `caseDirectory`: the agent-visible roles into `inputs/`
|
|
92
|
+
* and evaluator-only files into `evaluator-inputs/`, so the directory the target's files live in
|
|
93
|
+
* never holds an evaluator's file. `all` keeps the manifest order for scorers; `target` is the
|
|
94
|
+
* agent-visible subset.
|
|
95
|
+
*/
|
|
96
|
+
export declare function downloadCaseInputs(client: EvaluationClient, files: CaseFile[], caseDirectory: string): Promise<{
|
|
97
|
+
all: LocalFile[];
|
|
98
|
+
target: LocalFile[];
|
|
99
|
+
}>;
|
|
42
100
|
/** Copy the target's generated files next to the checkpoint and record their identities. */
|
|
43
101
|
export declare function stageOutputFiles(files: OutputFile[], directory: string): Promise<StagedOutputFile[]>;
|
|
102
|
+
/** How long an upload waits for Hue to verify one artifact, and how often it looks. Hue verifies
|
|
103
|
+
* within a two-minute lease, which can outlast one request's timeout. */
|
|
104
|
+
export declare const artifactSettling: {
|
|
105
|
+
settleMillis: number;
|
|
106
|
+
pollMillis: number;
|
|
107
|
+
maxPollMillis: number;
|
|
108
|
+
};
|
|
44
109
|
/**
|
|
45
110
|
* Publish staged files as verified artifacts. Reservations use stable keys derived from the
|
|
46
|
-
* execution and the bytes, so a resumed upload settles on the same artifact
|
|
111
|
+
* execution and the bytes, so a resumed upload settles on the same artifact, including one Hue
|
|
112
|
+
* finished verifying after an earlier attempt stopped waiting.
|
|
47
113
|
*/
|
|
48
|
-
export declare function uploadOutputFiles(client: EvaluationClient, executionId: string, files: StagedOutputFile[], save: () => Promise<void
|
|
114
|
+
export declare function uploadOutputFiles(client: EvaluationClient, executionId: string, files: StagedOutputFile[], save: () => Promise<void>, timing?: typeof artifactSettling): Promise<void>;
|
|
49
115
|
/** Project uploaded staged files as the verified local files handed to scorers. */
|
|
50
116
|
export declare function localOutputFiles(files: StagedOutputFile[]): LocalFile[];
|
package/dist/evals/files.js
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
2
|
+
import { constants } from "node:fs";
|
|
3
|
+
import { lstat, mkdir, open, readFile, rename, rm } from "node:fs/promises";
|
|
4
|
+
import { extname, join, resolve } from "node:path";
|
|
4
5
|
import { randomUUID } from "node:crypto";
|
|
5
|
-
import { HueApiError } from "./client.js";
|
|
6
|
+
import { ArtifactSizeError, HueApiError } from "./client.js";
|
|
6
7
|
/** Roles the target receives. Organization templates stay with grading, as in the managed protocol. */
|
|
7
8
|
export const targetFileRoles = [
|
|
8
9
|
"source",
|
|
@@ -37,25 +38,153 @@ export class OutputFileError extends Error {
|
|
|
37
38
|
this.name = "OutputFileError";
|
|
38
39
|
}
|
|
39
40
|
}
|
|
41
|
+
/**
|
|
42
|
+
* Thrown when a pinned file cannot be used: its downloaded bytes differ from the manifest's size
|
|
43
|
+
* or SHA-256 (`case_file_mismatch`), or a file meant for an agent working in a world is not
|
|
44
|
+
* named by one safe file name (`case_file_name_refused`). Raised before the case's execution
|
|
45
|
+
* starts, so no execution, world or target call is spent on it.
|
|
46
|
+
*/
|
|
47
|
+
export class CaseFileError extends Error {
|
|
48
|
+
code;
|
|
49
|
+
artifactId;
|
|
50
|
+
constructor(
|
|
51
|
+
/** Stable diagnostic code. */
|
|
52
|
+
code,
|
|
53
|
+
/** Hue artifact identity of the refused file. */
|
|
54
|
+
artifactId, message) {
|
|
55
|
+
super(`${message} (${code})`);
|
|
56
|
+
this.code = code;
|
|
57
|
+
this.artifactId = artifactId;
|
|
58
|
+
this.name = "CaseFileError";
|
|
59
|
+
}
|
|
60
|
+
}
|
|
40
61
|
const sha256 = (bytes) => createHash("sha256").update(bytes).digest("hex");
|
|
41
|
-
/**
|
|
62
|
+
/** A file name quoted for a message: at most 120 characters, controls escaped. */
|
|
63
|
+
const quoted = (name) => JSON.stringify([...name].slice(0, 120).join(""));
|
|
64
|
+
/** Names Windows reserves for devices, with or without an extension. */
|
|
65
|
+
const DEVICE_NAME = /^(?:con|prn|aux|nul|com[0-9¹²³]|lpt[0-9¹²³]|conin\$|conout\$)$/iu;
|
|
66
|
+
/** The longest file name the runner writes, in UTF-8 bytes: room is left for the temporary
|
|
67
|
+
* suffix of an atomic write and a copy counter within the usual 255-byte limit. */
|
|
68
|
+
const MAX_NAME_BYTES = 200;
|
|
69
|
+
/** The longest prefix of `name` that fits in `bytes` UTF-8 bytes, cut between code points. */
|
|
70
|
+
function truncateBytes(name, bytes) {
|
|
71
|
+
let kept = "";
|
|
72
|
+
let size = 0;
|
|
73
|
+
for (const character of name) {
|
|
74
|
+
size += Buffer.byteLength(character);
|
|
75
|
+
if (size > bytes)
|
|
76
|
+
break;
|
|
77
|
+
kept += character;
|
|
78
|
+
}
|
|
79
|
+
return kept;
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Whether `name` can be used unchanged as one file name: not empty, `.` or `..`; no `/`, `\`,
|
|
83
|
+
* C0 or C1 control character, or character Windows reserves (`: < > " | ? *`); no Windows device
|
|
84
|
+
* name (`CON`, `NUL`, `COM1`, … with any extension); no trailing dot or space, which Windows
|
|
85
|
+
* drops; at most 200 UTF-8 bytes. An absolute path or a path with a parent step always contains
|
|
86
|
+
* a separator, so it is refused too.
|
|
87
|
+
*/
|
|
88
|
+
export function isSafeFileName(name) {
|
|
89
|
+
if (typeof name !== "string" || !name || name === "." || name === "..")
|
|
90
|
+
return false;
|
|
91
|
+
// eslint-disable-next-line no-control-regex -- control characters are refused
|
|
92
|
+
if (/[/\\:<>"|?*\x00-\x1f\x7f-\x9f]/u.test(name))
|
|
93
|
+
return false;
|
|
94
|
+
if (name.endsWith(".") || name.endsWith(" "))
|
|
95
|
+
return false;
|
|
96
|
+
if (DEVICE_NAME.test(name.split(".")[0].trimEnd()))
|
|
97
|
+
return false;
|
|
98
|
+
return Buffer.byteLength(name) <= MAX_NAME_BYTES;
|
|
99
|
+
}
|
|
100
|
+
/** Refuse a pinned file whose name is not one safe file name; see {@link isSafeFileName}. */
|
|
101
|
+
export function assertSafeFileNames(files) {
|
|
102
|
+
for (const file of files)
|
|
103
|
+
if (!isSafeFileName(file.filename))
|
|
104
|
+
throw new CaseFileError("case_file_name_refused", file.artifactId, `Pinned file ${quoted(String(file.filename))} is not a safe file name`);
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* Reduce a declared file name to a single printable path segment, at most 200 UTF-8 bytes. A
|
|
108
|
+
* longer name keeps its extension, so `.pdf` stays `.pdf`, and its stem is shortened and marked
|
|
109
|
+
* with a short hash of the whole name, so two long names never shorten to the same one.
|
|
110
|
+
*/
|
|
42
111
|
export function safeFilename(name) {
|
|
43
112
|
const cleaned = [...name]
|
|
44
113
|
.filter((c) => c.charCodeAt(0) >= 32 && c.charCodeAt(0) !== 127 && c !== "/" && c !== "\\")
|
|
45
114
|
.join("")
|
|
46
115
|
.trim();
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
116
|
+
let kept = cleaned;
|
|
117
|
+
if (Buffer.byteLength(cleaned) > MAX_NAME_BYTES) {
|
|
118
|
+
const found = extname(cleaned);
|
|
119
|
+
const extension = Buffer.byteLength(found) <= 32 ? found : "";
|
|
120
|
+
const mark = `~${sha256(Buffer.from(cleaned)).slice(0, 8)}`;
|
|
121
|
+
const room = MAX_NAME_BYTES - Buffer.byteLength(extension) - mark.length;
|
|
122
|
+
kept = `${truncateBytes(cleaned.slice(0, cleaned.length - extension.length), room)}${mark}${extension}`;
|
|
123
|
+
}
|
|
124
|
+
return kept && kept !== "." && kept !== ".." ? kept : "file";
|
|
50
125
|
}
|
|
126
|
+
/** Create `path` if needed and require a directory this user owns that no one else can open. */
|
|
51
127
|
async function privateDirectory(path) {
|
|
52
128
|
const root = resolve(path);
|
|
53
129
|
await mkdir(root, { recursive: true, mode: 0o700 });
|
|
54
130
|
const info = await lstat(root);
|
|
55
|
-
|
|
56
|
-
|
|
131
|
+
const uid = process.getuid?.();
|
|
132
|
+
if (!info.isDirectory() ||
|
|
133
|
+
info.isSymbolicLink() ||
|
|
134
|
+
(uid !== undefined && info.uid !== uid) ||
|
|
135
|
+
(process.platform !== "win32" && (info.mode & 0o077) !== 0))
|
|
136
|
+
throw new Error("Use a private files directory (owned by this user, mode 0700, no symlink)");
|
|
57
137
|
return root;
|
|
58
138
|
}
|
|
139
|
+
// A symlink as the last path component fails to open, and a FIFO or device never blocks.
|
|
140
|
+
const AGENT_FILE_FLAGS = constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK;
|
|
141
|
+
/**
|
|
142
|
+
* Read a file an agent wrote without trusting its path: the last component must not be a
|
|
143
|
+
* symlink and the open file must be a regular file of at most `maxBytes`, checked before any byte
|
|
144
|
+
* is read, and, when `expected` is given, the very file a listing saw. The bytes come from that
|
|
145
|
+
* open file. Returns undefined for a missing file only when nothing was expected there; every
|
|
146
|
+
* other refusal is an `OutputFileError` naming `label`.
|
|
147
|
+
*/
|
|
148
|
+
export async function readAgentFile(path, label, maxBytes, expected) {
|
|
149
|
+
let file;
|
|
150
|
+
try {
|
|
151
|
+
file = await open(path, AGENT_FILE_FLAGS);
|
|
152
|
+
}
|
|
153
|
+
catch (error) {
|
|
154
|
+
const code = error.code;
|
|
155
|
+
if (code === "ENOENT" && !expected)
|
|
156
|
+
return undefined;
|
|
157
|
+
if (code === "ELOOP" || code === "EMLINK")
|
|
158
|
+
throw new OutputFileError(`${label} is a symbolic link`);
|
|
159
|
+
if (code === "ENOENT")
|
|
160
|
+
throw new OutputFileError(`${label} changed while it was collected`);
|
|
161
|
+
throw new OutputFileError(`${label} could not be read`);
|
|
162
|
+
}
|
|
163
|
+
try {
|
|
164
|
+
const info = await file.stat();
|
|
165
|
+
if (!info.isFile())
|
|
166
|
+
throw new OutputFileError(`${label} is not a regular file`);
|
|
167
|
+
if (expected && (info.dev !== expected.dev || info.ino !== expected.ino))
|
|
168
|
+
throw new OutputFileError(`${label} changed while it was collected`);
|
|
169
|
+
if (info.size > maxBytes)
|
|
170
|
+
throw new OutputFileError(`${label} exceeds ${Math.floor(maxBytes / 1024 / 1024)} MiB`);
|
|
171
|
+
// One byte past the size seen: a file that is still growing is refused, not read whole.
|
|
172
|
+
const bytes = Buffer.allocUnsafe(info.size + 1);
|
|
173
|
+
let total = 0;
|
|
174
|
+
while (total < bytes.length) {
|
|
175
|
+
const { bytesRead } = await file.read(bytes, total, bytes.length - total, total);
|
|
176
|
+
if (!bytesRead)
|
|
177
|
+
break;
|
|
178
|
+
total += bytesRead;
|
|
179
|
+
}
|
|
180
|
+
if (total !== info.size)
|
|
181
|
+
throw new OutputFileError(`${label} changed while it was collected`);
|
|
182
|
+
return bytes.subarray(0, total);
|
|
183
|
+
}
|
|
184
|
+
finally {
|
|
185
|
+
await file.close();
|
|
186
|
+
}
|
|
187
|
+
}
|
|
59
188
|
async function writePrivate(path, bytes) {
|
|
60
189
|
const temporary = `${path}.${randomUUID()}.tmp`;
|
|
61
190
|
const file = await open(temporary, "wx", 0o600);
|
|
@@ -74,6 +203,28 @@ async function verifiedBytes(path, expected) {
|
|
|
74
203
|
throw new Error(`Saved file ${path} no longer matches its verified identity`);
|
|
75
204
|
return bytes;
|
|
76
205
|
}
|
|
206
|
+
/** Download one pinned file and check its size and SHA-256 against the manifest. The download
|
|
207
|
+
* stops one byte past the pinned size. */
|
|
208
|
+
async function downloadVerified(client, file) {
|
|
209
|
+
const mismatch = () => new CaseFileError("case_file_mismatch", file.artifactId, `Downloaded file ${quoted(file.filename)} does not match its pinned size and SHA-256`);
|
|
210
|
+
let bytes;
|
|
211
|
+
try {
|
|
212
|
+
bytes = await client.downloadArtifact(file.artifactId, { maxBytes: file.byteSize });
|
|
213
|
+
}
|
|
214
|
+
catch (error) {
|
|
215
|
+
if (error instanceof ArtifactSizeError)
|
|
216
|
+
throw mismatch();
|
|
217
|
+
throw error;
|
|
218
|
+
}
|
|
219
|
+
if (bytes.byteLength !== file.byteSize || sha256(bytes) !== file.sha256)
|
|
220
|
+
throw mismatch();
|
|
221
|
+
return bytes;
|
|
222
|
+
}
|
|
223
|
+
/** Check pinned files against the manifest without keeping them: nothing is written to disk. */
|
|
224
|
+
export async function verifyCaseFiles(client, files) {
|
|
225
|
+
for (const file of files)
|
|
226
|
+
await downloadVerified(client, file);
|
|
227
|
+
}
|
|
77
228
|
/**
|
|
78
229
|
* Save verified copies of a case's pinned input files under `directory`. A file already
|
|
79
230
|
* present with the pinned identity is reused, so a resumed run does not download again.
|
|
@@ -91,12 +242,8 @@ export async function downloadCaseFiles(client, files, directory, primaryArtifac
|
|
|
91
242
|
catch {
|
|
92
243
|
present = false;
|
|
93
244
|
}
|
|
94
|
-
if (!present)
|
|
95
|
-
|
|
96
|
-
if (bytes.byteLength !== file.byteSize || sha256(bytes) !== file.sha256)
|
|
97
|
-
throw new Error(`Downloaded input ${file.filename} does not match its pinned identity`);
|
|
98
|
-
await writePrivate(path, bytes);
|
|
99
|
-
}
|
|
245
|
+
if (!present)
|
|
246
|
+
await writePrivate(path, await downloadVerified(client, file));
|
|
100
247
|
saved.push({
|
|
101
248
|
artifactId: file.artifactId,
|
|
102
249
|
role: file.role,
|
|
@@ -110,6 +257,26 @@ export async function downloadCaseFiles(client, files, directory, primaryArtifac
|
|
|
110
257
|
}
|
|
111
258
|
return saved;
|
|
112
259
|
}
|
|
260
|
+
/**
|
|
261
|
+
* Download a case's needed inputs under `caseDirectory`: the agent-visible roles into `inputs/`
|
|
262
|
+
* and evaluator-only files into `evaluator-inputs/`, so the directory the target's files live in
|
|
263
|
+
* never holds an evaluator's file. `all` keeps the manifest order for scorers; `target` is the
|
|
264
|
+
* agent-visible subset.
|
|
265
|
+
*/
|
|
266
|
+
export async function downloadCaseInputs(client, files, caseDirectory) {
|
|
267
|
+
const visible = (file) => targetFileRoles.includes(file.role);
|
|
268
|
+
const agentFiles = files.filter(visible);
|
|
269
|
+
const evaluatorFiles = files.filter((file) => !visible(file));
|
|
270
|
+
const target = agentFiles.length
|
|
271
|
+
? await downloadCaseFiles(client, agentFiles, join(caseDirectory, "inputs"))
|
|
272
|
+
: [];
|
|
273
|
+
const evaluator = evaluatorFiles.length
|
|
274
|
+
? await downloadCaseFiles(client, evaluatorFiles, join(caseDirectory, "evaluator-inputs"))
|
|
275
|
+
: [];
|
|
276
|
+
const saved = [...target, ...evaluator];
|
|
277
|
+
const all = files.map((file) => saved.find((entry) => entry.artifactId === file.artifactId && entry.role === file.role));
|
|
278
|
+
return { all, target };
|
|
279
|
+
}
|
|
113
280
|
/** Copy the target's generated files next to the checkpoint and record their identities. */
|
|
114
281
|
export async function stageOutputFiles(files, directory) {
|
|
115
282
|
if (!Array.isArray(files) || !files.length)
|
|
@@ -132,24 +299,17 @@ export async function stageOutputFiles(files, directory) {
|
|
|
132
299
|
names.add(filename);
|
|
133
300
|
if (!outputContentTypes.includes(file.contentType))
|
|
134
301
|
throw new OutputFileError(`Unsupported generated file type ${String(file.contentType)}`);
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
catch {
|
|
302
|
+
// A path is read once, as a regular file within the limit, and staged from those bytes.
|
|
303
|
+
const bytes = file.bytes ??
|
|
304
|
+
(await readAgentFile(file.path, `Generated file ${filename}`, outputFileLimits.bytes));
|
|
305
|
+
if (!bytes)
|
|
140
306
|
throw new OutputFileError(`Generated file ${filename} could not be read`);
|
|
141
|
-
}
|
|
142
307
|
if (!bytes.byteLength)
|
|
143
308
|
throw new OutputFileError(`Generated file ${filename} is empty`);
|
|
144
309
|
if (bytes.byteLength > outputFileLimits.bytes)
|
|
145
310
|
throw new OutputFileError(`Generated file ${filename} exceeds 25 MiB`);
|
|
146
311
|
const path = join(root, filename);
|
|
147
|
-
|
|
148
|
-
await writePrivate(path, bytes);
|
|
149
|
-
else {
|
|
150
|
-
await copyFile(file.path, path);
|
|
151
|
-
await verifiedBytes(path, { byteSize: bytes.byteLength, sha256: sha256(bytes) });
|
|
152
|
-
}
|
|
312
|
+
await writePrivate(path, bytes);
|
|
153
313
|
staged.push({
|
|
154
314
|
filename,
|
|
155
315
|
contentType: file.contentType,
|
|
@@ -161,11 +321,74 @@ export async function stageOutputFiles(files, directory) {
|
|
|
161
321
|
}
|
|
162
322
|
return staged;
|
|
163
323
|
}
|
|
324
|
+
/** How long an upload waits for Hue to verify one artifact, and how often it looks. Hue verifies
|
|
325
|
+
* within a two-minute lease, which can outlast one request's timeout. */
|
|
326
|
+
export const artifactSettling = { settleMillis: 180_000, pollMillis: 1000, maxPollMillis: 10_000 };
|
|
327
|
+
/** A completion failure that can mean Hue is still verifying: a lost or timed-out response, a
|
|
328
|
+
* verification already running (409), or a refusal to retry (429, 503). */
|
|
329
|
+
function mayStillVerify(error) {
|
|
330
|
+
return (error instanceof HueApiError &&
|
|
331
|
+
(error.status === undefined ||
|
|
332
|
+
error.status === 409 ||
|
|
333
|
+
error.status === 429 ||
|
|
334
|
+
error.status === 503));
|
|
335
|
+
}
|
|
336
|
+
/**
|
|
337
|
+
* Completes an artifact's verification and returns the state it settles in. When a completion
|
|
338
|
+
* times out, finds verification already running or is refused for now, the artifact is read after
|
|
339
|
+
* a pause: ready is returned; verifying is completed again, which Hue refuses while its lease
|
|
340
|
+
* holds and restarts once it lapsed; an artifact a retryable refusal (a lost response, 429 or 503)
|
|
341
|
+
* left unverified is completed again up to three times; any other state is returned for the
|
|
342
|
+
* caller to refuse (a resume uploads a rejected artifact again). All within `timing.settleMillis`.
|
|
343
|
+
*/
|
|
344
|
+
async function settleArtifact(client, id, timing) {
|
|
345
|
+
const deadline = Date.now() + timing.settleMillis;
|
|
346
|
+
let pause = timing.pollMillis;
|
|
347
|
+
let retries = 0;
|
|
348
|
+
for (;;) {
|
|
349
|
+
let refusedForNow;
|
|
350
|
+
let asked = 0;
|
|
351
|
+
try {
|
|
352
|
+
return (await client.completeArtifact(id)).state;
|
|
353
|
+
}
|
|
354
|
+
catch (error) {
|
|
355
|
+
if (!mayStillVerify(error) || Date.now() >= deadline)
|
|
356
|
+
throw error;
|
|
357
|
+
refusedForNow = error.status !== 409;
|
|
358
|
+
// A refusal that says how long to wait is waited out, within the settling window.
|
|
359
|
+
asked = (error.retryAfterSeconds ?? 0) * 1000;
|
|
360
|
+
}
|
|
361
|
+
await new Promise((resolve) => setTimeout(resolve, Math.max(0, Math.min(Math.max(pause, asked), deadline - Date.now()))));
|
|
362
|
+
pause = Math.min(pause * 2, timing.maxPollMillis);
|
|
363
|
+
let current;
|
|
364
|
+
try {
|
|
365
|
+
current = await client.getArtifact(id);
|
|
366
|
+
}
|
|
367
|
+
catch (error) {
|
|
368
|
+
if (!mayStillVerify(error))
|
|
369
|
+
throw error;
|
|
370
|
+
}
|
|
371
|
+
if (current?.state === "ready")
|
|
372
|
+
return current.state;
|
|
373
|
+
const unverified = current &&
|
|
374
|
+
["reserved", "rejected"].includes(current.state) &&
|
|
375
|
+
current.failureCode !== "mismatch";
|
|
376
|
+
if (current && current.state !== "verifying" && !(unverified && refusedForNow && retries++ < 3))
|
|
377
|
+
return current.state;
|
|
378
|
+
if (Date.now() >= deadline) {
|
|
379
|
+
const seconds = Math.round(timing.settleMillis / 1000);
|
|
380
|
+
throw new Error(current?.state === "verifying"
|
|
381
|
+
? `Hue was still verifying generated file ${id} after ${seconds} seconds`
|
|
382
|
+
: `Hue had not verified generated file ${id} after ${seconds} seconds (${current ? `it was ${current.state}` : "its state could not be read"})`);
|
|
383
|
+
}
|
|
384
|
+
}
|
|
385
|
+
}
|
|
164
386
|
/**
|
|
165
387
|
* Publish staged files as verified artifacts. Reservations use stable keys derived from the
|
|
166
|
-
* execution and the bytes, so a resumed upload settles on the same artifact
|
|
388
|
+
* execution and the bytes, so a resumed upload settles on the same artifact, including one Hue
|
|
389
|
+
* finished verifying after an earlier attempt stopped waiting.
|
|
167
390
|
*/
|
|
168
|
-
export async function uploadOutputFiles(client, executionId, files, save) {
|
|
391
|
+
export async function uploadOutputFiles(client, executionId, files, save, timing = artifactSettling) {
|
|
169
392
|
for (const file of files) {
|
|
170
393
|
if (file.artifactId)
|
|
171
394
|
continue;
|
|
@@ -179,7 +402,8 @@ export async function uploadOutputFiles(client, executionId, files, save) {
|
|
|
179
402
|
});
|
|
180
403
|
let state = reserved.state;
|
|
181
404
|
if (state !== "ready") {
|
|
182
|
-
|
|
405
|
+
// A verification already running holds the uploaded bytes; Hue refuses another upload then.
|
|
406
|
+
if (reserved.copyState !== "acknowledged" && state !== "verifying") {
|
|
183
407
|
const upload = await client.requestArtifactUpload(reserved.id);
|
|
184
408
|
try {
|
|
185
409
|
await client.uploadArtifactBytes(upload, bytes, file.contentType);
|
|
@@ -189,18 +413,7 @@ export async function uploadOutputFiles(client, executionId, files, save) {
|
|
|
189
413
|
// settled by verified completion; never rewrite a final object here.
|
|
190
414
|
}
|
|
191
415
|
}
|
|
192
|
-
|
|
193
|
-
for (;;) {
|
|
194
|
-
try {
|
|
195
|
-
state = (await client.completeArtifact(reserved.id)).state;
|
|
196
|
-
break;
|
|
197
|
-
}
|
|
198
|
-
catch (error) {
|
|
199
|
-
if (!(error instanceof HueApiError) || error.status !== 503 || attempt++ >= 2)
|
|
200
|
-
throw error;
|
|
201
|
-
await new Promise((resolve) => setTimeout(resolve, 1000 * attempt));
|
|
202
|
-
}
|
|
203
|
-
}
|
|
416
|
+
state = await settleArtifact(client, reserved.id, timing);
|
|
204
417
|
}
|
|
205
418
|
if (state !== "ready")
|
|
206
419
|
throw new Error(`Generated file ${file.filename} was not verified by Hue (${state})`);
|
package/dist/evals/json.js
CHANGED
|
@@ -4,27 +4,96 @@ export function aggregateBounds(bytes) {
|
|
|
4
4
|
return { bytes, nodes: Math.ceil(bytes / 2), depth: valueBounds.depth + 8 };
|
|
5
5
|
}
|
|
6
6
|
const isText = (item) => item.isWellFormed() && !item.includes("\u0000");
|
|
7
|
+
/** A string's code points: its UTF-16 length less one for each surrogate pair. */
|
|
8
|
+
function codePoints(text) {
|
|
9
|
+
let count = text.length;
|
|
10
|
+
for (let index = 0; index < text.length - 1; index++) {
|
|
11
|
+
const unit = text.charCodeAt(index);
|
|
12
|
+
if (unit < 0xd800 || unit > 0xdbff)
|
|
13
|
+
continue;
|
|
14
|
+
const next = text.charCodeAt(index + 1);
|
|
15
|
+
if (next >= 0xdc00 && next <= 0xdfff) {
|
|
16
|
+
count--;
|
|
17
|
+
index++;
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
return count;
|
|
21
|
+
}
|
|
22
|
+
/** Thrown within `json` when the byte bound is passed, which is checked last. */
|
|
23
|
+
class PastByteBound extends Error {
|
|
24
|
+
}
|
|
7
25
|
/** Reject lossy JSON serialization before creating requests/checkpoints. */
|
|
8
26
|
export function json(value, requested = valueBounds) {
|
|
9
27
|
const bounds = typeof requested === "number" ? { ...valueBounds, bytes: requested } : requested;
|
|
10
28
|
const ancestors = new Set();
|
|
11
29
|
let nodes = 0;
|
|
30
|
+
// The UTF-8 length of the JSON text, counted as the value is read, so a value too long to
|
|
31
|
+
// serialize (past the runtime's longest string) is refused before `JSON.stringify` is asked to.
|
|
32
|
+
let bytes = 0;
|
|
33
|
+
// Once the byte bound is passed, the value is read again only to check it (below).
|
|
34
|
+
let checking = false;
|
|
35
|
+
// Text the second read has found valid is not checked again, so a string referenced many times
|
|
36
|
+
// is scanned once: by content up to 16,383 UTF-16 units, and a longer string when it is the last
|
|
37
|
+
// checked of its length. V8 hashes those by length alone, so a set of them would compare each
|
|
38
|
+
// with every other of that length.
|
|
39
|
+
const checkedText = new Set();
|
|
40
|
+
const checkedLong = new Map();
|
|
41
|
+
const text = (item) => {
|
|
42
|
+
if (!checking)
|
|
43
|
+
return isText(item);
|
|
44
|
+
if (item.length <= 16_383 ? checkedText.has(item) : checkedLong.get(item.length) === item)
|
|
45
|
+
return true;
|
|
46
|
+
if (!isText(item))
|
|
47
|
+
return false;
|
|
48
|
+
if (item.length <= 16_383)
|
|
49
|
+
checkedText.add(item);
|
|
50
|
+
else
|
|
51
|
+
checkedLong.set(item.length, item);
|
|
52
|
+
return true;
|
|
53
|
+
};
|
|
54
|
+
const charge = (amount) => {
|
|
55
|
+
if (checking)
|
|
56
|
+
return;
|
|
57
|
+
bytes += amount;
|
|
58
|
+
if (bytes > bounds.bytes)
|
|
59
|
+
throw new PastByteBound();
|
|
60
|
+
};
|
|
61
|
+
// A string's JSON text is at least as long as the string, so one longer than what is left is
|
|
62
|
+
// refused before it is escaped.
|
|
63
|
+
const chargeText = (text) => {
|
|
64
|
+
if (checking)
|
|
65
|
+
return;
|
|
66
|
+
if (text.length + 2 > bounds.bytes - bytes)
|
|
67
|
+
throw new PastByteBound();
|
|
68
|
+
charge(Buffer.byteLength(JSON.stringify(text)));
|
|
69
|
+
};
|
|
12
70
|
const visit = (item, depth) => {
|
|
13
71
|
if (++nodes > bounds.nodes || depth > bounds.depth)
|
|
14
72
|
throw new RangeError("JSON exceeds depth/node limits");
|
|
15
|
-
if (item === null || typeof item === "boolean")
|
|
73
|
+
if (item === null || typeof item === "boolean") {
|
|
74
|
+
charge(item === false ? 5 : 4);
|
|
16
75
|
return item;
|
|
17
|
-
|
|
76
|
+
}
|
|
77
|
+
if (typeof item === "number" && Number.isFinite(item)) {
|
|
78
|
+
charge(JSON.stringify(item).length);
|
|
18
79
|
return item;
|
|
19
|
-
|
|
80
|
+
}
|
|
81
|
+
if (typeof item === "string" && text(item)) {
|
|
82
|
+
chargeText(item);
|
|
20
83
|
return item;
|
|
84
|
+
}
|
|
21
85
|
if (!item || typeof item !== "object" || ancestors.has(item))
|
|
22
86
|
throw new TypeError("Expected finite JSON without cycles or invalid Unicode");
|
|
23
87
|
ancestors.add(item);
|
|
24
88
|
try {
|
|
25
89
|
if (Array.isArray(item)) {
|
|
90
|
+
// Each element is a value: an array longer than the values left is refused before its
|
|
91
|
+
// keys are listed.
|
|
92
|
+
if (item.length > bounds.nodes - nodes)
|
|
93
|
+
throw new RangeError("JSON exceeds depth/node limits");
|
|
26
94
|
if (Object.keys(item).length !== item.length)
|
|
27
95
|
throw new TypeError("Sparse/extended arrays are not JSON");
|
|
96
|
+
charge(item.length ? item.length + 1 : 2);
|
|
28
97
|
return item.map((entry) => visit(entry, depth + 1));
|
|
29
98
|
}
|
|
30
99
|
if (Object.getPrototypeOf(item) !== Object.prototype && Object.getPrototypeOf(item) !== null)
|
|
@@ -32,9 +101,33 @@ export function json(value, requested = valueBounds) {
|
|
|
32
101
|
if (Object.getOwnPropertySymbols(item).length)
|
|
33
102
|
throw new TypeError("JSON cannot contain symbol properties");
|
|
34
103
|
const result = Object.create(null);
|
|
35
|
-
|
|
36
|
-
|
|
104
|
+
// Each member is a value: an object with more than the values left is refused before its
|
|
105
|
+
// keys are sorted.
|
|
106
|
+
const keys = Object.keys(item);
|
|
107
|
+
if (keys.length > bounds.nodes - nodes)
|
|
108
|
+
throw new RangeError("JSON exceeds depth/node limits");
|
|
109
|
+
if (!checking) {
|
|
110
|
+
// Keys whose JSON text alone (at least their code points, two quotes and a colon each)
|
|
111
|
+
// needs more bytes than are left pass the byte bound before they are sorted, as in the
|
|
112
|
+
// Python SDK, so both read such an object the same way. A key has at least half its
|
|
113
|
+
// UTF-16 length in code points, so a long one passes it without counting them.
|
|
114
|
+
let left = bounds.bytes - bytes;
|
|
115
|
+
for (const key of keys) {
|
|
116
|
+
left -= 3;
|
|
117
|
+
if (key.length / 2 > left)
|
|
118
|
+
throw new PastByteBound();
|
|
119
|
+
left -= codePoints(key);
|
|
120
|
+
if (left < 0)
|
|
121
|
+
throw new PastByteBound();
|
|
122
|
+
}
|
|
123
|
+
keys.sort();
|
|
124
|
+
}
|
|
125
|
+
charge(keys.length ? keys.length + 1 : 2);
|
|
126
|
+
for (const key of keys) {
|
|
127
|
+
if (!text(key))
|
|
37
128
|
throw new TypeError("Expected finite JSON without cycles or invalid Unicode");
|
|
129
|
+
chargeText(key);
|
|
130
|
+
charge(1);
|
|
38
131
|
const descriptor = Object.getOwnPropertyDescriptor(item, key);
|
|
39
132
|
if (!("value" in descriptor))
|
|
40
133
|
throw new TypeError("JSON cannot contain accessors");
|
|
@@ -47,10 +140,24 @@ export function json(value, requested = valueBounds) {
|
|
|
47
140
|
ancestors.delete(item);
|
|
48
141
|
}
|
|
49
142
|
};
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
143
|
+
try {
|
|
144
|
+
const result = visit(value, 0);
|
|
145
|
+
if (Buffer.byteLength(JSON.stringify(result)) > bounds.bytes)
|
|
146
|
+
throw new RangeError("JSON exceeds byte limit");
|
|
147
|
+
return result;
|
|
148
|
+
}
|
|
149
|
+
catch (error) {
|
|
150
|
+
if (!(error instanceof PastByteBound))
|
|
151
|
+
throw error;
|
|
152
|
+
}
|
|
153
|
+
// The byte bound is checked last, as it was before the length was counted as the value is
|
|
154
|
+
// read: a value past it that holds a value that is not JSON, or passes the value or depth bound,
|
|
155
|
+
// is refused for that. So the value is read again with each object's keys in their own order,
|
|
156
|
+
// and nothing is counted, escaped or sorted, nor text checked twice (`text`).
|
|
157
|
+
checking = true;
|
|
158
|
+
nodes = 0;
|
|
159
|
+
visit(value, 0);
|
|
160
|
+
throw new RangeError("JSON exceeds byte limit");
|
|
54
161
|
}
|
|
55
162
|
export function digest(value) {
|
|
56
163
|
return createHash("sha256")
|