@hue-run/sdk 0.4.2 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLI.md +192 -4
- package/ENVIRONMENTS.md +29 -8
- package/EVALUATIONS.md +125 -4
- package/README.md +27 -2
- package/dist/cli/eval-direct.d.ts +47 -0
- package/dist/cli/eval-direct.js +171 -0
- package/dist/cli/eval.d.ts +28 -0
- package/dist/cli/eval.js +985 -0
- package/dist/cli/login.d.ts +32 -0
- package/dist/cli/login.js +712 -0
- package/dist/cli/mcp.d.ts +76 -0
- package/dist/cli/mcp.js +466 -0
- package/dist/evals/client.d.ts +30 -1
- package/dist/evals/client.js +138 -1
- package/dist/evals/files.d.ts +50 -0
- package/dist/evals/files.js +223 -0
- package/dist/evals/local-worker.d.ts +49 -4
- package/dist/evals/local-worker.js +63 -12
- package/dist/evals/runner.d.ts +30 -10
- package/dist/evals/runner.js +216 -68
- package/dist/evals/scenarios.d.ts +79 -0
- package/dist/evals/scenarios.js +169 -0
- package/dist/evals/scorers.d.ts +13 -0
- package/dist/evals/scorers.js +21 -3
- package/dist/evals/simulation.d.ts +12 -1
- package/dist/evals/simulation.js +36 -0
- package/dist/evals/types.d.ts +174 -1
- package/dist/evals/types.js +17 -1
- package/dist/evals/verdicts.d.ts +161 -0
- package/dist/evals/verdicts.js +192 -0
- package/dist/evals.d.ts +9 -3
- package/dist/evals.js +5 -1
- package/dist/setup/cli.js +13 -3
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +1 -1
package/dist/evals/client.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { validateOptions } from "../config.js";
|
|
1
|
+
import { isLoopbackHost, validateOptions } from "../config.js";
|
|
2
2
|
import { json, uuid, valueBounds } from "./json.js";
|
|
3
3
|
import { attemptBindingRead, parsePrepareAttemptResultV2, parseRefreshedAttemptResultV2, parseRevocationResult, prepareAttemptInputV2, validateAttemptConnectionBundleV2, } from "./attempt.js";
|
|
4
4
|
/** Thrown for a failed evaluation API request; the message is fixed and never includes response text. */
|
|
@@ -83,6 +83,73 @@ export class EvaluationClient {
|
|
|
83
83
|
throw new HueApiError();
|
|
84
84
|
}
|
|
85
85
|
}
|
|
86
|
+
/** Verified bytes of one ready artifact in this project, bounded to the 25 MiB pilot file size. */
|
|
87
|
+
async downloadArtifact(id) {
|
|
88
|
+
let response;
|
|
89
|
+
try {
|
|
90
|
+
response = await fetch(`${this.baseUrl}/api/v1/artifacts/${uuid(id)}/download`, {
|
|
91
|
+
method: "GET",
|
|
92
|
+
headers: { Authorization: `Bearer ${this.apiKey}` },
|
|
93
|
+
redirect: "error",
|
|
94
|
+
signal: AbortSignal.timeout(Math.max(this.timeoutMillis, 120_000)),
|
|
95
|
+
});
|
|
96
|
+
}
|
|
97
|
+
catch {
|
|
98
|
+
throw new HueApiError();
|
|
99
|
+
}
|
|
100
|
+
if (!response.ok) {
|
|
101
|
+
await response.body?.cancel();
|
|
102
|
+
throw new HueApiError(response.status);
|
|
103
|
+
}
|
|
104
|
+
try {
|
|
105
|
+
const reader = response.body?.getReader();
|
|
106
|
+
if (!reader)
|
|
107
|
+
throw new Error("Missing response");
|
|
108
|
+
const chunks = [];
|
|
109
|
+
let size = 0;
|
|
110
|
+
try {
|
|
111
|
+
for (;;) {
|
|
112
|
+
const { done, value } = await reader.read();
|
|
113
|
+
if (done)
|
|
114
|
+
break;
|
|
115
|
+
size += value.byteLength;
|
|
116
|
+
if (size > 25 * 1024 * 1024)
|
|
117
|
+
throw new Error("Oversized artifact");
|
|
118
|
+
chunks.push(value);
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
finally {
|
|
122
|
+
await reader.cancel();
|
|
123
|
+
}
|
|
124
|
+
return new Uint8Array(Buffer.concat(chunks));
|
|
125
|
+
}
|
|
126
|
+
catch {
|
|
127
|
+
throw new HueApiError();
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
/** Stage bytes at the storage capability Hue issued. The Hue key is never sent to storage. */
|
|
131
|
+
async uploadArtifactBytes(upload, bytes, contentType) {
|
|
132
|
+
const uploadUrl = signedUploadUrl(upload.uploadUrl);
|
|
133
|
+
if (upload.method !== "PUT")
|
|
134
|
+
throw new HueApiError();
|
|
135
|
+
const headers = signedUploadHeaders(upload.headers, contentType);
|
|
136
|
+
let response;
|
|
137
|
+
try {
|
|
138
|
+
response = await fetch(uploadUrl, {
|
|
139
|
+
method: "PUT",
|
|
140
|
+
headers,
|
|
141
|
+
body: bytes,
|
|
142
|
+
redirect: "error",
|
|
143
|
+
signal: AbortSignal.timeout(Math.max(this.timeoutMillis, 120_000)),
|
|
144
|
+
});
|
|
145
|
+
}
|
|
146
|
+
catch {
|
|
147
|
+
throw new HueApiError();
|
|
148
|
+
}
|
|
149
|
+
await response.body?.cancel().catch(() => undefined);
|
|
150
|
+
if (!response.ok)
|
|
151
|
+
throw new HueApiError(response.status);
|
|
152
|
+
}
|
|
86
153
|
page(options = {}) {
|
|
87
154
|
const query = new URLSearchParams();
|
|
88
155
|
if (options.after)
|
|
@@ -253,6 +320,22 @@ export class EvaluationClient {
|
|
|
253
320
|
throw new HueApiError();
|
|
254
321
|
}
|
|
255
322
|
}
|
|
323
|
+
/** Reads one artifact reservation and its verification state. */
|
|
324
|
+
getArtifact(id) {
|
|
325
|
+
return this.request("GET", `/artifacts/${uuid(id)}`);
|
|
326
|
+
}
|
|
327
|
+
/** Reserves an artifact by declared identity; replaying the key returns the same reservation. */
|
|
328
|
+
reserveArtifact(input) {
|
|
329
|
+
return this.request("POST", "/artifacts", input);
|
|
330
|
+
}
|
|
331
|
+
/** Issues a short-lived storage capability for staging the reserved artifact's bytes. */
|
|
332
|
+
requestArtifactUpload(id) {
|
|
333
|
+
return this.request("POST", `/artifacts/${uuid(id)}/upload`, {});
|
|
334
|
+
}
|
|
335
|
+
/** Asks Hue to verify the staged bytes against the declared identity. */
|
|
336
|
+
completeArtifact(id) {
|
|
337
|
+
return this.request("POST", `/artifacts/${uuid(id)}/complete`, {});
|
|
338
|
+
}
|
|
256
339
|
/** Saves an execution's outcome and creates its immutable subject. */
|
|
257
340
|
completeExecution(id, input) {
|
|
258
341
|
return this.request("POST", `/experiment-executions/${uuid(id)}/complete`, input);
|
|
@@ -269,6 +352,10 @@ export class EvaluationClient {
|
|
|
269
352
|
getEvaluationRun(id) {
|
|
270
353
|
return this.request("GET", `/evaluation-runs/${uuid(id)}`);
|
|
271
354
|
}
|
|
355
|
+
/** Every evaluation run of the project, oldest first; a grading worker polls this for pending pins. */
|
|
356
|
+
listEvaluationRuns(page) {
|
|
357
|
+
return this.request("GET", `/evaluation-runs${this.page(page)}`);
|
|
358
|
+
}
|
|
272
359
|
/** Lists the subjects of an evaluation run. */
|
|
273
360
|
listEvaluationItems(id, page) {
|
|
274
361
|
return this.request("GET", `/evaluation-runs/${uuid(id)}/items${this.page(page)}`);
|
|
@@ -325,6 +412,14 @@ export class EvaluationClient {
|
|
|
325
412
|
completeLocalAgentRun(input) {
|
|
326
413
|
return this.request("POST", "/local-agent-worker/runs/complete", input);
|
|
327
414
|
}
|
|
415
|
+
/** Lists Scenarios (draft and published) of the project; requires a Tracing and evaluations key. */
|
|
416
|
+
listCaseConversions(page) {
|
|
417
|
+
return this.request("GET", `/case-conversions${this.page(page)}`);
|
|
418
|
+
}
|
|
419
|
+
/** Reads one Scenario with its immutable publication pins. */
|
|
420
|
+
getCaseConversion(id) {
|
|
421
|
+
return this.request("GET", `/case-conversions/${uuid(id)}`);
|
|
422
|
+
}
|
|
328
423
|
/** Creates the legacy execution-scoped generic MCP capability for one world. */
|
|
329
424
|
createSimulationMcpCapability(input) {
|
|
330
425
|
return this.request("POST", "/local-agent-worker/mcp-capability", input);
|
|
@@ -338,3 +433,45 @@ export class EvaluationClient {
|
|
|
338
433
|
export function createEvaluationClient(options) {
|
|
339
434
|
return new EvaluationClient(options);
|
|
340
435
|
}
|
|
436
|
+
function signedUploadUrl(value) {
|
|
437
|
+
// eslint-disable-next-line no-control-regex -- control characters are rejected deliberately
|
|
438
|
+
if (typeof value !== "string" || value.length > 8192 || /[\x00-\x20\x7f]/u.test(value))
|
|
439
|
+
throw new HueApiError();
|
|
440
|
+
let url;
|
|
441
|
+
try {
|
|
442
|
+
url = new URL(value);
|
|
443
|
+
}
|
|
444
|
+
catch {
|
|
445
|
+
throw new HueApiError();
|
|
446
|
+
}
|
|
447
|
+
if ((url.protocol !== "https:" && !(url.protocol === "http:" && isLoopbackHost(url.hostname))) ||
|
|
448
|
+
url.username ||
|
|
449
|
+
url.password ||
|
|
450
|
+
url.hash)
|
|
451
|
+
throw new HueApiError();
|
|
452
|
+
// Validate without rewriting the provider's signed capability.
|
|
453
|
+
return value;
|
|
454
|
+
}
|
|
455
|
+
function signedUploadHeaders(value, contentType) {
|
|
456
|
+
const headers = { "content-type": contentType };
|
|
457
|
+
if (value === undefined || value === null)
|
|
458
|
+
return headers;
|
|
459
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
460
|
+
throw new HueApiError();
|
|
461
|
+
for (const [name, raw] of Object.entries(value)) {
|
|
462
|
+
if (typeof raw !== "string" ||
|
|
463
|
+
!raw ||
|
|
464
|
+
raw.length > 255 ||
|
|
465
|
+
// eslint-disable-next-line no-control-regex -- control characters are rejected deliberately
|
|
466
|
+
/[\x00-\x1f\x7f]/u.test(raw))
|
|
467
|
+
throw new HueApiError();
|
|
468
|
+
const lower = name.toLowerCase();
|
|
469
|
+
if (lower === "content-type" && raw === contentType)
|
|
470
|
+
headers[lower] = raw;
|
|
471
|
+
else if (lower === "x-vercel-blob-access" && raw === "private")
|
|
472
|
+
headers[lower] = raw;
|
|
473
|
+
else
|
|
474
|
+
throw new HueApiError();
|
|
475
|
+
}
|
|
476
|
+
return headers;
|
|
477
|
+
}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import { type EvaluationClient } from "./client.js";
|
|
2
|
+
import type { CaseFile, LocalFile, OutputFile, SubjectFile } from "./types.js";
|
|
3
|
+
/** Roles the target receives. Organization templates stay with grading, as in the managed protocol. */
|
|
4
|
+
export declare const targetFileRoles: readonly CaseFile["role"][];
|
|
5
|
+
/** Hue's artifact policy: the pilot file size and the accepted document types. */
|
|
6
|
+
export declare const outputFileLimits: {
|
|
7
|
+
/** Maximum number of generated files per execution. */
|
|
8
|
+
readonly count: 32;
|
|
9
|
+
/** Maximum size of one generated file in bytes (25 MiB). */
|
|
10
|
+
readonly bytes: number;
|
|
11
|
+
};
|
|
12
|
+
/** Content types Hue accepts for generated files. */
|
|
13
|
+
export declare const outputContentTypes: readonly string[];
|
|
14
|
+
/** A generated file the runner copied next to its checkpoint before uploading it. */
|
|
15
|
+
export interface StagedOutputFile {
|
|
16
|
+
/** Sanitized file name, unique within the execution. */
|
|
17
|
+
filename: string;
|
|
18
|
+
/** Declared content type. */
|
|
19
|
+
contentType: string;
|
|
20
|
+
/** Size of the staged bytes. */
|
|
21
|
+
byteSize: number;
|
|
22
|
+
/** SHA-256 of the staged bytes, hex encoded. */
|
|
23
|
+
sha256: string;
|
|
24
|
+
/** Absolute path of the staged copy. */
|
|
25
|
+
path: string;
|
|
26
|
+
/** Whether the target declared this file as the primary document. */
|
|
27
|
+
primary: boolean;
|
|
28
|
+
/** Hue artifact identity once the upload was verified. */
|
|
29
|
+
artifactId?: string;
|
|
30
|
+
}
|
|
31
|
+
/** Thrown when the target's declared files cannot be used; recorded as the target's error. */
|
|
32
|
+
export declare class OutputFileError extends Error {
|
|
33
|
+
constructor(message: string);
|
|
34
|
+
}
|
|
35
|
+
/** Reduce a declared file name to a single printable path segment, at most 200 characters. */
|
|
36
|
+
export declare function safeFilename(name: string): string;
|
|
37
|
+
/**
|
|
38
|
+
* Save verified copies of a case's pinned input files under `directory`. A file already
|
|
39
|
+
* present with the pinned identity is reused, so a resumed run does not download again.
|
|
40
|
+
*/
|
|
41
|
+
export declare function downloadCaseFiles(client: EvaluationClient, files: SubjectFile[], directory: string, primaryArtifactId?: string | null): Promise<LocalFile[]>;
|
|
42
|
+
/** Copy the target's generated files next to the checkpoint and record their identities. */
|
|
43
|
+
export declare function stageOutputFiles(files: OutputFile[], directory: string): Promise<StagedOutputFile[]>;
|
|
44
|
+
/**
|
|
45
|
+
* Publish staged files as verified artifacts. Reservations use stable keys derived from the
|
|
46
|
+
* execution and the bytes, so a resumed upload settles on the same artifact.
|
|
47
|
+
*/
|
|
48
|
+
export declare function uploadOutputFiles(client: EvaluationClient, executionId: string, files: StagedOutputFile[], save: () => Promise<void>): Promise<void>;
|
|
49
|
+
/** Project uploaded staged files as the verified local files handed to scorers. */
|
|
50
|
+
export declare function localOutputFiles(files: StagedOutputFile[]): LocalFile[];
|
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { copyFile, lstat, mkdir, open, readFile, rename, rm } from "node:fs/promises";
|
|
3
|
+
import { join, resolve } from "node:path";
|
|
4
|
+
import { randomUUID } from "node:crypto";
|
|
5
|
+
import { HueApiError } from "./client.js";
|
|
6
|
+
/** Roles the target receives. Organization templates stay with grading, as in the managed protocol. */
|
|
7
|
+
export const targetFileRoles = [
|
|
8
|
+
"source",
|
|
9
|
+
"attached_template",
|
|
10
|
+
"attached_reference",
|
|
11
|
+
"original",
|
|
12
|
+
];
|
|
13
|
+
/** Hue's artifact policy: the pilot file size and the accepted document types. */
|
|
14
|
+
export const outputFileLimits = {
|
|
15
|
+
/** Maximum number of generated files per execution. */
|
|
16
|
+
count: 32,
|
|
17
|
+
/** Maximum size of one generated file in bytes (25 MiB). */
|
|
18
|
+
bytes: 25 * 1024 * 1024,
|
|
19
|
+
};
|
|
20
|
+
/** Content types Hue accepts for generated files. */
|
|
21
|
+
export const outputContentTypes = [
|
|
22
|
+
"application/pdf",
|
|
23
|
+
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
|
24
|
+
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
|
|
25
|
+
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
|
26
|
+
"application/json",
|
|
27
|
+
"text/plain",
|
|
28
|
+
"text/csv",
|
|
29
|
+
"image/png",
|
|
30
|
+
"image/jpeg",
|
|
31
|
+
"image/webp",
|
|
32
|
+
];
|
|
33
|
+
/** Thrown when the target's declared files cannot be used; recorded as the target's error. */
|
|
34
|
+
export class OutputFileError extends Error {
|
|
35
|
+
constructor(message) {
|
|
36
|
+
super(message);
|
|
37
|
+
this.name = "OutputFileError";
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
const sha256 = (bytes) => createHash("sha256").update(bytes).digest("hex");
|
|
41
|
+
/** Reduce a declared file name to a single printable path segment, at most 200 characters. */
|
|
42
|
+
export function safeFilename(name) {
|
|
43
|
+
const cleaned = [...name]
|
|
44
|
+
.filter((c) => c.charCodeAt(0) >= 32 && c.charCodeAt(0) !== 127 && c !== "/" && c !== "\\")
|
|
45
|
+
.join("")
|
|
46
|
+
.trim();
|
|
47
|
+
return cleaned && cleaned !== "." && cleaned !== ".."
|
|
48
|
+
? [...cleaned].slice(0, 200).join("")
|
|
49
|
+
: "file";
|
|
50
|
+
}
|
|
51
|
+
async function privateDirectory(path) {
|
|
52
|
+
const root = resolve(path);
|
|
53
|
+
await mkdir(root, { recursive: true, mode: 0o700 });
|
|
54
|
+
const info = await lstat(root);
|
|
55
|
+
if (!info.isDirectory() || info.isSymbolicLink())
|
|
56
|
+
throw new Error("Use a private files directory (no symlink)");
|
|
57
|
+
return root;
|
|
58
|
+
}
|
|
59
|
+
async function writePrivate(path, bytes) {
|
|
60
|
+
const temporary = `${path}.${randomUUID()}.tmp`;
|
|
61
|
+
const file = await open(temporary, "wx", 0o600);
|
|
62
|
+
try {
|
|
63
|
+
await file.writeFile(bytes);
|
|
64
|
+
await file.sync();
|
|
65
|
+
}
|
|
66
|
+
finally {
|
|
67
|
+
await file.close();
|
|
68
|
+
}
|
|
69
|
+
await rename(temporary, path);
|
|
70
|
+
}
|
|
71
|
+
async function verifiedBytes(path, expected) {
|
|
72
|
+
const bytes = await readFile(path);
|
|
73
|
+
if (bytes.byteLength !== expected.byteSize || sha256(bytes) !== expected.sha256)
|
|
74
|
+
throw new Error(`Saved file ${path} no longer matches its verified identity`);
|
|
75
|
+
return bytes;
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Save verified copies of a case's pinned input files under `directory`. A file already
|
|
79
|
+
* present with the pinned identity is reused, so a resumed run does not download again.
|
|
80
|
+
*/
|
|
81
|
+
export async function downloadCaseFiles(client, files, directory, primaryArtifactId) {
|
|
82
|
+
const root = await privateDirectory(directory);
|
|
83
|
+
const saved = [];
|
|
84
|
+
for (const [index, file] of files.entries()) {
|
|
85
|
+
const path = join(root, `${index + 1}-${safeFilename(file.filename)}`);
|
|
86
|
+
let present = false;
|
|
87
|
+
try {
|
|
88
|
+
await verifiedBytes(path, file);
|
|
89
|
+
present = true;
|
|
90
|
+
}
|
|
91
|
+
catch {
|
|
92
|
+
present = false;
|
|
93
|
+
}
|
|
94
|
+
if (!present) {
|
|
95
|
+
const bytes = await client.downloadArtifact(file.artifactId);
|
|
96
|
+
if (bytes.byteLength !== file.byteSize || sha256(bytes) !== file.sha256)
|
|
97
|
+
throw new Error(`Downloaded input ${file.filename} does not match its pinned identity`);
|
|
98
|
+
await writePrivate(path, bytes);
|
|
99
|
+
}
|
|
100
|
+
saved.push({
|
|
101
|
+
artifactId: file.artifactId,
|
|
102
|
+
role: file.role,
|
|
103
|
+
filename: file.filename,
|
|
104
|
+
contentType: file.contentType,
|
|
105
|
+
byteSize: file.byteSize,
|
|
106
|
+
sha256: file.sha256,
|
|
107
|
+
path,
|
|
108
|
+
...(file.role === "output" && file.artifactId === primaryArtifactId ? { primary: true } : {}),
|
|
109
|
+
});
|
|
110
|
+
}
|
|
111
|
+
return saved;
|
|
112
|
+
}
|
|
113
|
+
/** Copy the target's generated files next to the checkpoint and record their identities. */
|
|
114
|
+
export async function stageOutputFiles(files, directory) {
|
|
115
|
+
if (!Array.isArray(files) || !files.length)
|
|
116
|
+
throw new OutputFileError("No generated files");
|
|
117
|
+
if (files.length > outputFileLimits.count)
|
|
118
|
+
throw new OutputFileError(`At most ${outputFileLimits.count} generated files are supported`);
|
|
119
|
+
if (files.filter((file) => file.primary).length > 1)
|
|
120
|
+
throw new OutputFileError("Declare at most one primary generated file");
|
|
121
|
+
const root = await privateDirectory(directory);
|
|
122
|
+
await rm(root, { recursive: true, force: true });
|
|
123
|
+
await mkdir(root, { recursive: true, mode: 0o700 });
|
|
124
|
+
const staged = [];
|
|
125
|
+
const names = new Set();
|
|
126
|
+
for (const file of files) {
|
|
127
|
+
if (typeof file.filename !== "string" || !file.filename.trim())
|
|
128
|
+
throw new OutputFileError("Generated files need a filename");
|
|
129
|
+
const filename = safeFilename(file.filename);
|
|
130
|
+
if (names.has(filename))
|
|
131
|
+
throw new OutputFileError(`Duplicate generated file ${filename}`);
|
|
132
|
+
names.add(filename);
|
|
133
|
+
if (!outputContentTypes.includes(file.contentType))
|
|
134
|
+
throw new OutputFileError(`Unsupported generated file type ${String(file.contentType)}`);
|
|
135
|
+
let bytes;
|
|
136
|
+
try {
|
|
137
|
+
bytes = file.bytes ?? (await readFile(file.path));
|
|
138
|
+
}
|
|
139
|
+
catch {
|
|
140
|
+
throw new OutputFileError(`Generated file ${filename} could not be read`);
|
|
141
|
+
}
|
|
142
|
+
if (!bytes.byteLength)
|
|
143
|
+
throw new OutputFileError(`Generated file ${filename} is empty`);
|
|
144
|
+
if (bytes.byteLength > outputFileLimits.bytes)
|
|
145
|
+
throw new OutputFileError(`Generated file ${filename} exceeds 25 MiB`);
|
|
146
|
+
const path = join(root, filename);
|
|
147
|
+
if (file.bytes)
|
|
148
|
+
await writePrivate(path, bytes);
|
|
149
|
+
else {
|
|
150
|
+
await copyFile(file.path, path);
|
|
151
|
+
await verifiedBytes(path, { byteSize: bytes.byteLength, sha256: sha256(bytes) });
|
|
152
|
+
}
|
|
153
|
+
staged.push({
|
|
154
|
+
filename,
|
|
155
|
+
contentType: file.contentType,
|
|
156
|
+
byteSize: bytes.byteLength,
|
|
157
|
+
sha256: sha256(bytes),
|
|
158
|
+
path,
|
|
159
|
+
primary: file.primary === true,
|
|
160
|
+
});
|
|
161
|
+
}
|
|
162
|
+
return staged;
|
|
163
|
+
}
|
|
164
|
+
/**
|
|
165
|
+
* Publish staged files as verified artifacts. Reservations use stable keys derived from the
|
|
166
|
+
* execution and the bytes, so a resumed upload settles on the same artifact.
|
|
167
|
+
*/
|
|
168
|
+
export async function uploadOutputFiles(client, executionId, files, save) {
|
|
169
|
+
for (const file of files) {
|
|
170
|
+
if (file.artifactId)
|
|
171
|
+
continue;
|
|
172
|
+
const bytes = await verifiedBytes(file.path, file);
|
|
173
|
+
const reserved = await client.reserveArtifact({
|
|
174
|
+
idempotencyKey: `hue-sdk:${executionId}:${file.sha256}:${sha256(Buffer.from(file.filename)).slice(0, 16)}`,
|
|
175
|
+
filename: file.filename,
|
|
176
|
+
contentType: file.contentType,
|
|
177
|
+
byteSize: file.byteSize,
|
|
178
|
+
sha256: file.sha256,
|
|
179
|
+
});
|
|
180
|
+
let state = reserved.state;
|
|
181
|
+
if (state !== "ready") {
|
|
182
|
+
if (reserved.copyState !== "acknowledged") {
|
|
183
|
+
const upload = await client.requestArtifactUpload(reserved.id);
|
|
184
|
+
try {
|
|
185
|
+
await client.uploadArtifactBytes(upload, bytes, file.contentType);
|
|
186
|
+
}
|
|
187
|
+
catch {
|
|
188
|
+
// A lost staging acknowledgement or an immutable object already present can only be
|
|
189
|
+
// settled by verified completion; never rewrite a final object here.
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
let attempt = 0;
|
|
193
|
+
for (;;) {
|
|
194
|
+
try {
|
|
195
|
+
state = (await client.completeArtifact(reserved.id)).state;
|
|
196
|
+
break;
|
|
197
|
+
}
|
|
198
|
+
catch (error) {
|
|
199
|
+
if (!(error instanceof HueApiError) || error.status !== 503 || attempt++ >= 2)
|
|
200
|
+
throw error;
|
|
201
|
+
await new Promise((resolve) => setTimeout(resolve, 1000 * attempt));
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
if (state !== "ready")
|
|
206
|
+
throw new Error(`Generated file ${file.filename} was not verified by Hue (${state})`);
|
|
207
|
+
file.artifactId = reserved.id;
|
|
208
|
+
await save();
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
/** Project uploaded staged files as the verified local files handed to scorers. */
|
|
212
|
+
export function localOutputFiles(files) {
|
|
213
|
+
return files.map((file) => ({
|
|
214
|
+
artifactId: file.artifactId,
|
|
215
|
+
role: "output",
|
|
216
|
+
filename: file.filename,
|
|
217
|
+
contentType: file.contentType,
|
|
218
|
+
byteSize: file.byteSize,
|
|
219
|
+
sha256: file.sha256,
|
|
220
|
+
path: file.path,
|
|
221
|
+
...(file.primary ? { primary: true } : {}),
|
|
222
|
+
}));
|
|
223
|
+
}
|
|
@@ -3,8 +3,16 @@ import type { EnvironmentClient } from "../environment/client.js";
|
|
|
3
3
|
import type { EnvironmentTool } from "../environment/tools.js";
|
|
4
4
|
import type { ActualAgentManifestInputV2, AttemptConnectionBundleV2, RequestedAttemptProviderV2 } from "./attempt.js";
|
|
5
5
|
import type { EvaluationClient } from "./client.js";
|
|
6
|
-
import { type RunnerReport } from "./runner.js";
|
|
7
|
-
import type { ExperimentCase, JsonValue, LocalAgentRegistration, LocalScorer } from "./types.js";
|
|
6
|
+
import { type RunExperimentTargetContext, type RunnerReport } from "./runner.js";
|
|
7
|
+
import type { ExperimentCase, JsonValue, LocalAgentRegistration, LocalFile, LocalScorer, TargetResult } from "./types.js";
|
|
8
|
+
/** Capability strings a registration declares; Hue offers only matching cases. */
|
|
9
|
+
export declare const localAgentCapabilities: {
|
|
10
|
+
/** Cases pinned to a hosted synthetic world. */
|
|
11
|
+
readonly environment: "environment:v1";
|
|
12
|
+
/** Ordinary cases run directly on this machine: JSON inputs and pinned input files in,
|
|
13
|
+
* JSON output and generated files out. */
|
|
14
|
+
readonly direct: "direct:v1";
|
|
15
|
+
};
|
|
8
16
|
/** Candidate-visible context for one queued local agent execution. */
|
|
9
17
|
export interface LocalAgentTargetContext {
|
|
10
18
|
/** Frozen candidate configuration, cloned before invocation. */
|
|
@@ -35,6 +43,30 @@ export interface LocalAgentTargetContext {
|
|
|
35
43
|
* checkpoints, logs or adds this response to parity digests. */
|
|
36
44
|
connectionBundle?: AttemptConnectionBundleV2;
|
|
37
45
|
}
|
|
46
|
+
/** Candidate-visible context for one queued ordinary case run directly on this machine. */
|
|
47
|
+
export interface LocalAgentDirectContext {
|
|
48
|
+
/** Frozen candidate configuration, cloned before invocation. */
|
|
49
|
+
config: JsonValue;
|
|
50
|
+
/** Identity only. Expected outcomes and metadata are evaluator-private. */
|
|
51
|
+
item: Pick<ExperimentCase, "id" | "externalKey">;
|
|
52
|
+
/** Identity of this target execution. */
|
|
53
|
+
executionId: string;
|
|
54
|
+
/** Trace identities without mutable span or grading data. */
|
|
55
|
+
trace: {
|
|
56
|
+
/** OpenTelemetry trace identifier. */
|
|
57
|
+
traceId: string;
|
|
58
|
+
/** Root execution span identifier. */
|
|
59
|
+
spanId: string;
|
|
60
|
+
};
|
|
61
|
+
/** Verified copies of the case's input files meant for the agent. */
|
|
62
|
+
files: LocalFile[];
|
|
63
|
+
/** Private scratch directory for this case; return generated files with `withFiles`. */
|
|
64
|
+
outputDirectory: string;
|
|
65
|
+
/** Cooperative worker stop signal. */
|
|
66
|
+
signal?: AbortSignal;
|
|
67
|
+
}
|
|
68
|
+
/** Allowlist the candidate surface for an ordinary case; frozen expectations stay with grading. */
|
|
69
|
+
export declare function localAgentDirectContext(context: RunExperimentTargetContext, signal?: AbortSignal): LocalAgentDirectContext;
|
|
38
70
|
/** Fixed local callback, clients and durable queue-worker settings. */
|
|
39
71
|
export interface RunLocalAgentOptions {
|
|
40
72
|
/** Evaluation client for the worker project. */
|
|
@@ -76,11 +108,24 @@ export interface RunLocalAgentOptions {
|
|
|
76
108
|
/** Selected MCP surface. */
|
|
77
109
|
surfaceKey: "google.gmail/mcp" | "slack/mcp";
|
|
78
110
|
};
|
|
79
|
-
/**
|
|
80
|
-
|
|
111
|
+
/** Runs cases pinned to a hosted world (`environment:v1`): invokes the existing agent against
|
|
112
|
+
* isolated tools and candidate-safe context. */
|
|
113
|
+
target?(inputs: JsonValue, tools: Record<string, EnvironmentTool>, context: LocalAgentTargetContext): JsonValue | undefined | Promise<JsonValue | undefined>;
|
|
114
|
+
/** Runs ordinary cases without a world (`direct:v1`): the agent receives the case inputs
|
|
115
|
+
* and verified input files and returns JSON output and/or generated files. */
|
|
116
|
+
directTarget?(inputs: JsonValue, context: LocalAgentDirectContext): JsonValue | TargetResult | undefined | Promise<JsonValue | TargetResult | undefined>;
|
|
81
117
|
/** Called after the experiment and queue completion are acknowledged. */
|
|
82
118
|
onCompleted?(report: RunnerReport): void | Promise<void>;
|
|
83
119
|
}
|
|
120
|
+
/** The registration's capabilities: explicit values plus one per supplied callback. */
|
|
121
|
+
export declare function registeredCapabilities(options: {
|
|
122
|
+
/** Fixed agent key, revision and declared capabilities. */
|
|
123
|
+
agent: LocalAgentRegistration;
|
|
124
|
+
/** The environment-case callback, when supplied. */
|
|
125
|
+
target?: unknown;
|
|
126
|
+
/** The ordinary-case callback, when supplied. */
|
|
127
|
+
directTarget?: unknown;
|
|
128
|
+
}): string[];
|
|
84
129
|
/**
|
|
85
130
|
* Starts an outbound-only worker for one fixed local agent entry point. Hue chooses
|
|
86
131
|
* only the registered key/revision; no command or source is received from the cloud.
|
|
@@ -3,6 +3,26 @@ import { randomUUID } from "node:crypto";
|
|
|
3
3
|
import { CheckpointStore } from "./checkpoint.js";
|
|
4
4
|
import { pinRequestedAttemptV2, requestedAttemptV2, runEnvironmentTarget, } from "./environment-target.js";
|
|
5
5
|
import { runExperiment, OutcomeSerializationError, TargetOutcomeUncertainError, UncertainExecutionError, } from "./runner.js";
|
|
6
|
+
/** Capability strings a registration declares; Hue offers only matching cases. */
|
|
7
|
+
export const localAgentCapabilities = {
|
|
8
|
+
/** Cases pinned to a hosted synthetic world. */
|
|
9
|
+
environment: "environment:v1",
|
|
10
|
+
/** Ordinary cases run directly on this machine: JSON inputs and pinned input files in,
|
|
11
|
+
* JSON output and generated files out. */
|
|
12
|
+
direct: "direct:v1",
|
|
13
|
+
};
|
|
14
|
+
/** Allowlist the candidate surface for an ordinary case; frozen expectations stay with grading. */
|
|
15
|
+
export function localAgentDirectContext(context, signal) {
|
|
16
|
+
return {
|
|
17
|
+
config: structuredClone(context.config),
|
|
18
|
+
item: { id: context.item.id, externalKey: context.item.externalKey },
|
|
19
|
+
executionId: context.executionId,
|
|
20
|
+
trace: { traceId: context.span.traceId, spanId: context.span.spanId },
|
|
21
|
+
files: structuredClone(context.files),
|
|
22
|
+
outputDirectory: context.outputDirectory,
|
|
23
|
+
...(signal ? { signal } : {}),
|
|
24
|
+
};
|
|
25
|
+
}
|
|
6
26
|
/** Allowlist the candidate surface instead of forwarding the generic evaluation context. */
|
|
7
27
|
function localAgentTargetContext(context) {
|
|
8
28
|
return {
|
|
@@ -21,6 +41,23 @@ function localAgentTargetContext(context) {
|
|
|
21
41
|
: {}),
|
|
22
42
|
};
|
|
23
43
|
}
|
|
44
|
+
/** The registration's capabilities: explicit values plus one per supplied callback. */
|
|
45
|
+
export function registeredCapabilities(options) {
|
|
46
|
+
if (!options.target && !options.directTarget)
|
|
47
|
+
throw new TypeError("Supply target for environment cases, directTarget for ordinary cases, or both");
|
|
48
|
+
const declared = options.agent.capabilities ?? [];
|
|
49
|
+
if (declared.includes(localAgentCapabilities.direct) && !options.directTarget)
|
|
50
|
+
throw new TypeError("direct:v1 requires a directTarget callback");
|
|
51
|
+
if (declared.includes(localAgentCapabilities.environment) && !options.target)
|
|
52
|
+
throw new TypeError("environment:v1 requires a target callback");
|
|
53
|
+
return [
|
|
54
|
+
...new Set([
|
|
55
|
+
...declared,
|
|
56
|
+
...(options.target ? [localAgentCapabilities.environment] : []),
|
|
57
|
+
...(options.directTarget ? [localAgentCapabilities.direct] : []),
|
|
58
|
+
]),
|
|
59
|
+
];
|
|
60
|
+
}
|
|
24
61
|
function validInterval(value) {
|
|
25
62
|
const interval = value ?? 2_000;
|
|
26
63
|
if (!Number.isInteger(interval) || interval < 250 || interval > 60_000)
|
|
@@ -63,6 +100,7 @@ function needsAttention(error, seen = new Set()) {
|
|
|
63
100
|
* only the registered key/revision; no command or source is received from the cloud.
|
|
64
101
|
*/
|
|
65
102
|
export async function runLocalAgent(options) {
|
|
103
|
+
const capabilities = registeredCapabilities(options);
|
|
66
104
|
const requestedConfiguration = requestedAttemptV2(options);
|
|
67
105
|
const interval = validInterval(options.pollIntervalMillis);
|
|
68
106
|
const maxRuns = options.maxRuns ?? Number.POSITIVE_INFINITY;
|
|
@@ -91,7 +129,7 @@ export async function runLocalAgent(options) {
|
|
|
91
129
|
while (completed < maxRuns && !options.signal?.aborted) {
|
|
92
130
|
const agent = await options.client.registerLocalAgent({
|
|
93
131
|
...options.agent,
|
|
94
|
-
capabilities
|
|
132
|
+
capabilities,
|
|
95
133
|
scorerDigests: options.agent.scorerDigests ??
|
|
96
134
|
options.scorers?.map((item) => item.definition.sourceDigest) ??
|
|
97
135
|
[],
|
|
@@ -117,20 +155,33 @@ export async function runLocalAgent(options) {
|
|
|
117
155
|
experimentId: claim.experimentId,
|
|
118
156
|
checkpointDirectory: join(directory, `experiment-${claim.experimentId}`),
|
|
119
157
|
persistResultContent: true,
|
|
120
|
-
|
|
158
|
+
// Worker dispatch uses the case pin: directTarget never receives a world.
|
|
159
|
+
environmentEvidence: options.directTarget ? "when_pinned" : "required",
|
|
121
160
|
traceEvidence: { mode: "required" },
|
|
122
161
|
scorers: options.scorers,
|
|
123
162
|
concurrency: options.concurrency,
|
|
124
|
-
target: (inputs, context) =>
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
163
|
+
target: (inputs, context) => {
|
|
164
|
+
if (!context.item.environmentVersionId) {
|
|
165
|
+
// Hue offers ordinary cases only to registrations that declared direct:v1.
|
|
166
|
+
if (!options.directTarget)
|
|
167
|
+
throw new Error("This worker runs only cases pinned to a Hue environment");
|
|
168
|
+
return options.directTarget(structuredClone(inputs), localAgentDirectContext(context, options.signal));
|
|
169
|
+
}
|
|
170
|
+
if (!options.target)
|
|
171
|
+
throw new Error("This worker runs only cases without a Hue environment");
|
|
172
|
+
// Bound to the options object, as the previous direct `options.target(...)` call was.
|
|
173
|
+
const target = options.target.bind(options);
|
|
174
|
+
return runEnvironmentTarget({
|
|
175
|
+
client: options.client,
|
|
176
|
+
environmentClient: options.environmentClient,
|
|
177
|
+
hue: options.hue,
|
|
178
|
+
inputs,
|
|
179
|
+
context,
|
|
180
|
+
requested,
|
|
181
|
+
signal: options.signal,
|
|
182
|
+
target: (targetInputs, targetContext) => target(structuredClone(targetInputs), targetContext.tools, localAgentTargetContext(targetContext)),
|
|
183
|
+
});
|
|
184
|
+
},
|
|
134
185
|
});
|
|
135
186
|
// From this point onward the experiment outcome is authoritative. If reporting the
|
|
136
187
|
// queue completion fails, leave the claim intact for checkpointed recovery instead of
|