@hue-run/sdk 0.19.1 → 0.20.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLI.md +27 -10
- package/ENVIRONMENTS.md +5 -2
- package/EVALUATIONS.md +6 -1
- package/README.md +2 -2
- package/WORLDS.md +133 -0
- package/dist/cli/eval.d.ts +4 -0
- package/dist/cli/eval.js +434 -186
- package/dist/cli/listen.js +1 -1
- package/dist/environment/client.d.ts +2 -95
- package/dist/environment/client.js +2 -283
- package/dist/environment/tools.d.ts +2 -35
- package/dist/environment/tools.js +2 -30
- package/dist/environment/types.d.ts +2 -623
- package/dist/environment/types.js +2 -1
- package/dist/environment/world.d.ts +2 -65
- package/dist/environment/world.js +2 -171
- package/dist/environment.d.ts +7 -7
- package/dist/environment.js +3 -3
- package/dist/evals/attempt.d.ts +1 -1
- package/dist/evals/client.d.ts +3 -2
- package/dist/evals/client.js +2 -1
- package/dist/evals/environment-target.d.ts +3 -3
- package/dist/evals/environment-target.js +3 -3
- package/dist/evals/failure.js +1 -1
- package/dist/evals/local-worker.d.ts +8 -3
- package/dist/evals/local-worker.js +10 -2
- package/dist/evals/simulation.d.ts +6 -3
- package/dist/evals/simulation.js +2 -1
- package/dist/evals/types.d.ts +2 -2
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/world/client.d.ts +161 -0
- package/dist/world/client.js +387 -0
- package/dist/world/definitions.d.ts +47 -0
- package/dist/world/definitions.js +1 -0
- package/dist/world/model-types.d.ts +123 -0
- package/dist/world/model-types.js +1 -0
- package/dist/world/models.d.ts +42 -0
- package/dist/world/models.js +56 -0
- package/dist/world/readiness.d.ts +18 -0
- package/dist/world/readiness.js +69 -0
- package/dist/world/tools.d.ts +35 -0
- package/dist/world/tools.js +30 -0
- package/dist/world/types.d.ts +625 -0
- package/dist/world/types.js +1 -0
- package/dist/world/world.d.ts +65 -0
- package/dist/world/world.js +175 -0
- package/dist/world.d.ts +13 -0
- package/dist/world.js +5 -0
- package/package.json +8 -2
|
@@ -1,65 +1,2 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
* The world an agent acts on through provider mirrors, read from a create or replay response.
|
|
4
|
-
* Null for a world created while the gateway was off: that world has Hue-native tools and no
|
|
5
|
-
* mirror URLs. The handoff carries the world token; keep it out of logs and checkpoints.
|
|
6
|
-
*/
|
|
7
|
-
export declare function worldHandoff(run: EnvironmentRun): WorldHandoff | null;
|
|
8
|
-
/**
|
|
9
|
-
* The world's clock: what an agent reads for today's date in place of the wall clock, so a
|
|
10
|
-
* date-relative request (tomorrow, `newer_than:7d`, this month) lands on the dates the world
|
|
11
|
-
* holds however long after the recording the run starts. From the handoff's `now`, else from
|
|
12
|
-
* `HUE_WORLD_NOW` in the given environment (this process's by default, where `hue eval
|
|
13
|
-
* --command` and `agentEnvironment` set it). Null when neither names it.
|
|
14
|
-
*/
|
|
15
|
-
export declare function worldNow(source?: WorldHandoff | Record<string, string | undefined>): Date | null;
|
|
16
|
-
/** Variables that authenticate against Hue's control plane rather than a simulated provider. */
|
|
17
|
-
export declare const HUE_CONTROL_PLANE_VARIABLES: readonly string[];
|
|
18
|
-
/** Prefixes of the Hue control-plane credentials the server issues: project and setup keys,
|
|
19
|
-
* attempt grants, the project MCP key, invocation and installation tokens, and OAuth tokens and
|
|
20
|
-
* secrets. World tokens (`hue_world_`) and MCP capabilities (`hue_sim_`) are what the agent holds. */
|
|
21
|
-
export declare const HUE_CONTROL_PLANE_CREDENTIAL_PREFIXES: readonly string[];
|
|
22
|
-
/** True for a control-plane variable by name, or for a value carrying a Hue control-plane
|
|
23
|
-
* credential anywhere in it, including wrapped, percent-encoded or JSON-Unicode-escaped values. */
|
|
24
|
-
export declare function isHueControlPlaneCredential(name: string, value: string | undefined): boolean;
|
|
25
|
-
/** The parent's variables without Hue control-plane credentials; `undefined` values are dropped. */
|
|
26
|
-
export declare function stripHueControlPlaneCredentials(parent: Record<string, string | undefined>): Record<string, string>;
|
|
27
|
-
/** Options for {@link agentEnvironment}. */
|
|
28
|
-
export interface AgentEnvironmentOptions {
|
|
29
|
-
/** The environment to start from; defaults to this process's. */
|
|
30
|
-
parent?: Record<string, string | undefined>;
|
|
31
|
-
/** Keep Hue control-plane credentials in the child. Off by default: the agent gets world
|
|
32
|
-
* tokens only (plan section 10.3) and the project key stays with the runner. */
|
|
33
|
-
includeHueCredentials?: boolean;
|
|
34
|
-
/** Also set `HUE_MCP_URL`, `HUE_MCP_TOKEN` and `HUE_MCP_EXPIRES_AT` from the world's first
|
|
35
|
-
* MCP mirror, the names the `hue_sim_` bridge used, for one compatibility release. On by
|
|
36
|
-
* default; the canonical carriers are the world's own `env`. */
|
|
37
|
-
legacyMcpVariables?: boolean;
|
|
38
|
-
}
|
|
39
|
-
/** `environment` without any variable a world or its server owns. */
|
|
40
|
-
export declare function withoutWorldVariables(environment: Record<string, string>): Record<string, string>;
|
|
41
|
-
/**
|
|
42
|
-
* The environment for an agent child process running one case (the coordination briefs'
|
|
43
|
-
* recommended policy): the parent's variables minus Hue control-plane credentials and any
|
|
44
|
-
* world-scoped variable, then the world's carriers. Nothing here is logged.
|
|
45
|
-
*/
|
|
46
|
-
export declare function agentEnvironment(world: WorldHandoff, options?: AgentEnvironmentOptions): Record<string, string>;
|
|
47
|
-
/** The `{ url, token, expiresAt }` shape the `hue_sim_` capability had, projected from the
|
|
48
|
-
* world's first MCP mirror so an adapter written for the bridge keeps working through the
|
|
49
|
-
* compatibility release. Undefined for a world without an MCP surface. */
|
|
50
|
-
export declare function legacyMcpCapability(world: WorldHandoff): LegacyMcpCapability | undefined;
|
|
51
|
-
/** The owner-only `mcpServers` file {@link writeMcpConfig} wrote. */
|
|
52
|
-
export interface McpConfigFile {
|
|
53
|
-
/** The owner-only file; pass its path to the harness and dispose after the run. */
|
|
54
|
-
path: string;
|
|
55
|
-
/** Removes the file and its private directory; safe to call twice. */
|
|
56
|
-
dispose(): Promise<void>;
|
|
57
|
-
}
|
|
58
|
-
/**
|
|
59
|
-
* Writes the world's `mcpConfig` as an owner-only file in a private directory (plan section
|
|
60
|
-
* 4.9): the file carries the world token, so it is created with mode 0600 inside a 0700
|
|
61
|
-
* directory, is never logged, and `dispose` removes the directory after the run.
|
|
62
|
-
*/
|
|
63
|
-
export declare function writeMcpConfig(world: WorldHandoff, options?: {
|
|
64
|
-
directory?: string;
|
|
65
|
-
}): Promise<McpConfigFile>;
|
|
1
|
+
/** Compatibility entry point. Use the world module. */
|
|
2
|
+
export * from "../world/world.js";
|
|
@@ -1,171 +1,2 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
import { join } from "node:path";
|
|
4
|
-
/**
|
|
5
|
-
* The world an agent acts on through provider mirrors, read from a create or replay response.
|
|
6
|
-
* Null for a world created while the gateway was off: that world has Hue-native tools and no
|
|
7
|
-
* mirror URLs. The handoff carries the world token; keep it out of logs and checkpoints.
|
|
8
|
-
*/
|
|
9
|
-
export function worldHandoff(run) {
|
|
10
|
-
if (!run.token || !run.env || !run.mcpConfig || !run.surfaces)
|
|
11
|
-
return null;
|
|
12
|
-
const id = run.worldId ?? run.id;
|
|
13
|
-
return {
|
|
14
|
-
id,
|
|
15
|
-
token: run.token,
|
|
16
|
-
expiresAt: run.expiresAt,
|
|
17
|
-
lifecycle: run.lifecycle ?? "live",
|
|
18
|
-
completingUntil: run.completingUntil ?? null,
|
|
19
|
-
traceparent: run.traceparent ?? null,
|
|
20
|
-
baggage: run.baggage ?? `hue-world=${id}`,
|
|
21
|
-
now: run.now ?? run.env.HUE_WORLD_NOW ?? null,
|
|
22
|
-
surfaces: run.surfaces.map((surface) => ({ ...surface })),
|
|
23
|
-
env: { ...run.env },
|
|
24
|
-
mcpConfig: {
|
|
25
|
-
mcpServers: Object.fromEntries(Object.entries(run.mcpConfig.mcpServers).map(([name, server]) => [
|
|
26
|
-
name,
|
|
27
|
-
{ type: server.type, url: server.url, headers: { ...server.headers } },
|
|
28
|
-
])),
|
|
29
|
-
},
|
|
30
|
-
};
|
|
31
|
-
}
|
|
32
|
-
/**
|
|
33
|
-
* The world's clock: what an agent reads for today's date in place of the wall clock, so a
|
|
34
|
-
* date-relative request (tomorrow, `newer_than:7d`, this month) lands on the dates the world
|
|
35
|
-
* holds however long after the recording the run starts. From the handoff's `now`, else from
|
|
36
|
-
* `HUE_WORLD_NOW` in the given environment (this process's by default, where `hue eval
|
|
37
|
-
* --command` and `agentEnvironment` set it). Null when neither names it.
|
|
38
|
-
*/
|
|
39
|
-
export function worldNow(source = process.env) {
|
|
40
|
-
const text = "env" in source && typeof source.env === "object" && source.env !== null
|
|
41
|
-
? (source.now ?? source.env.HUE_WORLD_NOW)
|
|
42
|
-
: source.HUE_WORLD_NOW;
|
|
43
|
-
if (!text)
|
|
44
|
-
return null;
|
|
45
|
-
const date = new Date(text);
|
|
46
|
-
return Number.isNaN(date.getTime()) ? null : date;
|
|
47
|
-
}
|
|
48
|
-
/** Variables that authenticate against Hue's control plane rather than a simulated provider. */
|
|
49
|
-
export const HUE_CONTROL_PLANE_VARIABLES = [
|
|
50
|
-
"HUE_API_KEY",
|
|
51
|
-
"HUE_MCP_KEY",
|
|
52
|
-
"HUE_PROJECT_KEY",
|
|
53
|
-
"HUE_SERVICE_KEY",
|
|
54
|
-
];
|
|
55
|
-
/** Prefixes of the Hue control-plane credentials the server issues: project and setup keys,
|
|
56
|
-
* attempt grants, the project MCP key, invocation and installation tokens, and OAuth tokens and
|
|
57
|
-
* secrets. World tokens (`hue_world_`) and MCP capabilities (`hue_sim_`) are what the agent holds. */
|
|
58
|
-
export const HUE_CONTROL_PLANE_CREDENTIAL_PREFIXES = [
|
|
59
|
-
"hue_sk_",
|
|
60
|
-
"hue_attempt_",
|
|
61
|
-
"hue_mcp_",
|
|
62
|
-
"hue_setup_",
|
|
63
|
-
"hue_inv_",
|
|
64
|
-
"hue_install_",
|
|
65
|
-
"hue_at_",
|
|
66
|
-
"hue_rt_",
|
|
67
|
-
"hue_oauth_",
|
|
68
|
-
];
|
|
69
|
-
/** A Hue credential anywhere in a value, including JSON Unicode escapes, not inside a longer word. */
|
|
70
|
-
const HUE_CREDENTIAL_SHAPE = new RegExp(`(?:^|[^A-Za-z0-9_])(?:${HUE_CONTROL_PLANE_CREDENTIAL_PREFIXES.join("|")})`);
|
|
71
|
-
const ASCII_PERCENT_ESCAPE = /%[0-7][0-9a-f]/i;
|
|
72
|
-
const ASCII_JSON_UNICODE_ESCAPE = /\\u00([0-7][0-9a-fA-F])/g;
|
|
73
|
-
function decodeAsciiJsonUnicodeEscapes(value) {
|
|
74
|
-
return value.replace(ASCII_JSON_UNICODE_ESCAPE, (_, hex) => String.fromCharCode(Number.parseInt(hex, 16)));
|
|
75
|
-
}
|
|
76
|
-
function hasHueControlPlaneCredential(value) {
|
|
77
|
-
let candidates = new Set([value]);
|
|
78
|
-
for (let layer = 0; layer <= 4; layer++) {
|
|
79
|
-
const jsonDecodedCandidates = new Set();
|
|
80
|
-
for (const candidate of candidates) {
|
|
81
|
-
jsonDecodedCandidates.add(candidate);
|
|
82
|
-
jsonDecodedCandidates.add(decodeAsciiJsonUnicodeEscapes(candidate));
|
|
83
|
-
}
|
|
84
|
-
if ([...jsonDecodedCandidates].some((candidate) => HUE_CREDENTIAL_SHAPE.test(candidate)))
|
|
85
|
-
return true;
|
|
86
|
-
if (layer === 4)
|
|
87
|
-
return [...jsonDecodedCandidates].some((candidate) => ASCII_PERCENT_ESCAPE.test(candidate));
|
|
88
|
-
candidates = new Set([...jsonDecodedCandidates]
|
|
89
|
-
.filter((candidate) => ASCII_PERCENT_ESCAPE.test(candidate))
|
|
90
|
-
.map((candidate) => candidate.replace(/%([0-7][0-9a-f])/gi, (_, hex) => String.fromCharCode(Number.parseInt(hex, 16)))));
|
|
91
|
-
if (candidates.size === 0)
|
|
92
|
-
return false;
|
|
93
|
-
}
|
|
94
|
-
return false;
|
|
95
|
-
}
|
|
96
|
-
/** True for a control-plane variable by name, or for a value carrying a Hue control-plane
|
|
97
|
-
* credential anywhere in it, including wrapped, percent-encoded or JSON-Unicode-escaped values. */
|
|
98
|
-
export function isHueControlPlaneCredential(name, value) {
|
|
99
|
-
if (HUE_CONTROL_PLANE_VARIABLES.includes(name))
|
|
100
|
-
return true;
|
|
101
|
-
return typeof value === "string" && hasHueControlPlaneCredential(value);
|
|
102
|
-
}
|
|
103
|
-
/** The parent's variables without Hue control-plane credentials; `undefined` values are dropped. */
|
|
104
|
-
export function stripHueControlPlaneCredentials(parent) {
|
|
105
|
-
const child = {};
|
|
106
|
-
for (const [name, value] of Object.entries(parent)) {
|
|
107
|
-
if (value === undefined || isHueControlPlaneCredential(name, value))
|
|
108
|
-
continue;
|
|
109
|
-
child[name] = value;
|
|
110
|
-
}
|
|
111
|
-
return child;
|
|
112
|
-
}
|
|
113
|
-
/** Variables a world or its server owns: a parent's value belongs to another world (a runner
|
|
114
|
-
* started inside a case, a shell left from an earlier run) or to the server (a signing key), and
|
|
115
|
-
* would send this world's token to a mirror this world does not have. The whole `HUE_WORLD_`
|
|
116
|
-
* prefix is the world's. */
|
|
117
|
-
const WORLD_SCOPED = /^(?:HUE_WORLD_[A-Z0-9_]+|HUE_MCP_(?:CONFIG|URL|TOKEN|EXPIRES_AT)|HUE_SIM_[A-Z0-9_]+_(?:URL|ALIAS))$/;
|
|
118
|
-
/** `environment` without any variable a world or its server owns. */
|
|
119
|
-
export function withoutWorldVariables(environment) {
|
|
120
|
-
return Object.fromEntries(Object.entries(environment).filter(([name]) => !WORLD_SCOPED.test(name)));
|
|
121
|
-
}
|
|
122
|
-
/**
|
|
123
|
-
* The environment for an agent child process running one case (the coordination briefs'
|
|
124
|
-
* recommended policy): the parent's variables minus Hue control-plane credentials and any
|
|
125
|
-
* world-scoped variable, then the world's carriers. Nothing here is logged.
|
|
126
|
-
*/
|
|
127
|
-
export function agentEnvironment(world, options = {}) {
|
|
128
|
-
const parent = options.parent ?? process.env;
|
|
129
|
-
const child = withoutWorldVariables(options.includeHueCredentials
|
|
130
|
-
? Object.fromEntries(Object.entries(parent).filter((entry) => entry[1] !== undefined))
|
|
131
|
-
: stripHueControlPlaneCredentials(parent));
|
|
132
|
-
Object.assign(child, world.env);
|
|
133
|
-
if (options.legacyMcpVariables ?? true) {
|
|
134
|
-
const legacy = legacyMcpCapability(world);
|
|
135
|
-
if (legacy) {
|
|
136
|
-
child.HUE_MCP_URL = legacy.url;
|
|
137
|
-
child.HUE_MCP_TOKEN = legacy.token;
|
|
138
|
-
child.HUE_MCP_EXPIRES_AT = legacy.expiresAt;
|
|
139
|
-
}
|
|
140
|
-
}
|
|
141
|
-
return child;
|
|
142
|
-
}
|
|
143
|
-
/** The `{ url, token, expiresAt }` shape the `hue_sim_` capability had, projected from the
|
|
144
|
-
* world's first MCP mirror so an adapter written for the bridge keeps working through the
|
|
145
|
-
* compatibility release. Undefined for a world without an MCP surface. */
|
|
146
|
-
export function legacyMcpCapability(world) {
|
|
147
|
-
const server = Object.values(world.mcpConfig.mcpServers)[0];
|
|
148
|
-
if (!server)
|
|
149
|
-
return undefined;
|
|
150
|
-
return { url: server.url, token: world.token, expiresAt: world.expiresAt };
|
|
151
|
-
}
|
|
152
|
-
/**
|
|
153
|
-
* Writes the world's `mcpConfig` as an owner-only file in a private directory (plan section
|
|
154
|
-
* 4.9): the file carries the world token, so it is created with mode 0600 inside a 0700
|
|
155
|
-
* directory, is never logged, and `dispose` removes the directory after the run.
|
|
156
|
-
*/
|
|
157
|
-
export async function writeMcpConfig(world, options = {}) {
|
|
158
|
-
const directory = await mkdtemp(join(options.directory ?? tmpdir(), "hue-world-"));
|
|
159
|
-
const dispose = () => rm(directory, { recursive: true, force: true });
|
|
160
|
-
try {
|
|
161
|
-
await chmod(directory, 0o700);
|
|
162
|
-
const path = join(directory, "mcp.json");
|
|
163
|
-
await writeFile(path, `${JSON.stringify(world.mcpConfig, null, 2)}\n`, { mode: 0o600 });
|
|
164
|
-
return { path, dispose };
|
|
165
|
-
}
|
|
166
|
-
catch (error) {
|
|
167
|
-
// A half-written file would hold the token; never leave it behind.
|
|
168
|
-
await dispose();
|
|
169
|
-
throw error;
|
|
170
|
-
}
|
|
171
|
-
}
|
|
1
|
+
/** Compatibility entry point. Use the world module. */
|
|
2
|
+
export * from "../world/world.js";
|
package/dist/environment.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
export { createEnvironmentClient, EnvironmentClient, HueEnvironmentError, isTransientEnvironmentError, } from "./
|
|
2
|
-
export type { EnvironmentClientOptions } from "./
|
|
3
|
-
export { bindEnvironmentTools } from "./
|
|
4
|
-
export type { BindEnvironmentToolsOptions, EnvironmentTool } from "./
|
|
5
|
-
export type * from "./
|
|
6
|
-
export { agentEnvironment, HUE_CONTROL_PLANE_CREDENTIAL_PREFIXES, HUE_CONTROL_PLANE_VARIABLES, isHueControlPlaneCredential, legacyMcpCapability, stripHueControlPlaneCredentials, worldHandoff, worldNow, writeMcpConfig, } from "./
|
|
7
|
-
export type { AgentEnvironmentOptions, McpConfigFile } from "./
|
|
1
|
+
export { createEnvironmentClient, EnvironmentClient, HueEnvironmentError, isTransientEnvironmentError, } from "./world/client.js";
|
|
2
|
+
export type { EnvironmentClientOptions } from "./world/client.js";
|
|
3
|
+
export { bindEnvironmentTools } from "./world/tools.js";
|
|
4
|
+
export type { BindEnvironmentToolsOptions, EnvironmentTool } from "./world/tools.js";
|
|
5
|
+
export type * from "./world/types.js";
|
|
6
|
+
export { agentEnvironment, HUE_CONTROL_PLANE_CREDENTIAL_PREFIXES, HUE_CONTROL_PLANE_VARIABLES, isHueControlPlaneCredential, legacyMcpCapability, stripHueControlPlaneCredentials, worldHandoff, worldNow, writeMcpConfig, } from "./world/world.js";
|
|
7
|
+
export type { AgentEnvironmentOptions, McpConfigFile } from "./world/world.js";
|
package/dist/environment.js
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
export { createEnvironmentClient, EnvironmentClient, HueEnvironmentError, isTransientEnvironmentError, } from "./
|
|
2
|
-
export { bindEnvironmentTools } from "./
|
|
3
|
-
export { agentEnvironment, HUE_CONTROL_PLANE_CREDENTIAL_PREFIXES, HUE_CONTROL_PLANE_VARIABLES, isHueControlPlaneCredential, legacyMcpCapability, stripHueControlPlaneCredentials, worldHandoff, worldNow, writeMcpConfig, } from "./
|
|
1
|
+
export { createEnvironmentClient, EnvironmentClient, HueEnvironmentError, isTransientEnvironmentError, } from "./world/client.js";
|
|
2
|
+
export { bindEnvironmentTools } from "./world/tools.js";
|
|
3
|
+
export { agentEnvironment, HUE_CONTROL_PLANE_CREDENTIAL_PREFIXES, HUE_CONTROL_PLANE_VARIABLES, isHueControlPlaneCredential, legacyMcpCapability, stripHueControlPlaneCredentials, worldHandoff, worldNow, writeMcpConfig, } from "./world/world.js";
|
package/dist/evals/attempt.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { z } from "zod/v4";
|
|
2
|
-
import type { CoverageGap } from "../
|
|
2
|
+
import type { CoverageGap } from "../world/types.js";
|
|
3
3
|
import type { SimulationMcpCapability } from "./types.js";
|
|
4
4
|
/** Canonical caller-observed agent configuration after missing evidence is made explicit. */
|
|
5
5
|
export type ActualAgentManifestV2 = {
|
package/dist/evals/client.d.ts
CHANGED
|
@@ -216,7 +216,7 @@ export declare class EvaluationClient {
|
|
|
216
216
|
getEnvironmentSteps(executionId: string, page?: {
|
|
217
217
|
after?: number;
|
|
218
218
|
limit?: number;
|
|
219
|
-
}): Promise<import("../
|
|
219
|
+
}): Promise<import("../world/types.js").StepPage>;
|
|
220
220
|
/**
|
|
221
221
|
* Prepares the immutable provider-profile binding for one execution. The route identity is
|
|
222
222
|
* removed from the JSON body, and credential-bearing responses are validated against the
|
|
@@ -367,7 +367,8 @@ export declare class EvaluationClient {
|
|
|
367
367
|
listScoringResults(id: string, page?: PageOptions): Promise<Page<ScoringResultSummary>>;
|
|
368
368
|
/** Reads a stored evaluator result. */
|
|
369
369
|
getScoringResult(id: string): Promise<StoredScoringResult>;
|
|
370
|
-
/** Dispatches hosted judge jobs for `llm_judge` pins
|
|
370
|
+
/** Dispatches hosted judge jobs for `llm_judge` pins. Hue checks admission for new requests;
|
|
371
|
+
* replay an interrupted request with its original key and body even if the budget changed. */
|
|
371
372
|
createJudgeJobs(id: string, input: {
|
|
372
373
|
idempotencyKey: string;
|
|
373
374
|
jobs: {
|
package/dist/evals/client.js
CHANGED
|
@@ -837,7 +837,8 @@ export class EvaluationClient {
|
|
|
837
837
|
async getScoringResult(id) {
|
|
838
838
|
return productRunFields(await this.getResult(id), "result");
|
|
839
839
|
}
|
|
840
|
-
/** Dispatches hosted judge jobs for `llm_judge` pins
|
|
840
|
+
/** Dispatches hosted judge jobs for `llm_judge` pins. Hue checks admission for new requests;
|
|
841
|
+
* replay an interrupted request with its original key and body even if the budget changed. */
|
|
841
842
|
createJudgeJobs(id, input) {
|
|
842
843
|
return this.request("POST", `/evaluation-runs/${uuid(id)}/judge-jobs`, input);
|
|
843
844
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { HueClient } from "../client.js";
|
|
2
|
-
import { type EnvironmentClient } from "../
|
|
3
|
-
import { type EnvironmentTool } from "../
|
|
4
|
-
import type { EnvironmentRun, WorldHandoff } from "../
|
|
2
|
+
import { type EnvironmentClient } from "../world/client.js";
|
|
3
|
+
import { type EnvironmentTool } from "../world/tools.js";
|
|
4
|
+
import type { EnvironmentRun, WorldHandoff } from "../world/types.js";
|
|
5
5
|
import type { HueSpan } from "../types.js";
|
|
6
6
|
import { type ActualAgentManifestInputV2, type AttemptBaselineV2, type AttemptConnectionBundleV2, type RequestedAttemptProviderV2, type SurfaceBindingV2 } from "./attempt.js";
|
|
7
7
|
import type { EvaluationClient } from "./client.js";
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
|
-
import { HueEnvironmentError, isTransientEnvironmentError, } from "../
|
|
3
|
-
import { bindEnvironmentTools } from "../
|
|
4
|
-
import { legacyMcpCapability, worldHandoff } from "../
|
|
2
|
+
import { HueEnvironmentError, isTransientEnvironmentError, } from "../world/client.js";
|
|
3
|
+
import { bindEnvironmentTools } from "../world/tools.js";
|
|
4
|
+
import { legacyMcpCapability, worldHandoff } from "../world/world.js";
|
|
5
5
|
import { actualAgentManifestV2, attemptBaselineV2, projectMcpConnectionV2, requestedAttemptProvidersV2, validateAttemptConnectionBundleV2, } from "./attempt.js";
|
|
6
6
|
import { TargetCancelledError, TargetOutcomeUncertainError, TargetTimeoutError } from "./runner.js";
|
|
7
7
|
export function requestedAttemptV2(options) {
|
package/dist/evals/failure.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { HueConnectionError } from "../client.js";
|
|
2
|
-
import { HueEnvironmentError } from "../
|
|
2
|
+
import { HueEnvironmentError } from "../world/client.js";
|
|
3
3
|
import { HueApiError } from "./client.js";
|
|
4
4
|
const TIMEOUT_NAMES = new Set(["TimeoutError", "APIConnectionTimeoutError", "APITimeoutError"]);
|
|
5
5
|
/** Node's and Undici's codes for a call that timed out, connecting or waiting on the answer. */
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { HueClient } from "../client.js";
|
|
2
|
-
import { type EnvironmentClient } from "../
|
|
3
|
-
import type { EnvironmentTool } from "../
|
|
4
|
-
import type { WorldHandoff } from "../
|
|
2
|
+
import { type EnvironmentClient } from "../world/client.js";
|
|
3
|
+
import type { EnvironmentTool } from "../world/tools.js";
|
|
4
|
+
import type { WorldHandoff } from "../world/types.js";
|
|
5
5
|
import type { ActualAgentManifestInputV2, AttemptConnectionBundleV2, RequestedAttemptProviderV2 } from "./attempt.js";
|
|
6
6
|
import type { EvaluationClient } from "./client.js";
|
|
7
7
|
import { type RunExperimentTargetContext, type RunnerReport, type TelemetryNotAccepted } from "./runner.js";
|
|
@@ -161,6 +161,11 @@ export interface RunLocalAgentOptions {
|
|
|
161
161
|
/** Runs ordinary cases without a world (`direct:v1`): the agent receives the case inputs
|
|
162
162
|
* and verified input files and returns JSON output and/or generated files. */
|
|
163
163
|
directTarget?(inputs: JsonValue, context: LocalAgentDirectContext): JsonValue | TargetResult | undefined | Promise<JsonValue | TargetResult | undefined>;
|
|
164
|
+
/** Called after outputs upload, before queue completion. A failure stops the worker with
|
|
165
|
+
* its claim intact. A resumed claim may repeat this callback without invoking the agent. */
|
|
166
|
+
onUploaded?(report: RunnerReport, claim: LocalAgentClaim & {
|
|
167
|
+
workerId: string;
|
|
168
|
+
}): void | Promise<void>;
|
|
164
169
|
/** Called after the experiment and queue completion are acknowledged. */
|
|
165
170
|
onCompleted?(report: RunnerReport): void | Promise<void>;
|
|
166
171
|
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { resolve } from "node:path";
|
|
2
2
|
import { randomUUID } from "node:crypto";
|
|
3
|
-
import { isTransientEnvironmentError } from "../
|
|
3
|
+
import { isTransientEnvironmentError } from "../world/client.js";
|
|
4
4
|
import { HueApiError, isTransientApiError } from "./client.js";
|
|
5
5
|
import { HueExportError } from "../transport.js";
|
|
6
6
|
import { CheckpointStore, checkpointPath, checkpointSegment } from "./checkpoint.js";
|
|
@@ -252,6 +252,7 @@ export async function runLocalAgent(options) {
|
|
|
252
252
|
}, Math.min(15_000, Math.max(1_000, interval)));
|
|
253
253
|
let experimentFinished = false;
|
|
254
254
|
let queueCompleted = false;
|
|
255
|
+
let handoffFailed = false;
|
|
255
256
|
try {
|
|
256
257
|
const requested = requestedConfiguration
|
|
257
258
|
? pinRequestedAttemptV2(requestedConfiguration, (await options.client.getExperiment(experimentId)).config)
|
|
@@ -307,6 +308,13 @@ export async function runLocalAgent(options) {
|
|
|
307
308
|
// queue completion fails, leave the claim intact for checkpointed recovery instead of
|
|
308
309
|
// rewriting a successful experiment as an agent failure.
|
|
309
310
|
experimentFinished = true;
|
|
311
|
+
try {
|
|
312
|
+
await options.onUploaded?.(report, { ...claim, workerId });
|
|
313
|
+
}
|
|
314
|
+
catch (error) {
|
|
315
|
+
handoffFailed = true;
|
|
316
|
+
throw error;
|
|
317
|
+
}
|
|
310
318
|
await options.client.completeLocalAgentRun({
|
|
311
319
|
runId: claim.runId,
|
|
312
320
|
workerId,
|
|
@@ -320,7 +328,7 @@ export async function runLocalAgent(options) {
|
|
|
320
328
|
catch (error) {
|
|
321
329
|
// The run is completed in the queue: a failure after that is the caller's own
|
|
322
330
|
// (`onCompleted` threw) and is thrown as itself, not reported as a retry that cannot be.
|
|
323
|
-
if (queueCompleted)
|
|
331
|
+
if (queueCompleted || handoffFailed)
|
|
324
332
|
throw error;
|
|
325
333
|
if (experimentFinished && error instanceof HueApiError && error.status === 409) {
|
|
326
334
|
// The experiment finished, but Hue no longer accepts this worker's completion: the
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { HueClient } from "../client.js";
|
|
2
|
-
import { type EnvironmentClient } from "../
|
|
3
|
-
import type { EnvironmentTool } from "../
|
|
4
|
-
import type { EnvironmentIdentity, PublishableEnvironmentDefinition, WorldHandoff } from "../
|
|
2
|
+
import { type EnvironmentClient } from "../world/client.js";
|
|
3
|
+
import type { EnvironmentTool } from "../world/tools.js";
|
|
4
|
+
import type { EnvironmentIdentity, PublishableEnvironmentDefinition, WorldHandoff } from "../world/types.js";
|
|
5
5
|
import { type EvaluationClient } from "./client.js";
|
|
6
6
|
import type { ActualAgentManifestInputV2, AttemptConnectionBundleV2, RequestedAttemptProviderV2 } from "./attempt.js";
|
|
7
7
|
import { type RunnerReport, type TelemetryNotAccepted } from "./runner.js";
|
|
@@ -205,6 +205,9 @@ export interface RunSimulationOptions {
|
|
|
205
205
|
/** Invokes the existing local agent exactly once for this attempt; return the output, or
|
|
206
206
|
* `withFiles(output, files)` when it generated files. */
|
|
207
207
|
target(inputs: JsonValue, context: SimulationTargetContext): JsonValue | TargetResult | undefined | Promise<JsonValue | TargetResult | undefined>;
|
|
208
|
+
/** Called after outputs upload and before the attempt is marked completed. A failure leaves
|
|
209
|
+
* the uploaded attempt resumable. The callback may be repeated after an interruption. */
|
|
210
|
+
onUploaded?(report: SimulationReport): void | Promise<void>;
|
|
208
211
|
/** Receives bounded nonsecret lifecycle progress. */
|
|
209
212
|
onProgress?(event: SimulationProgress): void | Promise<void>;
|
|
210
213
|
}
|
package/dist/evals/simulation.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
|
-
import { HueEnvironmentError } from "../
|
|
2
|
+
import { HueEnvironmentError } from "../world/client.js";
|
|
3
3
|
import { HueApiError } from "./client.js";
|
|
4
4
|
import { CheckpointStore, checkpointPath, checkpointSegment } from "./checkpoint.js";
|
|
5
5
|
import { MAX_ENVIRONMENT_STEPS } from "./environment-evidence.js";
|
|
@@ -491,6 +491,7 @@ export async function runSimulation(options) {
|
|
|
491
491
|
}),
|
|
492
492
|
});
|
|
493
493
|
const complete = { ...report, experimentId, runUrl };
|
|
494
|
+
await options.onUploaded?.(complete);
|
|
494
495
|
attempt.stage = "completed";
|
|
495
496
|
attempt.report = complete;
|
|
496
497
|
await store.write("active-attempt", attempt);
|
package/dist/evals/types.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { JsonValue } from "../types.js";
|
|
2
|
-
import type { EnvironmentCoverage } from "../
|
|
2
|
+
import type { EnvironmentCoverage } from "../world/types.js";
|
|
3
3
|
export type { JsonValue } from "../types.js";
|
|
4
4
|
/** One page of a paginated list. */
|
|
5
5
|
export interface Page<T> {
|
|
@@ -947,7 +947,7 @@ export interface EnvironmentEvidenceSnapshot extends EnvironmentCoverage {
|
|
|
947
947
|
/** Sealed environment evidence with its full ordered journal. */
|
|
948
948
|
export interface EnvironmentEvidence extends EnvironmentEvidenceSnapshot {
|
|
949
949
|
/** Complete steps ordered by ordinal. */
|
|
950
|
-
steps: import("../
|
|
950
|
+
steps: import("../world/types.js").Step[];
|
|
951
951
|
}
|
|
952
952
|
/** A local scorer: its pinned definition and the callback bound to it. */
|
|
953
953
|
export interface LocalScorer {
|
package/dist/version.d.ts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
/** Package version shared by the instrumentation scope and the export User-Agent. */
|
|
2
|
-
export declare const sdkVersion = "0.
|
|
2
|
+
export declare const sdkVersion = "0.20.1";
|
package/dist/version.js
CHANGED
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
import { type WorldPreparationProgress } from "./readiness.js";
|
|
2
|
+
import type { ActionInput, ActionResult, CoverageGapInput, CoverageGapResult, CreateRunInput, Environment, PublishableEnvironmentDefinition, EnvironmentIdentity, EnvironmentPage, EnvironmentPageOptions, EnvironmentRun, EnvironmentSummary, EnvironmentVersion, EnvironmentVersionSummary, FinishRunInput, SealedRun, StepPage, StepPageOptions, WorldEvidence, WorldEvidenceOptions } from "./types.js";
|
|
3
|
+
import type { PublishableWorldDefinition, WorldVersion } from "./definitions.js";
|
|
4
|
+
/** Connection and retry options for {@link createEnvironmentClient}. */
|
|
5
|
+
export interface EnvironmentClientOptions {
|
|
6
|
+
/** Project service key sent as a bearer token; server-side only. */
|
|
7
|
+
apiKey: string;
|
|
8
|
+
/** Hue origin; defaults to `https://app.hue.run`. */
|
|
9
|
+
baseUrl?: string;
|
|
10
|
+
/** Per-request deadline in milliseconds. */
|
|
11
|
+
timeoutMillis?: number;
|
|
12
|
+
/** Attempts for idempotent run mutations, 1–10; defaults to 4. */
|
|
13
|
+
maxAttempts?: number;
|
|
14
|
+
}
|
|
15
|
+
/** Sanitized environment API failure that never includes response text or credentials. */
|
|
16
|
+
export declare class HueEnvironmentError extends Error {
|
|
17
|
+
/** HTTP status when Hue answered; absent for transport, timeout or parse failure. */
|
|
18
|
+
readonly status?: number | undefined;
|
|
19
|
+
/** Hue's `Retry-After` in milliseconds, bounded, when a 429 or 503 carried one. */
|
|
20
|
+
readonly retryAfterMs?: number | undefined;
|
|
21
|
+
/** The server's `X-Hue-Diagnostic` code, when it is a short snake-case value. */
|
|
22
|
+
readonly diagnostic?: string | undefined;
|
|
23
|
+
/** Bounded, sanitized progress for a World preparation refusal. */
|
|
24
|
+
readonly preparation?: WorldPreparationProgress | undefined;
|
|
25
|
+
constructor(
|
|
26
|
+
/** HTTP status when Hue answered; absent for transport, timeout or parse failure. */
|
|
27
|
+
status?: number | undefined,
|
|
28
|
+
/** Hue's `Retry-After` in milliseconds, bounded, when a 429 or 503 carried one. */
|
|
29
|
+
retryAfterMs?: number | undefined,
|
|
30
|
+
/** The server's `X-Hue-Diagnostic` code, when it is a short snake-case value. */
|
|
31
|
+
diagnostic?: string | undefined,
|
|
32
|
+
/** Bounded, sanitized progress for a World preparation refusal. */
|
|
33
|
+
preparation?: WorldPreparationProgress | undefined);
|
|
34
|
+
}
|
|
35
|
+
/** Starts one isolated instance from an immutable World version. */
|
|
36
|
+
export interface CreateInstanceInput extends Omit<CreateRunInput, "environmentVersionId"> {
|
|
37
|
+
/** Immutable World version to instantiate. */
|
|
38
|
+
worldVersionId: string;
|
|
39
|
+
}
|
|
40
|
+
/** A connection failure or a status the client retries; seal polling continues through these. */
|
|
41
|
+
export declare function isTransientEnvironmentError(error: unknown): boolean;
|
|
42
|
+
/** Typed client for authored Worlds, isolated World instances and immutable journals. */
|
|
43
|
+
export declare class WorldClient {
|
|
44
|
+
#private;
|
|
45
|
+
/** Validated Hue origin. */
|
|
46
|
+
readonly baseUrl: string;
|
|
47
|
+
private readonly timeoutMillis;
|
|
48
|
+
private readonly maxAttempts;
|
|
49
|
+
constructor(options: EnvironmentClientOptions);
|
|
50
|
+
/** Compatibility clients override only resource routing. */
|
|
51
|
+
protected resourcePath(path: string): string;
|
|
52
|
+
private send;
|
|
53
|
+
private page;
|
|
54
|
+
/** Creates an isolated instance from an immutable World version. */
|
|
55
|
+
createInstance(input: CreateInstanceInput, options?: {
|
|
56
|
+
signal?: AbortSignal;
|
|
57
|
+
}): Promise<EnvironmentRun>;
|
|
58
|
+
/** Reads current or sealed instance state. */
|
|
59
|
+
getInstance(instanceId: string, options?: {
|
|
60
|
+
signal?: AbortSignal;
|
|
61
|
+
}): Promise<{
|
|
62
|
+
id: string;
|
|
63
|
+
environmentVersionId: string;
|
|
64
|
+
executionId: string | null;
|
|
65
|
+
seed: string;
|
|
66
|
+
status: "open" | "completed" | "abandoned" | "expired";
|
|
67
|
+
stepCount: number;
|
|
68
|
+
maxSteps: number;
|
|
69
|
+
clockNs: string;
|
|
70
|
+
expiresAt: string;
|
|
71
|
+
createdAt: string;
|
|
72
|
+
sealedAt: string | null;
|
|
73
|
+
stateDigest: string;
|
|
74
|
+
finalState?: import("./types.js").JsonValue;
|
|
75
|
+
worldId?: string;
|
|
76
|
+
lifecycle?: import("./types.js").WorldLifecycle;
|
|
77
|
+
completingUntil?: string | null;
|
|
78
|
+
traceExternalId?: string | null;
|
|
79
|
+
surfaces?: import("./types.js").WorldSurface[];
|
|
80
|
+
flags?: import("./types.js").WorldFlag[];
|
|
81
|
+
connection?: null;
|
|
82
|
+
validity: "environment_incomplete" | "not_assessed";
|
|
83
|
+
coverageGap: import("./types.js").CoverageGap | null;
|
|
84
|
+
}>;
|
|
85
|
+
/** Completes or abandons one instance and freezes its evidence. */
|
|
86
|
+
finishInstance(instanceId: string, input: FinishRunInput): Promise<SealedRun>;
|
|
87
|
+
/** Creates a World identity using the client's resource routes. */
|
|
88
|
+
createWorld(input: EnvironmentIdentity): Promise<EnvironmentSummary>;
|
|
89
|
+
/** Lists active Worlds. */
|
|
90
|
+
listWorlds(page?: EnvironmentPageOptions): Promise<EnvironmentPage>;
|
|
91
|
+
/** Reads a World and its immutable version summaries. */
|
|
92
|
+
getWorld(id: string): Promise<Environment>;
|
|
93
|
+
/** Creates an environment identity; this non-idempotent registry write is not retried. */
|
|
94
|
+
createEnvironment(input: EnvironmentIdentity): Promise<EnvironmentSummary>;
|
|
95
|
+
/** Lists active environment identities. */
|
|
96
|
+
listEnvironments(page?: EnvironmentPageOptions): Promise<EnvironmentPage>;
|
|
97
|
+
/** Reads one environment and its immutable version summaries. */
|
|
98
|
+
getEnvironment(id: string): Promise<Environment>;
|
|
99
|
+
/** Publishes an immutable definition; this non-idempotent registry write is not retried. */
|
|
100
|
+
publishVersion(environmentId: string, definition: PublishableWorldDefinition): Promise<EnvironmentVersionSummary>;
|
|
101
|
+
/** Reads a full immutable environment version and generated action catalog. */
|
|
102
|
+
getVersion(id: string): Promise<WorldVersion>;
|
|
103
|
+
/** Creates or recovers one fresh isolated world using a stable idempotency key. */
|
|
104
|
+
createRun(input: CreateRunInput, options?: {
|
|
105
|
+
signal?: AbortSignal;
|
|
106
|
+
}): Promise<EnvironmentRun>;
|
|
107
|
+
/** Reads authoritative current or sealed world state. */
|
|
108
|
+
getRun(runId: string, options?: {
|
|
109
|
+
signal?: AbortSignal;
|
|
110
|
+
}): Promise<{
|
|
111
|
+
id: string;
|
|
112
|
+
environmentVersionId: string;
|
|
113
|
+
executionId: string | null;
|
|
114
|
+
seed: string;
|
|
115
|
+
status: "open" | "completed" | "abandoned" | "expired";
|
|
116
|
+
stepCount: number;
|
|
117
|
+
maxSteps: number;
|
|
118
|
+
clockNs: string;
|
|
119
|
+
expiresAt: string;
|
|
120
|
+
createdAt: string;
|
|
121
|
+
sealedAt: string | null;
|
|
122
|
+
stateDigest: string;
|
|
123
|
+
finalState?: import("./types.js").JsonValue;
|
|
124
|
+
worldId?: string;
|
|
125
|
+
lifecycle?: import("./types.js").WorldLifecycle;
|
|
126
|
+
completingUntil?: string | null;
|
|
127
|
+
traceExternalId?: string | null;
|
|
128
|
+
surfaces?: import("./types.js").WorldSurface[];
|
|
129
|
+
flags?: import("./types.js").WorldFlag[];
|
|
130
|
+
connection?: null;
|
|
131
|
+
validity: "environment_incomplete" | "not_assessed";
|
|
132
|
+
coverageGap: import("./types.js").CoverageGap | null;
|
|
133
|
+
}>;
|
|
134
|
+
/** Record a known coverage gap with durable identity; retries reuse the exact request. */
|
|
135
|
+
recordCoverageGap(runId: string, input: CoverageGapInput): Promise<CoverageGapResult>;
|
|
136
|
+
/** Invokes an action; repeating an invocation identity replays its recorded result. */
|
|
137
|
+
act(runId: string, input: ActionInput): Promise<ActionResult>;
|
|
138
|
+
/** Pages the immutable journal by step ordinal. */
|
|
139
|
+
listSteps(runId: string, page?: StepPageOptions): Promise<StepPage>;
|
|
140
|
+
/** Seals a world as completed or abandoned and freezes its evidence. */
|
|
141
|
+
finishRun(runId: string, input: FinishRunInput): Promise<SealedRun>;
|
|
142
|
+
/** Reads a sealed world's evaluator-only evidence with the project key; a world token never
|
|
143
|
+
* can. An open world answers 409 until it is sealed. */
|
|
144
|
+
getEvidence(runId: string, options?: WorldEvidenceOptions): Promise<WorldEvidence>;
|
|
145
|
+
}
|
|
146
|
+
/** Creates a typed simulated-environment client. */
|
|
147
|
+
/** Compatibility client using the original REST resource names. */
|
|
148
|
+
export declare class EnvironmentClient extends WorldClient {
|
|
149
|
+
/** Publishes the definition shapes supported by the compatibility client. */
|
|
150
|
+
publishVersion(environmentId: string, definition: PublishableEnvironmentDefinition): Promise<EnvironmentVersionSummary>;
|
|
151
|
+
/** Reads the definition using the compatibility client's original return type. */
|
|
152
|
+
getVersion(id: string): Promise<EnvironmentVersion>;
|
|
153
|
+
protected resourcePath(path: string): string;
|
|
154
|
+
}
|
|
155
|
+
/** Creates the compatibility client using the original REST resource names. */
|
|
156
|
+
export declare function createEnvironmentClient(options: EnvironmentClientOptions): EnvironmentClient;
|
|
157
|
+
/** Connection options for the World API client. */
|
|
158
|
+
export type WorldClientOptions = EnvironmentClientOptions;
|
|
159
|
+
/** Creates a World API client using canonical World and instance routes. */
|
|
160
|
+
export declare function createWorldClient(options: WorldClientOptions): WorldClient;
|
|
161
|
+
export { HueEnvironmentError as HueWorldError, isTransientEnvironmentError as isTransientWorldError, };
|