@hue-run/sdk 0.11.3 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLI.md +33 -18
- package/EVALUATIONS.md +25 -9
- package/README.md +6 -5
- package/dist/cli/eval.js +23 -5
- package/dist/cli/login.js +2 -1
- package/dist/cli/mcp.d.ts +5 -4
- package/dist/cli/mcp.js +245 -31
- package/dist/client.d.ts +3 -0
- package/dist/client.js +36 -24
- package/dist/environment/world.d.ts +4 -2
- package/dist/environment/world.js +13 -4
- package/dist/evals/client.d.ts +25 -5
- package/dist/evals/client.js +93 -15
- package/dist/evals/environment-target.d.ts +15 -8
- package/dist/evals/environment-target.js +61 -21
- package/dist/evals/local-worker.d.ts +38 -3
- package/dist/evals/local-worker.js +158 -34
- package/dist/evals/runner.d.ts +24 -0
- package/dist/evals/runner.js +154 -12
- package/dist/evals.d.ts +3 -3
- package/dist/evals.js +2 -2
- package/dist/transport.d.ts +7 -0
- package/dist/transport.js +76 -15
- package/dist/types.d.ts +7 -0
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +1 -1
|
@@ -53,16 +53,25 @@ export function stripHueControlPlaneCredentials(parent) {
|
|
|
53
53
|
}
|
|
54
54
|
return child;
|
|
55
55
|
}
|
|
56
|
+
/** Variables a world or its server owns: a parent's value belongs to another world (a runner
|
|
57
|
+
* started inside a case, a shell left from an earlier run) or to the server (a signing key), and
|
|
58
|
+
* would send this world's token to a mirror this world does not have. The whole `HUE_WORLD_`
|
|
59
|
+
* prefix is the world's. */
|
|
60
|
+
const WORLD_SCOPED = /^(?:HUE_WORLD_[A-Z0-9_]+|HUE_MCP_(?:CONFIG|URL|TOKEN|EXPIRES_AT)|HUE_SIM_[A-Z0-9_]+_(?:URL|ALIAS))$/;
|
|
61
|
+
/** `environment` without any variable a world or its server owns. */
|
|
62
|
+
export function withoutWorldVariables(environment) {
|
|
63
|
+
return Object.fromEntries(Object.entries(environment).filter(([name]) => !WORLD_SCOPED.test(name)));
|
|
64
|
+
}
|
|
56
65
|
/**
|
|
57
66
|
* The environment for an agent child process running one case (the coordination briefs'
|
|
58
|
-
* recommended policy): the parent's variables minus Hue control-plane credentials
|
|
59
|
-
* world
|
|
67
|
+
* recommended policy): the parent's variables minus Hue control-plane credentials and any
|
|
68
|
+
* world-scoped variable, then the world's carriers. Nothing here is logged.
|
|
60
69
|
*/
|
|
61
70
|
export function agentEnvironment(world, options = {}) {
|
|
62
71
|
const parent = options.parent ?? process.env;
|
|
63
|
-
const child = options.includeHueCredentials
|
|
72
|
+
const child = withoutWorldVariables(options.includeHueCredentials
|
|
64
73
|
? Object.fromEntries(Object.entries(parent).filter((entry) => entry[1] !== undefined))
|
|
65
|
-
: stripHueControlPlaneCredentials(parent);
|
|
74
|
+
: stripHueControlPlaneCredentials(parent));
|
|
66
75
|
Object.assign(child, world.env);
|
|
67
76
|
if (options.legacyMcpVariables ?? true) {
|
|
68
77
|
const legacy = legacyMcpCapability(world);
|
package/dist/evals/client.d.ts
CHANGED
|
@@ -9,6 +9,10 @@ export interface EvaluationClientOptions {
|
|
|
9
9
|
baseUrl?: string;
|
|
10
10
|
/** Per-request budget in milliseconds, 100–60000. Default 10000. */
|
|
11
11
|
timeoutMillis?: number;
|
|
12
|
+
/** Bounded attempts for a read or a mutation the server deduplicates by its key, after a
|
|
13
|
+
* connection failure, a timeout or a 408 or 5xx Hue did not time with `Retry-After`. Default 4,
|
|
14
|
+
* maximum 10; 1 sends every such request once. */
|
|
15
|
+
maxAttempts?: number;
|
|
12
16
|
}
|
|
13
17
|
/** Thrown for a failed evaluation API request; the message is fixed and never includes response text. */
|
|
14
18
|
export declare class HueApiError extends Error {
|
|
@@ -16,12 +20,22 @@ export declare class HueApiError extends Error {
|
|
|
16
20
|
readonly status?: number | undefined;
|
|
17
21
|
/** Seconds Hue asked the caller to wait (`Retry-After` on a 429 or 503), when it said. */
|
|
18
22
|
readonly retryAfterSeconds?: number | undefined;
|
|
23
|
+
/** Hue's `X-Hue-Diagnostic` code, which tells refusals of one status apart, when it sent one. */
|
|
24
|
+
readonly diagnostic?: string | undefined;
|
|
19
25
|
constructor(
|
|
20
26
|
/** HTTP status when Hue answered; absent for network, timeout and parsing failures. */
|
|
21
27
|
status?: number | undefined,
|
|
22
28
|
/** Seconds Hue asked the caller to wait (`Retry-After` on a 429 or 503), when it said. */
|
|
23
|
-
retryAfterSeconds?: number | undefined
|
|
29
|
+
retryAfterSeconds?: number | undefined,
|
|
30
|
+
/** Hue's `X-Hue-Diagnostic` code, which tells refusals of one status apart, when it sent one. */
|
|
31
|
+
diagnostic?: string | undefined);
|
|
24
32
|
}
|
|
33
|
+
/**
|
|
34
|
+
* Whether the same request, sent again, can succeed: a connection failure, a timeout, or a status
|
|
35
|
+
* Hue answers while it cannot act yet (408, 429, 500, 502, 503, 504). A refusal Hue decided on,
|
|
36
|
+
* any other 4xx, is not, and no number of attempts changes it.
|
|
37
|
+
*/
|
|
38
|
+
export declare function isTransientApiError(error: unknown): boolean;
|
|
25
39
|
/** An artifact download that ran past the size its caller expected; the rest is not read. */
|
|
26
40
|
export declare class ArtifactSizeError extends Error {
|
|
27
41
|
/** The size the caller expected, in bytes. */
|
|
@@ -32,22 +46,28 @@ export declare class ArtifactSizeError extends Error {
|
|
|
32
46
|
}
|
|
33
47
|
/**
|
|
34
48
|
* Typed client for Hue's evaluation REST API: datasets, scorers, experiments, executions, runs,
|
|
35
|
-
* results and hosted judge jobs.
|
|
36
|
-
*
|
|
37
|
-
*
|
|
38
|
-
*
|
|
49
|
+
* results and hosted judge jobs. A request Hue refused before acting on it with a short
|
|
50
|
+
* `Retry-After` (HTTP 429 or 503, at most 5 seconds) is sent again after that wait whatever its
|
|
51
|
+
* method, up to four times; a longer wait is the caller's error, carried as `retryAfterSeconds`.
|
|
52
|
+
* A read, or a mutation the server deduplicates by its idempotency key, is also sent again after
|
|
53
|
+
* a failure Hue did not time, a connection failure, a timeout or a 408 or 5xx without the header,
|
|
54
|
+
* up to `maxAttempts` times with a jittered backoff. A mutation without a key is never retried
|
|
55
|
+
* implicitly: callers retain stable idempotency keys for experiments and results. Responses are
|
|
56
|
+
* bounded to 4 MiB.
|
|
39
57
|
*/
|
|
40
58
|
export declare class EvaluationClient {
|
|
41
59
|
/** Validated Hue origin. */
|
|
42
60
|
readonly baseUrl: string;
|
|
43
61
|
private readonly apiKey;
|
|
44
62
|
private readonly timeoutMillis;
|
|
63
|
+
private readonly maxAttempts;
|
|
45
64
|
constructor(options: EvaluationClientOptions);
|
|
46
65
|
/** Sends one request to Hue, again after the wait while Hue refuses it with a short
|
|
47
66
|
* `Retry-After`. `init` runs for every attempt, so each has its own timeout; `signal` also ends
|
|
48
67
|
* each attempt, and a wait to send one again, whose refusal is then the caller's error. */
|
|
49
68
|
private send;
|
|
50
69
|
private request;
|
|
70
|
+
private requestOnce;
|
|
51
71
|
/** Verified bytes of one ready artifact in this project, bounded to the 25 MiB pilot file size.
|
|
52
72
|
* With `maxBytes`, the download stops one byte past it and throws `ArtifactSizeError`. */
|
|
53
73
|
downloadArtifact(id: string, options?: {
|
package/dist/evals/client.js
CHANGED
|
@@ -5,17 +5,51 @@ import { attemptBindingRead, parsePrepareAttemptResultV2, parseRefreshedAttemptR
|
|
|
5
5
|
export class HueApiError extends Error {
|
|
6
6
|
status;
|
|
7
7
|
retryAfterSeconds;
|
|
8
|
+
diagnostic;
|
|
8
9
|
constructor(
|
|
9
10
|
/** HTTP status when Hue answered; absent for network, timeout and parsing failures. */
|
|
10
11
|
status,
|
|
11
12
|
/** Seconds Hue asked the caller to wait (`Retry-After` on a 429 or 503), when it said. */
|
|
12
|
-
retryAfterSeconds
|
|
13
|
-
|
|
13
|
+
retryAfterSeconds,
|
|
14
|
+
/** Hue's `X-Hue-Diagnostic` code, which tells refusals of one status apart, when it sent one. */
|
|
15
|
+
diagnostic) {
|
|
16
|
+
super(status
|
|
17
|
+
? `Hue API request failed (HTTP ${status}${diagnostic ? `, ${diagnostic}` : ""})`
|
|
18
|
+
: "Hue API connection or response failed");
|
|
14
19
|
this.status = status;
|
|
15
20
|
this.retryAfterSeconds = retryAfterSeconds;
|
|
21
|
+
this.diagnostic = diagnostic;
|
|
16
22
|
this.name = "HueApiError";
|
|
17
23
|
}
|
|
18
24
|
}
|
|
25
|
+
const TRANSIENT = new Set([408, 429, 500, 502, 503, 504]);
|
|
26
|
+
/**
|
|
27
|
+
* Whether the same request, sent again, can succeed: a connection failure, a timeout, or a status
|
|
28
|
+
* Hue answers while it cannot act yet (408, 429, 500, 502, 503, 504). A refusal Hue decided on,
|
|
29
|
+
* any other 4xx, is not, and no number of attempts changes it.
|
|
30
|
+
*/
|
|
31
|
+
export function isTransientApiError(error) {
|
|
32
|
+
return (error instanceof HueApiError && (error.status === undefined || TRANSIENT.has(error.status)));
|
|
33
|
+
}
|
|
34
|
+
const DIAGNOSTIC = /^[a-z_]{1,64}$/;
|
|
35
|
+
function diagnosticOf(response) {
|
|
36
|
+
const value = response.headers.get("x-hue-diagnostic");
|
|
37
|
+
return value !== null && DIAGNOSTIC.test(value) ? value : undefined;
|
|
38
|
+
}
|
|
39
|
+
/** Whether a request may be sent again after a transient failure: every read, and a mutation the
|
|
40
|
+
* server deduplicates, either by the `idempotencyKey` in its body or because its route is
|
|
41
|
+
* idempotent by design (the local worker's register, claim, heartbeat, completion and capability
|
|
42
|
+
* routes). A mutation without a key is sent once; its caller resolves the outcome before asking
|
|
43
|
+
* again. */
|
|
44
|
+
function retryable(method, body, idempotent) {
|
|
45
|
+
if (idempotent !== undefined)
|
|
46
|
+
return idempotent;
|
|
47
|
+
if (method === "GET")
|
|
48
|
+
return true;
|
|
49
|
+
return (body !== null &&
|
|
50
|
+
typeof body === "object" &&
|
|
51
|
+
typeof body.idempotencyKey === "string");
|
|
52
|
+
}
|
|
19
53
|
/** An artifact download that ran past the size its caller expected; the rest is not read. */
|
|
20
54
|
export class ArtifactSizeError extends Error {
|
|
21
55
|
maxBytes;
|
|
@@ -128,16 +162,21 @@ function refusalRetryAfter(response) {
|
|
|
128
162
|
}
|
|
129
163
|
/**
|
|
130
164
|
* Typed client for Hue's evaluation REST API: datasets, scorers, experiments, executions, runs,
|
|
131
|
-
* results and hosted judge jobs.
|
|
132
|
-
*
|
|
133
|
-
*
|
|
134
|
-
*
|
|
165
|
+
* results and hosted judge jobs. A request Hue refused before acting on it with a short
|
|
166
|
+
* `Retry-After` (HTTP 429 or 503, at most 5 seconds) is sent again after that wait whatever its
|
|
167
|
+
* method, up to four times; a longer wait is the caller's error, carried as `retryAfterSeconds`.
|
|
168
|
+
* A read, or a mutation the server deduplicates by its idempotency key, is also sent again after
|
|
169
|
+
* a failure Hue did not time, a connection failure, a timeout or a 408 or 5xx without the header,
|
|
170
|
+
* up to `maxAttempts` times with a jittered backoff. A mutation without a key is never retried
|
|
171
|
+
* implicitly: callers retain stable idempotency keys for experiments and results. Responses are
|
|
172
|
+
* bounded to 4 MiB.
|
|
135
173
|
*/
|
|
136
174
|
export class EvaluationClient {
|
|
137
175
|
/** Validated Hue origin. */
|
|
138
176
|
baseUrl;
|
|
139
177
|
apiKey;
|
|
140
178
|
timeoutMillis;
|
|
179
|
+
maxAttempts;
|
|
141
180
|
constructor(options) {
|
|
142
181
|
const validated = validateOptions({
|
|
143
182
|
...options,
|
|
@@ -147,6 +186,10 @@ export class EvaluationClient {
|
|
|
147
186
|
this.baseUrl = validated.baseUrl;
|
|
148
187
|
this.apiKey = validated.apiKey;
|
|
149
188
|
this.timeoutMillis = validated.timeoutMillis;
|
|
189
|
+
const attempts = options.maxAttempts ?? 4;
|
|
190
|
+
if (!Number.isInteger(attempts) || attempts < 1 || attempts > 10)
|
|
191
|
+
throw new RangeError("maxAttempts must be 1–10");
|
|
192
|
+
this.maxAttempts = attempts;
|
|
150
193
|
}
|
|
151
194
|
/** Sends one request to Hue, again after the wait while Hue refuses it with a short
|
|
152
195
|
* `Retry-After`. `init` runs for every attempt, so each has its own timeout; `signal` also ends
|
|
@@ -183,11 +226,44 @@ export class EvaluationClient {
|
|
|
183
226
|
throw new HueApiError(response.status, askedRetryAfter(response));
|
|
184
227
|
}
|
|
185
228
|
}
|
|
186
|
-
async request(method, path, body, bounds = { ...valueBounds, bytes: 1024 * 1024 }, signal
|
|
187
|
-
|
|
229
|
+
async request(method, path, body, bounds = { ...valueBounds, bytes: 1024 * 1024 }, signal,
|
|
230
|
+
/** Overrides the retry decision `retryable` takes from the method and body. */
|
|
231
|
+
idempotent) {
|
|
232
|
+
// Serialized once, outside the attempts: a body this client cannot encode is a caller error
|
|
233
|
+
// no attempt fixes. Optional top-level fields are omitted intentionally; nested undefined
|
|
234
|
+
// remains invalid.
|
|
188
235
|
const payload = body === undefined
|
|
189
236
|
? undefined
|
|
190
237
|
: JSON.stringify(json(Object.fromEntries(Object.entries(body).filter(([, value]) => value !== undefined)), bounds));
|
|
238
|
+
const attempts = retryable(method, body, idempotent) ? this.maxAttempts : 1;
|
|
239
|
+
for (let attempt = 1;; attempt++) {
|
|
240
|
+
try {
|
|
241
|
+
return await this.requestOnce(method, path, payload, signal);
|
|
242
|
+
}
|
|
243
|
+
catch (error) {
|
|
244
|
+
// A refusal that carried `Retry-After` was already decided by `send`: sent again while
|
|
245
|
+
// the wait was short, the caller's error when it was long. This loop covers the failures
|
|
246
|
+
// Hue did not time: a lost connection, a timeout, a 5xx or 408 without the header.
|
|
247
|
+
if (!isTransientApiError(error) ||
|
|
248
|
+
error.retryAfterSeconds !== undefined ||
|
|
249
|
+
attempt >= attempts ||
|
|
250
|
+
signal?.aborted)
|
|
251
|
+
throw error;
|
|
252
|
+
const backoff = Math.min(100 * 2 ** (attempt - 1), 2000);
|
|
253
|
+
const wait = backoff + Math.random() * backoff;
|
|
254
|
+
await new Promise((resolve) => {
|
|
255
|
+
const done = () => {
|
|
256
|
+
clearTimeout(timer);
|
|
257
|
+
signal?.removeEventListener("abort", done);
|
|
258
|
+
resolve();
|
|
259
|
+
};
|
|
260
|
+
const timer = setTimeout(done, wait);
|
|
261
|
+
signal?.addEventListener("abort", done, { once: true });
|
|
262
|
+
});
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
async requestOnce(method, path, payload, signal) {
|
|
191
267
|
const response = await this.send(`${this.baseUrl}/api/v1${path}`, () => ({
|
|
192
268
|
method,
|
|
193
269
|
headers: {
|
|
@@ -200,7 +276,7 @@ export class EvaluationClient {
|
|
|
200
276
|
}), signal);
|
|
201
277
|
if (!response.ok) {
|
|
202
278
|
await response.body?.cancel();
|
|
203
|
-
throw new HueApiError(response.status, askedRetryAfter(response));
|
|
279
|
+
throw new HueApiError(response.status, askedRetryAfter(response), diagnosticOf(response));
|
|
204
280
|
}
|
|
205
281
|
try {
|
|
206
282
|
const reader = response.body?.getReader();
|
|
@@ -474,7 +550,9 @@ export class EvaluationClient {
|
|
|
474
550
|
async prepareAttempt(input) {
|
|
475
551
|
const request = prepareAttemptInputV2.parse(input);
|
|
476
552
|
const { executionId, ...body } = request;
|
|
477
|
-
|
|
553
|
+
// Sent once although keyed: a lost acknowledgement of a credential-bearing decision stays
|
|
554
|
+
// uncertain, and the worker never reacquires credentials.
|
|
555
|
+
const response = await this.request("POST", `/experiment-executions/${executionId}/prepare-attempt`, body, { ...valueBounds, bytes: 128_000 }, undefined, false);
|
|
478
556
|
try {
|
|
479
557
|
return parsePrepareAttemptResultV2(response, request);
|
|
480
558
|
}
|
|
@@ -662,21 +740,21 @@ export class EvaluationClient {
|
|
|
662
740
|
}
|
|
663
741
|
/** Register or refresh the fixed local agent key and revision. */
|
|
664
742
|
async registerLocalAgent(input) {
|
|
665
|
-
const agent = await this.request("POST", "/local-agent-worker/register", input);
|
|
743
|
+
const agent = await this.request("POST", "/local-agent-worker/register", input, undefined, undefined, true);
|
|
666
744
|
// Hue's response names the key `agentKey`; read either name.
|
|
667
745
|
return { ...agent, key: agent.key ?? agent.agentKey ?? input.key };
|
|
668
746
|
}
|
|
669
747
|
/** Claim a queued run for this agent and durable worker identity. */
|
|
670
748
|
claimLocalAgentRun(input) {
|
|
671
|
-
return this.request("POST", "/local-agent-worker/claim", input);
|
|
749
|
+
return this.request("POST", "/local-agent-worker/claim", input, undefined, undefined, true);
|
|
672
750
|
}
|
|
673
751
|
/** Refresh the lease of a claimed local run. */
|
|
674
752
|
heartbeatLocalAgentRun(input) {
|
|
675
|
-
return this.request("POST", "/local-agent-worker/runs/heartbeat", input);
|
|
753
|
+
return this.request("POST", "/local-agent-worker/runs/heartbeat", input, undefined, undefined, true);
|
|
676
754
|
}
|
|
677
755
|
/** Report acknowledged completion or an execution requiring attention. */
|
|
678
756
|
completeLocalAgentRun(input) {
|
|
679
|
-
return this.request("POST", "/local-agent-worker/runs/complete", input);
|
|
757
|
+
return this.request("POST", "/local-agent-worker/runs/complete", input, undefined, undefined, true);
|
|
680
758
|
}
|
|
681
759
|
/** Lists Scenarios (draft and published) of the project; requires a Read and write key. */
|
|
682
760
|
listCaseConversions(page) {
|
|
@@ -688,7 +766,7 @@ export class EvaluationClient {
|
|
|
688
766
|
}
|
|
689
767
|
/** Creates the legacy execution-scoped generic MCP capability for one world. */
|
|
690
768
|
createSimulationMcpCapability(input) {
|
|
691
|
-
return this.request("POST", "/local-agent-worker/mcp-capability", input);
|
|
769
|
+
return this.request("POST", "/local-agent-worker/mcp-capability", input, undefined, undefined, true);
|
|
692
770
|
}
|
|
693
771
|
}
|
|
694
772
|
/**
|
|
@@ -105,6 +105,9 @@ export interface RunEnvironmentTargetOptions {
|
|
|
105
105
|
export declare const MAX_GRACE_WAIT_MS = 10000;
|
|
106
106
|
export declare const SEAL_POLL_MS = 250;
|
|
107
107
|
export declare const SEAL_WAIT_MS = 30000;
|
|
108
|
+
/** How long past the world's `expiresAt` the target is given before it is stopped: room for a
|
|
109
|
+
* last call in flight at the deadline and for clock skew between the worker and Hue. */
|
|
110
|
+
export declare const DEADLINE_MARGIN_MS = 5000;
|
|
108
111
|
/** The seal wait's bounds. Only tests shorten them; the helpers always use the defaults. */
|
|
109
112
|
export interface SealTiming {
|
|
110
113
|
maxGraceWaitMs: number;
|
|
@@ -123,16 +126,20 @@ export declare function caseTraceparent(span: {
|
|
|
123
126
|
};
|
|
124
127
|
};
|
|
125
128
|
}): string;
|
|
126
|
-
/** What a deployment's credential-free gateway
|
|
127
|
-
*
|
|
128
|
-
* `x-hue-diagnostic`), or `unknown` for anything else. */
|
|
129
|
+
/** What a deployment's credential-free gateway probe said: `on` (the gateway answered, with its
|
|
130
|
+
* health or with a refusal carrying `x-hue-diagnostic`), `off` (the empty 404 the disabled
|
|
131
|
+
* handler answers, with no `x-hue-diagnostic`), or `unknown` for anything else. */
|
|
129
132
|
export type GatewayState = "on" | "off" | "unknown";
|
|
130
133
|
/**
|
|
131
|
-
* Whether the deployment serves the simulation gateway, from
|
|
132
|
-
*
|
|
133
|
-
*
|
|
134
|
-
*
|
|
135
|
-
*
|
|
134
|
+
* Whether the deployment serves the simulation gateway, from a credential-free probe of the
|
|
135
|
+
* gateway's reserved health path with no app named (`/api/sim/_hue/health`), so the answer does
|
|
136
|
+
* not depend on which apps the deployment mirrors. The gateway answers it on its own host with
|
|
137
|
+
* its health, 200 with `gateway: "simulation"`, and on the application host with a refusal that
|
|
138
|
+
* carries `x-hue-diagnostic` (`unmirrored_provider`: the reserved segment is no app); either is
|
|
139
|
+
* on, since only the enabled gateway writes that header. The disabled handler's empty 404 (no
|
|
140
|
+
* `x-hue-diagnostic`) is off, and anything else (a network failure, a timeout, a redirect, another
|
|
141
|
+
* status or body) is unknown. Probed only after a create was refused; on and off are remembered
|
|
142
|
+
* per origin, unknown is probed again next time.
|
|
136
143
|
*/
|
|
137
144
|
export declare function gatewayState(baseUrl: string, fetchImpl?: (url: string, init?: RequestInit) => Promise<Response>): Promise<GatewayState>;
|
|
138
145
|
/**
|
|
@@ -3,7 +3,7 @@ import { HueEnvironmentError, isTransientEnvironmentError, } from "../environmen
|
|
|
3
3
|
import { bindEnvironmentTools } from "../environment/tools.js";
|
|
4
4
|
import { legacyMcpCapability, worldHandoff } from "../environment/world.js";
|
|
5
5
|
import { actualAgentManifestV2, attemptBaselineV2, projectMcpConnectionV2, requestedAttemptProvidersV2, validateAttemptConnectionBundleV2, } from "./attempt.js";
|
|
6
|
-
import { TargetCancelledError, TargetOutcomeUncertainError } from "./runner.js";
|
|
6
|
+
import { TargetCancelledError, TargetOutcomeUncertainError, TargetTimeoutError } from "./runner.js";
|
|
7
7
|
export function requestedAttemptV2(options) {
|
|
8
8
|
const requested = options.requestedProviders !== undefined;
|
|
9
9
|
const selected = options.mcpSurface !== undefined;
|
|
@@ -47,6 +47,9 @@ export function pinRequestedAttemptV2(requested, config) {
|
|
|
47
47
|
export const MAX_GRACE_WAIT_MS = 10_000;
|
|
48
48
|
export const SEAL_POLL_MS = 250;
|
|
49
49
|
export const SEAL_WAIT_MS = 30_000;
|
|
50
|
+
/** How long past the world's `expiresAt` the target is given before it is stopped: room for a
|
|
51
|
+
* last call in flight at the deadline and for clock skew between the worker and Hue. */
|
|
52
|
+
export const DEADLINE_MARGIN_MS = 5_000;
|
|
50
53
|
const SEAL_TIMING = {
|
|
51
54
|
maxGraceWaitMs: MAX_GRACE_WAIT_MS,
|
|
52
55
|
pollMs: SEAL_POLL_MS,
|
|
@@ -145,21 +148,28 @@ export function caseTraceparent(span) {
|
|
|
145
148
|
}
|
|
146
149
|
const gatewayStates = new Map();
|
|
147
150
|
/**
|
|
148
|
-
* Whether the deployment serves the simulation gateway, from
|
|
149
|
-
*
|
|
150
|
-
*
|
|
151
|
-
*
|
|
152
|
-
*
|
|
151
|
+
* Whether the deployment serves the simulation gateway, from a credential-free probe of the
|
|
152
|
+
* gateway's reserved health path with no app named (`/api/sim/_hue/health`), so the answer does
|
|
153
|
+
* not depend on which apps the deployment mirrors. The gateway answers it on its own host with
|
|
154
|
+
* its health, 200 with `gateway: "simulation"`, and on the application host with a refusal that
|
|
155
|
+
* carries `x-hue-diagnostic` (`unmirrored_provider`: the reserved segment is no app); either is
|
|
156
|
+
* on, since only the enabled gateway writes that header. The disabled handler's empty 404 (no
|
|
157
|
+
* `x-hue-diagnostic`) is off, and anything else (a network failure, a timeout, a redirect, another
|
|
158
|
+
* status or body) is unknown. Probed only after a create was refused; on and off are remembered
|
|
159
|
+
* per origin, unknown is probed again next time.
|
|
153
160
|
*/
|
|
154
161
|
export function gatewayState(baseUrl, fetchImpl = fetch) {
|
|
155
162
|
const origin = new URL(baseUrl).origin;
|
|
156
163
|
const remembered = gatewayStates.get(origin);
|
|
157
164
|
if (remembered)
|
|
158
165
|
return remembered;
|
|
159
|
-
const probe = fetchImpl(`${origin}/api/sim/
|
|
166
|
+
const probe = fetchImpl(`${origin}/api/sim/_hue/health`, {
|
|
167
|
+
redirect: "error",
|
|
168
|
+
signal: AbortSignal.timeout(5_000),
|
|
169
|
+
})
|
|
160
170
|
.then(async (response) => {
|
|
161
171
|
if (response.status === 404)
|
|
162
|
-
return response.headers.has("x-hue-diagnostic") ? "
|
|
172
|
+
return response.headers.has("x-hue-diagnostic") ? "on" : "off";
|
|
163
173
|
if (!response.ok)
|
|
164
174
|
return "unknown";
|
|
165
175
|
const body = (await response.json());
|
|
@@ -307,20 +317,50 @@ export async function runEnvironmentTarget(options) {
|
|
|
307
317
|
if (options.signal?.aborted)
|
|
308
318
|
throw new TargetCancelledError();
|
|
309
319
|
await progress({ type: "target_started", environmentRunId: run.id });
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
320
|
+
// The world's deadline bounds the target: once the world expires its mirrors refuse every
|
|
321
|
+
// call, so an agent still running can achieve nothing more. The target is told through its
|
|
322
|
+
// signal a moment after the deadline and, whether or not it listens, its case ends as a
|
|
323
|
+
// `TargetTimeout` and the world is sealed abandoned; a promise that never settles is left
|
|
324
|
+
// behind rather than holding the run open for good.
|
|
325
|
+
const expiresAt = Date.parse(run.expiresAt);
|
|
326
|
+
const deadline = new AbortController();
|
|
327
|
+
let timer;
|
|
328
|
+
const timedOut = new Promise((_, reject) => {
|
|
329
|
+
if (!Number.isFinite(expiresAt))
|
|
330
|
+
return;
|
|
331
|
+
timer = setTimeout(() => {
|
|
332
|
+
// The timeout is decided before the target is told: a target that rejects with its
|
|
333
|
+
// own abort error on the signal must not win the race and be saved as its own error.
|
|
334
|
+
const error = new TargetTimeoutError(run.id, run.expiresAt);
|
|
335
|
+
reject(error);
|
|
336
|
+
deadline.abort(error);
|
|
337
|
+
}, Math.max(0, expiresAt + DEADLINE_MARGIN_MS - Date.now()));
|
|
323
338
|
});
|
|
339
|
+
let output;
|
|
340
|
+
try {
|
|
341
|
+
output = await Promise.race([
|
|
342
|
+
options.target(options.inputs, {
|
|
343
|
+
config: context.config,
|
|
344
|
+
item: context.item,
|
|
345
|
+
executionId: context.executionId,
|
|
346
|
+
environmentRunId: run.id,
|
|
347
|
+
trace: { traceId: context.span.traceId, spanId: context.span.spanId },
|
|
348
|
+
tools,
|
|
349
|
+
...(world ? { world } : {}),
|
|
350
|
+
...(mcp ? { mcp } : {}),
|
|
351
|
+
...(connectionBundle ? { connectionBundle } : {}),
|
|
352
|
+
files: structuredClone(context.files),
|
|
353
|
+
outputDirectory: context.outputDirectory,
|
|
354
|
+
signal: options.signal
|
|
355
|
+
? AbortSignal.any([options.signal, deadline.signal])
|
|
356
|
+
: deadline.signal,
|
|
357
|
+
}),
|
|
358
|
+
timedOut,
|
|
359
|
+
]);
|
|
360
|
+
}
|
|
361
|
+
finally {
|
|
362
|
+
clearTimeout(timer);
|
|
363
|
+
}
|
|
324
364
|
await seal(options.environmentClient, run.id, context.executionId, "completed", timing);
|
|
325
365
|
finalized = true;
|
|
326
366
|
await Promise.resolve(progress({ type: "world_sealed", environmentRunId: run.id })).catch(() => undefined);
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import type { HueClient } from "../client.js";
|
|
2
|
-
import type
|
|
2
|
+
import { type EnvironmentClient } from "../environment/client.js";
|
|
3
3
|
import type { EnvironmentTool } from "../environment/tools.js";
|
|
4
4
|
import type { WorldHandoff } from "../environment/types.js";
|
|
5
5
|
import type { ActualAgentManifestInputV2, AttemptConnectionBundleV2, RequestedAttemptProviderV2 } from "./attempt.js";
|
|
6
6
|
import type { EvaluationClient } from "./client.js";
|
|
7
7
|
import { type RunExperimentTargetContext, type RunnerReport, type TelemetryNotAccepted } from "./runner.js";
|
|
8
|
-
import type { ExperimentCase, JsonValue, LocalAgentRegistration, LocalFile, LocalScorer, TargetResult } from "./types.js";
|
|
8
|
+
import type { ExperimentCase, JsonValue, LocalAgentClaim, LocalAgentRegistration, LocalFile, LocalScorer, TargetResult } from "./types.js";
|
|
9
9
|
/** Capability strings a registration declares; Hue offers only matching cases. */
|
|
10
10
|
export declare const localAgentCapabilities: {
|
|
11
11
|
/** Cases pinned to a hosted synthetic world. */
|
|
@@ -58,6 +58,9 @@ export interface LocalAgentTargetContext {
|
|
|
58
58
|
/** Private directory for this case, removed after it; return generated files with
|
|
59
59
|
* `withFiles`. */
|
|
60
60
|
outputDirectory: string;
|
|
61
|
+
/** Ends when the worker stops or the world's deadline passes; a target that listens stops
|
|
62
|
+
* its work, and one that does not is left behind when its case ends as a `TargetTimeout`. */
|
|
63
|
+
signal?: AbortSignal;
|
|
61
64
|
}
|
|
62
65
|
/** Candidate-visible context for one queued ordinary case run directly on this machine. */
|
|
63
66
|
export interface LocalAgentDirectContext {
|
|
@@ -109,8 +112,27 @@ export interface RunLocalAgentOptions {
|
|
|
109
112
|
pollIntervalMillis?: number;
|
|
110
113
|
/** Stops polling cooperatively; does not cancel an active callback. */
|
|
111
114
|
signal?: AbortSignal;
|
|
112
|
-
/**
|
|
115
|
+
/** Stop after this many runs settle: completed, or given up as `attention`. Useful for
|
|
116
|
+
* one-shot jobs and deterministic acceptance. Omit to keep polling. */
|
|
113
117
|
maxRuns?: number;
|
|
118
|
+
/** How many times one run is resumed after a transient failure (a request Hue refused
|
|
119
|
+
* transiently past the client's own retries, a world seal not read back, a lost network,
|
|
120
|
+
* telemetry not accepted) before the worker gives it up as `attention`. Default 5, range
|
|
121
|
+
* 1–20. A failure that would recur (an uncertain target outcome, an unserializable output, a
|
|
122
|
+
* refusal Hue decided on, an input the SDK refuses) is attention at once. */
|
|
123
|
+
maxRunAttempts?: number;
|
|
124
|
+
/** The lifetime of each world the worker creates for a case, in seconds (1–86400); the
|
|
125
|
+
* server's default, an hour, applies when omitted. The target is stopped when the world
|
|
126
|
+
* expires: its case ends as a `TargetTimeout` error and the world is sealed abandoned. */
|
|
127
|
+
worldTtlSeconds?: number;
|
|
128
|
+
/** Called when a poll (registration or claim) fails transiently; the worker waits and polls
|
|
129
|
+
* again. A refusal it cannot recover from (a revoked key, a disabled agent) is thrown. */
|
|
130
|
+
onPollError?(error: unknown): void | Promise<void>;
|
|
131
|
+
/** Called when a claimed run's attempt fails: `retry` says the worker keeps the claim and
|
|
132
|
+
* resumes it after `waitMillis`; `attention` says the run was given up and needs a project
|
|
133
|
+
* member (requeue or cancel); `refused` says Hue no longer accepts this worker's outcome for
|
|
134
|
+
* the run (it was released or cancelled meanwhile). */
|
|
135
|
+
onRunFailed?(failure: LocalRunFailure): void | Promise<void>;
|
|
114
136
|
/** Emit a one-time `DeprecationWarning` when the deployment serves a legacy world; on by default. */
|
|
115
137
|
deprecationWarnings?: boolean;
|
|
116
138
|
/** Opt into the experiment's immutable V2 provider profile. These three values are
|
|
@@ -151,6 +173,19 @@ export declare function registeredCapabilities(options: {
|
|
|
151
173
|
/** The ordinary-case callback, when supplied. */
|
|
152
174
|
directTarget?: unknown;
|
|
153
175
|
}): string[];
|
|
176
|
+
/** Why a claimed run's attempt ended without completing, and what the worker did about it. */
|
|
177
|
+
export interface LocalRunFailure extends LocalAgentClaim {
|
|
178
|
+
/** What failed. */
|
|
179
|
+
error: unknown;
|
|
180
|
+
/** Failed attempts of this run in this process, this one included; 0 when not counted. */
|
|
181
|
+
attempt: number;
|
|
182
|
+
/** `retry`: the claim is kept and the run resumes after `waitMillis`. `attention`: the run
|
|
183
|
+
* was given up for a project member to requeue or cancel. `refused`: Hue no longer accepts
|
|
184
|
+
* this worker's outcome for the run, which was released or cancelled meanwhile. */
|
|
185
|
+
outcome: "retry" | "attention" | "refused";
|
|
186
|
+
/** The wait before the run resumes, for `retry`. */
|
|
187
|
+
waitMillis?: number;
|
|
188
|
+
}
|
|
154
189
|
/**
|
|
155
190
|
* Starts an outbound-only worker for one fixed local agent entry point. Hue chooses
|
|
156
191
|
* only the registered key/revision; no command or source is received from the cloud.
|