@hue-run/sdk 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLI.md +22 -13
- package/ENVIRONMENTS.md +61 -0
- package/README.md +47 -1
- package/dist/cli/eval.js +79 -24
- package/dist/client.js +3 -2
- package/dist/config.d.ts +3 -1
- package/dist/config.js +8 -1
- package/dist/environment/client.d.ts +16 -2
- package/dist/environment/client.js +46 -3
- package/dist/environment/types.d.ts +131 -2
- package/dist/environment/world.d.ts +50 -0
- package/dist/environment/world.js +105 -0
- package/dist/environment.d.ts +2 -0
- package/dist/environment.js +1 -0
- package/dist/evals/environment-target.d.ts +53 -2
- package/dist/evals/environment-target.js +114 -10
- package/dist/evals/local-worker.d.ts +12 -5
- package/dist/evals/local-worker.js +13 -5
- package/dist/evals/runner.d.ts +1 -1
- package/dist/evals/runner.js +2 -2
- package/dist/evals/simulation.d.ts +14 -5
- package/dist/evals/simulation.js +26 -8
- package/dist/live-spans.d.ts +25 -0
- package/dist/live-spans.js +115 -0
- package/dist/transport.d.ts +17 -0
- package/dist/transport.js +390 -61
- package/dist/types.d.ts +7 -1
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +1 -1
|
@@ -275,6 +275,80 @@ export interface Effect {
|
|
|
275
275
|
/** Changed field names. */
|
|
276
276
|
fields: string[];
|
|
277
277
|
}
|
|
278
|
+
/** Lifecycle of a world the simulation gateway serves. */
|
|
279
|
+
export type WorldLifecycle = "pending" | "live" | "completing" | "sealed";
|
|
280
|
+
/** Advisory flags on a world's status read. */
|
|
281
|
+
export type WorldFlag = "no_calls" | "fingerprint_differs";
|
|
282
|
+
/** One pinned provider surface of one provider instance, as the mirror URL an official client
|
|
283
|
+
* is pointed at. `alias` stays null until the provider's alias hosts are served. */
|
|
284
|
+
export interface WorldSurface {
|
|
285
|
+
/** Provider identity, such as `google.gmail`. */
|
|
286
|
+
provider: string;
|
|
287
|
+
/** Surface identity, such as `google.gmail/mcp` or `google.gmail/rest`. */
|
|
288
|
+
surface: string;
|
|
289
|
+
/** The provider instance of the environment version this surface belongs to. */
|
|
290
|
+
providerInstanceKey: string;
|
|
291
|
+
/** Mirror URL in the path form. */
|
|
292
|
+
url: string;
|
|
293
|
+
/** Alias-host URL, or null while unavailable. */
|
|
294
|
+
alias: string | null;
|
|
295
|
+
}
|
|
296
|
+
/** The headers an MCP client sends the mirror. */
|
|
297
|
+
export interface WorldMcpHeaders {
|
|
298
|
+
/** `Bearer <world token>`. */
|
|
299
|
+
Authorization: string;
|
|
300
|
+
}
|
|
301
|
+
/** One MCP server entry of the common `mcpServers` configuration shape. */
|
|
302
|
+
export interface WorldMcpServer {
|
|
303
|
+
/** Transport; the mirrors serve Streamable HTTP. */
|
|
304
|
+
type: "http";
|
|
305
|
+
/** Mirror URL of the MCP surface. */
|
|
306
|
+
url: string;
|
|
307
|
+
/** The world token as a bearer. */
|
|
308
|
+
headers: WorldMcpHeaders;
|
|
309
|
+
}
|
|
310
|
+
/** The `{ url, token, expiresAt }` shape the retired `hue_sim_` capability had. */
|
|
311
|
+
export interface LegacyMcpCapability {
|
|
312
|
+
/** MCP endpoint: the world's first MCP mirror. */
|
|
313
|
+
url: string;
|
|
314
|
+
/** The world token. */
|
|
315
|
+
token: string;
|
|
316
|
+
/** The world deadline as an ISO timestamp. */
|
|
317
|
+
expiresAt: string;
|
|
318
|
+
}
|
|
319
|
+
/** The common `mcpServers` shape, one server per MCP surface of each provider instance. */
|
|
320
|
+
export interface WorldMcpConfig {
|
|
321
|
+
/** Servers keyed by provider instance (suffixed by surface when an instance has several). */
|
|
322
|
+
mcpServers: Record<string, WorldMcpServer>;
|
|
323
|
+
}
|
|
324
|
+
/**
|
|
325
|
+
* What the World API hands an agent for one world: the world token in the provider's own
|
|
326
|
+
* credential slot, the mirror URLs, the environment carriers and the MCP configuration.
|
|
327
|
+
* Minted per response and never stored by Hue; keep it out of logs and checkpoints.
|
|
328
|
+
*/
|
|
329
|
+
export interface WorldHandoff {
|
|
330
|
+
/** World identity (the environment-run ID). */
|
|
331
|
+
id: string;
|
|
332
|
+
/** The `hue_world_…` credential; lives exactly as long as the world. */
|
|
333
|
+
token: string;
|
|
334
|
+
/** World deadline; the token outlives it by the 5 s completion grace. */
|
|
335
|
+
expiresAt: string;
|
|
336
|
+
/** Current lifecycle. */
|
|
337
|
+
lifecycle: WorldLifecycle;
|
|
338
|
+
/** Grace deadline while completing, else null. */
|
|
339
|
+
completingUntil: string | null;
|
|
340
|
+
/** The caller's trace context, echoed for the agent; null when none was supplied. */
|
|
341
|
+
traceparent: string | null;
|
|
342
|
+
/** `hue-world=<id>`, the OpenTelemetry baggage carrier value. */
|
|
343
|
+
baggage: string;
|
|
344
|
+
/** Mirror surfaces of every provider instance the environment version binds. */
|
|
345
|
+
surfaces: WorldSurface[];
|
|
346
|
+
/** `HUE_WORLD_ID`, `HUE_WORLD_TOKEN`, `BAGGAGE`, `TRACEPARENT` and one
|
|
347
|
+
* `HUE_SIM_<SURFACE ID>_URL` per surface. */
|
|
348
|
+
env: Record<string, string>;
|
|
349
|
+
/** The `mcpServers` configuration for the MCP surfaces. */
|
|
350
|
+
mcpConfig: WorldMcpConfig;
|
|
351
|
+
}
|
|
278
352
|
/** Newly created isolated world and its action catalog. */
|
|
279
353
|
export interface EnvironmentRun {
|
|
280
354
|
/** Environment-run identity. */
|
|
@@ -291,6 +365,26 @@ export interface EnvironmentRun {
|
|
|
291
365
|
expiresAt: string;
|
|
292
366
|
/** Closed generated action catalog. */
|
|
293
367
|
actions: ActionDefinition[];
|
|
368
|
+
/** World identity; present for a world the simulation gateway serves. */
|
|
369
|
+
worldId?: string;
|
|
370
|
+
/** World token; present for a world the simulation gateway serves. */
|
|
371
|
+
token?: string;
|
|
372
|
+
/** Lifecycle; present for a world the simulation gateway serves. */
|
|
373
|
+
lifecycle?: WorldLifecycle;
|
|
374
|
+
/** Grace deadline while completing. */
|
|
375
|
+
completingUntil?: string | null;
|
|
376
|
+
/** Baggage carrier value. */
|
|
377
|
+
baggage?: string;
|
|
378
|
+
/** The stored caller trace context, or null. */
|
|
379
|
+
traceparent?: string | null;
|
|
380
|
+
/** Mirror surfaces. */
|
|
381
|
+
surfaces?: WorldSurface[];
|
|
382
|
+
/** Environment carriers. */
|
|
383
|
+
env?: Record<string, string>;
|
|
384
|
+
/** MCP configuration. */
|
|
385
|
+
mcpConfig?: WorldMcpConfig;
|
|
386
|
+
/** Reserved for connection keys; null today. */
|
|
387
|
+
connection?: null;
|
|
294
388
|
}
|
|
295
389
|
/** Result of invoking one environment action. */
|
|
296
390
|
export interface ActionResult {
|
|
@@ -382,6 +476,20 @@ export interface RunState extends EnvironmentCoverage {
|
|
|
382
476
|
stateDigest: string;
|
|
383
477
|
/** Final state, available after sealing. */
|
|
384
478
|
finalState?: JsonValue;
|
|
479
|
+
/** World identity; present for a world the simulation gateway serves. */
|
|
480
|
+
worldId?: string;
|
|
481
|
+
/** Lifecycle; present for a world the simulation gateway serves. */
|
|
482
|
+
lifecycle?: WorldLifecycle;
|
|
483
|
+
/** Grace deadline while completing. */
|
|
484
|
+
completingUntil?: string | null;
|
|
485
|
+
/** The trace the linked execution declared, or null. */
|
|
486
|
+
traceExternalId?: string | null;
|
|
487
|
+
/** Mirror surfaces. */
|
|
488
|
+
surfaces?: WorldSurface[];
|
|
489
|
+
/** Advisory flags. A status read never returns the token. */
|
|
490
|
+
flags?: WorldFlag[];
|
|
491
|
+
/** Reserved for connection keys; null today. */
|
|
492
|
+
connection?: null;
|
|
385
493
|
}
|
|
386
494
|
/** One immutable journal entry. */
|
|
387
495
|
export interface Step {
|
|
@@ -423,9 +531,25 @@ export interface SealedRun {
|
|
|
423
531
|
stepCount: number;
|
|
424
532
|
/** Final state digest. */
|
|
425
533
|
stateDigest: string;
|
|
426
|
-
/** Seal timestamp. */
|
|
427
|
-
sealedAt: string;
|
|
534
|
+
/** Seal timestamp; null while a gateway world is `completing` (its seal follows the grace). */
|
|
535
|
+
sealedAt: string | null;
|
|
536
|
+
/** Lifecycle after finish, for a world the simulation gateway serves. */
|
|
537
|
+
lifecycle?: "completing" | "sealed";
|
|
538
|
+
/** Grace deadline while completing. */
|
|
539
|
+
completingUntil?: string | null;
|
|
540
|
+
}
|
|
541
|
+
/** One section of a sealed world's evidence, or everything. */
|
|
542
|
+
export type WorldEvidenceSection = "all" | "start" | "end" | "diff" | "ledger";
|
|
543
|
+
/** Options for reading a sealed world's evaluator-only evidence. */
|
|
544
|
+
export interface WorldEvidenceOptions {
|
|
545
|
+
/** One section, or everything (default). */
|
|
546
|
+
section?: WorldEvidenceSection;
|
|
547
|
+
/** Include ledger request and response bodies (default true). */
|
|
548
|
+
bodies?: boolean;
|
|
428
549
|
}
|
|
550
|
+
/** The evaluator-only evidence of a sealed world: start and end state, the diff, the call ledger,
|
|
551
|
+
* coverage and the fingerprint. Hue's shape is the authority; this client does not narrow it. */
|
|
552
|
+
export type WorldEvidence = Record<string, JsonValue>;
|
|
429
553
|
/** Options for creating one fresh isolated world. */
|
|
430
554
|
export interface CreateRunInput {
|
|
431
555
|
/** Stable idempotency key for recovering creation acknowledgement. */
|
|
@@ -440,6 +564,11 @@ export interface CreateRunInput {
|
|
|
440
564
|
maxSteps?: number;
|
|
441
565
|
/** Optional lease in seconds. */
|
|
442
566
|
ttlSeconds?: number;
|
|
567
|
+
/** The case span's W3C context (`00-<trace id>-<span id>-<flags>`); its trace ID must equal
|
|
568
|
+
* the trace the execution declared at start. The world span is parented on it. */
|
|
569
|
+
traceparent?: string;
|
|
570
|
+
/** The agent revision under test, 1 to 256 characters; part of the world's fingerprint. */
|
|
571
|
+
agentRevision?: string;
|
|
443
572
|
}
|
|
444
573
|
/** Request to invoke one action. */
|
|
445
574
|
export interface ActionInput {
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import type { EnvironmentRun, LegacyMcpCapability, WorldHandoff } from "./types.js";
|
|
2
|
+
/**
|
|
3
|
+
* The world an agent acts on through provider mirrors, read from a create or replay response.
|
|
4
|
+
* Null for a world created while the gateway was off: that world has Hue-native tools and no
|
|
5
|
+
* mirror URLs. The handoff carries the world token; keep it out of logs and checkpoints.
|
|
6
|
+
*/
|
|
7
|
+
export declare function worldHandoff(run: EnvironmentRun): WorldHandoff | null;
|
|
8
|
+
/** Variables that authenticate against Hue's control plane rather than a simulated provider. */
|
|
9
|
+
export declare const HUE_CONTROL_PLANE_VARIABLES: readonly string[];
|
|
10
|
+
/** True for a control-plane variable by name, or for any variable holding a Hue credential. */
|
|
11
|
+
export declare function isHueControlPlaneCredential(name: string, value: string | undefined): boolean;
|
|
12
|
+
/** The parent's variables without Hue control-plane credentials; `undefined` values are dropped. */
|
|
13
|
+
export declare function stripHueControlPlaneCredentials(parent: Record<string, string | undefined>): Record<string, string>;
|
|
14
|
+
/** Options for {@link agentEnvironment}. */
|
|
15
|
+
export interface AgentEnvironmentOptions {
|
|
16
|
+
/** The environment to start from; defaults to this process's. */
|
|
17
|
+
parent?: Record<string, string | undefined>;
|
|
18
|
+
/** Keep Hue control-plane credentials in the child. Off by default: the agent gets world
|
|
19
|
+
* tokens only (plan section 10.3) and the project key stays with the runner. */
|
|
20
|
+
includeHueCredentials?: boolean;
|
|
21
|
+
/** Also set `HUE_MCP_URL`, `HUE_MCP_TOKEN` and `HUE_MCP_EXPIRES_AT` from the world's first
|
|
22
|
+
* MCP mirror, the names the `hue_sim_` bridge used, for one compatibility release. On by
|
|
23
|
+
* default; the canonical carriers are the world's own `env`. */
|
|
24
|
+
legacyMcpVariables?: boolean;
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* The environment for an agent child process running one case (the coordination briefs'
|
|
28
|
+
* recommended policy): the parent's variables minus Hue control-plane credentials, then the
|
|
29
|
+
* world's carriers, which win over anything the parent set. Nothing here is logged.
|
|
30
|
+
*/
|
|
31
|
+
export declare function agentEnvironment(world: WorldHandoff, options?: AgentEnvironmentOptions): Record<string, string>;
|
|
32
|
+
/** The `{ url, token, expiresAt }` shape the `hue_sim_` capability had, projected from the
|
|
33
|
+
* world's first MCP mirror so an adapter written for the bridge keeps working through the
|
|
34
|
+
* compatibility release. Undefined for a world without an MCP surface. */
|
|
35
|
+
export declare function legacyMcpCapability(world: WorldHandoff): LegacyMcpCapability | undefined;
|
|
36
|
+
/** The owner-only `mcpServers` file {@link writeMcpConfig} wrote. */
|
|
37
|
+
export interface McpConfigFile {
|
|
38
|
+
/** The owner-only file; pass its path to the harness and dispose after the run. */
|
|
39
|
+
path: string;
|
|
40
|
+
/** Removes the file and its private directory; safe to call twice. */
|
|
41
|
+
dispose(): Promise<void>;
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* Writes the world's `mcpConfig` as an owner-only file in a private directory (plan section
|
|
45
|
+
* 4.9): the file carries the world token, so it is created with mode 0600 inside a 0700
|
|
46
|
+
* directory, is never logged, and `dispose` removes the directory after the run.
|
|
47
|
+
*/
|
|
48
|
+
export declare function writeMcpConfig(world: WorldHandoff, options?: {
|
|
49
|
+
directory?: string;
|
|
50
|
+
}): Promise<McpConfigFile>;
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import { chmod, mkdtemp, rm, writeFile } from "node:fs/promises";
|
|
2
|
+
import { tmpdir } from "node:os";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
/**
|
|
5
|
+
* The world an agent acts on through provider mirrors, read from a create or replay response.
|
|
6
|
+
* Null for a world created while the gateway was off: that world has Hue-native tools and no
|
|
7
|
+
* mirror URLs. The handoff carries the world token; keep it out of logs and checkpoints.
|
|
8
|
+
*/
|
|
9
|
+
export function worldHandoff(run) {
|
|
10
|
+
if (!run.token || !run.env || !run.mcpConfig || !run.surfaces)
|
|
11
|
+
return null;
|
|
12
|
+
const id = run.worldId ?? run.id;
|
|
13
|
+
return {
|
|
14
|
+
id,
|
|
15
|
+
token: run.token,
|
|
16
|
+
expiresAt: run.expiresAt,
|
|
17
|
+
lifecycle: run.lifecycle ?? "live",
|
|
18
|
+
completingUntil: run.completingUntil ?? null,
|
|
19
|
+
traceparent: run.traceparent ?? null,
|
|
20
|
+
baggage: run.baggage ?? `hue-world=${id}`,
|
|
21
|
+
surfaces: run.surfaces.map((surface) => ({ ...surface })),
|
|
22
|
+
env: { ...run.env },
|
|
23
|
+
mcpConfig: {
|
|
24
|
+
mcpServers: Object.fromEntries(Object.entries(run.mcpConfig.mcpServers).map(([name, server]) => [
|
|
25
|
+
name,
|
|
26
|
+
{ type: server.type, url: server.url, headers: { ...server.headers } },
|
|
27
|
+
])),
|
|
28
|
+
},
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
/** Variables that authenticate against Hue's control plane rather than a simulated provider. */
|
|
32
|
+
export const HUE_CONTROL_PLANE_VARIABLES = [
|
|
33
|
+
"HUE_API_KEY",
|
|
34
|
+
"HUE_MCP_KEY",
|
|
35
|
+
"HUE_PROJECT_KEY",
|
|
36
|
+
"HUE_SERVICE_KEY",
|
|
37
|
+
];
|
|
38
|
+
/** Hue's own credential shapes: project keys, attempt grants and the project MCP key. */
|
|
39
|
+
const HUE_CREDENTIAL_SHAPE = /^hue_(sk|attempt|mcp)_/;
|
|
40
|
+
/** True for a control-plane variable by name, or for any variable holding a Hue credential. */
|
|
41
|
+
export function isHueControlPlaneCredential(name, value) {
|
|
42
|
+
if (HUE_CONTROL_PLANE_VARIABLES.includes(name))
|
|
43
|
+
return true;
|
|
44
|
+
return typeof value === "string" && HUE_CREDENTIAL_SHAPE.test(value.trim());
|
|
45
|
+
}
|
|
46
|
+
/** The parent's variables without Hue control-plane credentials; `undefined` values are dropped. */
|
|
47
|
+
export function stripHueControlPlaneCredentials(parent) {
|
|
48
|
+
const child = {};
|
|
49
|
+
for (const [name, value] of Object.entries(parent)) {
|
|
50
|
+
if (value === undefined || isHueControlPlaneCredential(name, value))
|
|
51
|
+
continue;
|
|
52
|
+
child[name] = value;
|
|
53
|
+
}
|
|
54
|
+
return child;
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* The environment for an agent child process running one case (the coordination briefs'
|
|
58
|
+
* recommended policy): the parent's variables minus Hue control-plane credentials, then the
|
|
59
|
+
* world's carriers, which win over anything the parent set. Nothing here is logged.
|
|
60
|
+
*/
|
|
61
|
+
export function agentEnvironment(world, options = {}) {
|
|
62
|
+
const parent = options.parent ?? process.env;
|
|
63
|
+
const child = options.includeHueCredentials
|
|
64
|
+
? Object.fromEntries(Object.entries(parent).filter((entry) => entry[1] !== undefined))
|
|
65
|
+
: stripHueControlPlaneCredentials(parent);
|
|
66
|
+
Object.assign(child, world.env);
|
|
67
|
+
if (options.legacyMcpVariables ?? true) {
|
|
68
|
+
const legacy = legacyMcpCapability(world);
|
|
69
|
+
if (legacy) {
|
|
70
|
+
child.HUE_MCP_URL = legacy.url;
|
|
71
|
+
child.HUE_MCP_TOKEN = legacy.token;
|
|
72
|
+
child.HUE_MCP_EXPIRES_AT = legacy.expiresAt;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
return child;
|
|
76
|
+
}
|
|
77
|
+
/** The `{ url, token, expiresAt }` shape the `hue_sim_` capability had, projected from the
|
|
78
|
+
* world's first MCP mirror so an adapter written for the bridge keeps working through the
|
|
79
|
+
* compatibility release. Undefined for a world without an MCP surface. */
|
|
80
|
+
export function legacyMcpCapability(world) {
|
|
81
|
+
const server = Object.values(world.mcpConfig.mcpServers)[0];
|
|
82
|
+
if (!server)
|
|
83
|
+
return undefined;
|
|
84
|
+
return { url: server.url, token: world.token, expiresAt: world.expiresAt };
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* Writes the world's `mcpConfig` as an owner-only file in a private directory (plan section
|
|
88
|
+
* 4.9): the file carries the world token, so it is created with mode 0600 inside a 0700
|
|
89
|
+
* directory, is never logged, and `dispose` removes the directory after the run.
|
|
90
|
+
*/
|
|
91
|
+
export async function writeMcpConfig(world, options = {}) {
|
|
92
|
+
const directory = await mkdtemp(join(options.directory ?? tmpdir(), "hue-world-"));
|
|
93
|
+
const dispose = () => rm(directory, { recursive: true, force: true });
|
|
94
|
+
try {
|
|
95
|
+
await chmod(directory, 0o700);
|
|
96
|
+
const path = join(directory, "mcp.json");
|
|
97
|
+
await writeFile(path, `${JSON.stringify(world.mcpConfig, null, 2)}\n`, { mode: 0o600 });
|
|
98
|
+
return { path, dispose };
|
|
99
|
+
}
|
|
100
|
+
catch (error) {
|
|
101
|
+
// A half-written file would hold the token; never leave it behind.
|
|
102
|
+
await dispose();
|
|
103
|
+
throw error;
|
|
104
|
+
}
|
|
105
|
+
}
|
package/dist/environment.d.ts
CHANGED
|
@@ -3,3 +3,5 @@ export type { EnvironmentClientOptions } from "./environment/client.js";
|
|
|
3
3
|
export { bindEnvironmentTools } from "./environment/tools.js";
|
|
4
4
|
export type { BindEnvironmentToolsOptions, EnvironmentTool } from "./environment/tools.js";
|
|
5
5
|
export type * from "./environment/types.js";
|
|
6
|
+
export { agentEnvironment, HUE_CONTROL_PLANE_VARIABLES, isHueControlPlaneCredential, legacyMcpCapability, stripHueControlPlaneCredentials, worldHandoff, writeMcpConfig, } from "./environment/world.js";
|
|
7
|
+
export type { AgentEnvironmentOptions, McpConfigFile } from "./environment/world.js";
|
package/dist/environment.js
CHANGED
|
@@ -1,2 +1,3 @@
|
|
|
1
1
|
export { createEnvironmentClient, EnvironmentClient, HueEnvironmentError, } from "./environment/client.js";
|
|
2
2
|
export { bindEnvironmentTools } from "./environment/tools.js";
|
|
3
|
+
export { agentEnvironment, HUE_CONTROL_PLANE_VARIABLES, isHueControlPlaneCredential, legacyMcpCapability, stripHueControlPlaneCredentials, worldHandoff, writeMcpConfig, } from "./environment/world.js";
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { HueClient } from "../client.js";
|
|
2
|
-
import type
|
|
2
|
+
import { type EnvironmentClient } from "../environment/client.js";
|
|
3
3
|
import { type EnvironmentTool } from "../environment/tools.js";
|
|
4
|
+
import type { EnvironmentRun, WorldHandoff } from "../environment/types.js";
|
|
4
5
|
import type { HueSpan } from "../types.js";
|
|
5
6
|
import { type ActualAgentManifestInputV2, type AttemptBaselineV2, type AttemptConnectionBundleV2, type RequestedAttemptProviderV2, type SurfaceBindingV2 } from "./attempt.js";
|
|
6
7
|
import type { EvaluationClient } from "./client.js";
|
|
@@ -52,8 +53,16 @@ export interface EnvironmentTargetContext {
|
|
|
52
53
|
traceId: string;
|
|
53
54
|
spanId: string;
|
|
54
55
|
};
|
|
56
|
+
/** Hue-native tools of a world created while the gateway was off; empty for a gateway
|
|
57
|
+
* world, whose calls go to the provider mirrors in `world`. */
|
|
55
58
|
tools: Record<string, EnvironmentTool>;
|
|
56
|
-
|
|
59
|
+
/** The mirror URLs, world token, environment carriers and MCP configuration of a gateway
|
|
60
|
+
* world. Absent for a world created while the gateway was off. */
|
|
61
|
+
world?: WorldHandoff;
|
|
62
|
+
/** One MCP endpoint and bearer: for a gateway world, its first MCP mirror with the world
|
|
63
|
+
* token; otherwise the deprecated execution-scoped `hue_sim_` capability. Absent when a
|
|
64
|
+
* gateway world has no MCP surface. */
|
|
65
|
+
mcp?: SimulationMcpCapability;
|
|
57
66
|
connectionBundle?: AttemptConnectionBundleV2;
|
|
58
67
|
signal?: AbortSignal;
|
|
59
68
|
}
|
|
@@ -72,10 +81,52 @@ export interface RunEnvironmentTargetOptions {
|
|
|
72
81
|
requested?: PinnedAttemptV2;
|
|
73
82
|
maxSteps?: number;
|
|
74
83
|
ttlSeconds?: number;
|
|
84
|
+
/** The agent revision under test, sent on create for the world's fingerprint. */
|
|
85
|
+
agentRevision?: string;
|
|
86
|
+
/** Emit a one-time `DeprecationWarning` when a legacy path (Hue-native tools, the `hue_sim_`
|
|
87
|
+
* capability, the provider facade) is used; on by default. A harness that adapts to whatever
|
|
88
|
+
* the deployment serves, such as `hue eval`, turns it off. */
|
|
89
|
+
deprecationWarnings?: boolean;
|
|
75
90
|
signal?: AbortSignal;
|
|
76
91
|
onProgress?(event: EnvironmentTargetProgress): void | Promise<void>;
|
|
77
92
|
target(inputs: JsonValue, context: EnvironmentTargetContext): JsonValue | undefined | Promise<JsonValue | undefined>;
|
|
78
93
|
}
|
|
94
|
+
/** The W3C context of the case span, sent on create so the world span parents on it. The
|
|
95
|
+
* flags are the span's own: an unsampled case span is not exported, and the World API must not
|
|
96
|
+
* be told otherwise. */
|
|
97
|
+
export declare function caseTraceparent(span: {
|
|
98
|
+
traceId: string;
|
|
99
|
+
spanId: string;
|
|
100
|
+
span?: {
|
|
101
|
+
spanContext?(): {
|
|
102
|
+
traceFlags?: number;
|
|
103
|
+
};
|
|
104
|
+
};
|
|
105
|
+
}): string;
|
|
106
|
+
/** What a deployment's credential-free gateway health said: `on` (200 with
|
|
107
|
+
* `gateway: "simulation"`), `off` (the empty 404 the disabled handler answers, with no
|
|
108
|
+
* `x-hue-diagnostic`), or `unknown` for anything else. */
|
|
109
|
+
export type GatewayState = "on" | "off" | "unknown";
|
|
110
|
+
/**
|
|
111
|
+
* Whether the deployment serves the simulation gateway, from its credential-free health
|
|
112
|
+
* endpoint: 200 with `gateway: "simulation"` is on, the disabled handler's empty 404 (no
|
|
113
|
+
* `x-hue-diagnostic`) is off, and anything else (a network failure, a timeout, a redirect, a
|
|
114
|
+
* refusal carrying a diagnostic, another status or body) is unknown. Probed only after a create
|
|
115
|
+
* was refused; on and off are remembered per origin, unknown is probed again next time.
|
|
116
|
+
*/
|
|
117
|
+
export declare function gatewayState(baseUrl: string, fetchImpl?: (url: string, init?: RequestInit) => Promise<Response>): Promise<GatewayState>;
|
|
118
|
+
/**
|
|
119
|
+
* Creates the world with the World API fields. An older deployment whose simulation gateway is
|
|
120
|
+
* off refuses them on the legacy create (400); when the deployment's health says the gateway is
|
|
121
|
+
* off, the create is repeated once without them and the world is the legacy kind. A 400 from a
|
|
122
|
+
* deployment with the gateway on is a real refusal (a trace context that does not match the
|
|
123
|
+
* execution, for one) and is raised as it is, and so is a 400 whose deployment's health is
|
|
124
|
+
* unknown (unreachable, a timeout): the fields are never dropped on a guess. The stable
|
|
125
|
+
* idempotency key makes a repeat a replay, never a second world.
|
|
126
|
+
*/
|
|
127
|
+
export declare function createWorldForExecution(client: Pick<EnvironmentClient, "createRun" | "baseUrl">, input: Parameters<EnvironmentClient["createRun"]>[0], options?: {
|
|
128
|
+
gatewayState?: (baseUrl: string) => Promise<GatewayState>;
|
|
129
|
+
}): Promise<EnvironmentRun>;
|
|
79
130
|
/** One authoritative environment/provider lifecycle shared by direct simulations and
|
|
80
131
|
* outbound local workers. Credential-bearing connections stay in this call frame and
|
|
81
132
|
* are never returned to either runner's checkpoint state.
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { HueEnvironmentError } from "../environment/client.js";
|
|
2
3
|
import { bindEnvironmentTools } from "../environment/tools.js";
|
|
4
|
+
import { legacyMcpCapability, worldHandoff } from "../environment/world.js";
|
|
3
5
|
import { actualAgentManifestV2, attemptBaselineV2, projectMcpConnectionV2, requestedAttemptProvidersV2, validateAttemptConnectionBundleV2, } from "./attempt.js";
|
|
4
6
|
import { TargetCancelledError, TargetOutcomeUncertainError } from "./runner.js";
|
|
5
7
|
export function requestedAttemptV2(options) {
|
|
@@ -49,11 +51,91 @@ async function seal(client, runId, executionId, status) {
|
|
|
49
51
|
});
|
|
50
52
|
}
|
|
51
53
|
catch (error) {
|
|
54
|
+
// A gateway world answers 409 once it is completing, sealed or expired: each is the
|
|
55
|
+
// outcome the caller wanted or the one it can no longer change.
|
|
52
56
|
const recovered = await client.getRun(runId).catch(() => undefined);
|
|
53
|
-
if (recovered?.status !== status &&
|
|
57
|
+
if (recovered?.status !== status &&
|
|
58
|
+
recovered?.status !== "expired" &&
|
|
59
|
+
!(recovered?.status === "open" && recovered.lifecycle === "completing"))
|
|
54
60
|
throw new TargetOutcomeUncertainError(executionId, { cause: error });
|
|
55
61
|
}
|
|
56
62
|
}
|
|
63
|
+
const deprecations = new Set();
|
|
64
|
+
/** One warning per process per legacy path. */
|
|
65
|
+
function warnDeprecated(code, message) {
|
|
66
|
+
if (deprecations.has(code))
|
|
67
|
+
return;
|
|
68
|
+
deprecations.add(code);
|
|
69
|
+
process.emitWarning(message, { type: "DeprecationWarning", code });
|
|
70
|
+
}
|
|
71
|
+
/** The W3C context of the case span, sent on create so the world span parents on it. The
|
|
72
|
+
* flags are the span's own: an unsampled case span is not exported, and the World API must not
|
|
73
|
+
* be told otherwise. */
|
|
74
|
+
export function caseTraceparent(span) {
|
|
75
|
+
let flags = 1;
|
|
76
|
+
try {
|
|
77
|
+
const context = span.span?.spanContext?.();
|
|
78
|
+
if (context && typeof context.traceFlags === "number")
|
|
79
|
+
flags = context.traceFlags;
|
|
80
|
+
}
|
|
81
|
+
catch {
|
|
82
|
+
// A span that cannot report its context is treated as sampled, as before.
|
|
83
|
+
}
|
|
84
|
+
return `00-${span.traceId}-${span.spanId}-${(flags & 0xff).toString(16).padStart(2, "0")}`;
|
|
85
|
+
}
|
|
86
|
+
const gatewayStates = new Map();
|
|
87
|
+
/**
|
|
88
|
+
* Whether the deployment serves the simulation gateway, from its credential-free health
|
|
89
|
+
* endpoint: 200 with `gateway: "simulation"` is on, the disabled handler's empty 404 (no
|
|
90
|
+
* `x-hue-diagnostic`) is off, and anything else (a network failure, a timeout, a redirect, a
|
|
91
|
+
* refusal carrying a diagnostic, another status or body) is unknown. Probed only after a create
|
|
92
|
+
* was refused; on and off are remembered per origin, unknown is probed again next time.
|
|
93
|
+
*/
|
|
94
|
+
export function gatewayState(baseUrl, fetchImpl = fetch) {
|
|
95
|
+
const origin = new URL(baseUrl).origin;
|
|
96
|
+
const remembered = gatewayStates.get(origin);
|
|
97
|
+
if (remembered)
|
|
98
|
+
return remembered;
|
|
99
|
+
const probe = fetchImpl(`${origin}/api/sim/gmailmcp.googleapis.com/_hue/health`, { redirect: "error", signal: AbortSignal.timeout(5_000) })
|
|
100
|
+
.then(async (response) => {
|
|
101
|
+
if (response.status === 404)
|
|
102
|
+
return response.headers.has("x-hue-diagnostic") ? "unknown" : "off";
|
|
103
|
+
if (!response.ok)
|
|
104
|
+
return "unknown";
|
|
105
|
+
const body = (await response.json());
|
|
106
|
+
return body.gateway === "simulation" ? "on" : "unknown";
|
|
107
|
+
})
|
|
108
|
+
.catch(() => "unknown");
|
|
109
|
+
gatewayStates.set(origin, probe);
|
|
110
|
+
void probe.then((state) => {
|
|
111
|
+
if (state === "unknown" && gatewayStates.get(origin) === probe)
|
|
112
|
+
gatewayStates.delete(origin);
|
|
113
|
+
});
|
|
114
|
+
return probe;
|
|
115
|
+
}
|
|
116
|
+
/**
|
|
117
|
+
* Creates the world with the World API fields. An older deployment whose simulation gateway is
|
|
118
|
+
* off refuses them on the legacy create (400); when the deployment's health says the gateway is
|
|
119
|
+
* off, the create is repeated once without them and the world is the legacy kind. A 400 from a
|
|
120
|
+
* deployment with the gateway on is a real refusal (a trace context that does not match the
|
|
121
|
+
* execution, for one) and is raised as it is, and so is a 400 whose deployment's health is
|
|
122
|
+
* unknown (unreachable, a timeout): the fields are never dropped on a guess. The stable
|
|
123
|
+
* idempotency key makes a repeat a replay, never a second world.
|
|
124
|
+
*/
|
|
125
|
+
export async function createWorldForExecution(client, input, options = {}) {
|
|
126
|
+
try {
|
|
127
|
+
return await client.createRun(input);
|
|
128
|
+
}
|
|
129
|
+
catch (error) {
|
|
130
|
+
const { traceparent, agentRevision, ...legacy } = input;
|
|
131
|
+
if (!(error instanceof HueEnvironmentError) ||
|
|
132
|
+
error.status !== 400 ||
|
|
133
|
+
(traceparent === undefined && agentRevision === undefined) ||
|
|
134
|
+
(await (options.gatewayState ?? gatewayState)(client.baseUrl)) !== "off")
|
|
135
|
+
throw error;
|
|
136
|
+
return client.createRun(legacy);
|
|
137
|
+
}
|
|
138
|
+
}
|
|
57
139
|
/** One authoritative environment/provider lifecycle shared by direct simulations and
|
|
58
140
|
* outbound local workers. Credential-bearing connections stay in this call frame and
|
|
59
141
|
* are never returned to either runner's checkpoint state.
|
|
@@ -63,28 +145,42 @@ export async function runEnvironmentTarget(options) {
|
|
|
63
145
|
const environmentVersionId = context.item.environmentVersionId;
|
|
64
146
|
if (!environmentVersionId)
|
|
65
147
|
throw new Error("The simulation case has no pinned environment version");
|
|
66
|
-
|
|
148
|
+
// One stable idempotency key per case attempt: a replay after a lost acknowledgement gets
|
|
149
|
+
// the same world and the same token.
|
|
150
|
+
const run = await createWorldForExecution(options.environmentClient, {
|
|
67
151
|
idempotencyKey: `execution:${context.executionId}`,
|
|
68
152
|
environmentVersionId,
|
|
69
153
|
executionId: context.executionId,
|
|
70
154
|
maxSteps: options.maxSteps,
|
|
71
155
|
ttlSeconds: options.ttlSeconds,
|
|
156
|
+
traceparent: caseTraceparent(context.span),
|
|
157
|
+
agentRevision: options.agentRevision,
|
|
72
158
|
});
|
|
159
|
+
const world = worldHandoff(run);
|
|
73
160
|
const progress = (event) => options.onProgress?.(event);
|
|
74
161
|
let finalized = false;
|
|
75
162
|
try {
|
|
76
163
|
await progress({ type: "world_created", environmentRunId: run.id });
|
|
77
164
|
if (options.signal?.aborted)
|
|
78
165
|
throw new TargetCancelledError();
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
166
|
+
// A gateway world refuses Hue-native actions (409 simulation_world); its agent reaches
|
|
167
|
+
// the provider mirrors in `world` instead.
|
|
168
|
+
const tools = world
|
|
169
|
+
? {}
|
|
170
|
+
: bindEnvironmentTools({
|
|
171
|
+
hue: options.hue,
|
|
172
|
+
client: options.environmentClient,
|
|
173
|
+
run,
|
|
174
|
+
parentContext: context.span.context,
|
|
175
|
+
});
|
|
176
|
+
const warn = options.deprecationWarnings ?? true;
|
|
177
|
+
if (!world && warn)
|
|
178
|
+
warnDeprecated("HUE_NATIVE_SIMULATION_TOOLS", "Hue-native simulation tools and the hue_sim_ MCP capability are deprecated; worlds created through the simulation gateway hand the agent provider mirrors and a world token (context.world).");
|
|
85
179
|
let connectionBundle;
|
|
86
180
|
let mcp;
|
|
87
|
-
if (options.requested) {
|
|
181
|
+
if (options.requested && !world) {
|
|
182
|
+
if (warn)
|
|
183
|
+
warnDeprecated("HUE_PROVIDER_FACADE", "The provider facade (prepareAttempt) is deprecated; worlds created through the simulation gateway hand the agent provider mirrors and a world token (context.world).");
|
|
88
184
|
const actualManifest = actualAgentManifestV2.parse(typeof options.requested.actualAgentManifest === "function"
|
|
89
185
|
? await options.requested.actualAgentManifest({
|
|
90
186
|
config: structuredClone(context.config),
|
|
@@ -134,6 +230,13 @@ export async function runEnvironmentTarget(options) {
|
|
|
134
230
|
throw new TypeError("The prepared attempt has no selected MCP surface");
|
|
135
231
|
mcp = projected;
|
|
136
232
|
}
|
|
233
|
+
else if (world) {
|
|
234
|
+
// A gateway world is served by the provider mirrors; the facade's attempt preflight
|
|
235
|
+
// belongs to the legacy transport and is not run for it.
|
|
236
|
+
if (options.requested && warn)
|
|
237
|
+
warnDeprecated("HUE_PROVIDER_FACADE_IGNORED", "Provider-profile options (requestedProviders, mcpSurface, actualAgentManifest) are ignored for a world the simulation gateway serves; the agent receives the provider mirrors in context.world.");
|
|
238
|
+
mcp = legacyMcpCapability(world);
|
|
239
|
+
}
|
|
137
240
|
else {
|
|
138
241
|
mcp = await options.client.createSimulationMcpCapability({
|
|
139
242
|
runId: run.id,
|
|
@@ -150,7 +253,8 @@ export async function runEnvironmentTarget(options) {
|
|
|
150
253
|
environmentRunId: run.id,
|
|
151
254
|
trace: { traceId: context.span.traceId, spanId: context.span.spanId },
|
|
152
255
|
tools,
|
|
153
|
-
|
|
256
|
+
...(world ? { world } : {}),
|
|
257
|
+
...(mcp ? { mcp } : {}),
|
|
154
258
|
...(connectionBundle ? { connectionBundle } : {}),
|
|
155
259
|
signal: options.signal,
|
|
156
260
|
});
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { HueClient } from "../client.js";
|
|
2
2
|
import type { EnvironmentClient } from "../environment/client.js";
|
|
3
3
|
import type { EnvironmentTool } from "../environment/tools.js";
|
|
4
|
+
import type { WorldHandoff } from "../environment/types.js";
|
|
4
5
|
import type { ActualAgentManifestInputV2, AttemptConnectionBundleV2, RequestedAttemptProviderV2 } from "./attempt.js";
|
|
5
6
|
import type { EvaluationClient } from "./client.js";
|
|
6
7
|
import { type RunExperimentTargetContext, type RunnerReport } from "./runner.js";
|
|
@@ -30,13 +31,17 @@ export interface LocalAgentTargetContext {
|
|
|
30
31
|
/** Root execution span identifier. */
|
|
31
32
|
spanId: string;
|
|
32
33
|
};
|
|
33
|
-
/**
|
|
34
|
-
|
|
35
|
-
|
|
34
|
+
/** The mirror URLs, world token, environment carriers and MCP configuration of a gateway
|
|
35
|
+
* world; absent for a world created while the gateway was off. */
|
|
36
|
+
world?: WorldHandoff;
|
|
37
|
+
/** One MCP endpoint and bearer: the gateway world's first MCP mirror with the world token,
|
|
38
|
+
* or the deprecated execution-scoped `hue_sim_` capability of a legacy world. */
|
|
39
|
+
mcp?: {
|
|
40
|
+
/** MCP endpoint. */
|
|
36
41
|
url: string;
|
|
37
|
-
/**
|
|
42
|
+
/** Bearer for that endpoint, never the project service key. */
|
|
38
43
|
token: string;
|
|
39
|
-
/**
|
|
44
|
+
/** Expiry as an ISO timestamp. */
|
|
40
45
|
expiresAt: string;
|
|
41
46
|
};
|
|
42
47
|
/** Credential-bearing provider connections for this callback only. Hue never
|
|
@@ -89,6 +94,8 @@ export interface RunLocalAgentOptions {
|
|
|
89
94
|
signal?: AbortSignal;
|
|
90
95
|
/** Useful for one-shot jobs and deterministic acceptance. Omit to keep polling. */
|
|
91
96
|
maxRuns?: number;
|
|
97
|
+
/** Emit a one-time `DeprecationWarning` when the deployment serves a legacy world; on by default. */
|
|
98
|
+
deprecationWarnings?: boolean;
|
|
92
99
|
/** Opt into the experiment's immutable V2 provider profile. These three values are
|
|
93
100
|
* validated together before the worker polls; no endpoint or credential is supplied here. */
|
|
94
101
|
actualAgentManifest?: ActualAgentManifestInputV2 | ((context: {
|
|
@@ -31,11 +31,16 @@ function localAgentTargetContext(context) {
|
|
|
31
31
|
executionId: context.executionId,
|
|
32
32
|
environmentRunId: context.environmentRunId,
|
|
33
33
|
trace: { traceId: context.trace.traceId, spanId: context.trace.spanId },
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
34
|
+
...(context.world ? { world: structuredClone(context.world) } : {}),
|
|
35
|
+
...(context.mcp
|
|
36
|
+
? {
|
|
37
|
+
mcp: {
|
|
38
|
+
url: context.mcp.url,
|
|
39
|
+
token: context.mcp.token,
|
|
40
|
+
expiresAt: context.mcp.expiresAt,
|
|
41
|
+
},
|
|
42
|
+
}
|
|
43
|
+
: {}),
|
|
39
44
|
...(context.connectionBundle
|
|
40
45
|
? { connectionBundle: structuredClone(context.connectionBundle) }
|
|
41
46
|
: {}),
|
|
@@ -178,6 +183,9 @@ export async function runLocalAgent(options) {
|
|
|
178
183
|
inputs,
|
|
179
184
|
context,
|
|
180
185
|
requested,
|
|
186
|
+
// The registered revision is the agent revision under test.
|
|
187
|
+
agentRevision: options.agent.revision,
|
|
188
|
+
deprecationWarnings: options.deprecationWarnings,
|
|
181
189
|
signal: options.signal,
|
|
182
190
|
target: (targetInputs, targetContext) => target(structuredClone(targetInputs), targetContext.tools, localAgentTargetContext(targetContext)),
|
|
183
191
|
});
|