@hue-run/sdk 0.9.0 → 0.11.0

package/dist/client.js

@@ -8,12 +8,13 @@ import { LoggerProvider } from "@opentelemetry/sdk-logs";
 import { TracerProvider } from "@opentelemetry/sdk-trace";
 import { defaultResource, resourceFromAttributes } from "@opentelemetry/resources";
 import { HUE_SCOPE } from "./config.js";
-import { hostedServerAddresses, hostedToolActivity, hostedToolProvider } from "./provider-tools.js";
+import { hostedServerAddresses, hostedToolActivity, hostedToolProvider, providerErrorDescription, } from "./provider-tools.js";
+import { toolCatalogSummary } from "./tool-definitions.js";
 import { encodeContent, noopSpan, safeSpan } from "./safety.js";
 import { createHueTransport, HueExportError } from "./transport.js";
 import { verifyTrace } from "./receipt.js";
 import { sdkVersion } from "./version.js";
-import { MAX_FILE_DATA_BYTES } from "./config.js";
+import { MAX_BODY_BYTES, MAX_FILE_DATA_BYTES } from "./config.js";
 /**
  * Thrown by {@link HueClient.checkConnection} and {@link HueClient.verifyTrace} when Hue cannot be
  * reached, rejects the project key or answers unexpectedly. The message is fixed and safe to log;
@@ -149,6 +150,9 @@ class ContextualTracer {
     }
 }
 const propagator = new W3CTraceContextPropagator();
+/** A provider tool listing's bounds before its catalog summary: one export request, 64 levels and
+ * 65,536 values, as the Python SDK's content snapshot. */
+const catalogLimits = { bytes: MAX_BODY_BYTES, nodes: 65_536, depth: 64 };
 /**
  * Hue tracing client. Helpers create spans through a private tracer and local async context, never
  * through global OpenTelemetry registration, and they fail open: capture problems are counted in
@@ -517,11 +521,14 @@ export class HueClient {
      * `hue.tool()` call saw: OpenAI Responses `mcp_call`, `web_search_call`, `file_search_call` and
      * `code_interpreter_call` items, and Anthropic Messages `mcp_tool_use` / `server_tool_use`
      * blocks with their result blocks. Each becomes an `execute_tool {name}` child span of the active
-     * (or given) context with `gen_ai.tool.type` `extension` and `gen_ai.tool.call.id`; MCP calls add
+     * (or given) context with `gen_ai.tool.type` `extension`, `gen_ai.tool.call.id` and
+     * `hue.tool.call.position` (the item's 0-based position in the response); MCP calls add
      * `mcp.server.name` (the provider's label, or the `servers` entry for it). Arguments and results
-     * follow `captureContent`; a failed call carries `error.type` and ERROR status. An OpenAI
-     * `mcp_list_tools` item becomes a `tools/list` child span carrying that server's tools as
-     * `gen_ai.tool.definitions`. Call it inside `hue.model()` so the spans nest under the model call
+     * follow `captureContent`; a failed call carries `error.type` and ERROR status, and with content
+     * capture a failed OpenAI MCP call's status description is the provider's scrubbed, bounded error
+     * text. An OpenAI `mcp_list_tools` item becomes a `tools/list` child span carrying that server's
+     * tools as `gen_ai.tool.definitions`, or without content capture only `hue.tool.names` and
+     * `hue.tool.definitions.sha256`. Call it inside `hue.model()` so the spans nest under the model call
      * and `provider` defaults to its provider; pass `request` to record each server's host as
      * `server.address`. The spans have no duration of their own: the provider ran the tools inside
      * the model request. Unreadable items are skipped and counted; nothing is thrown.
@@ -564,9 +571,12 @@ export class HueClient {
                     attributes["server.address"] = address;
                 return attributes;
             };
-            const fail = (span, type) => {
+            const fail = (span, type, description) => {
                 span.setAttribute("error.type", type);
-                span.setStatus({ code: SpanStatusCode.ERROR });
+                span.setStatus({
+                    code: SpanStatusCode.ERROR,
+                    ...(description === undefined ? {} : { message: description }),
+                });
             };
             for (const call of activity.calls) {
                 const span = this.tracer.startSpan(`execute_tool ${call.name}`, {
@@ -576,6 +586,7 @@ export class HueClient {
                         "gen_ai.tool.name": call.name,
                         "gen_ai.tool.type": "extension",
                         ...(call.callId === undefined ? {} : { "gen_ai.tool.call.id": call.callId }),
+                        "hue.tool.call.position": call.position,
                         ...server(call.server),
                     },
                 }, parent);
@@ -584,7 +595,11 @@ export class HueClient {
                 if (call.result !== undefined)
                     this.setContent(span, "gen_ai.tool.call.result", call.result);
                 if (call.errorType !== undefined)
-                    fail(span, call.errorType);
+                    fail(span, call.errorType, 
+                    // The provider's error text is content: exported only when content is captured.
+                    this.captureContent && call.errorText !== undefined
+                        ? providerErrorDescription(call.errorText)
+                        : undefined);
                 span.end();
             }
             for (const listing of activity.listings) {
@@ -592,7 +607,18 @@ export class HueClient {
                     kind: SpanKind.INTERNAL,
                     attributes: { "mcp.method.name": "tools/list", ...server(listing.server) },
                 }, parent);
-                this.setContent(span, "gen_ai.tool.definitions", listing.definitions);
+                if (this.captureContent)
+                    this.setContent(span, "gen_ai.tool.definitions", listing.definitions);
+                // Names and the catalog digest are metadata: without content, the listing carries the
+                // summary metadata-only export gives any record's tool definitions, within the summary's
+                // bounds rather than one content field's.
+                else
+                    try {
+                        span.setAttributes(toolCatalogSummary(encodeContent(listing.definitions, catalogLimits)));
+                    }
+                    catch {
+                        this.transport.instrumentationFailure();
+                    }
                 if (listing.errorType !== undefined)
                     fail(span, listing.errorType);
                 span.end();

package/dist/evals/checkpoint.d.ts

@@ -1,6 +1,20 @@
-/** One owner per directory. A crash leaves .lock for explicit operator recovery. */
+/** The checkpoint belongs to a run started with a different identity. When only its content
+ * policy differs, `startedWith` holds the policy the unfinished run was started with. */
+export declare class CheckpointIdentityError extends Error {
+    readonly startedWith?: {
+        persistResultContent?: unknown;
+        captureContent?: unknown;
+    } | undefined;
+    constructor(startedWith?: {
+        persistResultContent?: unknown;
+        captureContent?: unknown;
+    } | undefined);
+}
+/** One owner per directory. A crash leaves .lock for explicit operator recovery; a forced exit of
+ * `hue eval`, which stops its agents first, releases it. */
 export declare class CheckpointStore {
     readonly directory: string;
+    private untrack;
     private constructor();
     static acquire(directory: string, identity: unknown): Promise<CheckpointStore>;
     read<T>(key: string): Promise<T | undefined>;

package/dist/evals/checkpoint.js

@@ -1,11 +1,40 @@
-import { constants } from "node:fs";
+import { constants, rmSync } from "node:fs";
 import { lstat, mkdir, open, rename, rm } from "node:fs/promises";
 import { join, resolve } from "node:path";
 import { randomUUID } from "node:crypto";
+import { onForcedExit } from "./exit-cleanup.js";
 import { digest } from "./json.js";
-/** One owner per directory. A crash leaves .lock for explicit operator recovery. */
+const CONTENT_POLICY = ["persistResultContent", "captureContent"];
+/** The checkpoint belongs to a run started with a different identity. When only its content
+ * policy differs, `startedWith` holds the policy the unfinished run was started with. */
+export class CheckpointIdentityError extends Error {
+    startedWith;
+    constructor(startedWith) {
+        super(startedWith
+            ? "Checkpoint content policy differs from the one this unfinished run was started with"
+            : "Checkpoint identity differs from this project, run, pins or content policy");
+        this.startedWith = startedWith;
+        this.name = "CheckpointIdentityError";
+    }
+}
+/** The prior policy when two identities differ only in their content policy. */
+function contentPolicyOnly(prior, identity) {
+    if (!prior || !identity || typeof prior !== "object" || typeof identity !== "object")
+        return undefined;
+    const before = prior;
+    const after = identity;
+    const keys = new Set([...Object.keys(before), ...Object.keys(after)]);
+    for (const key of keys)
+        if (!CONTENT_POLICY.includes(key) &&
+            JSON.stringify(before[key]) !== JSON.stringify(after[key]))
+            return undefined;
+    return Object.fromEntries(CONTENT_POLICY.map((key) => [key, before[key]]));
+}
+/** One owner per directory. A crash leaves .lock for explicit operator recovery; a forced exit of
+ * `hue eval`, which stops its agents first, releases it. */
 export class CheckpointStore {
     directory;
+    untrack = () => { };
     constructor(directory) {
         this.directory = directory;
     }
@@ -18,6 +47,7 @@ export class CheckpointStore {
         const store = new CheckpointStore(root);
         try {
             await mkdir(join(root, ".lock"), { mode: 0o700 });
+            store.untrack = onForcedExit(() => rmSync(join(root, ".lock"), { recursive: true, force: true }));
         }
         catch {
             throw new Error("Checkpoint directory is locked; confirm its owner stopped before explicitly removing .lock");
@@ -27,7 +57,7 @@ export class CheckpointStore {
             const expected = { format: 1, identity, digest: digest(identity) };
             const prior = await store.read("manifest");
             if (prior && (prior.format !== 1 || prior.digest !== expected.digest))
-                throw new Error("Checkpoint identity differs from this project, run, pins or content policy");
+                throw new CheckpointIdentityError(prior.format === 1 ? contentPolicyOnly(prior.identity, identity) : undefined);
             if (!prior)
                 await store.write("manifest", expected);
             return store;
@@ -85,5 +115,6 @@ export class CheckpointStore {
     }
     async release() {
         await rm(join(this.directory, ".lock"), { recursive: true });
+        this.untrack();
     }
 }

package/dist/evals/client.d.ts

@@ -14,14 +14,28 @@ export interface EvaluationClientOptions {
 export declare class HueApiError extends Error {
     /** HTTP status when Hue answered; absent for network, timeout and parsing failures. */
     readonly status?: number | undefined;
+    /** Seconds Hue asked the caller to wait (`Retry-After` on a 429 or 503), when it said. */
+    readonly retryAfterSeconds?: number | undefined;
     constructor(
     /** HTTP status when Hue answered; absent for network, timeout and parsing failures. */
-    status?: number | undefined);
+    status?: number | undefined, 
+    /** Seconds Hue asked the caller to wait (`Retry-After` on a 429 or 503), when it said. */
+    retryAfterSeconds?: number | undefined);
+}
+/** An artifact download that ran past the size its caller expected; the rest is not read. */
+export declare class ArtifactSizeError extends Error {
+    /** The size the caller expected, in bytes. */
+    readonly maxBytes: number;
+    constructor(
+    /** The size the caller expected, in bytes. */
+    maxBytes: number);
 }
 /**
  * Typed client for Hue's evaluation REST API: datasets, scorers, experiments, executions, runs,
  * results and hosted judge jobs. No implicit mutation retry: callers retain stable idempotency keys
- * for experiments and results. Responses are bounded to 4 MiB.
+ * for experiments and results. The one exception is a request Hue refused before acting on it with
+ * a short `Retry-After` (HTTP 429 or 503, at most 5 seconds): it is sent again after that wait, up
+ * to four times. Responses are bounded to 4 MiB.
  */
 export declare class EvaluationClient {
     /** Validated Hue origin. */
@@ -29,9 +43,15 @@ export declare class EvaluationClient {
     private readonly apiKey;
     private readonly timeoutMillis;
     constructor(options: EvaluationClientOptions);
+    /** Sends one request to Hue, again after the wait while Hue refuses it with a short
+     * `Retry-After`. `init` runs for every attempt, so each has its own timeout. */
+    private send;
     private request;
-    /** Verified bytes of one ready artifact in this project, bounded to the 25 MiB pilot file size. */
-    downloadArtifact(id: string): Promise<Uint8Array>;
+    /** Verified bytes of one ready artifact in this project, bounded to the 25 MiB pilot file size.
+     * With `maxBytes`, the download stops one byte past it and throws `ArtifactSizeError`. */
+    downloadArtifact(id: string, options?: {
+        maxBytes?: number;
+    }): Promise<Uint8Array>;
     /** Stage bytes at the storage capability Hue issued. The Hue key is never sent to storage. */
     uploadArtifactBytes(upload: ArtifactUpload, bytes: Uint8Array, contentType: string): Promise<void>;
     private page;

package/dist/evals/client.js

@@ -4,14 +4,29 @@ import { attemptBindingRead, parsePrepareAttemptResultV2, parseRefreshedAttemptR
 /** Thrown for a failed evaluation API request; the message is fixed and never includes response text. */
 export class HueApiError extends Error {
     status;
+    retryAfterSeconds;
     constructor(
     /** HTTP status when Hue answered; absent for network, timeout and parsing failures. */
-    status) {
+    status, 
+    /** Seconds Hue asked the caller to wait (`Retry-After` on a 429 or 503), when it said. */
+    retryAfterSeconds) {
         super(status ? `Hue API request failed (HTTP ${status})` : "Hue API connection or response failed");
         this.status = status;
+        this.retryAfterSeconds = retryAfterSeconds;
         this.name = "HueApiError";
     }
 }
+/** An artifact download that ran past the size its caller expected; the rest is not read. */
+export class ArtifactSizeError extends Error {
+    maxBytes;
+    constructor(
+    /** The size the caller expected, in bytes. */
+    maxBytes) {
+        super(`Artifact download exceeded the expected ${maxBytes} bytes`);
+        this.maxBytes = maxBytes;
+        this.name = "ArtifactSizeError";
+    }
+}
 const registryFieldAliases = [
     ["datasetId", "evalSetId"],
     ["datasetVersionId", "evalSetVersionId"],
@@ -91,10 +106,32 @@ function productRunFields(value, kind) {
         result.scoring = result.evaluation;
     return result;
 }
+/** Times a refused request is sent again before its refusal is the caller's error. */
+const REFUSAL_RETRIES = 4;
+/** A refusal asking for a longer wait fails at once rather than holding the caller. */
+const MAX_REFUSAL_RETRY_AFTER_SECONDS = 5;
+/** The whole seconds a 429 or 503 asks the caller to wait, up to a day, when it says. */
+function askedRetryAfter(response) {
+    if (response.status !== 429 && response.status !== 503)
+        return undefined;
+    const header = response.headers.get("retry-after")?.trim() ?? "";
+    return /^\d{1,6}$/.test(header) ? Math.min(Number(header), 86_400) : undefined;
+}
+/**
+ * The whole-second `Retry-After` of a 429 or 503 asking for at most 5 seconds. Hue sends one only
+ * when it refused the request before acting on it, such as a busy key check, so sending any method
+ * again is safe. A date, a longer wait or any other failure, a timeout included, is not retried.
+ */
+function refusalRetryAfter(response) {
+    const seconds = askedRetryAfter(response);
+    return seconds !== undefined && seconds <= MAX_REFUSAL_RETRY_AFTER_SECONDS ? seconds : undefined;
+}
 /**
  * Typed client for Hue's evaluation REST API: datasets, scorers, experiments, executions, runs,
  * results and hosted judge jobs. No implicit mutation retry: callers retain stable idempotency keys
- * for experiments and results. Responses are bounded to 4 MiB.
+ * for experiments and results. The one exception is a request Hue refused before acting on it with
+ * a short `Retry-After` (HTTP 429 or 503, at most 5 seconds): it is sent again after that wait, up
+ * to four times. Responses are bounded to 4 MiB.
  */
 export class EvaluationClient {
     /** Validated Hue origin. */
@@ -111,30 +148,43 @@ export class EvaluationClient {
         this.apiKey = validated.apiKey;
         this.timeoutMillis = validated.timeoutMillis;
     }
+    /** Sends one request to Hue, again after the wait while Hue refuses it with a short
+     * `Retry-After`. `init` runs for every attempt, so each has its own timeout. */
+    async send(url, init) {
+        for (let attempt = 0;; attempt++) {
+            let response;
+            try {
+                response = await fetch(url, init());
+            }
+            catch {
+                throw new HueApiError();
+            }
+            const seconds = attempt < REFUSAL_RETRIES ? refusalRetryAfter(response) : undefined;
+            if (seconds === undefined)
+                return response;
+            await response.body?.cancel().catch(() => undefined);
+            // Never sooner than asked; the jitter spreads out parallel requests refused together.
+            await new Promise((resolve) => setTimeout(resolve, seconds * 1000 * (1 + Math.random() * 0.5)));
+        }
+    }
     async request(method, path, body, bounds = { ...valueBounds, bytes: 1024 * 1024 }) {
         // Optional top-level fields are omitted intentionally; nested undefined remains invalid.
         const payload = body === undefined
             ? undefined
             : JSON.stringify(json(Object.fromEntries(Object.entries(body).filter(([, value]) => value !== undefined)), bounds));
-        let response;
-        try {
-            response = await fetch(`${this.baseUrl}/api/v1${path}`, {
-                method,
-                headers: {
-                    Authorization: `Bearer ${this.apiKey}`,
-                    ...(payload ? { "Content-Type": "application/json" } : {}),
-                },
-                body: payload,
-                redirect: "error",
-                signal: AbortSignal.timeout(this.timeoutMillis),
-            });
-        }
-        catch {
-            throw new HueApiError();
-        }
+        const response = await this.send(`${this.baseUrl}/api/v1${path}`, () => ({
+            method,
+            headers: {
+                Authorization: `Bearer ${this.apiKey}`,
+                ...(payload ? { "Content-Type": "application/json" } : {}),
+            },
+            body: payload,
+            redirect: "error",
+            signal: AbortSignal.timeout(this.timeoutMillis),
+        }));
         if (!response.ok) {
             await response.body?.cancel();
-            throw new HueApiError(response.status);
+            throw new HueApiError(response.status, askedRetryAfter(response));
         }
         try {
             const reader = response.body?.getReader();
@@ -162,23 +212,21 @@ export class EvaluationClient {
             throw new HueApiError();
         }
     }
-    /** Verified bytes of one ready artifact in this project, bounded to the 25 MiB pilot file size. */
-    async downloadArtifact(id) {
-        let response;
-        try {
-            response = await fetch(`${this.baseUrl}/api/v1/artifacts/${uuid(id)}/download`, {
-                method: "GET",
-                headers: { Authorization: `Bearer ${this.apiKey}` },
-                redirect: "error",
-                signal: AbortSignal.timeout(Math.max(this.timeoutMillis, 120_000)),
-            });
-        }
-        catch {
-            throw new HueApiError();
-        }
+    /** Verified bytes of one ready artifact in this project, bounded to the 25 MiB pilot file size.
+     * With `maxBytes`, the download stops one byte past it and throws `ArtifactSizeError`. */
+    async downloadArtifact(id, options = {}) {
+        const expected = options.maxBytes;
+        if (expected !== undefined && (!Number.isSafeInteger(expected) || expected < 0))
+            throw new RangeError("maxBytes must be a non-negative integer");
+        const response = await this.send(`${this.baseUrl}/api/v1/artifacts/${uuid(id)}/download`, () => ({
+            method: "GET",
+            headers: { Authorization: `Bearer ${this.apiKey}` },
+            redirect: "error",
+            signal: AbortSignal.timeout(Math.max(this.timeoutMillis, 120_000)),
+        }));
         if (!response.ok) {
             await response.body?.cancel();
-            throw new HueApiError(response.status);
+            throw new HueApiError(response.status, askedRetryAfter(response));
         }
         try {
             const reader = response.body?.getReader();
@@ -192,6 +240,8 @@ export class EvaluationClient {
                     if (done)
                         break;
                     size += value.byteLength;
+                    if (expected !== undefined && size > expected)
+                        throw new ArtifactSizeError(expected);
                     if (size > 25 * 1024 * 1024)
                         throw new Error("Oversized artifact");
                     chunks.push(value);
@@ -202,7 +252,9 @@ export class EvaluationClient {
             }
             return new Uint8Array(Buffer.concat(chunks));
         }
-        catch {
+        catch (error) {
+            if (error instanceof ArtifactSizeError)
+                throw error;
             throw new HueApiError();
         }
     }
@@ -227,7 +279,7 @@ export class EvaluationClient {
         }
         await response.body?.cancel().catch(() => undefined);
         if (!response.ok)
-            throw new HueApiError(response.status);
+            throw new HueApiError(response.status, askedRetryAfter(response));
     }
     page(options = {}) {
         const query = new URLSearchParams();
@@ -592,8 +644,10 @@ export class EvaluationClient {
         return this.request("GET", "/judge-budget");
     }
     /** Register or refresh the fixed local agent key and revision. */
-    registerLocalAgent(input) {
-        return this.request("POST", "/local-agent-worker/register", input);
+    async registerLocalAgent(input) {
+        const agent = await this.request("POST", "/local-agent-worker/register", input);
+        // Hue's response names the key `agentKey`; read either name.
+        return { ...agent, key: agent.key ?? agent.agentKey ?? input.key };
     }
     /** Claim a queued run for this agent and durable worker identity. */
     claimLocalAgentRun(input) {

package/dist/evals/environment-target.d.ts

@@ -5,7 +5,7 @@ import type { EnvironmentRun, WorldHandoff } from "../environment/types.js";
 import type { HueSpan } from "../types.js";
 import { type ActualAgentManifestInputV2, type AttemptBaselineV2, type AttemptConnectionBundleV2, type RequestedAttemptProviderV2, type SurfaceBindingV2 } from "./attempt.js";
 import type { EvaluationClient } from "./client.js";
-import type { ExperimentCase, JsonValue, SimulationMcpCapability } from "./types.js";
+import type { ExperimentCase, JsonValue, LocalFile, SimulationMcpCapability, TargetResult } from "./types.js";
 export type McpSurfaceKeyV2 = Extract<SurfaceBindingV2["surfaceKey"], `${string}/mcp`>;
 export type ActualAgentManifestResolverV2 = ActualAgentManifestInputV2 | ((context: {
     config: JsonValue;
@@ -64,6 +64,11 @@ export interface EnvironmentTargetContext {
      * gateway world has no MCP surface. */
     mcp?: SimulationMcpCapability;
     connectionBundle?: AttemptConnectionBundleV2;
+    /** Verified copies of the case's agent-visible input files; evaluator-only files are never
+     * among them. Empty when the case has none. */
+    files: LocalFile[];
+    /** Private directory for this case; return generated files with `withFiles`. */
+    outputDirectory: string;
     signal?: AbortSignal;
 }
 interface RunnerTargetContext {
@@ -71,6 +76,8 @@ interface RunnerTargetContext {
     item: ExperimentCase;
     executionId: string;
     span: HueSpan;
+    files: LocalFile[];
+    outputDirectory: string;
 }
 export interface RunEnvironmentTargetOptions {
     client: EvaluationClient;
@@ -88,14 +95,22 @@ export interface RunEnvironmentTargetOptions {
      * the deployment serves, such as `hue eval`, turns it off. */
     deprecationWarnings?: boolean;
     signal?: AbortSignal;
+    /** Test-only: shortens the seal wait. */
+    sealTiming?: Partial<SealTiming>;
     onProgress?(event: EnvironmentTargetProgress): void | Promise<void>;
-    target(inputs: JsonValue, context: EnvironmentTargetContext): JsonValue | undefined | Promise<JsonValue | undefined>;
+    target(inputs: JsonValue, context: EnvironmentTargetContext): JsonValue | TargetResult | undefined | Promise<JsonValue | TargetResult | undefined>;
 }
 /** The completion grace is five seconds; cap an unexpectedly distant timestamp and let reads
  * force the seal after the grace. */
 export declare const MAX_GRACE_WAIT_MS = 10000;
 export declare const SEAL_POLL_MS = 250;
 export declare const SEAL_WAIT_MS = 30000;
+/** The seal wait's bounds. Only tests shorten them; the helpers always use the defaults. */
+export interface SealTiming {
+    maxGraceWaitMs: number;
+    pollMs: number;
+    waitMs: number;
+}
 /** The W3C context of the case span, sent on create so the world span parents on it. The
  * flags are the span's own: an unsampled case span is not exported, and the World API must not
  * be told otherwise. */
@@ -136,5 +151,5 @@ export declare function createWorldForExecution(client: Pick<EnvironmentClient,
  * outbound local workers. Credential-bearing connections stay in this call frame and
  * are never returned to either runner's checkpoint state.
  */
-export declare function runEnvironmentTarget(options: RunEnvironmentTargetOptions): Promise<JsonValue | undefined>;
+export declare function runEnvironmentTarget(options: RunEnvironmentTargetOptions): Promise<JsonValue | TargetResult | undefined>;
 export {};

package/dist/evals/environment-target.js

@@ -47,8 +47,13 @@ export function pinRequestedAttemptV2(requested, config) {
 export const MAX_GRACE_WAIT_MS = 10_000;
 export const SEAL_POLL_MS = 250;
 export const SEAL_WAIT_MS = 30_000;
+const SEAL_TIMING = {
+    maxGraceWaitMs: MAX_GRACE_WAIT_MS,
+    pollMs: SEAL_POLL_MS,
+    waitMs: SEAL_WAIT_MS,
+};
 /** Finish, then wait for the authoritative run to leave open. */
-async function seal(client, runId, executionId, status) {
+async function seal(client, runId, executionId, status, timing) {
     let completingUntil;
     try {
         const finished = await client.finishRun(runId, {
@@ -70,7 +75,7 @@ async function seal(client, runId, executionId, status) {
     }
     if (completingUntil !== undefined) {
         try {
-            await awaitSeal(client, runId, completingUntil);
+            await awaitSeal(client, runId, completingUntil, timing);
         }
         catch (error) {
             throw new TargetOutcomeUncertainError(executionId, { cause: error });
@@ -78,19 +83,21 @@ async function seal(client, runId, executionId, status) {
     }
 }
 /** Wait out a completing world's grace, then read until the server seals it. */
-async function awaitSeal(client, runId, completingUntil) {
+async function awaitSeal(client, runId, completingUntil, timing) {
     const graceEnd = Date.parse(completingUntil ?? "");
     let wait = Number.isFinite(graceEnd)
-        ? Math.min(Math.max(0, graceEnd - Date.now()), MAX_GRACE_WAIT_MS)
+        ? Math.min(Math.max(0, graceEnd - Date.now()), timing.maxGraceWaitMs)
         : 0;
-    const deadline = performance.now() + wait + SEAL_WAIT_MS;
+    const deadline = performance.now() + wait + timing.waitMs;
     for (;;) {
         await new Promise((resolve) => setTimeout(resolve, wait));
         const remaining = deadline - performance.now();
         if (remaining <= 0)
             throw new Error(`World ${runId} was not sealed after its completion grace`);
+        // The deadline ends the read and the client's retries inside it.
         const controller = new AbortController();
         const timer = setTimeout(() => controller.abort(), remaining);
+        wait = timing.pollMs;
         try {
             if ((await client.getRun(runId, { signal: controller.signal })).status !== "open")
                 return;
@@ -102,13 +109,15 @@ async function awaitSeal(client, runId, completingUntil) {
                 throw new Error(`World ${runId} was not sealed after its completion grace`, {
                     cause: error,
                 });
+            // A 429 or 503 that outlasted the client's retries still says how long to wait.
+            wait = Math.max(wait, error.retryAfterMs ?? 0);
         }
         finally {
             clearTimeout(timer);
         }
         if (performance.now() >= deadline)
             throw new Error(`World ${runId} was not sealed after its completion grace`);
-        wait = SEAL_POLL_MS;
+        wait = Math.min(wait, deadline - performance.now());
     }
 }
 const deprecations = new Set();
@@ -208,6 +217,7 @@ export async function runEnvironmentTarget(options) {
         agentRevision: options.agentRevision,
     });
     const world = worldHandoff(run);
+    const timing = { ...SEAL_TIMING, ...options.sealTiming };
     const progress = (event) => options.onProgress?.(event);
     let finalized = false;
     try {
@@ -268,7 +278,7 @@ export async function runEnvironmentTarget(options) {
                     : {}),
             });
             if (prepared.status === "environment_incomplete") {
-                await seal(options.environmentClient, run.id, context.executionId, "completed");
+                await seal(options.environmentClient, run.id, context.executionId, "completed", timing);
                 finalized = true;
                 await Promise.resolve(progress({ type: "world_sealed", environmentRunId: run.id })).catch(() => undefined);
                 return undefined;
@@ -307,9 +317,11 @@ export async function runEnvironmentTarget(options) {
             ...(world ? { world } : {}),
             ...(mcp ? { mcp } : {}),
             ...(connectionBundle ? { connectionBundle } : {}),
+            files: structuredClone(context.files),
+            outputDirectory: context.outputDirectory,
             signal: options.signal,
         });
-        await seal(options.environmentClient, run.id, context.executionId, "completed");
+        await seal(options.environmentClient, run.id, context.executionId, "completed", timing);
         finalized = true;
         await Promise.resolve(progress({ type: "world_sealed", environmentRunId: run.id })).catch(() => undefined);
         return output;
@@ -329,7 +341,7 @@ export async function runEnvironmentTarget(options) {
         }
         if (environmentIncomplete) {
             try {
-                await seal(options.environmentClient, run.id, context.executionId, "completed");
+                await seal(options.environmentClient, run.id, context.executionId, "completed", timing);
             }
             catch (finalizationError) {
                 throw new TargetOutcomeUncertainError(context.executionId, {
@@ -341,7 +353,7 @@ export async function runEnvironmentTarget(options) {
             return undefined;
         }
         try {
-            await seal(options.environmentClient, run.id, context.executionId, "abandoned");
+            await seal(options.environmentClient, run.id, context.executionId, "abandoned", timing);
         }
         catch (finalizationError) {
             throw new TargetOutcomeUncertainError(context.executionId, {

package/dist/evals/exit-cleanup.d.ts

@@ -0,0 +1,4 @@
+/** Register a synchronous removal; the returned function unregisters it. */
+export declare function onForcedExit(cleanup: () => void): () => void;
+/** Run every registered removal, newest first: files before the lock that guards them. */
+export declare function runForcedExitCleanups(): void;

package/dist/evals/exit-cleanup.js

@@ -0,0 +1,26 @@
+/**
+ * What this process holds on disk while it runs — a world case's files, a checkpoint lock, a
+ * token-bearing MCP configuration — each with a synchronous removal. Normal paths remove them
+ * themselves and unregister; a forced exit (a second Ctrl+C in `hue eval`) cannot wait for them,
+ * so it runs whatever is still registered, newest first, before `process.exit`.
+ */
+const cleanups = new Set();
+/** Register a synchronous removal; the returned function unregisters it. */
+export function onForcedExit(cleanup) {
+    cleanups.add(cleanup);
+    return () => {
+        cleanups.delete(cleanup);
+    };
+}
+/** Run every registered removal, newest first: files before the lock that guards them. */
+export function runForcedExitCleanups() {
+    for (const cleanup of [...cleanups].reverse()) {
+        try {
+            cleanup();
+        }
+        catch {
+            // Best effort on the way out; the rest still run.
+        }
+    }
+    cleanups.clear();
+}