@hue-run/sdk 0.11.4 → 0.12.0

package/dist/evals/environment-target.js

@@ -3,7 +3,7 @@ import { HueEnvironmentError, isTransientEnvironmentError, } from "../environmen
 import { bindEnvironmentTools } from "../environment/tools.js";
 import { legacyMcpCapability, worldHandoff } from "../environment/world.js";
 import { actualAgentManifestV2, attemptBaselineV2, projectMcpConnectionV2, requestedAttemptProvidersV2, validateAttemptConnectionBundleV2, } from "./attempt.js";
-import { TargetCancelledError, TargetOutcomeUncertainError } from "./runner.js";
+import { TargetCancelledError, TargetOutcomeUncertainError, TargetTimeoutError } from "./runner.js";
 export function requestedAttemptV2(options) {
     const requested = options.requestedProviders !== undefined;
     const selected = options.mcpSurface !== undefined;
@@ -47,6 +47,9 @@ export function pinRequestedAttemptV2(requested, config) {
 export const MAX_GRACE_WAIT_MS = 10_000;
 export const SEAL_POLL_MS = 250;
 export const SEAL_WAIT_MS = 30_000;
+/** How long past the world's `expiresAt` the target is given before it is stopped: room for a
+ * last call in flight at the deadline and for clock skew between the worker and Hue. */
+export const DEADLINE_MARGIN_MS = 5_000;
 const SEAL_TIMING = {
     maxGraceWaitMs: MAX_GRACE_WAIT_MS,
     pollMs: SEAL_POLL_MS,
@@ -145,21 +148,28 @@ export function caseTraceparent(span) {
 }
 const gatewayStates = new Map();
 /**
- * Whether the deployment serves the simulation gateway, from its credential-free health
- * endpoint: 200 with `gateway: "simulation"` is on, the disabled handler's empty 404 (no
- * `x-hue-diagnostic`) is off, and anything else (a network failure, a timeout, a redirect, a
- * refusal carrying a diagnostic, another status or body) is unknown. Probed only after a create
- * was refused; on and off are remembered per origin, unknown is probed again next time.
+ * Whether the deployment serves the simulation gateway, from a credential-free probe of the
+ * gateway's reserved health path with no app named (`/api/sim/_hue/health`), so the answer does
+ * not depend on which apps the deployment mirrors. The gateway answers it on its own host with
+ * its health, 200 with `gateway: "simulation"`, and on the application host with a refusal that
+ * carries `x-hue-diagnostic` (`unmirrored_provider`: the reserved segment is no app); either is
+ * on, since only the enabled gateway writes that header. The disabled handler's empty 404 (no
+ * `x-hue-diagnostic`) is off, and anything else (a network failure, a timeout, a redirect, another
+ * status or body) is unknown. Probed only after a create was refused; on and off are remembered
+ * per origin, unknown is probed again next time.
  */
 export function gatewayState(baseUrl, fetchImpl = fetch) {
     const origin = new URL(baseUrl).origin;
     const remembered = gatewayStates.get(origin);
     if (remembered)
         return remembered;
-    const probe = fetchImpl(`${origin}/api/sim/gmailmcp.googleapis.com/_hue/health`, { redirect: "error", signal: AbortSignal.timeout(5_000) })
+    const probe = fetchImpl(`${origin}/api/sim/_hue/health`, {
+        redirect: "error",
+        signal: AbortSignal.timeout(5_000),
+    })
         .then(async (response) => {
         if (response.status === 404)
-            return response.headers.has("x-hue-diagnostic") ? "unknown" : "off";
+            return response.headers.has("x-hue-diagnostic") ? "on" : "off";
         if (!response.ok)
             return "unknown";
         const body = (await response.json());
@@ -307,20 +317,50 @@ export async function runEnvironmentTarget(options) {
         if (options.signal?.aborted)
             throw new TargetCancelledError();
         await progress({ type: "target_started", environmentRunId: run.id });
-        const output = await options.target(options.inputs, {
-            config: context.config,
-            item: context.item,
-            executionId: context.executionId,
-            environmentRunId: run.id,
-            trace: { traceId: context.span.traceId, spanId: context.span.spanId },
-            tools,
-            ...(world ? { world } : {}),
-            ...(mcp ? { mcp } : {}),
-            ...(connectionBundle ? { connectionBundle } : {}),
-            files: structuredClone(context.files),
-            outputDirectory: context.outputDirectory,
-            signal: options.signal,
+        // The world's deadline bounds the target: once the world expires its mirrors refuse every
+        // call, so an agent still running can achieve nothing more. The target is told through its
+        // signal a moment after the deadline and, whether or not it listens, its case ends as a
+        // `TargetTimeout` and the world is sealed abandoned; a promise that never settles is left
+        // behind rather than holding the run open for good.
+        const expiresAt = Date.parse(run.expiresAt);
+        const deadline = new AbortController();
+        let timer;
+        const timedOut = new Promise((_, reject) => {
+            if (!Number.isFinite(expiresAt))
+                return;
+            timer = setTimeout(() => {
+                // The timeout is decided before the target is told: a target that rejects with its
+                // own abort error on the signal must not win the race and be saved as its own error.
+                const error = new TargetTimeoutError(run.id, run.expiresAt);
+                reject(error);
+                deadline.abort(error);
+            }, Math.max(0, expiresAt + DEADLINE_MARGIN_MS - Date.now()));
         });
+        let output;
+        try {
+            output = await Promise.race([
+                options.target(options.inputs, {
+                    config: context.config,
+                    item: context.item,
+                    executionId: context.executionId,
+                    environmentRunId: run.id,
+                    trace: { traceId: context.span.traceId, spanId: context.span.spanId },
+                    tools,
+                    ...(world ? { world } : {}),
+                    ...(mcp ? { mcp } : {}),
+                    ...(connectionBundle ? { connectionBundle } : {}),
+                    files: structuredClone(context.files),
+                    outputDirectory: context.outputDirectory,
+                    signal: options.signal
+                        ? AbortSignal.any([options.signal, deadline.signal])
+                        : deadline.signal,
+                }),
+                timedOut,
+            ]);
+        }
+        finally {
+            clearTimeout(timer);
+        }
         await seal(options.environmentClient, run.id, context.executionId, "completed", timing);
         finalized = true;
         await Promise.resolve(progress({ type: "world_sealed", environmentRunId: run.id })).catch(() => undefined);

package/dist/evals/local-worker.d.ts

@@ -1,11 +1,11 @@
 import type { HueClient } from "../client.js";
-import type { EnvironmentClient } from "../environment/client.js";
+import { type EnvironmentClient } from "../environment/client.js";
 import type { EnvironmentTool } from "../environment/tools.js";
 import type { WorldHandoff } from "../environment/types.js";
 import type { ActualAgentManifestInputV2, AttemptConnectionBundleV2, RequestedAttemptProviderV2 } from "./attempt.js";
 import type { EvaluationClient } from "./client.js";
 import { type RunExperimentTargetContext, type RunnerReport, type TelemetryNotAccepted } from "./runner.js";
-import type { ExperimentCase, JsonValue, LocalAgentRegistration, LocalFile, LocalScorer, TargetResult } from "./types.js";
+import type { ExperimentCase, JsonValue, LocalAgentClaim, LocalAgentRegistration, LocalFile, LocalScorer, TargetResult } from "./types.js";
 /** Capability strings a registration declares; Hue offers only matching cases. */
 export declare const localAgentCapabilities: {
     /** Cases pinned to a hosted synthetic world. */
@@ -58,6 +58,9 @@ export interface LocalAgentTargetContext {
     /** Private directory for this case, removed after it; return generated files with
      * `withFiles`. */
     outputDirectory: string;
+    /** Ends when the worker stops or the world's deadline passes; a target that listens stops
+     * its work, and one that does not is left behind when its case ends as a `TargetTimeout`. */
+    signal?: AbortSignal;
 }
 /** Candidate-visible context for one queued ordinary case run directly on this machine. */
 export interface LocalAgentDirectContext {
@@ -109,8 +112,27 @@ export interface RunLocalAgentOptions {
     pollIntervalMillis?: number;
     /** Stops polling cooperatively; does not cancel an active callback. */
     signal?: AbortSignal;
-    /** Useful for one-shot jobs and deterministic acceptance. Omit to keep polling. */
+    /** Stop after this many runs settle: completed, or given up as `attention`. Useful for
+     * one-shot jobs and deterministic acceptance. Omit to keep polling. */
     maxRuns?: number;
+    /** How many times one run is resumed after a transient failure (a request Hue refused
+     * transiently past the client's own retries, a world seal not read back, a lost network,
+     * telemetry not accepted) before the worker gives it up as `attention`. Default 5, range
+     * 1–20. A failure that would recur (an uncertain target outcome, an unserializable output, a
+     * refusal Hue decided on, an input the SDK refuses) is attention at once. */
+    maxRunAttempts?: number;
+    /** The lifetime of each world the worker creates for a case, in seconds (1–86400); the
+     * server's default, an hour, applies when omitted. The target is stopped when the world
+     * expires: its case ends as a `TargetTimeout` error and the world is sealed abandoned. */
+    worldTtlSeconds?: number;
+    /** Called when a poll (registration or claim) fails transiently; the worker waits and polls
+     * again. A refusal it cannot recover from (a revoked key, a disabled agent) is thrown. */
+    onPollError?(error: unknown): void | Promise<void>;
+    /** Called when a claimed run's attempt fails: `retry` says the worker keeps the claim and
+     * resumes it after `waitMillis`; `attention` says the run was given up and needs a project
+     * member (requeue or cancel); `refused` says Hue no longer accepts this worker's outcome for
+     * the run (it was released or cancelled meanwhile). */
+    onRunFailed?(failure: LocalRunFailure): void | Promise<void>;
     /** Emit a one-time `DeprecationWarning` when the deployment serves a legacy world; on by default. */
     deprecationWarnings?: boolean;
     /** Opt into the experiment's immutable V2 provider profile. These three values are
@@ -151,6 +173,19 @@ export declare function registeredCapabilities(options: {
     /** The ordinary-case callback, when supplied. */
     directTarget?: unknown;
 }): string[];
+/** Why a claimed run's attempt ended without completing, and what the worker did about it. */
+export interface LocalRunFailure extends LocalAgentClaim {
+    /** What failed. */
+    error: unknown;
+    /** Failed attempts of this run in this process, this one included; 0 when not counted. */
+    attempt: number;
+    /** `retry`: the claim is kept and the run resumes after `waitMillis`. `attention`: the run
+     * was given up for a project member to requeue or cancel. `refused`: Hue no longer accepts
+     * this worker's outcome for the run, which was released or cancelled meanwhile. */
+    outcome: "retry" | "attention" | "refused";
+    /** The wait before the run resumes, for `retry`. */
+    waitMillis?: number;
+}
 /**
  * Starts an outbound-only worker for one fixed local agent entry point. Hue chooses
  * only the registered key/revision; no command or source is received from the cloud.

package/dist/evals/local-worker.js

@@ -1,8 +1,11 @@
 import { resolve } from "node:path";
 import { randomUUID } from "node:crypto";
+import { isTransientEnvironmentError } from "../environment/client.js";
+import { HueApiError, isTransientApiError } from "./client.js";
+import { HueExportError } from "../transport.js";
 import { CheckpointStore, checkpointPath, checkpointSegment } from "./checkpoint.js";
 import { pinRequestedAttemptV2, requestedAttemptV2, runEnvironmentTarget, } from "./environment-target.js";
-import { runExperiment, OutcomeSerializationError, TargetOutcomeUncertainError, UncertainExecutionError, } from "./runner.js";
+import { runExperiment, } from "./runner.js";
 /** Capability strings a registration declares; Hue offers only matching cases. */
 export const localAgentCapabilities = {
     /** Cases pinned to a hosted synthetic world. */
@@ -50,6 +53,7 @@ function localAgentTargetContext(context) {
             : {}),
         files: structuredClone(context.files),
         outputDirectory: context.outputDirectory,
+        ...(context.signal ? { signal: context.signal } : {}),
     };
 }
 /** The registration's capabilities: explicit values plus one per supplied callback. */
@@ -98,15 +102,22 @@ function wait(milliseconds, signal) {
         signal?.addEventListener("abort", aborted, { once: true });
     });
 }
-function needsAttention(error, seen = new Set()) {
-    if (error instanceof TargetOutcomeUncertainError ||
-        error instanceof UncertainExecutionError ||
-        error instanceof OutcomeSerializationError)
+/** A failure worth resuming: one that changes on its own. Hue or a world not answering or
+ * refusing transiently past the client's retries, telemetry not accepted at the flush, or a run
+ * of such failures across concurrent cases. Everything else, an outcome unsafe to resume (the
+ * target may have run without a saved outcome, an output that cannot be saved), a saved outcome
+ * whose trace was never acknowledged (a resume does not export it again), a refusal Hue decided
+ * on, an input the SDK refuses (a case file whose bytes differ from its manifest), would recur on
+ * the next attempt and is given up as attention at once. */
+function resumable(error, seen = new Set()) {
+    if (isTransientApiError(error) ||
+        isTransientEnvironmentError(error) ||
+        error instanceof HueExportError)
         return true;
-    if (!(error instanceof AggregateError) || seen.has(error))
+    if (!(error instanceof AggregateError) || seen.has(error) || !error.errors.length)
         return false;
     seen.add(error);
-    return error.errors.some((nested) => needsAttention(nested, seen));
+    return error.errors.every((nested) => resumable(nested, seen));
 }
 /**
  * Starts an outbound-only worker for one fixed local agent entry point. Hue chooses
@@ -119,10 +130,33 @@ export async function runLocalAgent(options) {
     const maxRuns = options.maxRuns ?? Number.POSITIVE_INFINITY;
     if (!(maxRuns === Number.POSITIVE_INFINITY || (Number.isInteger(maxRuns) && maxRuns > 0)))
         throw new RangeError("maxRuns must be a positive integer");
+    const maxRunAttempts = options.maxRunAttempts ?? 5;
+    if (!Number.isInteger(maxRunAttempts) || maxRunAttempts < 1 || maxRunAttempts > 20)
+        throw new RangeError("maxRunAttempts must be 1–20");
+    if (options.worldTtlSeconds !== undefined &&
+        (!Number.isInteger(options.worldTtlSeconds) ||
+            options.worldTtlSeconds < 1 ||
+            options.worldTtlSeconds > 86_400))
+        throw new RangeError("worldTtlSeconds must be 1–86400");
     if (options.environmentClient.baseUrl !== options.client.baseUrl)
         throw new Error("Environments and evaluations must use the same Hue origin");
     const directory = resolve(options.checkpointDirectory);
-    const project = await options.client.checkConnection();
+    // The first read is polled like the claims that follow: Hue not answering at startup is not
+    // a reason for the worker to exit; a refusal Hue decided on still is.
+    let project;
+    for (let failures = 0;;) {
+        try {
+            project = await options.client.checkConnection();
+            break;
+        }
+        catch (error) {
+            if (!isTransientApiError(error) || options.signal?.aborted)
+                throw error;
+            failures++;
+            await options.onPollError?.(error);
+            await wait(Math.min(interval * 2 ** Math.min(failures, 6), 60_000), options.signal);
+        }
+    }
     const store = await CheckpointStore.acquire(directory, {
         kind: "local-agent-worker",
         projectId: project.id,
@@ -138,28 +172,87 @@ export async function runLocalAgent(options) {
             workerId = randomUUID();
             await store.write("worker-id", workerId);
         }
-        let completed = 0;
-        while (completed < maxRuns && !options.signal?.aborted) {
-            const agent = await options.client.registerLocalAgent({
-                ...options.agent,
-                capabilities,
-                scorerDigests: options.agent.scorerDigests ??
-                    options.scorers?.map((item) => item.definition.sourceDigest) ??
-                    [],
-            });
-            const claim = await options.client.claimLocalAgentRun({ agentId: agent.id, workerId });
+        let settled = 0;
+        /** Consecutive poll failures, for the backoff between polls; reset by a poll that answers. */
+        let pollFailures = 0;
+        /** Consecutive failed attempts per claimed run; a run is given up at `maxRunAttempts`. */
+        const attempts = new Map();
+        const bump = (claim) => {
+            const attempt = (attempts.get(claim.runId) ?? 0) + 1;
+            attempts.set(claim.runId, attempt);
+            return attempt;
+        };
+        /** The wait before the run is resumed: the poll interval doubled per attempt, at most a minute. */
+        const backoff = (attempt) => Math.min(interval * 2 ** Math.min(attempt, 6), 60_000);
+        /** Gives the run up as attention with the failure's name; a refusal to record it (the run
+         * was released or cancelled meanwhile) is reported as such and changes nothing else. */
+        const giveUp = async (claim, error) => {
+            const failureType = error instanceof Error ? error.name.slice(0, 200) : "WorkerError";
+            let outcome = "attention";
+            try {
+                await options.client.completeLocalAgentRun({
+                    runId: claim.runId,
+                    workerId,
+                    state: "attention",
+                    failureType,
+                });
+            }
+            catch (completion) {
+                if (completion instanceof HueApiError && completion.status === 409)
+                    outcome = "refused";
+                // A transient failure past the client's retries leaves the run claimed; the next claim
+                // returns it and the attempt count gives it up again. A refusal Hue decided on (a
+                // revoked key, a disabled agent) is what needs fixing, and is thrown as itself.
+                else if (isTransientApiError(completion))
+                    return false;
+                else
+                    throw completion;
+            }
+            const attempt = attempts.get(claim.runId) ?? 0;
+            attempts.delete(claim.runId);
+            settled++;
+            await options.onRunFailed?.({ ...claim, error, attempt, outcome });
+            return true;
+        };
+        while (settled < maxRuns && !options.signal?.aborted) {
+            let claim;
+            try {
+                const agent = await options.client.registerLocalAgent({
+                    ...options.agent,
+                    capabilities,
+                    scorerDigests: options.agent.scorerDigests ??
+                        options.scorers?.map((item) => item.definition.sourceDigest) ??
+                        [],
+                });
+                claim = await options.client.claimLocalAgentRun({ agentId: agent.id, workerId });
+                pollFailures = 0;
+            }
+            catch (error) {
+                // A transient failure past the client's own retries (Hue restarting, a network blip)
+                // does not end the worker: it waits, longer each time, and polls again. A refusal Hue
+                // decided on (a revoked key, a disabled agent, a changed registration) is thrown.
+                if (!isTransientApiError(error))
+                    throw error;
+                pollFailures++;
+                await options.onPollError?.(error);
+                await wait(Math.min(interval * 2 ** Math.min(pollFailures, 6), 60_000), options.signal);
+                continue;
+            }
             if (!claim) {
                 await wait(interval, options.signal);
                 continue;
             }
+            // A claim naming an experiment id the checkpoint store refuses is a protocol violation,
+            // not an operational failure: it is thrown here, before the attempt, and never retried.
+            const experimentId = checkpointSegment(claim.experimentId, "experiment id");
             const heartbeat = setInterval(() => {
                 void options.client
                     .heartbeatLocalAgentRun({ runId: claim.runId, workerId })
                     .catch(() => undefined);
             }, Math.min(15_000, Math.max(1_000, interval)));
             let experimentFinished = false;
+            let queueCompleted = false;
             try {
-                const experimentId = checkpointSegment(claim.experimentId, "experiment id");
                 const requested = requestedConfiguration
                     ? pinRequestedAttemptV2(requestedConfiguration, (await options.client.getExperiment(experimentId)).config)
                     : undefined;
@@ -201,6 +294,7 @@ export async function runLocalAgent(options) {
                             inputs,
                             context,
                             requested,
+                            ttlSeconds: options.worldTtlSeconds,
                             // The registered revision is the agent revision under test.
                             agentRevision: options.agent.revision,
                             deprecationWarnings: options.deprecationWarnings,
@@ -218,24 +312,54 @@ export async function runLocalAgent(options) {
                     workerId,
                     state: "completed",
                 });
+                queueCompleted = true;
+                attempts.delete(claim.runId);
+                settled++;
                 await options.onCompleted?.(report);
-                completed++;
             }
             catch (error) {
-                // A durable outcome can still need completion/result uploads. Leave operational
-                // failures claimed so the same worker can resume them through its checkpoints.
-                // Attention is terminal in the queue and is reserved for explicit unsafe-to-resume
-                // outcomes that require operator intervention.
-                if (!experimentFinished && needsAttention(error))
-                    await options.client
-                        .completeLocalAgentRun({
-                        runId: claim.runId,
-                        workerId,
-                        state: "attention",
-                        failureType: error instanceof Error ? error.name.slice(0, 200) : "WorkerError",
-                    })
-                        .catch(() => undefined);
-                throw error;
+                // The run is completed in the queue: a failure after that is the caller's own
+                // (`onCompleted` threw) and is thrown as itself, not reported as a retry that cannot be.
+                if (queueCompleted)
+                    throw error;
+                if (experimentFinished && error instanceof HueApiError && error.status === 409) {
+                    // The experiment finished, but Hue no longer accepts this worker's completion: the
+                    // run was released or cancelled meanwhile. Its outcome is recorded; nothing to resume.
+                    attempts.delete(claim.runId);
+                    settled++;
+                    await options.onRunFailed?.({ ...claim, error, attempt: 0, outcome: "refused" });
+                }
+                else if (!experimentFinished && !resumable(error)) {
+                    // A failure that would recur: unsafe to resume, refused by Hue, or an input the SDK
+                    // refuses. A project member decides; the worker goes on to other runs. When Hue cannot
+                    // record that for `maxRunAttempts` polls, the worker stops with the error: it can
+                    // reach neither an outcome nor a record of one.
+                    if (!(await giveUp(claim, error))) {
+                        const attempt = bump(claim);
+                        if (attempt >= maxRunAttempts)
+                            throw error;
+                        await wait(backoff(attempt), options.signal);
+                    }
+                }
+                else {
+                    // A transient failure: the outcome so far is checkpointed, and the claim stays with
+                    // this worker, so the next claim returns the same run and the experiment resumes where
+                    // it stopped. After `maxRunAttempts` such failures the run is given up as attention;
+                    // when even that cannot be recorded, the worker stops with the error.
+                    const attempt = bump(claim);
+                    if (attempt >= maxRunAttempts) {
+                        if (!(await giveUp(claim, error))) {
+                            if (attempt >= 2 * maxRunAttempts)
+                                throw error;
+                            await wait(backoff(attempt), options.signal);
+                        }
+                    }
+                    else {
+                        const waitMillis = backoff(attempt);
+                        await options.onRunFailed?.({ ...claim, error, attempt, outcome: "retry", waitMillis });
+                        await wait(waitMillis, options.signal);
+                    }
+                }
             }
             finally {
                 clearInterval(heartbeat);

package/dist/evals/runner.d.ts

@@ -29,6 +29,30 @@ export declare class OutcomeSerializationError extends Error {
 export declare class TargetCancelledError extends Error {
     constructor();
 }
+/** Thrown by the environment target when the target did not return before its world's deadline.
+ * The world is sealed abandoned and the attempt saved as an `error` of type `TargetTimeout`, so a
+ * hung agent ends its case instead of holding the run open. */
+export declare class TargetTimeoutError extends Error {
+    /** The world whose deadline passed. */
+    readonly environmentRunId: string;
+    /** The world's `expiresAt`. */
+    readonly expiresAt: string;
+    constructor(
+    /** The world whose deadline passed. */
+    environmentRunId: string, 
+    /** The world's `expiresAt`. */
+    expiresAt: string);
+}
+/** The target's outcome is saved but the telemetry it requires was never acknowledged on an
+ * earlier attempt. Restore or export the trace, or complete with omitted evidence through the
+ * client; the target is never rerun. It is this case's own state, not the run's. */
+export declare class TraceExportUnacknowledgedError extends Error {
+    /** The execution whose saved outcome waits for its trace. */
+    readonly executionId: string;
+    constructor(
+    /** The execution whose saved outcome waits for its trace. */
+    executionId: string);
+}
 /** Thrown when the target or world may have committed but acknowledgement is unavailable. */
 export declare class TargetOutcomeUncertainError extends Error {
     /** Execution whose target outcome must never be replayed automatically. */