@hue-run/sdk 0.11.4 → 0.12.1

package/dist/evals/local-worker.js

@@ -1,8 +1,11 @@
 import { resolve } from "node:path";
 import { randomUUID } from "node:crypto";
+import { isTransientEnvironmentError } from "../environment/client.js";
+import { HueApiError, isTransientApiError } from "./client.js";
+import { HueExportError } from "../transport.js";
 import { CheckpointStore, checkpointPath, checkpointSegment } from "./checkpoint.js";
 import { pinRequestedAttemptV2, requestedAttemptV2, runEnvironmentTarget, } from "./environment-target.js";
-import { runExperiment, OutcomeSerializationError, TargetOutcomeUncertainError, UncertainExecutionError, } from "./runner.js";
+import { runExperiment, } from "./runner.js";
 /** Capability strings a registration declares; Hue offers only matching cases. */
 export const localAgentCapabilities = {
     /** Cases pinned to a hosted synthetic world. */
@@ -50,6 +53,7 @@ function localAgentTargetContext(context) {
             : {}),
         files: structuredClone(context.files),
         outputDirectory: context.outputDirectory,
+        ...(context.signal ? { signal: context.signal } : {}),
     };
 }
 /** The registration's capabilities: explicit values plus one per supplied callback. */
@@ -98,15 +102,22 @@ function wait(milliseconds, signal) {
         signal?.addEventListener("abort", aborted, { once: true });
     });
 }
-function needsAttention(error, seen = new Set()) {
-    if (error instanceof TargetOutcomeUncertainError ||
-        error instanceof UncertainExecutionError ||
-        error instanceof OutcomeSerializationError)
+/** A failure worth resuming: one that changes on its own. Hue or a world not answering or
+ * refusing transiently past the client's retries, telemetry not accepted at the flush, or a run
+ * of such failures across concurrent cases. Everything else, an outcome unsafe to resume (the
+ * target may have run without a saved outcome, an output that cannot be saved), a saved outcome
+ * whose trace was never acknowledged (a resume does not export it again), a refusal Hue decided
+ * on, an input the SDK refuses (a case file whose bytes differ from its manifest), would recur on
+ * the next attempt and is given up as attention at once. */
+function resumable(error, seen = new Set()) {
+    if (isTransientApiError(error) ||
+        isTransientEnvironmentError(error) ||
+        error instanceof HueExportError)
         return true;
-    if (!(error instanceof AggregateError) || seen.has(error))
+    if (!(error instanceof AggregateError) || seen.has(error) || !error.errors.length)
         return false;
     seen.add(error);
-    return error.errors.some((nested) => needsAttention(nested, seen));
+    return error.errors.every((nested) => resumable(nested, seen));
 }
 /**
  * Starts an outbound-only worker for one fixed local agent entry point. Hue chooses
@@ -119,10 +130,33 @@ export async function runLocalAgent(options) {
     const maxRuns = options.maxRuns ?? Number.POSITIVE_INFINITY;
     if (!(maxRuns === Number.POSITIVE_INFINITY || (Number.isInteger(maxRuns) && maxRuns > 0)))
         throw new RangeError("maxRuns must be a positive integer");
+    const maxRunAttempts = options.maxRunAttempts ?? 5;
+    if (!Number.isInteger(maxRunAttempts) || maxRunAttempts < 1 || maxRunAttempts > 20)
+        throw new RangeError("maxRunAttempts must be 1–20");
+    if (options.worldTtlSeconds !== undefined &&
+        (!Number.isInteger(options.worldTtlSeconds) ||
+            options.worldTtlSeconds < 1 ||
+            options.worldTtlSeconds > 86_400))
+        throw new RangeError("worldTtlSeconds must be 1–86400");
     if (options.environmentClient.baseUrl !== options.client.baseUrl)
         throw new Error("Environments and evaluations must use the same Hue origin");
     const directory = resolve(options.checkpointDirectory);
-    const project = await options.client.checkConnection();
+    // The first read is polled like the claims that follow: Hue not answering at startup is not
+    // a reason for the worker to exit; a refusal Hue decided on still is.
+    let project;
+    for (let failures = 0;;) {
+        try {
+            project = await options.client.checkConnection();
+            break;
+        }
+        catch (error) {
+            if (!isTransientApiError(error) || options.signal?.aborted)
+                throw error;
+            failures++;
+            await options.onPollError?.(error);
+            await wait(Math.min(interval * 2 ** Math.min(failures, 6), 60_000), options.signal);
+        }
+    }
     const store = await CheckpointStore.acquire(directory, {
         kind: "local-agent-worker",
         projectId: project.id,
@@ -138,28 +172,87 @@ export async function runLocalAgent(options) {
             workerId = randomUUID();
             await store.write("worker-id", workerId);
         }
-        let completed = 0;
-        while (completed < maxRuns && !options.signal?.aborted) {
-            const agent = await options.client.registerLocalAgent({
-                ...options.agent,
-                capabilities,
-                scorerDigests: options.agent.scorerDigests ??
-                    options.scorers?.map((item) => item.definition.sourceDigest) ??
-                    [],
-            });
-            const claim = await options.client.claimLocalAgentRun({ agentId: agent.id, workerId });
+        let settled = 0;
+        /** Consecutive poll failures, for the backoff between polls; reset by a poll that answers. */
+        let pollFailures = 0;
+        /** Consecutive failed attempts per claimed run; a run is given up at `maxRunAttempts`. */
+        const attempts = new Map();
+        const bump = (claim) => {
+            const attempt = (attempts.get(claim.runId) ?? 0) + 1;
+            attempts.set(claim.runId, attempt);
+            return attempt;
+        };
+        /** The wait before the run is resumed: the poll interval doubled per attempt, at most a minute. */
+        const backoff = (attempt) => Math.min(interval * 2 ** Math.min(attempt, 6), 60_000);
+        /** Gives the run up as attention with the failure's name; a refusal to record it (the run
+         * was released or cancelled meanwhile) is reported as such and changes nothing else. */
+        const giveUp = async (claim, error) => {
+            const failureType = error instanceof Error ? error.name.slice(0, 200) : "WorkerError";
+            let outcome = "attention";
+            try {
+                await options.client.completeLocalAgentRun({
+                    runId: claim.runId,
+                    workerId,
+                    state: "attention",
+                    failureType,
+                });
+            }
+            catch (completion) {
+                if (completion instanceof HueApiError && completion.status === 409)
+                    outcome = "refused";
+                // A transient failure past the client's retries leaves the run claimed; the next claim
+                // returns it and the attempt count gives it up again. A refusal Hue decided on (a
+                // revoked key, a disabled agent) is what needs fixing, and is thrown as itself.
+                else if (isTransientApiError(completion))
+                    return false;
+                else
+                    throw completion;
+            }
+            const attempt = attempts.get(claim.runId) ?? 0;
+            attempts.delete(claim.runId);
+            settled++;
+            await options.onRunFailed?.({ ...claim, error, attempt, outcome });
+            return true;
+        };
+        while (settled < maxRuns && !options.signal?.aborted) {
+            let claim;
+            try {
+                const agent = await options.client.registerLocalAgent({
+                    ...options.agent,
+                    capabilities,
+                    scorerDigests: options.agent.scorerDigests ??
+                        options.scorers?.map((item) => item.definition.sourceDigest) ??
+                        [],
+                });
+                claim = await options.client.claimLocalAgentRun({ agentId: agent.id, workerId });
+                pollFailures = 0;
+            }
+            catch (error) {
+                // A transient failure past the client's own retries (Hue restarting, a network blip)
+                // does not end the worker: it waits, longer each time, and polls again. A refusal Hue
+                // decided on (a revoked key, a disabled agent, a changed registration) is thrown.
+                if (!isTransientApiError(error))
+                    throw error;
+                pollFailures++;
+                await options.onPollError?.(error);
+                await wait(Math.min(interval * 2 ** Math.min(pollFailures, 6), 60_000), options.signal);
+                continue;
+            }
             if (!claim) {
                 await wait(interval, options.signal);
                 continue;
             }
+            // A claim naming an experiment id the checkpoint store refuses is a protocol violation,
+            // not an operational failure: it is thrown here, before the attempt, and never retried.
+            const experimentId = checkpointSegment(claim.experimentId, "experiment id");
             const heartbeat = setInterval(() => {
                 void options.client
                     .heartbeatLocalAgentRun({ runId: claim.runId, workerId })
                     .catch(() => undefined);
             }, Math.min(15_000, Math.max(1_000, interval)));
             let experimentFinished = false;
+            let queueCompleted = false;
             try {
-                const experimentId = checkpointSegment(claim.experimentId, "experiment id");
                 const requested = requestedConfiguration
                     ? pinRequestedAttemptV2(requestedConfiguration, (await options.client.getExperiment(experimentId)).config)
                     : undefined;
@@ -201,6 +294,7 @@ export async function runLocalAgent(options) {
                             inputs,
                             context,
                             requested,
+                            ttlSeconds: options.worldTtlSeconds,
                             // The registered revision is the agent revision under test.
                             agentRevision: options.agent.revision,
                             deprecationWarnings: options.deprecationWarnings,
@@ -218,24 +312,54 @@ export async function runLocalAgent(options) {
                     workerId,
                     state: "completed",
                 });
+                queueCompleted = true;
+                attempts.delete(claim.runId);
+                settled++;
                 await options.onCompleted?.(report);
-                completed++;
             }
             catch (error) {
-                // A durable outcome can still need completion/result uploads. Leave operational
-                // failures claimed so the same worker can resume them through its checkpoints.
-                // Attention is terminal in the queue and is reserved for explicit unsafe-to-resume
-                // outcomes that require operator intervention.
-                if (!experimentFinished && needsAttention(error))
-                    await options.client
-                        .completeLocalAgentRun({
-                        runId: claim.runId,
-                        workerId,
-                        state: "attention",
-                        failureType: error instanceof Error ? error.name.slice(0, 200) : "WorkerError",
-                    })
-                        .catch(() => undefined);
-                throw error;
+                // The run is completed in the queue: a failure after that is the caller's own
+                // (`onCompleted` threw) and is thrown as itself, not reported as a retry that cannot be.
+                if (queueCompleted)
+                    throw error;
+                if (experimentFinished && error instanceof HueApiError && error.status === 409) {
+                    // The experiment finished, but Hue no longer accepts this worker's completion: the
+                    // run was released or cancelled meanwhile. Its outcome is recorded; nothing to resume.
+                    attempts.delete(claim.runId);
+                    settled++;
+                    await options.onRunFailed?.({ ...claim, error, attempt: 0, outcome: "refused" });
+                }
+                else if (!experimentFinished && !resumable(error)) {
+                    // A failure that would recur: unsafe to resume, refused by Hue, or an input the SDK
+                    // refuses. A project member decides; the worker goes on to other runs. When Hue cannot
+                    // record that for `maxRunAttempts` polls, the worker stops with the error: it can
+                    // reach neither an outcome nor a record of one.
+                    if (!(await giveUp(claim, error))) {
+                        const attempt = bump(claim);
+                        if (attempt >= maxRunAttempts)
+                            throw error;
+                        await wait(backoff(attempt), options.signal);
+                    }
+                }
+                else {
+                    // A transient failure: the outcome so far is checkpointed, and the claim stays with
+                    // this worker, so the next claim returns the same run and the experiment resumes where
+                    // it stopped. After `maxRunAttempts` such failures the run is given up as attention;
+                    // when even that cannot be recorded, the worker stops with the error.
+                    const attempt = bump(claim);
+                    if (attempt >= maxRunAttempts) {
+                        if (!(await giveUp(claim, error))) {
+                            if (attempt >= 2 * maxRunAttempts)
+                                throw error;
+                            await wait(backoff(attempt), options.signal);
+                        }
+                    }
+                    else {
+                        const waitMillis = backoff(attempt);
+                        await options.onRunFailed?.({ ...claim, error, attempt, outcome: "retry", waitMillis });
+                        await wait(waitMillis, options.signal);
+                    }
+                }
             }
             finally {
                 clearInterval(heartbeat);

package/dist/evals/runner.d.ts

@@ -29,6 +29,30 @@ export declare class OutcomeSerializationError extends Error {
 export declare class TargetCancelledError extends Error {
     constructor();
 }
+/** Thrown by the environment target when the target did not return before its world's deadline.
+ * The world is sealed abandoned and the attempt saved as an `error` of type `TargetTimeout`, so a
+ * hung agent ends its case instead of holding the run open. */
+export declare class TargetTimeoutError extends Error {
+    /** The world whose deadline passed. */
+    readonly environmentRunId: string;
+    /** The world's `expiresAt`. */
+    readonly expiresAt: string;
+    constructor(
+    /** The world whose deadline passed. */
+    environmentRunId: string, 
+    /** The world's `expiresAt`. */
+    expiresAt: string);
+}
+/** The target's outcome is saved but the telemetry it requires was never acknowledged on an
+ * earlier attempt. Restore or export the trace, or complete with omitted evidence through the
+ * client; the target is never rerun. It is this case's own state, not the run's. */
+export declare class TraceExportUnacknowledgedError extends Error {
+    /** The execution whose saved outcome waits for its trace. */
+    readonly executionId: string;
+    constructor(
+    /** The execution whose saved outcome waits for its trace. */
+    executionId: string);
+}
 /** Thrown when the target or world may have committed but acknowledgement is unavailable. */
 export declare class TargetOutcomeUncertainError extends Error {
     /** Execution whose target outcome must never be replayed automatically. */
@@ -99,9 +123,13 @@ export interface RunExperimentOptions extends RunnerOptions {
      * rejects. `"fail_case"` completes the execution as failed instead, with the evidence omitted
      * under the reason `telemetry_not_accepted` and without its output or generated files, so no
      * scorer can pass it; reports it in {@link RunnerReport.telemetryNotAccepted} and goes on with
-     * the other cases.
+     * the other cases. `"pending"` completes the execution in its true state with
+     * `traceEvidence: "pending"` and the count of spans the case ended: Hue scores the sealed world
+     * at once and attaches the trace once it holds every span, or records the evidence omitted a
+     * day later; reports it in {@link RunnerReport.tracePending} and goes on. A saved outcome whose
+     * export the receipt does not accept on resume completes the same way.
      */
-    traceNotAccepted?: "stop" | "fail_case";
+    traceNotAccepted?: "stop" | "fail_case" | "pending";
     /** Called when this call completes a case as failed under `traceNotAccepted: "fail_case"`,
      * before the run goes on, so the failure is known even if a later case stops the run. A call
      * that resumes an already completed case lists it in the report without calling again. */
@@ -128,6 +156,21 @@ export interface RunnerReport {
     /** Cases completed as failed because their telemetry was not accepted, under
      * `traceNotAccepted: "fail_case"`; absent when there were none. */
     telemetryNotAccepted?: TelemetryNotAccepted[];
+    /** Cases completed with their trace evidence pending, under `traceNotAccepted: "pending"`;
+     * absent when there were none. */
+    tracePending?: TracePending[];
+}
+/** A case completed with its trace still to arrive. */
+export interface TracePending {
+    /** Frozen case ID. */
+    caseId: string;
+    /** The case's external key. */
+    caseKey: string;
+    /** Execution completed pending. */
+    executionId: string;
+    /** How many spans the case ended, which Hue holds the arriving trace to; absent when the
+     * checkpoint recorded no count. */
+    spanCount?: number;
 }
 /** Export issue counts, never content: which signal, what happened, the HTTP status when Hue
  * answered, and how many records. */

package/dist/evals/runner.js

@@ -6,6 +6,7 @@ import { ROOT_CONTEXT } from "@opentelemetry/api";
 import { HueExportError } from "../transport.js";
 import { HueApiError } from "./client.js";
 import { loadEnvironmentEvidence } from "./environment-evidence.js";
+import { serviceFailureType } from "./failure.js";
 import { CheckpointStore } from "./checkpoint.js";
 import { onForcedExit } from "./exit-cleanup.js";
 import { assertSafeFileNames, downloadCaseFiles, downloadCaseInputs, localOutputFiles, verifyCaseFiles, OutputFileError, stageOutputFiles, targetFileRoles, uploadOutputFiles, } from "./files.js";
@@ -65,6 +66,36 @@ export class TargetCancelledError extends Error {
         this.name = "TargetCancelledError";
     }
 }
+/** Thrown by the environment target when the target did not return before its world's deadline.
+ * The world is sealed abandoned and the attempt saved as an `error` of type `TargetTimeout`, so a
+ * hung agent ends its case instead of holding the run open. */
+export class TargetTimeoutError extends Error {
+    environmentRunId;
+    expiresAt;
+    constructor(
+    /** The world whose deadline passed. */
+    environmentRunId, 
+    /** The world's `expiresAt`. */
+    expiresAt) {
+        super(`Target did not finish before its world ${environmentRunId} expired at ${expiresAt}`);
+        this.environmentRunId = environmentRunId;
+        this.expiresAt = expiresAt;
+        this.name = "TargetTimeoutError";
+    }
+}
+/** The target's outcome is saved but the telemetry it requires was never acknowledged on an
+ * earlier attempt. Restore or export the trace, or complete with omitted evidence through the
+ * client; the target is never rerun. It is this case's own state, not the run's. */
+export class TraceExportUnacknowledgedError extends Error {
+    executionId;
+    constructor(
+    /** The execution whose saved outcome waits for its trace. */
+    executionId) {
+        super("Target outcome is saved but trace export acknowledgement is unavailable. Restore/export the trace or explicitly complete with omitted evidence through the client; never rerun the target.");
+        this.executionId = executionId;
+        this.name = "TraceExportUnacknowledgedError";
+    }
+}
 /** Thrown when the target or world may have committed but acknowledgement is unavailable. */
 export class TargetOutcomeUncertainError extends Error {
     executionId;
@@ -177,25 +208,96 @@ async function allPages(page, maximum = 5000) {
     } while (after !== undefined);
     return items;
 }
+/** A failure the next case would meet too: Hue not answering or refusing (a transient failure
+ * past the client's retries, a lost connection, a refusal such as a finished experiment or a
+ * world that cannot be sealed), telemetry not accepted by the one transport the cases share, or
+ * a target outcome left uncertain because its world could not be sealed or read back. Starting
+ * more cases into it only piles up the same failure; the run is resumed later instead. */
+function systemic(error) {
+    // A saved outcome still waiting for its trace (`TraceExportUnacknowledgedError`) is not
+    // systemic: the flush failed on an earlier attempt, telemetry may have recovered since, and
+    // the cases beside it can complete now; only a live export failure stops new cases.
+    return (error instanceof HueApiError ||
+        error instanceof HueExportError ||
+        error instanceof TargetOutcomeUncertainError);
+}
+/** Runs every case, `concurrency` at a time. One case's own failure (an attempt without a saved
+ * outcome, an output that cannot be saved, inputs the SDK refuses) does not stop the cases
+ * beside or after it: each is attempted, and the failures are thrown together at the end, so a
+ * resume has only the failed cases left to publish. A systemic failure stops new cases from
+ * starting, while the cases in flight finish; the run resumes when Hue answers again. */
 async function pool(items, concurrency, execute) {
     let position = 0;
     const failures = [];
+    let stop = false;
     await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, async () => {
-        while (position < items.length && !failures.length) {
+        while (position < items.length && !stop) {
             const item = items[position++];
             try {
                 await execute(item);
             }
             catch (error) {
                 failures.push(error);
+                if (systemic(error))
+                    stop = true;
             }
         }
     }));
     if (failures.length === 1)
         throw failures[0];
-    if (failures.length)
-        throw new AggregateError(failures, "Multiple case operations failed; resume uses saved outcomes");
+    if (failures.length) {
+        const first = failures[0];
+        throw new AggregateError(failures, `${failures.length} case operations failed; resume uses saved outcomes. First: ${first instanceof Error ? first.message : String(first)}`);
+    }
+}
+/** Whether Hue holds the case's telemetry, read from the trace receipt: its root span and at
+ * least as many spans as the case ended; false when they are not there by the wait's end or the
+ * receipt cannot be read, and for a checkpoint an older runner saved without the span. */
+/** Turn a prepared completion into a pending one: the trace is declared but still to arrive,
+ * and the span count the checkpoint recorded at the flush goes with it, so Hue freezes the trace
+ * only once it holds every span the case ended, never a root span alone. */
+function pendingCompletion(prepared) {
+    prepared.complete.traceEvidence = "pending";
+    delete prepared.complete.omissionReason;
+    if (prepared.trace?.spans !== undefined)
+        prepared.complete.traceSpanCount = prepared.trace.spans;
+}
+/** The reverse, once the receipt accepted the trace after all: required evidence, no count. */
+function acceptedCompletion(prepared) {
+    prepared.complete.traceEvidence = "required";
+    delete prepared.complete.traceSpanCount;
 }
+async function traceLanded(hue, trace) {
+    // Without the count of spans the case ended, the receipt cannot tell a lost child span from a
+    // landed trace, and does not accept.
+    if (!trace || trace.spans === undefined)
+        return false;
+    const deadline = Date.now() + RECEIPT_WAIT_MS;
+    try {
+        for (;;) {
+            // The root span present and every span the case ended held: a lost child span is not
+            // hidden behind a root that landed. The receipt answers as soon as the root is there, so
+            // spans still arriving behind it are polled for until the wait ends.
+            const verification = await hue.verifyTrace(trace.traceId, {
+                expectedSpanIds: [trace.spanId],
+                timeoutMillis: Math.max(250, deadline - Date.now()),
+            });
+            if (verification.verified && (verification.receipt?.spanCount ?? 0) >= trace.spans)
+                return true;
+            if (!verification.verified || Date.now() + RECEIPT_POLL_MS >= deadline)
+                return false;
+            await new Promise((resolve) => setTimeout(resolve, RECEIPT_POLL_MS));
+        }
+    }
+    catch {
+        return false;
+    }
+}
+/** How long a receipt is polled for the root span after a flush reported a failure: ingestion
+ * of an accepted batch is not instant, and a span that is not there by then was not delivered. */
+const RECEIPT_WAIT_MS = 5_000;
+/** How often the receipt is read again while the root is there but spans are still arriving. */
+const RECEIPT_POLL_MS = 500;
 async function scoresFor(versions, context, options, executionId, hasEnvironment = true) {
     const scores = [];
     let environmentUnavailable = false;
@@ -329,8 +431,16 @@ export async function runExperiment(options) {
     try {
         const filesRoot = resolve(options.filesDirectory ?? join(options.checkpointDirectory, "files"));
         const sanitize = (message) => message.slice(0, 4000).toWellFormed().replaceAll("\u0000", "");
+        // A target stopped at its world's deadline is a `TargetTimeout`, told apart from an error the
+        // target raised itself: the agent did not finish, rather than finishing wrongly. A service the
+        // agent called that refused, dropped the connection or timed out is named apart from the
+        // agent's own error (`failure.ts`), so Hue counts it as infrastructure, not as the agent.
         const errorPayload = (error) => ({
-            type: error instanceof OutputTooLargeError ? "OutputTooLarge" : "TargetError",
+            type: error instanceof OutputTooLargeError
+                ? "OutputTooLarge"
+                : error instanceof TargetTimeoutError
+                    ? "TargetTimeout"
+                    : (serviceFailureType(error) ?? "TargetError"),
             ...(options.persistResultContent && error instanceof Error
                 ? { message: sanitize(error.message) }
                 : {}),
@@ -378,6 +488,7 @@ export async function runExperiment(options) {
                 complete,
                 scores,
                 exportState: "pending",
+                ...(saved.trace ? { trace: saved.trace } : {}),
             };
             await store.write(file, prepared);
             return prepared;
@@ -533,6 +644,7 @@ export async function runExperiment(options) {
                     const uploading = {
                         stage: "uploading",
                         executionId: execution.id,
+                        trace: { traceId: span.traceId, spanId: span.spanId },
                         state,
                         hasOutput: output !== undefined,
                         ...(options.persistResultContent && output !== undefined ? { output } : {}),
@@ -553,13 +665,50 @@ export async function runExperiment(options) {
                 // End root before waiting for both OTLP signals. Export failure leaves the prepared checkpoint intact.
                 let exportError;
                 try {
-                    await options.hue.flush();
-                    // Another concurrent flush may already have surfaced this failure. OTLP
-                    // partial acknowledgements do not identify individual rejected records.
-                    if (options.hue.transport.getFailureSequence() !== failureSequenceBefore)
-                        throw new HueExportError(options.hue.transport
-                            .getIssues()
-                            .filter((issue) => issue.sequence > failureSequenceBefore && issue.kind !== "warning"), options.hue.transport.getReport());
+                    // The flush itself throws for any failure since the last flush; the issues it recorded
+                    // are read below, where they are attributed.
+                    await options.hue.flush().catch((error) => {
+                        if (!(error instanceof HueExportError))
+                            throw error;
+                    });
+                    // The transport is shared by every case in flight, so a failure the flush reports may
+                    // be another case's. Each issue names the traces of the records it concerned: one that
+                    // names this trace is this case's failure, final because its evidence is incomplete;
+                    // one that names only others is not this case's; one that names no trace (a processor
+                    // flush that failed) may be anyone's, and this case's trace receipt decides: its root
+                    // span landed, or the case stays pending for the receipt to decide again on resume.
+                    const decided = checkpoint;
+                    const traceId = decided.trace?.traceId;
+                    // Every span the case ended has reached the exporter by now; the receipt is held to
+                    // that count, so a lost child span is not hidden behind a root span that landed. The
+                    // count is saved before anything is decided, so a process that stops here leaves a
+                    // pending checkpoint the receipt can be held to; a transport that does not count (an
+                    // older or borrowed one) leaves no count, and the receipt then cannot accept.
+                    if (decided.trace && traceId !== undefined) {
+                        const spans = options.hue.transport.spansEnded?.(traceId);
+                        if (spans !== undefined) {
+                            decided.trace.spans = spans;
+                            await store.write(file, decided);
+                        }
+                    }
+                    if (options.hue.transport.getFailureSequence() !== failureSequenceBefore) {
+                        const retained = options.hue.transport.getIssues();
+                        const issues = retained.filter((issue) => issue.sequence > failureSequenceBefore && issue.kind !== "warning");
+                        const named = issues.filter((issue) => issue.traceIds && (traceId === undefined || issue.traceIds.includes(traceId)));
+                        // An issue naming no trace, and a failure whose issue the bounded history has already
+                        // rolled past (the oldest retained issue is younger than the first this case could
+                        // have caused), may be anyone's.
+                        const unattributed = issues.filter((issue) => !issue.traceIds);
+                        const evicted = issues.length === 0 || (retained[0]?.sequence ?? 0) > failureSequenceBefore + 1;
+                        if (named.length) {
+                            checkpoint.exportState = "failed";
+                            await store.write(file, checkpoint);
+                            throw new HueExportError([...named, ...unattributed], options.hue.transport.getReport());
+                        }
+                        if ((unattributed.length || evicted) &&
+                            !(await traceLanded(options.hue, decided.trace)))
+                            throw new HueExportError(unattributed, options.hue.transport.getReport());
+                    }
                 }
                 catch (error) {
                     exportError = error;
@@ -575,7 +724,15 @@ export async function runExperiment(options) {
                     if (options.traceEvidence.mode !== "omit")
                         throw error;
                 }
-                if (exportError !== undefined && options.traceEvidence.mode !== "omit") {
+                if (exportError !== undefined &&
+                    options.traceEvidence.mode !== "omit" &&
+                    options.traceNotAccepted === "pending") {
+                    // The outcome stands as the target gave it; Hue attaches the trace once it holds
+                    // every span the case ended, which the checkpoint's count tells it.
+                    pendingCompletion(checkpoint);
+                    await store.write(file, checkpoint);
+                }
+                else if (exportError !== undefined && options.traceEvidence.mode !== "omit") {
                     if (options.traceNotAccepted !== "fail_case")
                         throw exportError;
                     // Required evidence that Hue did not accept fails the case rather than leaving its
@@ -606,8 +763,35 @@ export async function runExperiment(options) {
             }
             const prepared = checkpoint;
             await options.client.getExecution(prepared.executionId);
-            if (prepared.exportState !== "accepted" && prepared.complete.traceEvidence !== "omit")
-                throw new Error("Target outcome is saved but trace export acknowledgement is unavailable. Restore/export the trace or explicitly complete with omitted evidence through the client; never rerun the target.");
+            // A saved outcome no flush decided on is asked about through its trace receipt: the root
+            // span there says the case's export landed before the process stopped, or that a later
+            // flush of the same transport delivered it. An export failure that named this trace is
+            // final: the evidence is incomplete whatever the receipt holds.
+            if (prepared.exportState === "pending" &&
+                prepared.complete.traceEvidence !== "omit" &&
+                (await traceLanded(options.hue, prepared.trace))) {
+                prepared.exportState = "accepted";
+                // A completion made pending by an earlier receipt read is required again: the trace
+                // landed, so Hue freezes it at completion as for any accepted case.
+                if (prepared.complete.traceEvidence === "pending")
+                    acceptedCompletion(prepared);
+                await store.write(file, prepared);
+            }
+            // Under the pending policy a saved outcome the receipt does not accept, whether no flush
+            // decided it or an export failure named its trace, completes with the trace pending rather
+            // than being refused: Hue attaches the trace when it arrives in full.
+            if (options.traceNotAccepted === "pending" &&
+                prepared.exportState !== "accepted" &&
+                prepared.exportState !== "not_accepted" &&
+                prepared.complete.traceEvidence !== "omit" &&
+                prepared.complete.traceEvidence !== "pending") {
+                pendingCompletion(prepared);
+                await store.write(file, prepared);
+            }
+            if (prepared.exportState !== "accepted" &&
+                prepared.complete.traceEvidence !== "omit" &&
+                prepared.complete.traceEvidence !== "pending")
+                throw new TraceExportUnacknowledgedError(prepared.executionId);
             const save = () => store.write(file, prepared);
             const notAccepted = prepared.exportState === "not_accepted"
                 ? {
@@ -628,6 +812,15 @@ export async function runExperiment(options) {
             }
             if (notAccepted)
                 (report.telemetryNotAccepted ??= []).push(notAccepted);
+            if (prepared.complete.traceEvidence === "pending")
+                (report.tracePending ??= []).push({
+                    caseId: item.id,
+                    caseKey: item.externalKey,
+                    executionId: prepared.executionId,
+                    ...(prepared.complete.traceSpanCount !== undefined
+                        ? { spanCount: prepared.complete.traceSpanCount }
+                        : {}),
+                });
             const results = await uploadScores(options, experiment.evaluation.id, prepared.scores, save, versions);
             report.subjectIds.push(prepared.completion.subjectId);
             report.resultIds.push(...results);

package/dist/evals/simulation.d.ts

@@ -160,7 +160,7 @@ export interface RunSimulationOptions {
     };
     /** What a case does when required telemetry is not accepted; see
      * `traceNotAccepted` of {@link runExperiment}. Defaults to `"stop"`. */
-    traceNotAccepted?: "stop" | "fail_case";
+    traceNotAccepted?: "stop" | "fail_case" | "pending";
     /** Called as each case is failed for its telemetry; see `onTelemetryNotAccepted` of
      * {@link runExperiment}. */
     onTelemetryNotAccepted?(entry: TelemetryNotAccepted): void | Promise<void>;