@hue-run/sdk 0.9.0 → 0.11.0

package/dist/evals/local-worker.d.ts

@@ -4,7 +4,7 @@ import type { EnvironmentTool } from "../environment/tools.js";
 import type { WorldHandoff } from "../environment/types.js";
 import type { ActualAgentManifestInputV2, AttemptConnectionBundleV2, RequestedAttemptProviderV2 } from "./attempt.js";
 import type { EvaluationClient } from "./client.js";
-import { type RunExperimentTargetContext, type RunnerReport } from "./runner.js";
+import { type RunExperimentTargetContext, type RunnerReport, type TelemetryNotAccepted } from "./runner.js";
 import type { ExperimentCase, JsonValue, LocalAgentRegistration, LocalFile, LocalScorer, TargetResult } from "./types.js";
 /** Capability strings a registration declares; Hue offers only matching cases. */
 export declare const localAgentCapabilities: {
@@ -13,6 +13,10 @@ export declare const localAgentCapabilities: {
     /** Ordinary cases run directly on this machine: JSON inputs and pinned input files in,
      * JSON output and generated files out. */
     readonly direct: "direct:v1";
+    /** Cases pinned to a world that also carry agent-visible input files: `target` receives them
+     * with the world and may return generated files. Declare it with `input:<extension>` for each
+     * accepted file type; it requires `target`. */
+    readonly environmentFiles: "environment-files:v1";
 };
 /** Candidate-visible context for one queued local agent execution. */
 export interface LocalAgentTargetContext {
@@ -47,6 +51,13 @@ export interface LocalAgentTargetContext {
     /** Credential-bearing provider connections for this callback only. Hue never
      * checkpoints, logs or adds this response to parity digests. */
     connectionBundle?: AttemptConnectionBundleV2;
+    /** Verified copies of the case's agent-visible input files; evaluator-only files are
+     * withheld. Hue offers a world case with such files only to a registration declaring
+     * `environment-files:v1`. Empty when the case has none. */
+    files: LocalFile[];
+    /** Private directory for this case, removed after it; return generated files with
+     * `withFiles`. */
+    outputDirectory: string;
 }
 /** Candidate-visible context for one queued ordinary case run directly on this machine. */
 export interface LocalAgentDirectContext {
@@ -88,6 +99,12 @@ export interface RunLocalAgentOptions {
     scorers?: LocalScorer[];
     /** Cases in flight, between 1 and 16; defaults to 1. */
     concurrency?: number;
+    /** What a case does when its telemetry is not accepted; see
+     * `traceNotAccepted` of {@link runExperiment}. Defaults to `"stop"`. */
+    traceNotAccepted?: "stop" | "fail_case";
+    /** Called as each case is failed for its telemetry; see `onTelemetryNotAccepted` of
+     * {@link runExperiment}. */
+    onTelemetryNotAccepted?(entry: TelemetryNotAccepted): void | Promise<void>;
     /** Polling interval in milliseconds, 250–60000; defaults to 2000. */
     pollIntervalMillis?: number;
     /** Stops polling cooperatively; does not cancel an active callback. */
@@ -116,8 +133,9 @@ export interface RunLocalAgentOptions {
         surfaceKey: "google.gmail/mcp" | "slack/mcp";
     };
     /** Runs cases pinned to a hosted world (`environment:v1`): invokes the existing agent against
-     * isolated tools and candidate-safe context. */
-    target?(inputs: JsonValue, tools: Record<string, EnvironmentTool>, context: LocalAgentTargetContext): JsonValue | undefined | Promise<JsonValue | undefined>;
+     * isolated tools and candidate-safe context. Return the output, or `withFiles(output, files)`
+     * when it generated files. */
+    target?(inputs: JsonValue, tools: Record<string, EnvironmentTool>, context: LocalAgentTargetContext): JsonValue | TargetResult | undefined | Promise<JsonValue | TargetResult | undefined>;
     /** Runs ordinary cases without a world (`direct:v1`): the agent receives the case inputs
      * and verified input files and returns JSON output and/or generated files. */
     directTarget?(inputs: JsonValue, context: LocalAgentDirectContext): JsonValue | TargetResult | undefined | Promise<JsonValue | TargetResult | undefined>;

package/dist/evals/local-worker.js

@@ -10,6 +10,10 @@ export const localAgentCapabilities = {
     /** Ordinary cases run directly on this machine: JSON inputs and pinned input files in,
      * JSON output and generated files out. */
     direct: "direct:v1",
+    /** Cases pinned to a world that also carry agent-visible input files: `target` receives them
+     * with the world and may return generated files. Declare it with `input:<extension>` for each
+     * accepted file type; it requires `target`. */
+    environmentFiles: "environment-files:v1",
 };
 /** Allowlist the candidate surface for an ordinary case; frozen expectations stay with grading. */
 export function localAgentDirectContext(context, signal) {
@@ -44,6 +48,8 @@ function localAgentTargetContext(context) {
         ...(context.connectionBundle
             ? { connectionBundle: structuredClone(context.connectionBundle) }
             : {}),
+        files: structuredClone(context.files),
+        outputDirectory: context.outputDirectory,
     };
 }
 /** The registration's capabilities: explicit values plus one per supplied callback. */
@@ -55,6 +61,8 @@ export function registeredCapabilities(options) {
         throw new TypeError("direct:v1 requires a directTarget callback");
     if (declared.includes(localAgentCapabilities.environment) && !options.target)
         throw new TypeError("environment:v1 requires a target callback");
+    if (declared.includes(localAgentCapabilities.environmentFiles) && !options.target)
+        throw new TypeError("environment-files:v1 requires a target callback");
     return [
         ...new Set([
             ...declared,
@@ -163,6 +171,12 @@ export async function runLocalAgent(options) {
                     // Worker dispatch uses the case pin: directTarget never receives a world.
                     environmentEvidence: options.directTarget ? "when_pinned" : "required",
                     traceEvidence: { mode: "required" },
+                    ...(options.traceNotAccepted ? { traceNotAccepted: options.traceNotAccepted } : {}),
+                    ...(options.onTelemetryNotAccepted
+                        ? {
+                            onTelemetryNotAccepted: (entry) => options.onTelemetryNotAccepted(entry),
+                        }
+                        : {}),
                     scorers: options.scorers,
                     concurrency: options.concurrency,
                     target: (inputs, context) => {

package/dist/evals/runner.d.ts

@@ -93,6 +93,19 @@ export interface RunExperimentOptions extends RunnerOptions {
         /** Why evidence is omitted, up to 4000 characters. */
         reason: string;
     };
+    /**
+     * What a case does when evidence is required and its trace or logs are not fully accepted.
+     * `"stop"` (the default) keeps the saved outcome, leaves the execution started for recovery and
+     * rejects. `"fail_case"` completes the execution as failed instead, with the evidence omitted
+     * under the reason `telemetry_not_accepted` and without its output or generated files, so no
+     * scorer can pass it; reports it in {@link RunnerReport.telemetryNotAccepted} and goes on with
+     * the other cases.
+     */
+    traceNotAccepted?: "stop" | "fail_case";
+    /** Called when this call completes a case as failed under `traceNotAccepted: "fail_case"`,
+     * before the run goes on, so the failure is known even if a later case stops the run. A call
+     * that resumes an already completed case lists it in the report without calling again. */
+    onTelemetryNotAccepted?(entry: TelemetryNotAccepted): void | Promise<void>;
     /** Runs the application for one frozen case; return the output, `withFiles(output, files)`
      * when it generated files, or `undefined` when unavailable. */
     target(inputs: JsonValue, context: RunExperimentTargetContext): JsonValue | TargetResult | undefined | Promise<JsonValue | TargetResult | undefined>;
@@ -112,7 +125,39 @@ export interface RunnerReport {
     resultIds: string[];
     /** Pins without a local implementation, left pending for their authorized executor. */
     deferredScorerVersionIds: string[];
+    /** Cases completed as failed because their telemetry was not accepted, under
+     * `traceNotAccepted: "fail_case"`; absent when there were none. */
+    telemetryNotAccepted?: TelemetryNotAccepted[];
 }
+/** Export issue counts, never content: which signal, what happened, the HTTP status when Hue
+ * answered, and how many records. */
+export interface TelemetryIssueCount {
+    /** Signal the records belong to. */
+    signal: "traces" | "logs";
+    /** `rejected` by Hue, `failed` to deliver, `dropped` from the queue, or an `invalid` record. */
+    kind: "rejected" | "failed" | "dropped" | "invalid";
+    /** HTTP status when the issue came from Hue's answer. */
+    status?: number;
+    /** Records affected. */
+    count: number;
+}
+/** A case whose trace or logs were not fully accepted, completed as failed. */
+export interface TelemetryNotAccepted {
+    /** Experiment case (item) ID. */
+    caseId: string;
+    /** The case's caller-chosen key. */
+    caseKey: string;
+    /** Execution completed as failed. */
+    executionId: string;
+    /** Sanitized issues the transport reported while the case's telemetry was exported. */
+    issues: TelemetryIssueCount[];
+}
+/** Stable reason, and first word of the omission reason, of a case failed for its telemetry. */
+export declare const TELEMETRY_NOT_ACCEPTED = "telemetry_not_accepted";
+/** The issues of a failed export, summed by signal, kind and status. */
+export declare function telemetryIssueCounts(error: unknown): TelemetryIssueCount[];
+/** `telemetry_not_accepted`, then the issue counts, for example `traces failed 1 (HTTP 400)`. */
+export declare function describeTelemetryIssues(issues: TelemetryIssueCount[]): string;
 /**
  * Runs every case of a frozen experiment through `target` on this machine, completes each
  * execution, scores it with local scorers and uploads the results, checkpointing so an interrupted

package/dist/evals/runner.js

@@ -1,13 +1,15 @@
 import { randomUUID } from "node:crypto";
-import { mkdir } from "node:fs/promises";
+import { rmSync } from "node:fs";
+import { mkdir, rm } from "node:fs/promises";
 import { join, resolve } from "node:path";
 import { ROOT_CONTEXT } from "@opentelemetry/api";
 import { HueExportError } from "../transport.js";
 import { HueApiError } from "./client.js";
 import { loadEnvironmentEvidence } from "./environment-evidence.js";
 import { CheckpointStore } from "./checkpoint.js";
-import { downloadCaseFiles, localOutputFiles, OutputFileError, stageOutputFiles, targetFileRoles, uploadOutputFiles, } from "./files.js";
-import { json, uuid } from "./json.js";
+import { onForcedExit } from "./exit-cleanup.js";
+import { assertSafeFileNames, downloadCaseFiles, downloadCaseInputs, localOutputFiles, verifyCaseFiles, OutputFileError, stageOutputFiles, targetFileRoles, uploadOutputFiles, } from "./files.js";
+import { json, uuid, valueBounds } from "./json.js";
 import { executableHere, persistedScore, scoreLocally, validateScorerBindings } from "./scorers.js";
 import { TargetResult, } from "./types.js";
 /**
@@ -39,6 +41,23 @@ export class OutcomeSerializationError extends Error {
         this.name = "OutcomeSerializationError";
     }
 }
+/** The bound `json` names when a valid output is over what Hue stores for one case. */
+const outputLimits = new Map([
+    ["JSON exceeds byte limit", "bytes"],
+    ["JSON exceeds depth/node limits", "structure"],
+]);
+/** `200000` as `200,000`, without depending on the runtime's locale data. */
+const grouped = (value) => String(value).replace(/\B(?=(\d{3})+(?!\d))/g, ",");
+/** A target's output beyond what Hue stores for one case. Recorded as that case's error (type
+ * `OutputTooLarge`), never thrown: the other cases keep running. */
+class OutputTooLargeError extends Error {
+    constructor(limit) {
+        super(limit === "bytes"
+            ? `The output is larger than ${grouped(valueBounds.bytes)} bytes of JSON, the most Hue stores for one case; return a large result as a generated file`
+            : `The output has more than ${grouped(valueBounds.nodes)} JSON values or nests deeper than ${valueBounds.depth} levels, the most Hue stores for one case`);
+        this.name = "OutputTooLargeError";
+    }
+}
 /** Thrown when cooperative caller cancellation stops target execution. */
 export class TargetCancelledError extends Error {
     constructor() {
@@ -72,6 +91,59 @@ function neededInputFiles(files, versions, options) {
         ? files
         : files.filter((file) => targetFileRoles.includes(file.role));
 }
+/** Remove a case's files; a failure is reported as a warning and never fails the case. */
+async function removeCaseFiles(directory) {
+    try {
+        await rm(directory, { recursive: true, force: true });
+    }
+    catch {
+        process.emitWarning(`Could not remove the case files in ${directory}`, {
+            code: "HUE_CASE_FILES_NOT_REMOVED",
+        });
+    }
+}
+/** Everything of a world case's directory except the staged outputs. */
+const WORK_PARTS = ["inputs", "evaluator-inputs", "work"];
+async function releaseWorldFiles(directory, keepOutputs) {
+    if (!keepOutputs)
+        return removeCaseFiles(directory);
+    for (const part of WORK_PARTS)
+        await removeCaseFiles(join(directory, part));
+}
+function releaseWorldFilesSync(directory, keepOutputs) {
+    for (const path of keepOutputs ? WORK_PARTS.map((part) => join(directory, part)) : [directory])
+        rmSync(path, { recursive: true, force: true });
+}
+/** Stable reason, and first word of the omission reason, of a case failed for its telemetry. */
+export const TELEMETRY_NOT_ACCEPTED = "telemetry_not_accepted";
+/** The issues of a failed export, summed by signal, kind and status. */
+export function telemetryIssueCounts(error) {
+    if (!(error instanceof HueExportError))
+        return [];
+    const counts = new Map();
+    for (const issue of error.issues) {
+        if (issue.kind === "warning")
+            continue;
+        const key = `${issue.signal}:${issue.kind}:${issue.status ?? ""}`;
+        const entry = counts.get(key) ?? {
+            signal: issue.signal,
+            kind: issue.kind,
+            ...(issue.status === undefined ? {} : { status: issue.status }),
+            count: 0,
+        };
+        entry.count += issue.count;
+        counts.set(key, entry);
+    }
+    // A zero-count entry, such as the processor's own flush failure, only repeats a signal that
+    // already has counted records.
+    const counted = new Set([...counts.values()].filter((entry) => entry.count).map((e) => e.signal));
+    return [...counts.values()].filter((entry) => entry.count || !counted.has(entry.signal));
+}
+/** `telemetry_not_accepted`, then the issue counts, for example `traces failed 1 (HTTP 400)`. */
+export function describeTelemetryIssues(issues) {
+    const parts = issues.map((issue) => `${issue.signal} ${issue.kind} ${issue.count}${issue.status === undefined ? "" : ` (HTTP ${issue.status})`}`);
+    return parts.length ? `${TELEMETRY_NOT_ACCEPTED}: ${parts.join(", ")}` : TELEMETRY_NOT_ACCEPTED;
+}
 function settings(options) {
     if (options.environmentEvidence !== undefined &&
         options.environmentEvidence !== "required" &&
@@ -258,7 +330,7 @@ export async function runExperiment(options) {
         const filesRoot = resolve(options.filesDirectory ?? join(options.checkpointDirectory, "files"));
         const sanitize = (message) => message.slice(0, 4000).toWellFormed().replaceAll("\u0000", "");
         const errorPayload = (error) => ({
-            type: "TargetError",
+            type: error instanceof OutputTooLargeError ? "OutputTooLarge" : "TargetError",
             ...(options.persistResultContent && error instanceof Error
                 ? { message: sanitize(error.message) }
                 : {}),
@@ -269,7 +341,7 @@ export async function runExperiment(options) {
         output) {
             const needed = neededInputFiles(frozenCase.inputFiles, versions, options);
             const inputs = needed.length
-                ? await downloadCaseFiles(options.client, needed, join(caseDirectory, "inputs"))
+                ? (await downloadCaseInputs(options.client, needed, caseDirectory)).all
                 : [];
             await uploadOutputFiles(options.client, saved.executionId, saved.files, () => store.write(file, saved));
             const outputs = localOutputFiles(saved.files);
@@ -310,10 +382,25 @@ export async function runExperiment(options) {
             await store.write(file, prepared);
             return prepared;
         }
-        await pool(items, concurrency, async (item) => {
+        // A case pinned to a world keeps its files apart and removes them when it ends: the agent's
+        // copies of its inputs, its work, the evaluator's downloads and the staged outputs.
+        const worldDirectoryOf = (itemId) => join(filesRoot, `world-case-${uuid(itemId)}`);
+        const runCase = async (item, files) => {
             const file = `case-${uuid(item.id)}`;
-            const caseDirectory = join(filesRoot, `case-${uuid(item.id)}`);
-            let checkpoint = await store.read(file);
+            const worldDirectory = worldDirectoryOf(item.id);
+            const caseDirectoryOf = (frozen) => frozen.environmentVersionId ? worldDirectory : join(filesRoot, `case-${uuid(item.id)}`);
+            let checkpoint;
+            try {
+                checkpoint = await store.read(file);
+            }
+            catch (error) {
+                // A checkpoint that is unsafe or fails its integrity check is never resumed from, so its
+                // staged outputs are not kept; an I/O failure may pass, so they are.
+                if (!error.code)
+                    files.keepOutputs = false;
+                throw error;
+            }
+            files.keepOutputs = checkpoint?.stage === "uploading";
             if (checkpoint && checkpoint.stage !== "prepared" && checkpoint.stage !== "uploading") {
                 if (checkpoint.stage === "serialization_failed")
                     throw new OutcomeSerializationError(checkpoint.executionId);
@@ -331,7 +418,8 @@ export async function runExperiment(options) {
                 if (checkpoint.hasOutput && checkpoint.output === undefined)
                     throw new UncertainExecutionError(item.id, checkpoint.executionId);
                 const frozenCase = await options.client.getExperimentCase(experiment.id, item.id);
-                checkpoint = await prepare(file, checkpoint, frozenCase, caseDirectory, checkpoint.output);
+                checkpoint = await prepare(file, checkpoint, frozenCase, caseDirectoryOf(frozenCase), checkpoint.output);
+                files.keepOutputs = false;
             }
             if (!checkpoint) {
                 if (item.execution)
@@ -343,10 +431,20 @@ export async function runExperiment(options) {
                 // target failures and cannot consume a case's execution slot.
                 const targetInputs = json(frozenCase.inputs);
                 const targetConfig = json(experiment.config);
-                // Pinned input files are verified on disk before an execution exists for the same reason.
+                // Pinned input files are verified before an execution exists for the same reason. Only
+                // the agent-visible ones are saved now: evaluator-only files a local code evaluator needs
+                // are checked and dropped, and saved for scoring after the target finished, so they are
+                // not on disk while it runs.
+                const caseDirectory = caseDirectoryOf(frozenCase);
+                const isTargetFile = (entry) => targetFileRoles.includes(entry.role);
                 const needed = neededInputFiles(frozenCase.inputFiles, versions, options);
-                const inputFiles = needed.length
-                    ? await downloadCaseFiles(options.client, needed, join(caseDirectory, "inputs"))
+                const agentFiles = needed.filter(isTargetFile);
+                // An agent working in a world receives its files by name: each must be one safe name.
+                if (frozenCase.environmentVersionId)
+                    assertSafeFileNames(agentFiles);
+                await verifyCaseFiles(options.client, needed.filter((entry) => !isTargetFile(entry)));
+                const inputFiles = agentFiles.length
+                    ? (await downloadCaseInputs(options.client, agentFiles, caseDirectory)).target
                     : [];
                 const outputDirectory = join(caseDirectory, "work");
                 await mkdir(outputDirectory, { recursive: true, mode: 0o700 });
@@ -373,7 +471,7 @@ export async function runExperiment(options) {
                             item: structuredClone(frozenCase),
                             span,
                             executionId: execution.id,
-                            files: structuredClone(inputFiles.filter((entry) => targetFileRoles.includes(entry.role))),
+                            files: structuredClone(inputFiles),
                             outputDirectory,
                         });
                         if (result instanceof TargetResult) {
@@ -395,12 +493,22 @@ export async function runExperiment(options) {
                         try {
                             output = json(output);
                         }
-                        catch {
-                            await store.write(file, {
-                                stage: "serialization_failed",
-                                executionId: execution.id,
-                            });
-                            throw new OutcomeSerializationError(execution.id);
+                        catch (error) {
+                            // An output over the size, count or depth bounds is the target's own failure, as
+                            // a file it cannot deliver is: the case completes as an error and the run goes
+                            // on. Output that is not JSON at all still stops the run for inspection.
+                            const limit = error instanceof RangeError ? outputLimits.get(error.message) : undefined;
+                            if (limit === undefined) {
+                                await store.write(file, {
+                                    stage: "serialization_failed",
+                                    executionId: execution.id,
+                                });
+                                throw new OutcomeSerializationError(execution.id);
+                            }
+                            state = "error";
+                            output = undefined;
+                            targetError = new OutputTooLargeError(limit);
+                            options.hue.recordError(span.span, targetError);
                         }
                     }
                     if (output !== undefined)
@@ -433,6 +541,7 @@ export async function runExperiment(options) {
                     };
                     // Without persisted result content a restart cannot reconstruct the outcome; the
                     // saved stage then reports the execution as uncertain instead of guessing.
+                    files.keepOutputs = true;
                     await store.write(file, uploading);
                     return prepare(file, uploading, frozenCase, caseDirectory, output);
                 }, {
@@ -440,7 +549,9 @@ export async function runExperiment(options) {
                     input: frozenCase.inputs,
                     attributes: { "hue.experiment.id": experiment.id, "hue.dataset.case.id": item.id },
                 });
+                files.keepOutputs = false;
                 // End root before waiting for both OTLP signals. Export failure leaves the prepared checkpoint intact.
+                let exportError;
                 try {
                     await options.hue.flush();
                     // Another concurrent flush may already have surfaced this failure. OTLP
@@ -449,29 +560,96 @@ export async function runExperiment(options) {
                         throw new HueExportError(options.hue.transport
                             .getIssues()
                             .filter((issue) => issue.sequence > failureSequenceBefore && issue.kind !== "warning"), options.hue.transport.getReport());
-                    checkpoint.exportState = "accepted";
-                    await store.write(file, checkpoint);
+                }
+                catch (error) {
+                    exportError = error;
+                }
+                try {
+                    if (exportError === undefined) {
+                        checkpoint.exportState = "accepted";
+                        await store.write(file, checkpoint);
+                    }
                 }
                 catch (error) {
                     // The explicitly chosen omission policy can complete without acknowledged telemetry.
                     if (options.traceEvidence.mode !== "omit")
                         throw error;
                 }
+                if (exportError !== undefined && options.traceEvidence.mode !== "omit") {
+                    if (options.traceNotAccepted !== "fail_case")
+                        throw exportError;
+                    // Required evidence that Hue did not accept fails the case rather than leaving its
+                    // execution started: the outcome is kept, the evidence is declared omitted. Only an
+                    // export failure takes this path; a checkpoint that cannot be saved is raised above.
+                    const failed = checkpoint;
+                    const issues = telemetryIssueCounts(exportError);
+                    // Without its evidence the case fails whatever it answered: the output and generated
+                    // files are not attached, so no scorer can pass it, and local scores taken of them go.
+                    const { traceEvidence: _required, output: _output, artifactIds: _artifacts, primaryArtifactId: _primary, ...complete } = failed.complete;
+                    failed.scores = [];
+                    if (complete.state === "succeeded") {
+                        complete.state = "error";
+                        complete.error = {
+                            type: "TelemetryNotAccepted",
+                            ...(options.persistResultContent ? { message: describeTelemetryIssues(issues) } : {}),
+                        };
+                    }
+                    failed.complete = {
+                        ...complete,
+                        traceEvidence: "omit",
+                        omissionReason: describeTelemetryIssues(issues).slice(0, 4000),
+                    };
+                    failed.exportState = "not_accepted";
+                    failed.telemetryIssues = issues;
+                    await store.write(file, failed);
+                }
             }
             const prepared = checkpoint;
             await options.client.getExecution(prepared.executionId);
             if (prepared.exportState !== "accepted" && prepared.complete.traceEvidence !== "omit")
                 throw new Error("Target outcome is saved but trace export acknowledgement is unavailable. Restore/export the trace or explicitly complete with omitted evidence through the client; never rerun the target.");
             const save = () => store.write(file, prepared);
+            const notAccepted = prepared.exportState === "not_accepted"
+                ? {
+                    caseId: item.id,
+                    caseKey: item.externalKey,
+                    executionId: prepared.executionId,
+                    issues: prepared.telemetryIssues ?? [],
+                }
+                : undefined;
             if (!prepared.completion) {
                 prepared.completion = await options.client.completeExecution(prepared.executionId, prepared.complete);
                 for (const score of prepared.scores)
                     score.payload.evaluationItemId = prepared.completion.evaluationItemId;
                 await save();
+                // Once per failed case: a resumed call finds the completion and only reports it.
+                if (notAccepted)
+                    await options.onTelemetryNotAccepted?.(notAccepted);
             }
+            if (notAccepted)
+                (report.telemetryNotAccepted ??= []).push(notAccepted);
             const results = await uploadScores(options, experiment.evaluation.id, prepared.scores, save, versions);
             report.subjectIds.push(prepared.completion.subjectId);
             report.resultIds.push(...results);
+        };
+        await pool(items, concurrency, async (item) => {
+            // A world case's directory goes however the case ends, a forced exit included. An
+            // interrupted upload keeps only the staged outputs it resumes from; inputs are downloaded
+            // anew for scoring. A direct case has no such directory, so nothing is removed for it.
+            const worldDirectory = worldDirectoryOf(item.id);
+            const files = { keepOutputs: true };
+            const untrack = onForcedExit(() => releaseWorldFilesSync(worldDirectory, files.keepOutputs));
+            try {
+                await runCase(item, files);
+                await releaseWorldFiles(worldDirectory, false);
+            }
+            catch (error) {
+                await releaseWorldFiles(worldDirectory, files.keepOutputs);
+                throw error;
+            }
+            finally {
+                untrack();
+            }
         });
         let finish = await store.read("finish");
         if (!finish) {

package/dist/evals/scenarios.d.ts

@@ -1,4 +1,4 @@
-import type { EvaluationClient } from "./client.js";
+import { type EvaluationClient } from "./client.js";
 import type { CaseConversion, CaseConversionSummary, Page, PageOptions } from "./types.js";
 /** Immutable pins resolved from a published Scenario or a saved eval set. */
 export interface ScenarioPins {
@@ -59,11 +59,13 @@ export interface NameMatch<T extends NamedCandidate> {
 /** Case-insensitive exact matches first, then unique prefix/substring matches. */
 export declare function matchByName<T extends NamedCandidate>(candidates: T[], name: string): NameMatch<T>;
 /**
- * Resolves a published Scenario's immutable pins from its ID, its Hue URL or its name. A name
- * matches the dataset name of published Scenarios case-insensitively: exact matches first, then
- * a unique prefix or substring.
+ * Resolves a published Scenario's immutable pins from its ID, the ID of the eval set case it
+ * published, a Hue URL naming either (`/scenarios/<id>`, `/case-conversions/<id>` or
+ * `/cases/<id>`) or its name. A name matches the dataset name of published Scenarios
+ * case-insensitively: exact matches first, then a unique prefix or substring.
  *
- * @throws Error when no Scenario matches, several match, or the Scenario is an unpublished draft.
+ * @throws Error when no Scenario matches, several match, or the Scenario is an unpublished draft;
+ * {@link HueApiError} with status 404 when neither a Scenario nor a published case has the ID.
  */
 export declare function resolveScenarioPins(client: ScenarioClient, selector: string): Promise<ScenarioPins>;
 /**

package/dist/evals/scenarios.js

@@ -1,6 +1,10 @@
+import { HueApiError } from "./client.js";
 const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
-/** Scenarios listed while resolving a name; bounds the registry reads of one selection. */
+/** Published Scenarios a name is matched against; bounds the registry reads of one selection. */
 const MAX_LISTED = 200;
+/** Scenarios, drafts included, one selection lists at most: a case ID is found among them, and a
+ * mistyped one ends after a bounded number of reads. */
+const MAX_SCANNED = 1000;
 /** Lists Scenarios of the project; requires a Read and write key. */
 export function listScenarios(client, page) {
     return client.listCaseConversions(page);
@@ -53,17 +57,57 @@ export function matchByName(candidates, name) {
         exact: false,
     };
 }
-async function listPublishedScenarios(client) {
-    const items = [];
+/** Published Scenario summaries in listing order, page by page, ending after
+ * {@link MAX_SCANNED} Scenarios, at an empty page or when the listing repeats a cursor. */
+async function* publishedScenarios(client) {
+    const cursors = new Set();
+    let listed = 0;
     let after;
     for (;;) {
         const page = await client.listCaseConversions({ after, limit: 100 });
-        items.push(...page.items.filter((item) => item.status === "published"));
-        if (!page.nextCursor || items.length >= MAX_LISTED)
-            return items.slice(0, MAX_LISTED);
+        for (const item of page.items) {
+            if (++listed > MAX_SCANNED)
+                return;
+            if (item.status === "published")
+                yield item;
+        }
+        if (!page.items.length || !page.nextCursor || cursors.has(page.nextCursor))
+            return;
+        cursors.add(page.nextCursor);
         after = page.nextCursor;
     }
 }
+async function listPublishedScenarios(client) {
+    const items = [];
+    for await (const item of publishedScenarios(client))
+        if (items.push(item) >= MAX_LISTED)
+            break;
+    return items;
+}
+/** A listed Scenario, or undefined when it can no longer be read (removed since the listing). */
+async function readListed(client, id) {
+    try {
+        return await client.getCaseConversion(id);
+    }
+    catch (error) {
+        if (error instanceof HueApiError && error.status === 404)
+            return undefined;
+        throw error;
+    }
+}
+/**
+ * The published Scenario whose eval set case has this ID, the one Hue shows on the case page.
+ * Published Scenarios are read in listing order until one matches, past the bound a name search
+ * uses; undefined when none does within {@link MAX_SCANNED}.
+ */
+async function scenarioOfPublishedCase(client, caseId) {
+    for await (const summary of publishedScenarios(client)) {
+        const scenario = await readListed(client, summary.id);
+        if (scenario?.publication?.caseId.toLowerCase() === caseId)
+            return scenario;
+    }
+    return undefined;
+}
 const describe = (candidates) => candidates.map((candidate) => `${candidate.name} (${candidate.id})`).join(", ");
 async function pinsFromScenario(client, scenario, dataset) {
     if (!scenario.publication)
@@ -84,16 +128,29 @@ async function pinsFromScenario(client, scenario, dataset) {
     };
 }
 /**
- * Resolves a published Scenario's immutable pins from its ID, its Hue URL or its name. A name
- * matches the dataset name of published Scenarios case-insensitively: exact matches first, then
- * a unique prefix or substring.
+ * Resolves a published Scenario's immutable pins from its ID, the ID of the eval set case it
+ * published, a Hue URL naming either (`/scenarios/<id>`, `/case-conversions/<id>` or
+ * `/cases/<id>`) or its name. A name matches the dataset name of published Scenarios
+ * case-insensitively: exact matches first, then a unique prefix or substring.
  *
- * @throws Error when no Scenario matches, several match, or the Scenario is an unpublished draft.
+ * @throws Error when no Scenario matches, several match, or the Scenario is an unpublished draft;
+ * {@link HueApiError} with status 404 when neither a Scenario nor a published case has the ID.
  */
 export async function resolveScenarioPins(client, selector) {
-    const parsed = parseScenarioSelector(selector);
+    const parsed = parseScenarioSelector(selector, ["scenarios", "case-conversions", "cases"]);
     if (parsed.kind === "id") {
-        const scenario = await client.getCaseConversion(parsed.id);
+        let scenario;
+        try {
+            scenario = await client.getCaseConversion(parsed.id);
+        }
+        catch (error) {
+            if (!(error instanceof HueApiError && error.status === 404))
+                throw error;
+            const published = await scenarioOfPublishedCase(client, parsed.id);
+            if (!published)
+                throw error;
+            scenario = published;
+        }
         if (!scenario.publication)
             throw new Error(`Scenario ${scenario.id} is a draft without published pins; publish it in Hue first`);
         return pinsFromScenario(client, scenario, await client.getDataset(scenario.publication.datasetId));
@@ -102,8 +159,8 @@ export async function resolveScenarioPins(client, selector) {
     const datasets = new Map();
     const candidates = [];
     for (const summary of published) {
-        const scenario = await client.getCaseConversion(summary.id);
-        if (!scenario.publication)
+        const scenario = await readListed(client, summary.id);
+        if (!scenario?.publication)
             continue;
         let dataset = datasets.get(scenario.publication.datasetId);
         if (!dataset) {