@hue-run/sdk 0.8.1 → 0.10.0

package/dist/evals/local-worker.js

@@ -10,6 +10,10 @@ export const localAgentCapabilities = {
     /** Ordinary cases run directly on this machine: JSON inputs and pinned input files in,
      * JSON output and generated files out. */
     direct: "direct:v1",
+    /** Cases pinned to a world that also carry agent-visible input files: `target` receives them
+     * with the world and may return generated files. Declare it with `input:<extension>` for each
+     * accepted file type; it requires `target`. */
+    environmentFiles: "environment-files:v1",
 };
 /** Allowlist the candidate surface for an ordinary case; frozen expectations stay with grading. */
 export function localAgentDirectContext(context, signal) {
@@ -44,6 +48,8 @@ function localAgentTargetContext(context) {
         ...(context.connectionBundle
             ? { connectionBundle: structuredClone(context.connectionBundle) }
             : {}),
+        files: structuredClone(context.files),
+        outputDirectory: context.outputDirectory,
     };
 }
 /** The registration's capabilities: explicit values plus one per supplied callback. */
@@ -55,6 +61,8 @@ export function registeredCapabilities(options) {
         throw new TypeError("direct:v1 requires a directTarget callback");
     if (declared.includes(localAgentCapabilities.environment) && !options.target)
         throw new TypeError("environment:v1 requires a target callback");
+    if (declared.includes(localAgentCapabilities.environmentFiles) && !options.target)
+        throw new TypeError("environment-files:v1 requires a target callback");
     return [
         ...new Set([
             ...declared,
@@ -163,6 +171,12 @@ export async function runLocalAgent(options) {
                     // Worker dispatch uses the case pin: directTarget never receives a world.
                     environmentEvidence: options.directTarget ? "when_pinned" : "required",
                     traceEvidence: { mode: "required" },
+                    ...(options.traceNotAccepted ? { traceNotAccepted: options.traceNotAccepted } : {}),
+                    ...(options.onTelemetryNotAccepted
+                        ? {
+                            onTelemetryNotAccepted: (entry) => options.onTelemetryNotAccepted(entry),
+                        }
+                        : {}),
                     scorers: options.scorers,
                     concurrency: options.concurrency,
                     target: (inputs, context) => {

package/dist/evals/runner.d.ts

@@ -93,6 +93,19 @@ export interface RunExperimentOptions extends RunnerOptions {
         /** Why evidence is omitted, up to 4000 characters. */
         reason: string;
     };
+    /**
+     * What a case does when evidence is required and its trace or logs are not fully accepted.
+     * `"stop"` (the default) keeps the saved outcome, leaves the execution started for recovery and
+     * rejects. `"fail_case"` completes the execution as failed instead, with the evidence omitted
+     * under the reason `telemetry_not_accepted` and without its output or generated files, so no
+     * scorer can pass it; reports it in {@link RunnerReport.telemetryNotAccepted} and goes on with
+     * the other cases.
+     */
+    traceNotAccepted?: "stop" | "fail_case";
+    /** Called when this call completes a case as failed under `traceNotAccepted: "fail_case"`,
+     * before the run goes on, so the failure is known even if a later case stops the run. A call
+     * that resumes an already completed case lists it in the report without calling again. */
+    onTelemetryNotAccepted?(entry: TelemetryNotAccepted): void | Promise<void>;
     /** Runs the application for one frozen case; return the output, `withFiles(output, files)`
      * when it generated files, or `undefined` when unavailable. */
     target(inputs: JsonValue, context: RunExperimentTargetContext): JsonValue | TargetResult | undefined | Promise<JsonValue | TargetResult | undefined>;
@@ -112,7 +125,39 @@ export interface RunnerReport {
     resultIds: string[];
     /** Pins without a local implementation, left pending for their authorized executor. */
     deferredScorerVersionIds: string[];
+    /** Cases completed as failed because their telemetry was not accepted, under
+     * `traceNotAccepted: "fail_case"`; absent when there were none. */
+    telemetryNotAccepted?: TelemetryNotAccepted[];
 }
+/** Export issue counts, never content: which signal, what happened, the HTTP status when Hue
+ * answered, and how many records. */
+export interface TelemetryIssueCount {
+    /** Signal the records belong to. */
+    signal: "traces" | "logs";
+    /** `rejected` by Hue, `failed` to deliver, `dropped` from the queue, or an `invalid` record. */
+    kind: "rejected" | "failed" | "dropped" | "invalid";
+    /** HTTP status when the issue came from Hue's answer. */
+    status?: number;
+    /** Records affected. */
+    count: number;
+}
+/** A case whose trace or logs were not fully accepted, completed as failed. */
+export interface TelemetryNotAccepted {
+    /** Experiment case (item) ID. */
+    caseId: string;
+    /** The case's caller-chosen key. */
+    caseKey: string;
+    /** Execution completed as failed. */
+    executionId: string;
+    /** Sanitized issues the transport reported while the case's telemetry was exported. */
+    issues: TelemetryIssueCount[];
+}
+/** Stable reason, and first word of the omission reason, of a case failed for its telemetry. */
+export declare const TELEMETRY_NOT_ACCEPTED = "telemetry_not_accepted";
+/** The issues of a failed export, summed by signal, kind and status. */
+export declare function telemetryIssueCounts(error: unknown): TelemetryIssueCount[];
+/** `telemetry_not_accepted`, then the issue counts, for example `traces failed 1 (HTTP 400)`. */
+export declare function describeTelemetryIssues(issues: TelemetryIssueCount[]): string;
 /**
  * Runs every case of a frozen experiment through `target` on this machine, completes each
  * execution, scores it with local scorers and uploads the results, checkpointing so an interrupted

package/dist/evals/runner.js

@@ -1,12 +1,14 @@
 import { randomUUID } from "node:crypto";
-import { mkdir } from "node:fs/promises";
+import { rmSync } from "node:fs";
+import { mkdir, rm } from "node:fs/promises";
 import { join, resolve } from "node:path";
 import { ROOT_CONTEXT } from "@opentelemetry/api";
 import { HueExportError } from "../transport.js";
 import { HueApiError } from "./client.js";
 import { loadEnvironmentEvidence } from "./environment-evidence.js";
 import { CheckpointStore } from "./checkpoint.js";
-import { downloadCaseFiles, localOutputFiles, OutputFileError, stageOutputFiles, targetFileRoles, uploadOutputFiles, } from "./files.js";
+import { onForcedExit } from "./exit-cleanup.js";
+import { assertSafeFileNames, downloadCaseFiles, downloadCaseInputs, localOutputFiles, verifyCaseFiles, OutputFileError, stageOutputFiles, targetFileRoles, uploadOutputFiles, } from "./files.js";
 import { json, uuid } from "./json.js";
 import { executableHere, persistedScore, scoreLocally, validateScorerBindings } from "./scorers.js";
 import { TargetResult, } from "./types.js";
@@ -72,6 +74,59 @@ function neededInputFiles(files, versions, options) {
         ? files
         : files.filter((file) => targetFileRoles.includes(file.role));
 }
+/** Remove a case's files; a failure is reported as a warning and never fails the case. */
+async function removeCaseFiles(directory) {
+    try {
+        await rm(directory, { recursive: true, force: true });
+    }
+    catch {
+        process.emitWarning(`Could not remove the case files in ${directory}`, {
+            code: "HUE_CASE_FILES_NOT_REMOVED",
+        });
+    }
+}
+/** Everything of a world case's directory except the staged outputs. */
+const WORK_PARTS = ["inputs", "evaluator-inputs", "work"];
+async function releaseWorldFiles(directory, keepOutputs) {
+    if (!keepOutputs)
+        return removeCaseFiles(directory);
+    for (const part of WORK_PARTS)
+        await removeCaseFiles(join(directory, part));
+}
+function releaseWorldFilesSync(directory, keepOutputs) {
+    for (const path of keepOutputs ? WORK_PARTS.map((part) => join(directory, part)) : [directory])
+        rmSync(path, { recursive: true, force: true });
+}
+/** Stable reason, and first word of the omission reason, of a case failed for its telemetry. */
+export const TELEMETRY_NOT_ACCEPTED = "telemetry_not_accepted";
+/** The issues of a failed export, summed by signal, kind and status. */
+export function telemetryIssueCounts(error) {
+    if (!(error instanceof HueExportError))
+        return [];
+    const counts = new Map();
+    for (const issue of error.issues) {
+        if (issue.kind === "warning")
+            continue;
+        const key = `${issue.signal}:${issue.kind}:${issue.status ?? ""}`;
+        const entry = counts.get(key) ?? {
+            signal: issue.signal,
+            kind: issue.kind,
+            ...(issue.status === undefined ? {} : { status: issue.status }),
+            count: 0,
+        };
+        entry.count += issue.count;
+        counts.set(key, entry);
+    }
+    // A zero-count entry, such as the processor's own flush failure, only repeats a signal that
+    // already has counted records.
+    const counted = new Set([...counts.values()].filter((entry) => entry.count).map((e) => e.signal));
+    return [...counts.values()].filter((entry) => entry.count || !counted.has(entry.signal));
+}
+/** `telemetry_not_accepted`, then the issue counts, for example `traces failed 1 (HTTP 400)`. */
+export function describeTelemetryIssues(issues) {
+    const parts = issues.map((issue) => `${issue.signal} ${issue.kind} ${issue.count}${issue.status === undefined ? "" : ` (HTTP ${issue.status})`}`);
+    return parts.length ? `${TELEMETRY_NOT_ACCEPTED}: ${parts.join(", ")}` : TELEMETRY_NOT_ACCEPTED;
+}
 function settings(options) {
     if (options.environmentEvidence !== undefined &&
         options.environmentEvidence !== "required" &&
@@ -269,7 +324,7 @@ export async function runExperiment(options) {
         output) {
             const needed = neededInputFiles(frozenCase.inputFiles, versions, options);
             const inputs = needed.length
-                ? await downloadCaseFiles(options.client, needed, join(caseDirectory, "inputs"))
+                ? (await downloadCaseInputs(options.client, needed, caseDirectory)).all
                 : [];
             await uploadOutputFiles(options.client, saved.executionId, saved.files, () => store.write(file, saved));
             const outputs = localOutputFiles(saved.files);
@@ -310,10 +365,25 @@ export async function runExperiment(options) {
             await store.write(file, prepared);
             return prepared;
         }
-        await pool(items, concurrency, async (item) => {
+        // A case pinned to a world keeps its files apart and removes them when it ends: the agent's
+        // copies of its inputs, its work, the evaluator's downloads and the staged outputs.
+        const worldDirectoryOf = (itemId) => join(filesRoot, `world-case-${uuid(itemId)}`);
+        const runCase = async (item, files) => {
             const file = `case-${uuid(item.id)}`;
-            const caseDirectory = join(filesRoot, `case-${uuid(item.id)}`);
-            let checkpoint = await store.read(file);
+            const worldDirectory = worldDirectoryOf(item.id);
+            const caseDirectoryOf = (frozen) => frozen.environmentVersionId ? worldDirectory : join(filesRoot, `case-${uuid(item.id)}`);
+            let checkpoint;
+            try {
+                checkpoint = await store.read(file);
+            }
+            catch (error) {
+                // A checkpoint that is unsafe or fails its integrity check is never resumed from, so its
+                // staged outputs are not kept; an I/O failure may pass, so they are.
+                if (!error.code)
+                    files.keepOutputs = false;
+                throw error;
+            }
+            files.keepOutputs = checkpoint?.stage === "uploading";
             if (checkpoint && checkpoint.stage !== "prepared" && checkpoint.stage !== "uploading") {
                 if (checkpoint.stage === "serialization_failed")
                     throw new OutcomeSerializationError(checkpoint.executionId);
@@ -331,7 +401,8 @@ export async function runExperiment(options) {
                 if (checkpoint.hasOutput && checkpoint.output === undefined)
                     throw new UncertainExecutionError(item.id, checkpoint.executionId);
                 const frozenCase = await options.client.getExperimentCase(experiment.id, item.id);
-                checkpoint = await prepare(file, checkpoint, frozenCase, caseDirectory, checkpoint.output);
+                checkpoint = await prepare(file, checkpoint, frozenCase, caseDirectoryOf(frozenCase), checkpoint.output);
+                files.keepOutputs = false;
             }
             if (!checkpoint) {
                 if (item.execution)
@@ -343,10 +414,20 @@ export async function runExperiment(options) {
                 // target failures and cannot consume a case's execution slot.
                 const targetInputs = json(frozenCase.inputs);
                 const targetConfig = json(experiment.config);
-                // Pinned input files are verified on disk before an execution exists for the same reason.
+                // Pinned input files are verified before an execution exists for the same reason. Only
+                // the agent-visible ones are saved now: evaluator-only files a local code evaluator needs
+                // are checked and dropped, and saved for scoring after the target finished, so they are
+                // not on disk while it runs.
+                const caseDirectory = caseDirectoryOf(frozenCase);
+                const isTargetFile = (entry) => targetFileRoles.includes(entry.role);
                 const needed = neededInputFiles(frozenCase.inputFiles, versions, options);
-                const inputFiles = needed.length
-                    ? await downloadCaseFiles(options.client, needed, join(caseDirectory, "inputs"))
+                const agentFiles = needed.filter(isTargetFile);
+                // An agent working in a world receives its files by name: each must be one safe name.
+                if (frozenCase.environmentVersionId)
+                    assertSafeFileNames(agentFiles);
+                await verifyCaseFiles(options.client, needed.filter((entry) => !isTargetFile(entry)));
+                const inputFiles = agentFiles.length
+                    ? (await downloadCaseInputs(options.client, agentFiles, caseDirectory)).target
                     : [];
                 const outputDirectory = join(caseDirectory, "work");
                 await mkdir(outputDirectory, { recursive: true, mode: 0o700 });
@@ -373,7 +454,7 @@ export async function runExperiment(options) {
                             item: structuredClone(frozenCase),
                             span,
                             executionId: execution.id,
-                            files: structuredClone(inputFiles.filter((entry) => targetFileRoles.includes(entry.role))),
+                            files: structuredClone(inputFiles),
                             outputDirectory,
                         });
                         if (result instanceof TargetResult) {
@@ -433,6 +514,7 @@ export async function runExperiment(options) {
                     };
                     // Without persisted result content a restart cannot reconstruct the outcome; the
                     // saved stage then reports the execution as uncertain instead of guessing.
+                    files.keepOutputs = true;
                     await store.write(file, uploading);
                     return prepare(file, uploading, frozenCase, caseDirectory, output);
                 }, {
@@ -440,7 +522,9 @@ export async function runExperiment(options) {
                     input: frozenCase.inputs,
                     attributes: { "hue.experiment.id": experiment.id, "hue.dataset.case.id": item.id },
                 });
+                files.keepOutputs = false;
                 // End root before waiting for both OTLP signals. Export failure leaves the prepared checkpoint intact.
+                let exportError;
                 try {
                     await options.hue.flush();
                     // Another concurrent flush may already have surfaced this failure. OTLP
@@ -449,29 +533,96 @@ export async function runExperiment(options) {
                         throw new HueExportError(options.hue.transport
                             .getIssues()
                             .filter((issue) => issue.sequence > failureSequenceBefore && issue.kind !== "warning"), options.hue.transport.getReport());
-                    checkpoint.exportState = "accepted";
-                    await store.write(file, checkpoint);
+                }
+                catch (error) {
+                    exportError = error;
+                }
+                try {
+                    if (exportError === undefined) {
+                        checkpoint.exportState = "accepted";
+                        await store.write(file, checkpoint);
+                    }
                 }
                 catch (error) {
                     // The explicitly chosen omission policy can complete without acknowledged telemetry.
                     if (options.traceEvidence.mode !== "omit")
                         throw error;
                 }
+                if (exportError !== undefined && options.traceEvidence.mode !== "omit") {
+                    if (options.traceNotAccepted !== "fail_case")
+                        throw exportError;
+                    // Required evidence that Hue did not accept fails the case rather than leaving its
+                    // execution started: the outcome is kept, the evidence is declared omitted. Only an
+                    // export failure takes this path; a checkpoint that cannot be saved is raised above.
+                    const failed = checkpoint;
+                    const issues = telemetryIssueCounts(exportError);
+                    // Without its evidence the case fails whatever it answered: the output and generated
+                    // files are not attached, so no scorer can pass it, and local scores taken of them go.
+                    const { traceEvidence: _required, output: _output, artifactIds: _artifacts, primaryArtifactId: _primary, ...complete } = failed.complete;
+                    failed.scores = [];
+                    if (complete.state === "succeeded") {
+                        complete.state = "error";
+                        complete.error = {
+                            type: "TelemetryNotAccepted",
+                            ...(options.persistResultContent ? { message: describeTelemetryIssues(issues) } : {}),
+                        };
+                    }
+                    failed.complete = {
+                        ...complete,
+                        traceEvidence: "omit",
+                        omissionReason: describeTelemetryIssues(issues).slice(0, 4000),
+                    };
+                    failed.exportState = "not_accepted";
+                    failed.telemetryIssues = issues;
+                    await store.write(file, failed);
+                }
             }
             const prepared = checkpoint;
             await options.client.getExecution(prepared.executionId);
             if (prepared.exportState !== "accepted" && prepared.complete.traceEvidence !== "omit")
                 throw new Error("Target outcome is saved but trace export acknowledgement is unavailable. Restore/export the trace or explicitly complete with omitted evidence through the client; never rerun the target.");
             const save = () => store.write(file, prepared);
+            const notAccepted = prepared.exportState === "not_accepted"
+                ? {
+                    caseId: item.id,
+                    caseKey: item.externalKey,
+                    executionId: prepared.executionId,
+                    issues: prepared.telemetryIssues ?? [],
+                }
+                : undefined;
             if (!prepared.completion) {
                 prepared.completion = await options.client.completeExecution(prepared.executionId, prepared.complete);
                 for (const score of prepared.scores)
                     score.payload.evaluationItemId = prepared.completion.evaluationItemId;
                 await save();
+                // Once per failed case: a resumed call finds the completion and only reports it.
+                if (notAccepted)
+                    await options.onTelemetryNotAccepted?.(notAccepted);
             }
+            if (notAccepted)
+                (report.telemetryNotAccepted ??= []).push(notAccepted);
             const results = await uploadScores(options, experiment.evaluation.id, prepared.scores, save, versions);
             report.subjectIds.push(prepared.completion.subjectId);
             report.resultIds.push(...results);
+        };
+        await pool(items, concurrency, async (item) => {
+            // A world case's directory goes however the case ends, a forced exit included. An
+            // interrupted upload keeps only the staged outputs it resumes from; inputs are downloaded
+            // anew for scoring. A direct case has no such directory, so nothing is removed for it.
+            const worldDirectory = worldDirectoryOf(item.id);
+            const files = { keepOutputs: true };
+            const untrack = onForcedExit(() => releaseWorldFilesSync(worldDirectory, files.keepOutputs));
+            try {
+                await runCase(item, files);
+                await releaseWorldFiles(worldDirectory, false);
+            }
+            catch (error) {
+                await releaseWorldFiles(worldDirectory, files.keepOutputs);
+                throw error;
+            }
+            finally {
+                untrack();
+            }
         });
         let finish = await store.read("finish");
         if (!finish) {

package/dist/evals/scenarios.d.ts

@@ -1,4 +1,4 @@
-import type { EvaluationClient } from "./client.js";
+import { type EvaluationClient } from "./client.js";
 import type { CaseConversion, CaseConversionSummary, Page, PageOptions } from "./types.js";
 /** Immutable pins resolved from a published Scenario or a saved eval set. */
 export interface ScenarioPins {
@@ -10,7 +10,7 @@ export interface ScenarioPins {
     datasetId: string;
     /** Pinned dataset version; frozen only when `saved` is true. */
     datasetVersionId: string;
-    /** Pinned scorer versions; a Scenario pins exactly one. */
+    /** Pinned scorer versions; a Scenario's publication lists them, its outcome scorer first. */
     scorerVersionIds: string[];
     /** Pinned simulated-world version, or `null` when the selection does not pin one. */
     environmentVersionId: string | null;
@@ -59,11 +59,13 @@ export interface NameMatch<T extends NamedCandidate> {
 /** Case-insensitive exact matches first, then unique prefix/substring matches. */
 export declare function matchByName<T extends NamedCandidate>(candidates: T[], name: string): NameMatch<T>;
 /**
- * Resolves a published Scenario's immutable pins from its ID, its Hue URL or its name. A name
- * matches the dataset name of published Scenarios case-insensitively: exact matches first, then
- * a unique prefix or substring.
+ * Resolves a published Scenario's immutable pins from its ID, the ID of the eval set case it
+ * published, a Hue URL naming either (`/scenarios/<id>`, `/case-conversions/<id>` or
+ * `/cases/<id>`) or its name. A name matches the dataset name of published Scenarios
+ * case-insensitively: exact matches first, then a unique prefix or substring.
  *
- * @throws Error when no Scenario matches, several match, or the Scenario is an unpublished draft.
+ * @throws Error when no Scenario matches, several match, or the Scenario is an unpublished draft;
+ * {@link HueApiError} with status 404 when neither a Scenario nor a published case has the ID.
  */
 export declare function resolveScenarioPins(client: ScenarioClient, selector: string): Promise<ScenarioPins>;
 /**

package/dist/evals/scenarios.js

@@ -1,6 +1,10 @@
+import { HueApiError } from "./client.js";
 const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
-/** Scenarios listed while resolving a name; bounds the registry reads of one selection. */
+/** Published Scenarios a name is matched against; bounds the registry reads of one selection. */
 const MAX_LISTED = 200;
+/** Scenarios, drafts included, one selection lists at most: a case ID is found among them, and a
+ * mistyped one ends after a bounded number of reads. */
+const MAX_SCANNED = 1000;
 /** Lists Scenarios of the project; requires a Read and write key. */
 export function listScenarios(client, page) {
     return client.listCaseConversions(page);
@@ -53,17 +57,57 @@ export function matchByName(candidates, name) {
         exact: false,
     };
 }
-async function listPublishedScenarios(client) {
-    const items = [];
+/** Published Scenario summaries in listing order, page by page, ending after
+ * {@link MAX_SCANNED} Scenarios, at an empty page or when the listing repeats a cursor. */
+async function* publishedScenarios(client) {
+    const cursors = new Set();
+    let listed = 0;
     let after;
     for (;;) {
         const page = await client.listCaseConversions({ after, limit: 100 });
-        items.push(...page.items.filter((item) => item.status === "published"));
-        if (!page.nextCursor || items.length >= MAX_LISTED)
-            return items.slice(0, MAX_LISTED);
+        for (const item of page.items) {
+            if (++listed > MAX_SCANNED)
+                return;
+            if (item.status === "published")
+                yield item;
+        }
+        if (!page.items.length || !page.nextCursor || cursors.has(page.nextCursor))
+            return;
+        cursors.add(page.nextCursor);
         after = page.nextCursor;
     }
 }
+async function listPublishedScenarios(client) {
+    const items = [];
+    for await (const item of publishedScenarios(client))
+        if (items.push(item) >= MAX_LISTED)
+            break;
+    return items;
+}
+/** A listed Scenario, or undefined when it can no longer be read (removed since the listing). */
+async function readListed(client, id) {
+    try {
+        return await client.getCaseConversion(id);
+    }
+    catch (error) {
+        if (error instanceof HueApiError && error.status === 404)
+            return undefined;
+        throw error;
+    }
+}
+/**
+ * The published Scenario whose eval set case has this ID, the one Hue shows on the case page.
+ * Published Scenarios are read in listing order until one matches, past the bound a name search
+ * uses; undefined when none does within {@link MAX_SCANNED}.
+ */
+async function scenarioOfPublishedCase(client, caseId) {
+    for await (const summary of publishedScenarios(client)) {
+        const scenario = await readListed(client, summary.id);
+        if (scenario?.publication?.caseId.toLowerCase() === caseId)
+            return scenario;
+    }
+    return undefined;
+}
 const describe = (candidates) => candidates.map((candidate) => `${candidate.name} (${candidate.id})`).join(", ");
 async function pinsFromScenario(client, scenario, dataset) {
     if (!scenario.publication)
@@ -74,23 +118,39 @@ async function pinsFromScenario(client, scenario, dataset) {
         name: dataset.name,
         datasetId: scenario.publication.datasetId,
         datasetVersionId: version.id,
-        scorerVersionIds: [scenario.publication.scorerVersionId],
+        // Publications list every pin, the outcome scorer first; older ones carry the single field.
+        scorerVersionIds: scenario.publication.scorerVersionIds?.length
+            ? [...new Set(scenario.publication.scorerVersionIds)]
+            : [scenario.publication.scorerVersionId],
         environmentVersionId: scenario.publication.environmentVersionId,
         saved: version.frozenAt !== null,
         revision: version.revision,
     };
 }
 /**
- * Resolves a published Scenario's immutable pins from its ID, its Hue URL or its name. A name
- * matches the dataset name of published Scenarios case-insensitively: exact matches first, then
- * a unique prefix or substring.
+ * Resolves a published Scenario's immutable pins from its ID, the ID of the eval set case it
+ * published, a Hue URL naming either (`/scenarios/<id>`, `/case-conversions/<id>` or
+ * `/cases/<id>`) or its name. A name matches the dataset name of published Scenarios
+ * case-insensitively: exact matches first, then a unique prefix or substring.
  *
- * @throws Error when no Scenario matches, several match, or the Scenario is an unpublished draft.
+ * @throws Error when no Scenario matches, several match, or the Scenario is an unpublished draft;
+ * {@link HueApiError} with status 404 when neither a Scenario nor a published case has the ID.
  */
 export async function resolveScenarioPins(client, selector) {
-    const parsed = parseScenarioSelector(selector);
+    const parsed = parseScenarioSelector(selector, ["scenarios", "case-conversions", "cases"]);
     if (parsed.kind === "id") {
-        const scenario = await client.getCaseConversion(parsed.id);
+        let scenario;
+        try {
+            scenario = await client.getCaseConversion(parsed.id);
+        }
+        catch (error) {
+            if (!(error instanceof HueApiError && error.status === 404))
+                throw error;
+            const published = await scenarioOfPublishedCase(client, parsed.id);
+            if (!published)
+                throw error;
+            scenario = published;
+        }
         if (!scenario.publication)
             throw new Error(`Scenario ${scenario.id} is a draft without published pins; publish it in Hue first`);
         return pinsFromScenario(client, scenario, await client.getDataset(scenario.publication.datasetId));
@@ -99,8 +159,8 @@ export async function resolveScenarioPins(client, selector) {
     const datasets = new Map();
     const candidates = [];
     for (const summary of published) {
-        const scenario = await client.getCaseConversion(summary.id);
-        if (!scenario.publication)
+        const scenario = await readListed(client, summary.id);
+        if (!scenario?.publication)
             continue;
         let dataset = datasets.get(scenario.publication.datasetId);
         if (!dataset) {

package/dist/evals/simulation.d.ts

@@ -4,8 +4,8 @@ import type { EnvironmentTool } from "../environment/tools.js";
 import type { EnvironmentIdentity, PublishableEnvironmentDefinition, WorldHandoff } from "../environment/types.js";
 import { type EvaluationClient } from "./client.js";
 import type { ActualAgentManifestInputV2, AttemptConnectionBundleV2, RequestedAttemptProviderV2 } from "./attempt.js";
-import { type RunnerReport } from "./runner.js";
-import type { ExperimentCase, Identity, JsonValue, LocalScorer, ScorerDefinition, SimulationMcpCapability } from "./types.js";
+import { type RunnerReport, type TelemetryNotAccepted } from "./runner.js";
+import type { ExperimentCase, Identity, JsonValue, LocalFile, LocalScorer, ScorerDefinition, SimulationMcpCapability, TargetResult } from "./types.js";
 /** Repository-authored scorer identity and its public definition or local binding. */
 export interface RepositorySimulationScorer extends Identity {
     /** Scorer definition to publish, or a local callback bound to its source digest. */
@@ -121,6 +121,12 @@ export interface SimulationTargetContext {
     /** Credential-bearing provider connections for this callback only. Hue never
      * checkpoints, logs or adds this response to parity digests. */
     connectionBundle?: AttemptConnectionBundleV2;
+    /** Verified copies of the case's agent-visible input files (`source`, templates,
+     * references, originals); evaluator-only files are withheld. Empty when the case has none. */
+    files: LocalFile[];
+    /** Private directory for this case, removed after it; return generated files with
+     * `withFiles`. */
+    outputDirectory: string;
     /** Cancellation is cooperative: pass this signal into the real agent/provider call. */
     signal?: AbortSignal;
 }
@@ -152,6 +158,12 @@ export interface RunSimulationOptions {
         /** Bounded explanation for the omission. */
         reason: string;
     };
+    /** What a case does when required telemetry is not accepted; see
+     * `traceNotAccepted` of {@link runExperiment}. Defaults to `"stop"`. */
+    traceNotAccepted?: "stop" | "fail_case";
+    /** Called as each case is failed for its telemetry; see `onTelemetryNotAccepted` of
+     * {@link runExperiment}. */
+    onTelemetryNotAccepted?(entry: TelemetryNotAccepted): void | Promise<void>;
     /** Local scorer callbacks bound by their declared source digests. */
     localScorers?: LocalScorer[];
     /** Cases in flight, 1–64; defaults to 1. */
@@ -190,8 +202,9 @@ export interface RunSimulationOptions {
         /** Selected MCP surface. */
         surfaceKey: "google.gmail/mcp" | "slack/mcp";
     };
-    /** Invokes the existing local agent exactly once for this attempt. */
-    target(inputs: JsonValue, context: SimulationTargetContext): JsonValue | undefined | Promise<JsonValue | undefined>;
+    /** Invokes the existing local agent exactly once for this attempt; return the output, or
+     * `withFiles(output, files)` when it generated files. */
+    target(inputs: JsonValue, context: SimulationTargetContext): JsonValue | TargetResult | undefined | Promise<JsonValue | TargetResult | undefined>;
     /** Receives bounded nonsecret lifecycle progress. */
     onProgress?(event: SimulationProgress): void | Promise<void>;
 }

package/dist/evals/simulation.js

@@ -430,6 +430,12 @@ export async function runSimulation(options) {
             checkpointDirectory: join(store.directory, `experiment-${experimentId}`),
             persistResultContent: options.persistResultContent,
             traceEvidence: options.traceEvidence,
+            ...(options.traceNotAccepted ? { traceNotAccepted: options.traceNotAccepted } : {}),
+            ...(options.onTelemetryNotAccepted
+                ? {
+                    onTelemetryNotAccepted: (entry) => options.onTelemetryNotAccepted(entry),
+                }
+                : {}),
             environmentEvidence: "required",
             scorers: bindings,
             concurrency: options.concurrency,
@@ -474,6 +480,8 @@ export async function runSimulation(options) {
                     ...(targetContext.connectionBundle
                         ? { connectionBundle: structuredClone(targetContext.connectionBundle) }
                         : {}),
+                    files: structuredClone(targetContext.files),
+                    outputDirectory: targetContext.outputDirectory,
                     signal: targetContext.signal,
                 }),
             }),

package/dist/evals/types.d.ts

@@ -264,6 +264,10 @@ export interface StoredResult extends ResultSummary {
     error: TypedError | null;
     /** Source digest of the local scorer that produced it, when applicable. */
     sourceDigest: string | null;
+    /** True only for a `skipped` result Hue's own outcome scoring recorded because the evaluator
+     * does not apply to the case (it has no outcome criteria or conversion rubric to grade); a
+     * submitted result never reads true. Absent from servers that predate it. */
+    notApplicable?: boolean;
 }
 /** A hosted judge job and its charge accounting. */
 export interface JudgeJob {
@@ -879,8 +883,13 @@ export interface CaseConversionPublication {
     environmentVersionId: string;
     /** Scorer identity of the published outcome checks. */
     scorerId: string;
-    /** Immutable scorer version pinned by the Scenario. */
+    /** Immutable scorer version pinned by the Scenario for its outcome checks. */
     scorerVersionId: string;
+    /**
+     * Every immutable scorer version the Scenario pins, `scorerVersionId` first. Absent on
+     * Scenarios published before publications listed their pins.
+     */
+    scorerVersionIds?: string[];
 }
 /** A Scenario read by {@link EvaluationClient.getCaseConversion}. Extra server fields are ignored. */
 export interface CaseConversion extends Partial<Omit<CaseConversionSummary, "id" | "status">> {