@hue-run/sdk 0.8.1 → 0.10.0

package/dist/evals/environment-target.d.ts

@@ -5,7 +5,7 @@ import type { EnvironmentRun, WorldHandoff } from "../environment/types.js";
 import type { HueSpan } from "../types.js";
 import { type ActualAgentManifestInputV2, type AttemptBaselineV2, type AttemptConnectionBundleV2, type RequestedAttemptProviderV2, type SurfaceBindingV2 } from "./attempt.js";
 import type { EvaluationClient } from "./client.js";
-import type { ExperimentCase, JsonValue, SimulationMcpCapability } from "./types.js";
+import type { ExperimentCase, JsonValue, LocalFile, SimulationMcpCapability, TargetResult } from "./types.js";
 export type McpSurfaceKeyV2 = Extract<SurfaceBindingV2["surfaceKey"], `${string}/mcp`>;
 export type ActualAgentManifestResolverV2 = ActualAgentManifestInputV2 | ((context: {
     config: JsonValue;
@@ -64,6 +64,11 @@ export interface EnvironmentTargetContext {
      * gateway world has no MCP surface. */
     mcp?: SimulationMcpCapability;
     connectionBundle?: AttemptConnectionBundleV2;
+    /** Verified copies of the case's agent-visible input files; evaluator-only files are never
+     * among them. Empty when the case has none. */
+    files: LocalFile[];
+    /** Private directory for this case; return generated files with `withFiles`. */
+    outputDirectory: string;
     signal?: AbortSignal;
 }
 interface RunnerTargetContext {
@@ -71,6 +76,8 @@ interface RunnerTargetContext {
     item: ExperimentCase;
     executionId: string;
     span: HueSpan;
+    files: LocalFile[];
+    outputDirectory: string;
 }
 export interface RunEnvironmentTargetOptions {
     client: EvaluationClient;
@@ -88,8 +95,21 @@ export interface RunEnvironmentTargetOptions {
      * the deployment serves, such as `hue eval`, turns it off. */
     deprecationWarnings?: boolean;
     signal?: AbortSignal;
+    /** Test-only: shortens the seal wait. */
+    sealTiming?: Partial<SealTiming>;
     onProgress?(event: EnvironmentTargetProgress): void | Promise<void>;
-    target(inputs: JsonValue, context: EnvironmentTargetContext): JsonValue | undefined | Promise<JsonValue | undefined>;
+    target(inputs: JsonValue, context: EnvironmentTargetContext): JsonValue | TargetResult | undefined | Promise<JsonValue | TargetResult | undefined>;
+}
+/** The completion grace is five seconds; cap an unexpectedly distant timestamp and let reads
+ * force the seal after the grace. */
+export declare const MAX_GRACE_WAIT_MS = 10000;
+export declare const SEAL_POLL_MS = 250;
+export declare const SEAL_WAIT_MS = 30000;
+/** The seal wait's bounds. Only tests shorten them; the helpers always use the defaults. */
+export interface SealTiming {
+    maxGraceWaitMs: number;
+    pollMs: number;
+    waitMs: number;
 }
 /** The W3C context of the case span, sent on create so the world span parents on it. The
  * flags are the span's own: an unsampled case span is not exported, and the World API must not
@@ -131,5 +151,5 @@ export declare function createWorldForExecution(client: Pick<EnvironmentClient,
  * outbound local workers. Credential-bearing connections stay in this call frame and
  * are never returned to either runner's checkpoint state.
  */
-export declare function runEnvironmentTarget(options: RunEnvironmentTargetOptions): Promise<JsonValue | undefined>;
+export declare function runEnvironmentTarget(options: RunEnvironmentTargetOptions): Promise<JsonValue | TargetResult | undefined>;
 export {};

package/dist/evals/environment-target.js

@@ -1,5 +1,5 @@
 import { randomUUID } from "node:crypto";
-import { HueEnvironmentError } from "../environment/client.js";
+import { HueEnvironmentError, isTransientEnvironmentError, } from "../environment/client.js";
 import { bindEnvironmentTools } from "../environment/tools.js";
 import { legacyMcpCapability, worldHandoff } from "../environment/world.js";
 import { actualAgentManifestV2, attemptBaselineV2, projectMcpConnectionV2, requestedAttemptProvidersV2, validateAttemptConnectionBundleV2, } from "./attempt.js";
@@ -42,22 +42,82 @@ export function pinRequestedAttemptV2(requested, config) {
     }
     return { ...requested, expectedAgentManifestDigest: baseline.data.expectedAgentManifestDigest };
 }
-/** Confirm a seal from the authoritative run after a lost acknowledgement. */
-async function seal(client, runId, executionId, status) {
+/** The completion grace is five seconds; cap an unexpectedly distant timestamp and let reads
+ * force the seal after the grace. */
+export const MAX_GRACE_WAIT_MS = 10_000;
+export const SEAL_POLL_MS = 250;
+export const SEAL_WAIT_MS = 30_000;
+const SEAL_TIMING = {
+    maxGraceWaitMs: MAX_GRACE_WAIT_MS,
+    pollMs: SEAL_POLL_MS,
+    waitMs: SEAL_WAIT_MS,
+};
+/** Finish, then wait for the authoritative run to leave open. */
+async function seal(client, runId, executionId, status, timing) {
+    let completingUntil;
     try {
-        await client.finishRun(runId, {
+        const finished = await client.finishRun(runId, {
             idempotencyKey: `execution:${executionId}:${status}`,
             status,
         });
+        if (finished.lifecycle === "completing")
+            completingUntil = finished.completingUntil ?? null;
     }
     catch (error) {
-        // A gateway world answers 409 once it is completing, sealed or expired: each is the
-        // outcome the caller wanted or the one it can no longer change.
+        // A gateway world answers 409 once it is completing, sealed or expired.
         const recovered = await client.getRun(runId).catch(() => undefined);
         if (recovered?.status !== status &&
             recovered?.status !== "expired" &&
             !(recovered?.status === "open" && recovered.lifecycle === "completing"))
             throw new TargetOutcomeUncertainError(executionId, { cause: error });
+        if (recovered?.status === "open")
+            completingUntil = recovered.completingUntil ?? null;
+    }
+    if (completingUntil !== undefined) {
+        try {
+            await awaitSeal(client, runId, completingUntil, timing);
+        }
+        catch (error) {
+            throw new TargetOutcomeUncertainError(executionId, { cause: error });
+        }
+    }
+}
+/** Wait out a completing world's grace, then read until the server seals it. */
+async function awaitSeal(client, runId, completingUntil, timing) {
+    const graceEnd = Date.parse(completingUntil ?? "");
+    let wait = Number.isFinite(graceEnd)
+        ? Math.min(Math.max(0, graceEnd - Date.now()), timing.maxGraceWaitMs)
+        : 0;
+    const deadline = performance.now() + wait + timing.waitMs;
+    for (;;) {
+        await new Promise((resolve) => setTimeout(resolve, wait));
+        const remaining = deadline - performance.now();
+        if (remaining <= 0)
+            throw new Error(`World ${runId} was not sealed after its completion grace`);
+        // The deadline ends the read and the client's retries inside it.
+        const controller = new AbortController();
+        const timer = setTimeout(() => controller.abort(), remaining);
+        wait = timing.pollMs;
+        try {
+            if ((await client.getRun(runId, { signal: controller.signal })).status !== "open")
+                return;
+        }
+        catch (error) {
+            if (!isTransientEnvironmentError(error))
+                throw error;
+            if (performance.now() >= deadline)
+                throw new Error(`World ${runId} was not sealed after its completion grace`, {
+                    cause: error,
+                });
+            // A 429 or 503 that outlasted the client's retries still says how long to wait.
+            wait = Math.max(wait, error.retryAfterMs ?? 0);
+        }
+        finally {
+            clearTimeout(timer);
+        }
+        if (performance.now() >= deadline)
+            throw new Error(`World ${runId} was not sealed after its completion grace`);
+        wait = Math.min(wait, deadline - performance.now());
     }
 }
 const deprecations = new Set();
@@ -157,6 +217,7 @@ export async function runEnvironmentTarget(options) {
         agentRevision: options.agentRevision,
     });
     const world = worldHandoff(run);
+    const timing = { ...SEAL_TIMING, ...options.sealTiming };
     const progress = (event) => options.onProgress?.(event);
     let finalized = false;
     try {
@@ -217,7 +278,7 @@ export async function runEnvironmentTarget(options) {
                     : {}),
             });
             if (prepared.status === "environment_incomplete") {
-                await seal(options.environmentClient, run.id, context.executionId, "completed");
+                await seal(options.environmentClient, run.id, context.executionId, "completed", timing);
                 finalized = true;
                 await Promise.resolve(progress({ type: "world_sealed", environmentRunId: run.id })).catch(() => undefined);
                 return undefined;
@@ -256,9 +317,11 @@ export async function runEnvironmentTarget(options) {
             ...(world ? { world } : {}),
             ...(mcp ? { mcp } : {}),
             ...(connectionBundle ? { connectionBundle } : {}),
+            files: structuredClone(context.files),
+            outputDirectory: context.outputDirectory,
             signal: options.signal,
         });
-        await seal(options.environmentClient, run.id, context.executionId, "completed");
+        await seal(options.environmentClient, run.id, context.executionId, "completed", timing);
         finalized = true;
         await Promise.resolve(progress({ type: "world_sealed", environmentRunId: run.id })).catch(() => undefined);
         return output;
@@ -278,7 +341,7 @@ export async function runEnvironmentTarget(options) {
         }
         if (environmentIncomplete) {
             try {
-                await seal(options.environmentClient, run.id, context.executionId, "completed");
+                await seal(options.environmentClient, run.id, context.executionId, "completed", timing);
             }
             catch (finalizationError) {
                 throw new TargetOutcomeUncertainError(context.executionId, {
@@ -290,7 +353,7 @@ export async function runEnvironmentTarget(options) {
             return undefined;
         }
         try {
-            await seal(options.environmentClient, run.id, context.executionId, "abandoned");
+            await seal(options.environmentClient, run.id, context.executionId, "abandoned", timing);
         }
         catch (finalizationError) {
             throw new TargetOutcomeUncertainError(context.executionId, {

package/dist/evals/exit-cleanup.d.ts

@@ -0,0 +1,4 @@
+/** Register a synchronous removal; the returned function unregisters it. */
+export declare function onForcedExit(cleanup: () => void): () => void;
+/** Run every registered removal, newest first: files before the lock that guards them. */
+export declare function runForcedExitCleanups(): void;

package/dist/evals/exit-cleanup.js

@@ -0,0 +1,26 @@
+/**
+ * What this process holds on disk while it runs — a world case's files, a checkpoint lock, a
+ * token-bearing MCP configuration — each with a synchronous removal. Normal paths remove them
+ * themselves and unregister; a forced exit (a second Ctrl+C in `hue eval`) cannot wait for them,
+ * so it runs whatever is still registered, newest first, before `process.exit`.
+ */
+const cleanups = new Set();
+/** Register a synchronous removal; the returned function unregisters it. */
+export function onForcedExit(cleanup) {
+    cleanups.add(cleanup);
+    return () => {
+        cleanups.delete(cleanup);
+    };
+}
+/** Run every registered removal, newest first: files before the lock that guards them. */
+export function runForcedExitCleanups() {
+    for (const cleanup of [...cleanups].reverse()) {
+        try {
+            cleanup();
+        }
+        catch {
+            // Best effort on the way out; the rest still run.
+        }
+    }
+    cleanups.clear();
+}

package/dist/evals/files.d.ts

@@ -32,13 +32,71 @@ export interface StagedOutputFile {
 export declare class OutputFileError extends Error {
     constructor(message: string);
 }
-/** Reduce a declared file name to a single printable path segment, at most 200 characters. */
+/** Stable codes of a pinned file the runner refuses to hand over. */
+export type CaseFileErrorCode = "case_file_mismatch" | "case_file_name_refused";
+/**
+ * Thrown when a pinned file cannot be used: its downloaded bytes differ from the manifest's size
+ * or SHA-256 (`case_file_mismatch`), or a file meant for an agent working in a world is not
+ * named by one safe file name (`case_file_name_refused`). Raised before the case's execution
+ * starts, so no execution, world or target call is spent on it.
+ */
+export declare class CaseFileError extends Error {
+    /** Stable diagnostic code. */
+    readonly code: CaseFileErrorCode;
+    /** Hue artifact identity of the refused file. */
+    readonly artifactId: string;
+    constructor(
+    /** Stable diagnostic code. */
+    code: CaseFileErrorCode, 
+    /** Hue artifact identity of the refused file. */
+    artifactId: string, message: string);
+}
+/**
+ * Whether `name` can be used unchanged as one file name: not empty, `.` or `..`; no `/`, `\`,
+ * C0 or C1 control character, or character Windows reserves (`: < > " | ? *`); no Windows device
+ * name (`CON`, `NUL`, `COM1`, … with any extension); no trailing dot or space, which Windows
+ * drops; at most 200 UTF-8 bytes. An absolute path or a path with a parent step always contains
+ * a separator, so it is refused too.
+ */
+export declare function isSafeFileName(name: string): boolean;
+/** Refuse a pinned file whose name is not one safe file name; see {@link isSafeFileName}. */
+export declare function assertSafeFileNames(files: readonly Pick<CaseFile, "artifactId" | "filename">[]): void;
+/**
+ * Reduce a declared file name to a single printable path segment, at most 200 UTF-8 bytes. A
+ * longer name keeps its extension, so `.pdf` stays `.pdf`, and its stem is shortened and marked
+ * with a short hash of the whole name, so two long names never shorten to the same one.
+ */
 export declare function safeFilename(name: string): string;
+/** The device and inode a listing saw for a file, compared with the file actually opened. */
+export interface FileIdentity {
+    dev: number;
+    ino: number;
+}
+/**
+ * Read a file an agent wrote without trusting its path: the last component must not be a
+ * symlink and the open file must be a regular file of at most `maxBytes`, checked before any byte
+ * is read, and, when `expected` is given, the very file a listing saw. The bytes come from that
+ * open file. Returns undefined for a missing file only when nothing was expected there; every
+ * other refusal is an `OutputFileError` naming `label`.
+ */
+export declare function readAgentFile(path: string, label: string, maxBytes: number, expected?: FileIdentity): Promise<Uint8Array | undefined>;
+/** Check pinned files against the manifest without keeping them: nothing is written to disk. */
+export declare function verifyCaseFiles(client: EvaluationClient, files: readonly SubjectFile[]): Promise<void>;
 /**
  * Save verified copies of a case's pinned input files under `directory`. A file already
  * present with the pinned identity is reused, so a resumed run does not download again.
  */
 export declare function downloadCaseFiles(client: EvaluationClient, files: SubjectFile[], directory: string, primaryArtifactId?: string | null): Promise<LocalFile[]>;
+/**
+ * Download a case's needed inputs under `caseDirectory`: the agent-visible roles into `inputs/`
+ * and evaluator-only files into `evaluator-inputs/`, so the directory the target's files live in
+ * never holds an evaluator's file. `all` keeps the manifest order for scorers; `target` is the
+ * agent-visible subset.
+ */
+export declare function downloadCaseInputs(client: EvaluationClient, files: CaseFile[], caseDirectory: string): Promise<{
+    all: LocalFile[];
+    target: LocalFile[];
+}>;
 /** Copy the target's generated files next to the checkpoint and record their identities. */
 export declare function stageOutputFiles(files: OutputFile[], directory: string): Promise<StagedOutputFile[]>;
 /**

package/dist/evals/files.js

@@ -1,8 +1,9 @@
 import { createHash } from "node:crypto";
-import { copyFile, lstat, mkdir, open, readFile, rename, rm } from "node:fs/promises";
-import { join, resolve } from "node:path";
+import { constants } from "node:fs";
+import { lstat, mkdir, open, readFile, rename, rm } from "node:fs/promises";
+import { extname, join, resolve } from "node:path";
 import { randomUUID } from "node:crypto";
-import { HueApiError } from "./client.js";
+import { ArtifactSizeError, HueApiError } from "./client.js";
 /** Roles the target receives. Organization templates stay with grading, as in the managed protocol. */
 export const targetFileRoles = [
     "source",
@@ -37,25 +38,153 @@ export class OutputFileError extends Error {
         this.name = "OutputFileError";
     }
 }
+/**
+ * Thrown when a pinned file cannot be used: its downloaded bytes differ from the manifest's size
+ * or SHA-256 (`case_file_mismatch`), or a file meant for an agent working in a world is not
+ * named by one safe file name (`case_file_name_refused`). Raised before the case's execution
+ * starts, so no execution, world or target call is spent on it.
+ */
+export class CaseFileError extends Error {
+    code;
+    artifactId;
+    constructor(
+    /** Stable diagnostic code. */
+    code, 
+    /** Hue artifact identity of the refused file. */
+    artifactId, message) {
+        super(`${message} (${code})`);
+        this.code = code;
+        this.artifactId = artifactId;
+        this.name = "CaseFileError";
+    }
+}
 const sha256 = (bytes) => createHash("sha256").update(bytes).digest("hex");
-/** Reduce a declared file name to a single printable path segment, at most 200 characters. */
+/** A file name quoted for a message: at most 120 characters, controls escaped. */
+const quoted = (name) => JSON.stringify([...name].slice(0, 120).join(""));
+/** Names Windows reserves for devices, with or without an extension. */
+const DEVICE_NAME = /^(?:con|prn|aux|nul|com[0-9¹²³]|lpt[0-9¹²³]|conin\$|conout\$)$/iu;
+/** The longest file name the runner writes, in UTF-8 bytes: room is left for the temporary
+ * suffix of an atomic write and a copy counter within the usual 255-byte limit. */
+const MAX_NAME_BYTES = 200;
+/** The longest prefix of `name` that fits in `bytes` UTF-8 bytes, cut between code points. */
+function truncateBytes(name, bytes) {
+    let kept = "";
+    let size = 0;
+    for (const character of name) {
+        size += Buffer.byteLength(character);
+        if (size > bytes)
+            break;
+        kept += character;
+    }
+    return kept;
+}
+/**
+ * Whether `name` can be used unchanged as one file name: not empty, `.` or `..`; no `/`, `\`,
+ * C0 or C1 control character, or character Windows reserves (`: < > " | ? *`); no Windows device
+ * name (`CON`, `NUL`, `COM1`, … with any extension); no trailing dot or space, which Windows
+ * drops; at most 200 UTF-8 bytes. An absolute path or a path with a parent step always contains
+ * a separator, so it is refused too.
+ */
+export function isSafeFileName(name) {
+    if (typeof name !== "string" || !name || name === "." || name === "..")
+        return false;
+    // eslint-disable-next-line no-control-regex -- control characters are refused
+    if (/[/\\:<>"|?*\x00-\x1f\x7f-\x9f]/u.test(name))
+        return false;
+    if (name.endsWith(".") || name.endsWith(" "))
+        return false;
+    if (DEVICE_NAME.test(name.split(".")[0].trimEnd()))
+        return false;
+    return Buffer.byteLength(name) <= MAX_NAME_BYTES;
+}
+/** Refuse a pinned file whose name is not one safe file name; see {@link isSafeFileName}. */
+export function assertSafeFileNames(files) {
+    for (const file of files)
+        if (!isSafeFileName(file.filename))
+            throw new CaseFileError("case_file_name_refused", file.artifactId, `Pinned file ${quoted(String(file.filename))} is not a safe file name`);
+}
+/**
+ * Reduce a declared file name to a single printable path segment, at most 200 UTF-8 bytes. A
+ * longer name keeps its extension, so `.pdf` stays `.pdf`, and its stem is shortened and marked
+ * with a short hash of the whole name, so two long names never shorten to the same one.
+ */
 export function safeFilename(name) {
     const cleaned = [...name]
         .filter((c) => c.charCodeAt(0) >= 32 && c.charCodeAt(0) !== 127 && c !== "/" && c !== "\\")
         .join("")
         .trim();
-    return cleaned && cleaned !== "." && cleaned !== ".."
-        ? [...cleaned].slice(0, 200).join("")
-        : "file";
+    let kept = cleaned;
+    if (Buffer.byteLength(cleaned) > MAX_NAME_BYTES) {
+        const found = extname(cleaned);
+        const extension = Buffer.byteLength(found) <= 32 ? found : "";
+        const mark = `~${sha256(Buffer.from(cleaned)).slice(0, 8)}`;
+        const room = MAX_NAME_BYTES - Buffer.byteLength(extension) - mark.length;
+        kept = `${truncateBytes(cleaned.slice(0, cleaned.length - extension.length), room)}${mark}${extension}`;
+    }
+    return kept && kept !== "." && kept !== ".." ? kept : "file";
 }
+/** Create `path` if needed and require a directory this user owns that no one else can open. */
 async function privateDirectory(path) {
     const root = resolve(path);
     await mkdir(root, { recursive: true, mode: 0o700 });
     const info = await lstat(root);
-    if (!info.isDirectory() || info.isSymbolicLink())
-        throw new Error("Use a private files directory (no symlink)");
+    const uid = process.getuid?.();
+    if (!info.isDirectory() ||
+        info.isSymbolicLink() ||
+        (uid !== undefined && info.uid !== uid) ||
+        (process.platform !== "win32" && (info.mode & 0o077) !== 0))
+        throw new Error("Use a private files directory (owned by this user, mode 0700, no symlink)");
     return root;
 }
+// A symlink as the last path component fails to open, and a FIFO or device never blocks.
+const AGENT_FILE_FLAGS = constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK;
+/**
+ * Read a file an agent wrote without trusting its path: the last component must not be a
+ * symlink and the open file must be a regular file of at most `maxBytes`, checked before any byte
+ * is read, and, when `expected` is given, the very file a listing saw. The bytes come from that
+ * open file. Returns undefined for a missing file only when nothing was expected there; every
+ * other refusal is an `OutputFileError` naming `label`.
+ */
+export async function readAgentFile(path, label, maxBytes, expected) {
+    let file;
+    try {
+        file = await open(path, AGENT_FILE_FLAGS);
+    }
+    catch (error) {
+        const code = error.code;
+        if (code === "ENOENT" && !expected)
+            return undefined;
+        if (code === "ELOOP" || code === "EMLINK")
+            throw new OutputFileError(`${label} is a symbolic link`);
+        if (code === "ENOENT")
+            throw new OutputFileError(`${label} changed while it was collected`);
+        throw new OutputFileError(`${label} could not be read`);
+    }
+    try {
+        const info = await file.stat();
+        if (!info.isFile())
+            throw new OutputFileError(`${label} is not a regular file`);
+        if (expected && (info.dev !== expected.dev || info.ino !== expected.ino))
+            throw new OutputFileError(`${label} changed while it was collected`);
+        if (info.size > maxBytes)
+            throw new OutputFileError(`${label} exceeds ${Math.floor(maxBytes / 1024 / 1024)} MiB`);
+        // One byte past the size seen: a file that is still growing is refused, not read whole.
+        const bytes = Buffer.allocUnsafe(info.size + 1);
+        let total = 0;
+        while (total < bytes.length) {
+            const { bytesRead } = await file.read(bytes, total, bytes.length - total, total);
+            if (!bytesRead)
+                break;
+            total += bytesRead;
+        }
+        if (total !== info.size)
+            throw new OutputFileError(`${label} changed while it was collected`);
+        return bytes.subarray(0, total);
+    }
+    finally {
+        await file.close();
+    }
+}
 async function writePrivate(path, bytes) {
     const temporary = `${path}.${randomUUID()}.tmp`;
     const file = await open(temporary, "wx", 0o600);
@@ -74,6 +203,28 @@ async function verifiedBytes(path, expected) {
         throw new Error(`Saved file ${path} no longer matches its verified identity`);
     return bytes;
 }
+/** Download one pinned file and check its size and SHA-256 against the manifest. The download
+ * stops one byte past the pinned size. */
+async function downloadVerified(client, file) {
+    const mismatch = () => new CaseFileError("case_file_mismatch", file.artifactId, `Downloaded file ${quoted(file.filename)} does not match its pinned size and SHA-256`);
+    let bytes;
+    try {
+        bytes = await client.downloadArtifact(file.artifactId, { maxBytes: file.byteSize });
+    }
+    catch (error) {
+        if (error instanceof ArtifactSizeError)
+            throw mismatch();
+        throw error;
+    }
+    if (bytes.byteLength !== file.byteSize || sha256(bytes) !== file.sha256)
+        throw mismatch();
+    return bytes;
+}
+/** Check pinned files against the manifest without keeping them: nothing is written to disk. */
+export async function verifyCaseFiles(client, files) {
+    for (const file of files)
+        await downloadVerified(client, file);
+}
 /**
  * Save verified copies of a case's pinned input files under `directory`. A file already
  * present with the pinned identity is reused, so a resumed run does not download again.
@@ -91,12 +242,8 @@ export async function downloadCaseFiles(client, files, directory, primaryArtifac
         catch {
             present = false;
         }
-        if (!present) {
-            const bytes = await client.downloadArtifact(file.artifactId);
-            if (bytes.byteLength !== file.byteSize || sha256(bytes) !== file.sha256)
-                throw new Error(`Downloaded input ${file.filename} does not match its pinned identity`);
-            await writePrivate(path, bytes);
-        }
+        if (!present)
+            await writePrivate(path, await downloadVerified(client, file));
         saved.push({
             artifactId: file.artifactId,
             role: file.role,
@@ -110,6 +257,26 @@ export async function downloadCaseFiles(client, files, directory, primaryArtifac
     }
     return saved;
 }
+/**
+ * Download a case's needed inputs under `caseDirectory`: the agent-visible roles into `inputs/`
+ * and evaluator-only files into `evaluator-inputs/`, so the directory the target's files live in
+ * never holds an evaluator's file. `all` keeps the manifest order for scorers; `target` is the
+ * agent-visible subset.
+ */
+export async function downloadCaseInputs(client, files, caseDirectory) {
+    const visible = (file) => targetFileRoles.includes(file.role);
+    const agentFiles = files.filter(visible);
+    const evaluatorFiles = files.filter((file) => !visible(file));
+    const target = agentFiles.length
+        ? await downloadCaseFiles(client, agentFiles, join(caseDirectory, "inputs"))
+        : [];
+    const evaluator = evaluatorFiles.length
+        ? await downloadCaseFiles(client, evaluatorFiles, join(caseDirectory, "evaluator-inputs"))
+        : [];
+    const saved = [...target, ...evaluator];
+    const all = files.map((file) => saved.find((entry) => entry.artifactId === file.artifactId && entry.role === file.role));
+    return { all, target };
+}
 /** Copy the target's generated files next to the checkpoint and record their identities. */
 export async function stageOutputFiles(files, directory) {
     if (!Array.isArray(files) || !files.length)
@@ -132,24 +299,17 @@ export async function stageOutputFiles(files, directory) {
         names.add(filename);
         if (!outputContentTypes.includes(file.contentType))
             throw new OutputFileError(`Unsupported generated file type ${String(file.contentType)}`);
-        let bytes;
-        try {
-            bytes = file.bytes ?? (await readFile(file.path));
-        }
-        catch {
+        // A path is read once, as a regular file within the limit, and staged from those bytes.
+        const bytes = file.bytes ??
+            (await readAgentFile(file.path, `Generated file ${filename}`, outputFileLimits.bytes));
+        if (!bytes)
             throw new OutputFileError(`Generated file ${filename} could not be read`);
-        }
         if (!bytes.byteLength)
             throw new OutputFileError(`Generated file ${filename} is empty`);
         if (bytes.byteLength > outputFileLimits.bytes)
             throw new OutputFileError(`Generated file ${filename} exceeds 25 MiB`);
         const path = join(root, filename);
-        if (file.bytes)
-            await writePrivate(path, bytes);
-        else {
-            await copyFile(file.path, path);
-            await verifiedBytes(path, { byteSize: bytes.byteLength, sha256: sha256(bytes) });
-        }
+        await writePrivate(path, bytes);
         staged.push({
             filename,
             contentType: file.contentType,

package/dist/evals/local-worker.d.ts

@@ -4,7 +4,7 @@ import type { EnvironmentTool } from "../environment/tools.js";
 import type { WorldHandoff } from "../environment/types.js";
 import type { ActualAgentManifestInputV2, AttemptConnectionBundleV2, RequestedAttemptProviderV2 } from "./attempt.js";
 import type { EvaluationClient } from "./client.js";
-import { type RunExperimentTargetContext, type RunnerReport } from "./runner.js";
+import { type RunExperimentTargetContext, type RunnerReport, type TelemetryNotAccepted } from "./runner.js";
 import type { ExperimentCase, JsonValue, LocalAgentRegistration, LocalFile, LocalScorer, TargetResult } from "./types.js";
 /** Capability strings a registration declares; Hue offers only matching cases. */
 export declare const localAgentCapabilities: {
@@ -13,6 +13,10 @@ export declare const localAgentCapabilities: {
     /** Ordinary cases run directly on this machine: JSON inputs and pinned input files in,
      * JSON output and generated files out. */
     readonly direct: "direct:v1";
+    /** Cases pinned to a world that also carry agent-visible input files: `target` receives them
+     * with the world and may return generated files. Declare it with `input:<extension>` for each
+     * accepted file type; it requires `target`. */
+    readonly environmentFiles: "environment-files:v1";
 };
 /** Candidate-visible context for one queued local agent execution. */
 export interface LocalAgentTargetContext {
@@ -47,6 +51,13 @@ export interface LocalAgentTargetContext {
     /** Credential-bearing provider connections for this callback only. Hue never
      * checkpoints, logs or adds this response to parity digests. */
     connectionBundle?: AttemptConnectionBundleV2;
+    /** Verified copies of the case's agent-visible input files; evaluator-only files are
+     * withheld. Hue offers a world case with such files only to a registration declaring
+     * `environment-files:v1`. Empty when the case has none. */
+    files: LocalFile[];
+    /** Private directory for this case, removed after it; return generated files with
+     * `withFiles`. */
+    outputDirectory: string;
 }
 /** Candidate-visible context for one queued ordinary case run directly on this machine. */
 export interface LocalAgentDirectContext {
@@ -88,6 +99,12 @@ export interface RunLocalAgentOptions {
     scorers?: LocalScorer[];
     /** Cases in flight, between 1 and 16; defaults to 1. */
     concurrency?: number;
+    /** What a case does when its telemetry is not accepted; see
+     * `traceNotAccepted` of {@link runExperiment}. Defaults to `"stop"`. */
+    traceNotAccepted?: "stop" | "fail_case";
+    /** Called as each case is failed for its telemetry; see `onTelemetryNotAccepted` of
+     * {@link runExperiment}. */
+    onTelemetryNotAccepted?(entry: TelemetryNotAccepted): void | Promise<void>;
     /** Polling interval in milliseconds, 250–60000; defaults to 2000. */
     pollIntervalMillis?: number;
     /** Stops polling cooperatively; does not cancel an active callback. */
@@ -116,8 +133,9 @@ export interface RunLocalAgentOptions {
         surfaceKey: "google.gmail/mcp" | "slack/mcp";
     };
     /** Runs cases pinned to a hosted world (`environment:v1`): invokes the existing agent against
-     * isolated tools and candidate-safe context. */
-    target?(inputs: JsonValue, tools: Record<string, EnvironmentTool>, context: LocalAgentTargetContext): JsonValue | undefined | Promise<JsonValue | undefined>;
+     * isolated tools and candidate-safe context. Return the output, or `withFiles(output, files)`
+     * when it generated files. */
+    target?(inputs: JsonValue, tools: Record<string, EnvironmentTool>, context: LocalAgentTargetContext): JsonValue | TargetResult | undefined | Promise<JsonValue | TargetResult | undefined>;
     /** Runs ordinary cases without a world (`direct:v1`): the agent receives the case inputs
      * and verified input files and returns JSON output and/or generated files. */
     directTarget?(inputs: JsonValue, context: LocalAgentDirectContext): JsonValue | TargetResult | undefined | Promise<JsonValue | TargetResult | undefined>;