@hue-run/sdk 0.9.0 → 0.10.0

package/dist/evals/files.js

@@ -1,8 +1,9 @@
 import { createHash } from "node:crypto";
-import { copyFile, lstat, mkdir, open, readFile, rename, rm } from "node:fs/promises";
-import { join, resolve } from "node:path";
+import { constants } from "node:fs";
+import { lstat, mkdir, open, readFile, rename, rm } from "node:fs/promises";
+import { extname, join, resolve } from "node:path";
 import { randomUUID } from "node:crypto";
-import { HueApiError } from "./client.js";
+import { ArtifactSizeError, HueApiError } from "./client.js";
 /** Roles the target receives. Organization templates stay with grading, as in the managed protocol. */
 export const targetFileRoles = [
     "source",
@@ -37,25 +38,153 @@ export class OutputFileError extends Error {
         this.name = "OutputFileError";
     }
 }
+/**
+ * Thrown when a pinned file cannot be used: its downloaded bytes differ from the manifest's size
+ * or SHA-256 (`case_file_mismatch`), or a file meant for an agent working in a world is not
+ * named by one safe file name (`case_file_name_refused`). Raised before the case's execution
+ * starts, so no execution, world or target call is spent on it.
+ */
+export class CaseFileError extends Error {
+    code;
+    artifactId;
+    constructor(
+    /** Stable diagnostic code. */
+    code, 
+    /** Hue artifact identity of the refused file. */
+    artifactId, message) {
+        super(`${message} (${code})`);
+        this.code = code;
+        this.artifactId = artifactId;
+        this.name = "CaseFileError";
+    }
+}
 const sha256 = (bytes) => createHash("sha256").update(bytes).digest("hex");
-/** Reduce a declared file name to a single printable path segment, at most 200 characters. */
+/** A file name quoted for a message: at most 120 characters, controls escaped. */
+const quoted = (name) => JSON.stringify([...name].slice(0, 120).join(""));
+/** Names Windows reserves for devices, with or without an extension. */
+const DEVICE_NAME = /^(?:con|prn|aux|nul|com[0-9¹²³]|lpt[0-9¹²³]|conin\$|conout\$)$/iu;
+/** The longest file name the runner writes, in UTF-8 bytes: room is left for the temporary
+ * suffix of an atomic write and a copy counter within the usual 255-byte limit. */
+const MAX_NAME_BYTES = 200;
+/** The longest prefix of `name` that fits in `bytes` UTF-8 bytes, cut between code points. */
+function truncateBytes(name, bytes) {
+    let kept = "";
+    let size = 0;
+    for (const character of name) {
+        size += Buffer.byteLength(character);
+        if (size > bytes)
+            break;
+        kept += character;
+    }
+    return kept;
+}
+/**
+ * Whether `name` can be used unchanged as one file name: not empty, `.` or `..`; no `/`, `\`,
+ * C0 or C1 control character, or character Windows reserves (`: < > " | ? *`); no Windows device
+ * name (`CON`, `NUL`, `COM1`, … with any extension); no trailing dot or space, which Windows
+ * drops; at most 200 UTF-8 bytes. An absolute path or a path with a parent step always contains
+ * a separator, so it is refused too.
+ */
+export function isSafeFileName(name) {
+    if (typeof name !== "string" || !name || name === "." || name === "..")
+        return false;
+    // eslint-disable-next-line no-control-regex -- control characters are refused
+    if (/[/\\:<>"|?*\x00-\x1f\x7f-\x9f]/u.test(name))
+        return false;
+    if (name.endsWith(".") || name.endsWith(" "))
+        return false;
+    if (DEVICE_NAME.test(name.split(".")[0].trimEnd()))
+        return false;
+    return Buffer.byteLength(name) <= MAX_NAME_BYTES;
+}
+/** Refuse a pinned file whose name is not one safe file name; see {@link isSafeFileName}. */
+export function assertSafeFileNames(files) {
+    for (const file of files)
+        if (!isSafeFileName(file.filename))
+            throw new CaseFileError("case_file_name_refused", file.artifactId, `Pinned file ${quoted(String(file.filename))} is not a safe file name`);
+}
+/**
+ * Reduce a declared file name to a single printable path segment, at most 200 UTF-8 bytes. A
+ * longer name keeps its extension, so `.pdf` stays `.pdf`, and its stem is shortened and marked
+ * with a short hash of the whole name, so two long names never shorten to the same one.
+ */
 export function safeFilename(name) {
     const cleaned = [...name]
         .filter((c) => c.charCodeAt(0) >= 32 && c.charCodeAt(0) !== 127 && c !== "/" && c !== "\\")
         .join("")
         .trim();
-    return cleaned && cleaned !== "." && cleaned !== ".."
-        ? [...cleaned].slice(0, 200).join("")
-        : "file";
+    let kept = cleaned;
+    if (Buffer.byteLength(cleaned) > MAX_NAME_BYTES) {
+        const found = extname(cleaned);
+        const extension = Buffer.byteLength(found) <= 32 ? found : "";
+        const mark = `~${sha256(Buffer.from(cleaned)).slice(0, 8)}`;
+        const room = MAX_NAME_BYTES - Buffer.byteLength(extension) - mark.length;
+        kept = `${truncateBytes(cleaned.slice(0, cleaned.length - extension.length), room)}${mark}${extension}`;
+    }
+    return kept && kept !== "." && kept !== ".." ? kept : "file";
 }
+/** Create `path` if needed and require a directory this user owns that no one else can open. */
 async function privateDirectory(path) {
     const root = resolve(path);
     await mkdir(root, { recursive: true, mode: 0o700 });
     const info = await lstat(root);
-    if (!info.isDirectory() || info.isSymbolicLink())
-        throw new Error("Use a private files directory (no symlink)");
+    const uid = process.getuid?.();
+    if (!info.isDirectory() ||
+        info.isSymbolicLink() ||
+        (uid !== undefined && info.uid !== uid) ||
+        (process.platform !== "win32" && (info.mode & 0o077) !== 0))
+        throw new Error("Use a private files directory (owned by this user, mode 0700, no symlink)");
     return root;
 }
+// A symlink as the last path component fails to open, and a FIFO or device never blocks.
+const AGENT_FILE_FLAGS = constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK;
+/**
+ * Read a file an agent wrote without trusting its path: the last component must not be a
+ * symlink and the open file must be a regular file of at most `maxBytes`, checked before any byte
+ * is read, and, when `expected` is given, the very file a listing saw. The bytes come from that
+ * open file. Returns undefined for a missing file only when nothing was expected there; every
+ * other refusal is an `OutputFileError` naming `label`.
+ */
+export async function readAgentFile(path, label, maxBytes, expected) {
+    let file;
+    try {
+        file = await open(path, AGENT_FILE_FLAGS);
+    }
+    catch (error) {
+        const code = error.code;
+        if (code === "ENOENT" && !expected)
+            return undefined;
+        if (code === "ELOOP" || code === "EMLINK")
+            throw new OutputFileError(`${label} is a symbolic link`);
+        if (code === "ENOENT")
+            throw new OutputFileError(`${label} changed while it was collected`);
+        throw new OutputFileError(`${label} could not be read`);
+    }
+    try {
+        const info = await file.stat();
+        if (!info.isFile())
+            throw new OutputFileError(`${label} is not a regular file`);
+        if (expected && (info.dev !== expected.dev || info.ino !== expected.ino))
+            throw new OutputFileError(`${label} changed while it was collected`);
+        if (info.size > maxBytes)
+            throw new OutputFileError(`${label} exceeds ${Math.floor(maxBytes / 1024 / 1024)} MiB`);
+        // One byte past the size seen: a file that is still growing is refused, not read whole.
+        const bytes = Buffer.allocUnsafe(info.size + 1);
+        let total = 0;
+        while (total < bytes.length) {
+            const { bytesRead } = await file.read(bytes, total, bytes.length - total, total);
+            if (!bytesRead)
+                break;
+            total += bytesRead;
+        }
+        if (total !== info.size)
+            throw new OutputFileError(`${label} changed while it was collected`);
+        return bytes.subarray(0, total);
+    }
+    finally {
+        await file.close();
+    }
+}
 async function writePrivate(path, bytes) {
     const temporary = `${path}.${randomUUID()}.tmp`;
     const file = await open(temporary, "wx", 0o600);
@@ -74,6 +203,28 @@ async function verifiedBytes(path, expected) {
         throw new Error(`Saved file ${path} no longer matches its verified identity`);
     return bytes;
 }
+/** Download one pinned file and check its size and SHA-256 against the manifest. The download
+ * stops one byte past the pinned size. */
+async function downloadVerified(client, file) {
+    const mismatch = () => new CaseFileError("case_file_mismatch", file.artifactId, `Downloaded file ${quoted(file.filename)} does not match its pinned size and SHA-256`);
+    let bytes;
+    try {
+        bytes = await client.downloadArtifact(file.artifactId, { maxBytes: file.byteSize });
+    }
+    catch (error) {
+        if (error instanceof ArtifactSizeError)
+            throw mismatch();
+        throw error;
+    }
+    if (bytes.byteLength !== file.byteSize || sha256(bytes) !== file.sha256)
+        throw mismatch();
+    return bytes;
+}
+/** Check pinned files against the manifest without keeping them: nothing is written to disk. */
+export async function verifyCaseFiles(client, files) {
+    for (const file of files)
+        await downloadVerified(client, file);
+}
 /**
  * Save verified copies of a case's pinned input files under `directory`. A file already
  * present with the pinned identity is reused, so a resumed run does not download again.
@@ -91,12 +242,8 @@ export async function downloadCaseFiles(client, files, directory, primaryArtifac
         catch {
             present = false;
         }
-        if (!present) {
-            const bytes = await client.downloadArtifact(file.artifactId);
-            if (bytes.byteLength !== file.byteSize || sha256(bytes) !== file.sha256)
-                throw new Error(`Downloaded input ${file.filename} does not match its pinned identity`);
-            await writePrivate(path, bytes);
-        }
+        if (!present)
+            await writePrivate(path, await downloadVerified(client, file));
         saved.push({
             artifactId: file.artifactId,
             role: file.role,
@@ -110,6 +257,26 @@ export async function downloadCaseFiles(client, files, directory, primaryArtifac
     }
     return saved;
 }
+/**
+ * Download a case's needed inputs under `caseDirectory`: the agent-visible roles into `inputs/`
+ * and evaluator-only files into `evaluator-inputs/`, so the directory the target's files live in
+ * never holds an evaluator's file. `all` keeps the manifest order for scorers; `target` is the
+ * agent-visible subset.
+ */
+export async function downloadCaseInputs(client, files, caseDirectory) {
+    const visible = (file) => targetFileRoles.includes(file.role);
+    const agentFiles = files.filter(visible);
+    const evaluatorFiles = files.filter((file) => !visible(file));
+    const target = agentFiles.length
+        ? await downloadCaseFiles(client, agentFiles, join(caseDirectory, "inputs"))
+        : [];
+    const evaluator = evaluatorFiles.length
+        ? await downloadCaseFiles(client, evaluatorFiles, join(caseDirectory, "evaluator-inputs"))
+        : [];
+    const saved = [...target, ...evaluator];
+    const all = files.map((file) => saved.find((entry) => entry.artifactId === file.artifactId && entry.role === file.role));
+    return { all, target };
+}
 /** Copy the target's generated files next to the checkpoint and record their identities. */
 export async function stageOutputFiles(files, directory) {
     if (!Array.isArray(files) || !files.length)
@@ -132,24 +299,17 @@ export async function stageOutputFiles(files, directory) {
         names.add(filename);
         if (!outputContentTypes.includes(file.contentType))
             throw new OutputFileError(`Unsupported generated file type ${String(file.contentType)}`);
-        let bytes;
-        try {
-            bytes = file.bytes ?? (await readFile(file.path));
-        }
-        catch {
+        // A path is read once, as a regular file within the limit, and staged from those bytes.
+        const bytes = file.bytes ??
+            (await readAgentFile(file.path, `Generated file ${filename}`, outputFileLimits.bytes));
+        if (!bytes)
             throw new OutputFileError(`Generated file ${filename} could not be read`);
-        }
         if (!bytes.byteLength)
             throw new OutputFileError(`Generated file ${filename} is empty`);
         if (bytes.byteLength > outputFileLimits.bytes)
             throw new OutputFileError(`Generated file ${filename} exceeds 25 MiB`);
         const path = join(root, filename);
-        if (file.bytes)
-            await writePrivate(path, bytes);
-        else {
-            await copyFile(file.path, path);
-            await verifiedBytes(path, { byteSize: bytes.byteLength, sha256: sha256(bytes) });
-        }
+        await writePrivate(path, bytes);
         staged.push({
             filename,
             contentType: file.contentType,

package/dist/evals/local-worker.d.ts

@@ -4,7 +4,7 @@ import type { EnvironmentTool } from "../environment/tools.js";
 import type { WorldHandoff } from "../environment/types.js";
 import type { ActualAgentManifestInputV2, AttemptConnectionBundleV2, RequestedAttemptProviderV2 } from "./attempt.js";
 import type { EvaluationClient } from "./client.js";
-import { type RunExperimentTargetContext, type RunnerReport } from "./runner.js";
+import { type RunExperimentTargetContext, type RunnerReport, type TelemetryNotAccepted } from "./runner.js";
 import type { ExperimentCase, JsonValue, LocalAgentRegistration, LocalFile, LocalScorer, TargetResult } from "./types.js";
 /** Capability strings a registration declares; Hue offers only matching cases. */
 export declare const localAgentCapabilities: {
@@ -13,6 +13,10 @@ export declare const localAgentCapabilities: {
     /** Ordinary cases run directly on this machine: JSON inputs and pinned input files in,
      * JSON output and generated files out. */
     readonly direct: "direct:v1";
+    /** Cases pinned to a world that also carry agent-visible input files: `target` receives them
+     * with the world and may return generated files. Declare it with `input:<extension>` for each
+     * accepted file type; it requires `target`. */
+    readonly environmentFiles: "environment-files:v1";
 };
 /** Candidate-visible context for one queued local agent execution. */
 export interface LocalAgentTargetContext {
@@ -47,6 +51,13 @@ export interface LocalAgentTargetContext {
     /** Credential-bearing provider connections for this callback only. Hue never
      * checkpoints, logs or adds this response to parity digests. */
     connectionBundle?: AttemptConnectionBundleV2;
+    /** Verified copies of the case's agent-visible input files; evaluator-only files are
+     * withheld. Hue offers a world case with such files only to a registration declaring
+     * `environment-files:v1`. Empty when the case has none. */
+    files: LocalFile[];
+    /** Private directory for this case, removed after it; return generated files with
+     * `withFiles`. */
+    outputDirectory: string;
 }
 /** Candidate-visible context for one queued ordinary case run directly on this machine. */
 export interface LocalAgentDirectContext {
@@ -88,6 +99,12 @@ export interface RunLocalAgentOptions {
     scorers?: LocalScorer[];
     /** Cases in flight, between 1 and 16; defaults to 1. */
     concurrency?: number;
+    /** What a case does when its telemetry is not accepted; see
+     * `traceNotAccepted` of {@link runExperiment}. Defaults to `"stop"`. */
+    traceNotAccepted?: "stop" | "fail_case";
+    /** Called as each case is failed for its telemetry; see `onTelemetryNotAccepted` of
+     * {@link runExperiment}. */
+    onTelemetryNotAccepted?(entry: TelemetryNotAccepted): void | Promise<void>;
     /** Polling interval in milliseconds, 250–60000; defaults to 2000. */
     pollIntervalMillis?: number;
     /** Stops polling cooperatively; does not cancel an active callback. */
@@ -116,8 +133,9 @@ export interface RunLocalAgentOptions {
         surfaceKey: "google.gmail/mcp" | "slack/mcp";
     };
     /** Runs cases pinned to a hosted world (`environment:v1`): invokes the existing agent against
-     * isolated tools and candidate-safe context. */
-    target?(inputs: JsonValue, tools: Record<string, EnvironmentTool>, context: LocalAgentTargetContext): JsonValue | undefined | Promise<JsonValue | undefined>;
+     * isolated tools and candidate-safe context. Return the output, or `withFiles(output, files)`
+     * when it generated files. */
+    target?(inputs: JsonValue, tools: Record<string, EnvironmentTool>, context: LocalAgentTargetContext): JsonValue | TargetResult | undefined | Promise<JsonValue | TargetResult | undefined>;
     /** Runs ordinary cases without a world (`direct:v1`): the agent receives the case inputs
      * and verified input files and returns JSON output and/or generated files. */
     directTarget?(inputs: JsonValue, context: LocalAgentDirectContext): JsonValue | TargetResult | undefined | Promise<JsonValue | TargetResult | undefined>;

package/dist/evals/local-worker.js

@@ -10,6 +10,10 @@ export const localAgentCapabilities = {
     /** Ordinary cases run directly on this machine: JSON inputs and pinned input files in,
      * JSON output and generated files out. */
     direct: "direct:v1",
+    /** Cases pinned to a world that also carry agent-visible input files: `target` receives them
+     * with the world and may return generated files. Declare it with `input:<extension>` for each
+     * accepted file type; it requires `target`. */
+    environmentFiles: "environment-files:v1",
 };
 /** Allowlist the candidate surface for an ordinary case; frozen expectations stay with grading. */
 export function localAgentDirectContext(context, signal) {
@@ -44,6 +48,8 @@ function localAgentTargetContext(context) {
         ...(context.connectionBundle
             ? { connectionBundle: structuredClone(context.connectionBundle) }
             : {}),
+        files: structuredClone(context.files),
+        outputDirectory: context.outputDirectory,
     };
 }
 /** The registration's capabilities: explicit values plus one per supplied callback. */
@@ -55,6 +61,8 @@ export function registeredCapabilities(options) {
         throw new TypeError("direct:v1 requires a directTarget callback");
     if (declared.includes(localAgentCapabilities.environment) && !options.target)
         throw new TypeError("environment:v1 requires a target callback");
+    if (declared.includes(localAgentCapabilities.environmentFiles) && !options.target)
+        throw new TypeError("environment-files:v1 requires a target callback");
     return [
         ...new Set([
             ...declared,
@@ -163,6 +171,12 @@ export async function runLocalAgent(options) {
                     // Worker dispatch uses the case pin: directTarget never receives a world.
                     environmentEvidence: options.directTarget ? "when_pinned" : "required",
                     traceEvidence: { mode: "required" },
+                    ...(options.traceNotAccepted ? { traceNotAccepted: options.traceNotAccepted } : {}),
+                    ...(options.onTelemetryNotAccepted
+                        ? {
+                            onTelemetryNotAccepted: (entry) => options.onTelemetryNotAccepted(entry),
+                        }
+                        : {}),
                     scorers: options.scorers,
                     concurrency: options.concurrency,
                     target: (inputs, context) => {

package/dist/evals/runner.d.ts

@@ -93,6 +93,19 @@ export interface RunExperimentOptions extends RunnerOptions {
         /** Why evidence is omitted, up to 4000 characters. */
         reason: string;
     };
+    /**
+     * What a case does when evidence is required and its trace or logs are not fully accepted.
+     * `"stop"` (the default) keeps the saved outcome, leaves the execution started for recovery and
+     * rejects. `"fail_case"` completes the execution as failed instead, with the evidence omitted
+     * under the reason `telemetry_not_accepted` and without its output or generated files, so no
+     * scorer can pass it; reports it in {@link RunnerReport.telemetryNotAccepted} and goes on with
+     * the other cases.
+     */
+    traceNotAccepted?: "stop" | "fail_case";
+    /** Called when this call completes a case as failed under `traceNotAccepted: "fail_case"`,
+     * before the run goes on, so the failure is known even if a later case stops the run. A call
+     * that resumes an already completed case lists it in the report without calling again. */
+    onTelemetryNotAccepted?(entry: TelemetryNotAccepted): void | Promise<void>;
     /** Runs the application for one frozen case; return the output, `withFiles(output, files)`
      * when it generated files, or `undefined` when unavailable. */
     target(inputs: JsonValue, context: RunExperimentTargetContext): JsonValue | TargetResult | undefined | Promise<JsonValue | TargetResult | undefined>;
@@ -112,7 +125,39 @@ export interface RunnerReport {
     resultIds: string[];
     /** Pins without a local implementation, left pending for their authorized executor. */
     deferredScorerVersionIds: string[];
+    /** Cases completed as failed because their telemetry was not accepted, under
+     * `traceNotAccepted: "fail_case"`; absent when there were none. */
+    telemetryNotAccepted?: TelemetryNotAccepted[];
 }
+/** Export issue counts, never content: which signal, what happened, the HTTP status when Hue
+ * answered, and how many records. */
+export interface TelemetryIssueCount {
+    /** Signal the records belong to. */
+    signal: "traces" | "logs";
+    /** `rejected` by Hue, `failed` to deliver, `dropped` from the queue, or an `invalid` record. */
+    kind: "rejected" | "failed" | "dropped" | "invalid";
+    /** HTTP status when the issue came from Hue's answer. */
+    status?: number;
+    /** Records affected. */
+    count: number;
+}
+/** A case whose trace or logs were not fully accepted, completed as failed. */
+export interface TelemetryNotAccepted {
+    /** Experiment case (item) ID. */
+    caseId: string;
+    /** The case's caller-chosen key. */
+    caseKey: string;
+    /** Execution completed as failed. */
+    executionId: string;
+    /** Sanitized issues the transport reported while the case's telemetry was exported. */
+    issues: TelemetryIssueCount[];
+}
+/** Stable reason, and first word of the omission reason, of a case failed for its telemetry. */
+export declare const TELEMETRY_NOT_ACCEPTED = "telemetry_not_accepted";
+/** The issues of a failed export, summed by signal, kind and status. */
+export declare function telemetryIssueCounts(error: unknown): TelemetryIssueCount[];
+/** `telemetry_not_accepted`, then the issue counts, for example `traces failed 1 (HTTP 400)`. */
+export declare function describeTelemetryIssues(issues: TelemetryIssueCount[]): string;
 /**
  * Runs every case of a frozen experiment through `target` on this machine, completes each
  * execution, scores it with local scorers and uploads the results, checkpointing so an interrupted