@hue-run/sdk 0.9.0 → 0.10.0

package/dist/cli/eval-direct.js

@@ -1,6 +1,7 @@
-import { copyFile, mkdir, readdir, readFile, stat, writeFile } from "node:fs/promises";
+import { constants } from "node:fs";
+import { copyFile, lstat, mkdir, opendir, writeFile } from "node:fs/promises";
 import { basename, extname, join } from "node:path";
-import { outputContentTypes } from "../evals/files.js";
+import { outputContentTypes, outputFileLimits, readAgentFile, safeFilename, } from "../evals/files.js";
 import { withFiles } from "../evals/types.js";
 /**
  * Direct (file) cases for `hue eval`: a case is a task plus pinned input files, and the agent's
@@ -43,6 +44,11 @@ for (const type of Object.values(outputExtensions))
         throw new Error(`outputExtensions maps to a content type Hue does not accept: ${type}`);
 const HELPER_FILES = new Set(["manifest.json", "result.json", "output.json"]);
 const SUMMARY_FILES = ["summary.txt", "summary.md", "final.txt", "resumen.txt"];
+/** The most a helper (`manifest.json`, `result.json`, `summary.txt`, …) may hold: the same bound
+ * as the command's stdout. */
+const HELPER_BYTES = 4 * 1024 * 1024;
+/** Entries a collection visits at most, hidden and skipped ones included. */
+const MAX_ENTRIES = 1024;
 /** Writes the case directory: inputs, descriptor, verified file copies and an empty output folder. */
 export async function stageDirectCase(root, input) {
     const caseDirectory = join(root, "case");
@@ -50,21 +56,25 @@ export async function stageDirectCase(root, input) {
     await mkdir(outputDirectory, { recursive: true, mode: 0o700 });
     const inputsPath = join(caseDirectory, "inputs.json");
     await writeFile(inputsPath, JSON.stringify(input.inputs, null, 2), { mode: 0o600 });
+    // Names that differ only in case or Unicode normalization are the same file on macOS and
+    // Windows filesystems; keep both inputs under distinct names, and never overwrite one.
     const used = new Set();
+    const key = (folder, name) => `${folder}/${name.normalize("NFC").toLowerCase()}`;
     const files = [];
     for (const file of input.files) {
         const folder = join(caseDirectory, "files", file.role);
         await mkdir(folder, { recursive: true, mode: 0o700 });
         // Two inputs may share a filename (two incident reports, say); keep both.
-        const name = basename(file.filename);
+        const name = safeFilename(basename(file.filename));
         const extension = extname(name);
         const stem = name.slice(0, name.length - extension.length);
-        let target = join(folder, name);
-        for (let copy = 2; used.has(target); copy++)
-            target = join(folder, `${stem} (${copy})${extension}`);
-        used.add(target);
-        await copyFile(file.path, target);
-        files.push({ role: file.role, filename: basename(target), path: target });
+        let target = name;
+        for (let copy = 2; used.has(key(folder, target)); copy++)
+            target = `${stem} (${copy})${extension}`;
+        used.add(key(folder, target));
+        const path = join(folder, target);
+        await copyFile(file.path, path, constants.COPYFILE_EXCL);
+        files.push({ role: file.role, filename: target, path });
     }
     await writeFile(join(caseDirectory, "case.json"), JSON.stringify({
         id: input.item.id,
@@ -76,33 +86,119 @@ export async function stageDirectCase(root, input) {
     }, null, 2), { mode: 0o600 });
     return { caseDirectory, inputsPath, outputDirectory, files };
 }
-async function readJsonFile(path) {
+const identityOf = (info) => ({
+    dev: info.dev,
+    ino: info.ino,
+});
+const sameIdentity = (a, b) => a.dev === b.dev && a.ino === b.ino;
+/** A real directory (not a symlink) and its identity, or undefined when `path` is absent. A
+ * parent that is no longer a directory (ENOTDIR) is a change, never an absent directory: a case
+ * directory replaced by a file must not read as "wrote nothing". */
+async function directoryIdentity(path, label) {
+    let info;
     try {
-        return JSON.parse(await readFile(path, "utf8"));
+        info = await lstat(path);
     }
     catch (error) {
         if (error.code === "ENOENT")
             return undefined;
-        throw new Error(`${basename(path)} in the output directory is not valid JSON`);
+        throw unexamined(label, error);
     }
+    if (info.isSymbolicLink() || !info.isDirectory())
+        throw new Error(`${label} is not a directory; a symbolic link or file is never collected`);
+    return identityOf(info);
 }
-/** Regular files anywhere under the output directory, named by their `/`-joined relative path.
- * Hidden and lock entries (dot names, `~$…`) and symlinks are skipped at every level. */
-async function listOutputFiles(directory, prefix = "") {
-    const entries = await readdir(directory, { withFileTypes: true }).catch(() => []);
+/** The error for an entry that vanished or changed kind while it was being collected, or could
+ * not be examined at all. */
+function unexamined(label, error) {
+    const code = error.code;
+    return new Error(code === "ENOENT" || code === "ENOTDIR" || code === "ELOOP"
+        ? `${label} changed while it was collected`
+        : `${label} could not be read`);
+}
+/**
+ * Regular files anywhere under the output directory, named by their `/`-joined relative path,
+ * with the device and inode each had when listed. Hidden and lock entries (dot names, `~$…`),
+ * symlinks and anything that is neither a regular file nor a directory are skipped at every
+ * level, except that a top-level helper name must be a regular file when present. The walk stops
+ * past 32 documents or 1024 entries, and an entry that vanished or a directory that changed while
+ * it was listed is refused.
+ */
+async function listOutputFiles(root, rootIdentity, seams) {
     const found = [];
-    for (const entry of entries) {
-        if (entry.name.startsWith(".") || entry.name.startsWith("~$"))
-            continue;
-        const path = join(directory, entry.name);
-        const name = `${prefix}${entry.name}`;
-        if (entry.isDirectory())
-            found.push(...(await listOutputFiles(path, `${name}/`)));
-        else if (entry.isFile())
-            found.push({ name, path });
+    const directories = [
+        { path: root, label: "output/", identity: rootIdentity },
+    ];
+    let visited = 0;
+    let documents = 0;
+    for (let index = 0; index < directories.length; index++) {
+        const directory = directories[index];
+        const prefix = index === 0 ? "" : `${directory.label.slice("output/".length)}`;
+        let entries;
+        try {
+            entries = await opendir(directory.path);
+        }
+        catch (error) {
+            throw unexamined(directory.label, error);
+        }
+        for await (const entry of entries) {
+            if (++visited > MAX_ENTRIES)
+                throw new Error(`The output directory holds more than ${MAX_ENTRIES} entries`);
+            const name = `${prefix}${entry.name}`;
+            const helper = index === 0 && (HELPER_FILES.has(name) || SUMMARY_FILES.includes(name));
+            if (!helper && (entry.name.startsWith(".") || entry.name.startsWith("~$")))
+                continue;
+            const path = join(directory.path, entry.name);
+            if (helper && !entry.isFile())
+                throw new Error(`output/${name} is not a regular file; the agent must write it itself`);
+            if (entry.isDirectory() || entry.isFile())
+                await seams.beforeEntry?.(name);
+            if (entry.isDirectory()) {
+                const identity = await directoryIdentity(path, `output/${name}`);
+                if (!identity)
+                    throw new Error(`output/${name} changed while it was collected`);
+                directories.push({ path, label: `output/${name}/`, identity });
+            }
+            else if (entry.isFile()) {
+                let info;
+                try {
+                    info = await lstat(path);
+                }
+                catch (error) {
+                    throw unexamined(`output/${name}`, error);
+                }
+                if (!info.isFile())
+                    throw new Error(`output/${name} changed while it was collected`);
+                found.push({ name, path, identity: identityOf(info) });
+                if (!helper && ++documents > outputFileLimits.count)
+                    throw new Error(`The agent wrote more than ${outputFileLimits.count} files`);
+            }
+        }
+    }
+    // Every directory must still be the one listed: none was swapped for a link meanwhile.
+    for (const directory of directories) {
+        const now = await directoryIdentity(directory.path, directory.label);
+        if (!now || !sameIdentity(now, directory.identity))
+            throw new Error(`${directory.label} changed while it was collected`);
     }
     return found;
 }
+async function readHelper(listed) {
+    if (!listed)
+        return undefined;
+    const bytes = await readAgentFile(listed.path, `output/${listed.name}`, HELPER_BYTES, listed.identity);
+    return Buffer.from(bytes).toString("utf8");
+}
+function parseHelper(listed, text) {
+    if (text === undefined)
+        return undefined;
+    try {
+        return JSON.parse(text);
+    }
+    catch {
+        throw new Error(`${listed.name} in the output directory is not valid JSON`);
+    }
+}
 /** Artifact filenames cannot contain path separators or controls. Encode only those forbidden
  * characters plus `%`, then represent `/` as `%2F`; distinct output paths stay distinct. */
 function artifactFilename(relativePath) {
@@ -120,9 +216,18 @@ function artifactFilename(relativePath) {
  * becomes a generated file; an unsupported extension is the agent's error rather than a silently
  * dropped document. `fallbackOutput` (for example the command's stdout) is used when no JSON
  * output or summary file was written.
+ *
+ * Nothing here trusts a path the agent could have changed: the output directory must be a real
+ * directory, every file is opened without following a final symlink or blocking on a FIFO, must
+ * be the regular file the listing saw and within its size limit, and the result carries the bytes
+ * read from that open file. `seams` are for tests only.
  */
-export async function collectDirectOutputs(outputDirectory, fallbackOutput) {
-    const manifest = await readJsonFile(join(outputDirectory, "manifest.json"));
+export async function collectDirectOutputs(outputDirectory, fallbackOutput, seams = {}) {
+    const rootIdentity = await directoryIdentity(outputDirectory, "The output directory");
+    const listed = rootIdentity ? await listOutputFiles(outputDirectory, rootIdentity, seams) : [];
+    await seams.afterListing?.();
+    const top = (name) => listed.find((entry) => entry.name === name);
+    const manifest = parseHelper(top("manifest.json"), await readHelper(top("manifest.json")));
     const declaredPrimary = manifest && typeof manifest === "object" && !Array.isArray(manifest)
         ? manifest.primary
         : undefined;
@@ -130,11 +235,11 @@ export async function collectDirectOutputs(outputDirectory, fallbackOutput) {
         ? manifest.output
         : undefined;
     let output = declaredOutput ??
-        (await readJsonFile(join(outputDirectory, "result.json"))) ??
-        (await readJsonFile(join(outputDirectory, "output.json")));
+        parseHelper(top("result.json"), await readHelper(top("result.json"))) ??
+        parseHelper(top("output.json"), await readHelper(top("output.json")));
     if (output === undefined)
         for (const name of SUMMARY_FILES) {
-            const text = await readFile(join(outputDirectory, name), "utf8").catch(() => undefined);
+            const text = await readHelper(top(name));
             if (text !== undefined) {
                 output = { summary: text };
                 break;
@@ -143,7 +248,7 @@ export async function collectDirectOutputs(outputDirectory, fallbackOutput) {
     if (output === undefined)
         output = fallbackOutput;
     // Helper and summary names are reserved at the top level only; a nested one is a document.
-    const entries = (await listOutputFiles(outputDirectory)).filter((entry) => !HELPER_FILES.has(entry.name) && !SUMMARY_FILES.includes(entry.name));
+    const entries = listed.filter((entry) => !HELPER_FILES.has(entry.name) && !SUMMARY_FILES.includes(entry.name));
     // Directory order differs between filesystems; keep uploads stable.
     entries.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
     const unsupported = entries.filter((entry) => !outputExtensions[extname(entry.name).toLowerCase()]);
@@ -151,10 +256,11 @@ export async function collectDirectOutputs(outputDirectory, fallbackOutput) {
         throw new Error(`The agent wrote files Hue does not accept as generated documents: ${unsupported.map((entry) => entry.name).join(", ")} (accepted: ${Object.keys(outputExtensions).join(" ")})`);
     const files = [];
     for (const entry of entries) {
-        if ((await stat(entry.path)).size === 0)
+        const bytes = (await readAgentFile(entry.path, `Generated file ${entry.name}`, outputFileLimits.bytes, entry.identity));
+        if (!bytes.byteLength)
             throw new Error(`The agent wrote an empty file: ${entry.name}`);
         files.push({
-            path: entry.path,
+            bytes,
             filename: artifactFilename(entry.name),
             contentType: outputExtensions[extname(entry.name).toLowerCase()],
         });
@@ -167,5 +273,11 @@ export async function collectDirectOutputs(outputDirectory, fallbackOutput) {
     }
     else if (files.length === 1)
         files[0].primary = true;
+    // The directory must still be the one listed when every byte has been read.
+    if (rootIdentity) {
+        const now = await directoryIdentity(outputDirectory, "The output directory");
+        if (!now || !sameIdentity(now, rootIdentity))
+            throw new Error("The output directory changed while it was collected");
+    }
     return withFiles(output, files);
 }

package/dist/cli/eval.d.ts

@@ -1,9 +1,10 @@
 import type { ExperimentCase, LocalFile } from "../evals/types.js";
 import { TargetResult } from "../evals/types.js";
 import { type SimulationTargetContext } from "../evals/simulation.js";
+import { type ExperimentVerdicts } from "../evals/verdicts.js";
 import type { JsonValue } from "../types.js";
 /** Adapter contract: the module's `default` or `runMyAgent` export. */
-export type EvalAdapter = (inputs: JsonValue, context: SimulationTargetContext) => JsonValue | undefined | Promise<JsonValue | undefined>;
+export type EvalAdapter = (inputs: JsonValue, context: SimulationTargetContext) => JsonValue | TargetResult | undefined | Promise<JsonValue | TargetResult | undefined>;
 /**
  * Context of a direct (file) case: pinned input files in, generated documents out. The same
  * adapter module serves both modes; `mode` tells it which context it received.
@@ -20,6 +21,32 @@ export interface DirectTargetContext {
     signal?: AbortSignal;
 }
 export type DirectEvalAdapter = (inputs: JsonValue, context: DirectTargetContext) => JsonValue | TargetResult | undefined | Promise<JsonValue | TargetResult | undefined>;
+export interface Output {
+    /** Human progress lines: stdout normally, stderr with --json. */
+    log(line: string): void;
+    /** Diagnostics; always stderr. */
+    error(line: string): void;
+}
+/**
+ * Spawns the agent command once in its own process group and returns its trimmed stdout. A
+ * timeout or Ctrl+C stops the agent the shell started, not only the shell: a survivor would still
+ * hold a world token and could write after Hue recorded the case as failed. The group is signalled
+ * even once the shell has exited, since a compound command's agent can outlive it, and a stopped
+ * command settles only when nothing in the group is left. A command that exits normally but
+ * leaves processes behind has them stopped the same way before it settles, so none can touch its
+ * outputs while they are collected. Once the group is found empty it is never signalled again,
+ * since its ID could be reused. Windows has no process group to signal, so the child alone is
+ * stopped there.
+ */
+export declare function spawnAgentCommand(command: string, options: {
+    cwd?: string;
+    env: Record<string, string | undefined>;
+    stdin?: string;
+    timeoutSeconds: number;
+    signal?: AbortSignal;
+}): Promise<string>;
+export declare function renderTable(verdicts: ExperimentVerdicts, output: Output): void;
+export declare function explain(error: unknown): string;
 /**
  * `hue eval`: runs a local adapter or command against a Scenario or eval set through
  * `runSimulation`, or registers it as an outbound worker through `runLocalAgent`.