@hue-run/sdk 0.4.2 → 0.5.0

package/dist/cli/eval-direct.js

@@ -0,0 +1,171 @@
+import { copyFile, mkdir, readdir, readFile, stat, writeFile } from "node:fs/promises";
+import { basename, extname, join } from "node:path";
+import { outputContentTypes } from "../evals/files.js";
+import { withFiles } from "../evals/types.js";
+/**
+ * Direct (file) cases for `hue eval`: a case is a task plus pinned input files, and the agent's
+ * answer is one or more generated documents. The agent command receives a private case directory
+ * instead of stdin/stdout JSON, the same layout Hue's document workers use:
+ *
+ *   <case dir>/inputs.json            the case inputs
+ *   <case dir>/case.json              id, external key, run config and the staged file list
+ *   <case dir>/files/<role>/<name>    verified copies of the agent-visible pinned files
+ *   <case dir>/output/                where the command writes what it produced
+ *
+ * Everything under `output/` becomes the execution's generated files. Optional helpers there:
+ * `manifest.json` (`{ "primary": "<filename>", "output": <json> }`), `result.json` (the JSON
+ * output) and `summary.txt` / `summary.md` (recorded as `{ "summary": "…" }`).
+ */
+export const directCaseEnvironment = [
+    "HUE_CASE_DIR",
+    "HUE_CASE_INPUTS",
+    "HUE_CASE_OUTPUT_DIR",
+    "HUE_CASE_ID",
+    "HUE_CASE_KEY",
+    "HUE_EXECUTION_ID",
+];
+/** Generated-file extensions Hue accepts and the content type recorded for each. */
+export const outputExtensions = {
+    ".pdf": "application/pdf",
+    ".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+    ".pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+    ".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+    ".json": "application/json",
+    ".txt": "text/plain",
+    ".csv": "text/csv",
+    ".png": "image/png",
+    ".jpg": "image/jpeg",
+    ".jpeg": "image/jpeg",
+    ".webp": "image/webp",
+};
+for (const type of Object.values(outputExtensions))
+    if (!outputContentTypes.includes(type))
+        throw new Error(`outputExtensions maps to a content type Hue does not accept: ${type}`);
+const HELPER_FILES = new Set(["manifest.json", "result.json", "output.json"]);
+const SUMMARY_FILES = ["summary.txt", "summary.md", "final.txt", "resumen.txt"];
+/** Writes the case directory: inputs, descriptor, verified file copies and an empty output folder. */
+export async function stageDirectCase(root, input) {
+    const caseDirectory = join(root, "case");
+    const outputDirectory = join(caseDirectory, "output");
+    await mkdir(outputDirectory, { recursive: true, mode: 0o700 });
+    const inputsPath = join(caseDirectory, "inputs.json");
+    await writeFile(inputsPath, JSON.stringify(input.inputs, null, 2), { mode: 0o600 });
+    const used = new Set();
+    const files = [];
+    for (const file of input.files) {
+        const folder = join(caseDirectory, "files", file.role);
+        await mkdir(folder, { recursive: true, mode: 0o700 });
+        // Two inputs may share a filename (two incident reports, say); keep both.
+        const name = basename(file.filename);
+        const extension = extname(name);
+        const stem = name.slice(0, name.length - extension.length);
+        let target = join(folder, name);
+        for (let copy = 2; used.has(target); copy++)
+            target = join(folder, `${stem} (${copy})${extension}`);
+        used.add(target);
+        await copyFile(file.path, target);
+        files.push({ role: file.role, filename: basename(target), path: target });
+    }
+    await writeFile(join(caseDirectory, "case.json"), JSON.stringify({
+        id: input.item.id,
+        externalKey: input.item.externalKey,
+        executionId: input.executionId,
+        config: input.config,
+        files,
+        outputDirectory,
+    }, null, 2), { mode: 0o600 });
+    return { caseDirectory, inputsPath, outputDirectory, files };
+}
+async function readJsonFile(path) {
+    try {
+        return JSON.parse(await readFile(path, "utf8"));
+    }
+    catch (error) {
+        if (error.code === "ENOENT")
+            return undefined;
+        throw new Error(`${basename(path)} in the output directory is not valid JSON`);
+    }
+}
+/** Regular files anywhere under the output directory, named by their `/`-joined relative path.
+ * Hidden and lock entries (dot names, `~$…`) and symlinks are skipped at every level. */
+async function listOutputFiles(directory, prefix = "") {
+    const entries = await readdir(directory, { withFileTypes: true }).catch(() => []);
+    const found = [];
+    for (const entry of entries) {
+        if (entry.name.startsWith(".") || entry.name.startsWith("~$"))
+            continue;
+        const path = join(directory, entry.name);
+        const name = `${prefix}${entry.name}`;
+        if (entry.isDirectory())
+            found.push(...(await listOutputFiles(path, `${name}/`)));
+        else if (entry.isFile())
+            found.push({ name, path });
+    }
+    return found;
+}
+/** Artifact filenames cannot contain path separators or controls. Encode only those forbidden
+ * characters plus `%`, then represent `/` as `%2F`; distinct output paths stay distinct. */
+function artifactFilename(relativePath) {
+    return relativePath
+        .split("/")
+        .map((segment) => segment
+        .replaceAll("%", "%25")
+        .replaceAll("\\", "%5C")
+        // eslint-disable-next-line no-control-regex -- artifact filenames forbid controls
+        .replace(/[\x00-\x1f\x7f]/gu, (character) => encodeURIComponent(character)))
+        .join("%2F");
+}
+/**
+ * Turns the output folder into the target's result. Every regular file, in subdirectories too,
+ * becomes a generated file; an unsupported extension is the agent's error rather than a silently
+ * dropped document. `fallbackOutput` (for example the command's stdout) is used when no JSON
+ * output or summary file was written.
+ */
+export async function collectDirectOutputs(outputDirectory, fallbackOutput) {
+    const manifest = await readJsonFile(join(outputDirectory, "manifest.json"));
+    const declaredPrimary = manifest && typeof manifest === "object" && !Array.isArray(manifest)
+        ? manifest.primary
+        : undefined;
+    const declaredOutput = manifest && typeof manifest === "object" && !Array.isArray(manifest)
+        ? manifest.output
+        : undefined;
+    let output = declaredOutput ??
+        (await readJsonFile(join(outputDirectory, "result.json"))) ??
+        (await readJsonFile(join(outputDirectory, "output.json")));
+    if (output === undefined)
+        for (const name of SUMMARY_FILES) {
+            const text = await readFile(join(outputDirectory, name), "utf8").catch(() => undefined);
+            if (text !== undefined) {
+                output = { summary: text };
+                break;
+            }
+        }
+    if (output === undefined)
+        output = fallbackOutput;
+    // Helper and summary names are reserved at the top level only; a nested one is a document.
+    const entries = (await listOutputFiles(outputDirectory)).filter((entry) => !HELPER_FILES.has(entry.name) && !SUMMARY_FILES.includes(entry.name));
+    // Directory order differs between filesystems; keep uploads stable.
+    entries.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
+    const unsupported = entries.filter((entry) => !outputExtensions[extname(entry.name).toLowerCase()]);
+    if (unsupported.length)
+        throw new Error(`The agent wrote files Hue does not accept as generated documents: ${unsupported.map((entry) => entry.name).join(", ")} (accepted: ${Object.keys(outputExtensions).join(" ")})`);
+    const files = [];
+    for (const entry of entries) {
+        if ((await stat(entry.path)).size === 0)
+            throw new Error(`The agent wrote an empty file: ${entry.name}`);
+        files.push({
+            path: entry.path,
+            filename: artifactFilename(entry.name),
+            contentType: outputExtensions[extname(entry.name).toLowerCase()],
+        });
+    }
+    if (typeof declaredPrimary === "string") {
+        const primaryIndex = entries.findIndex((entry) => entry.name === declaredPrimary);
+        if (primaryIndex < 0)
+            throw new Error(`manifest.json names a primary file that was not written: ${declaredPrimary}`);
+        files[primaryIndex].primary = true;
+    }
+    else if (files.length === 1)
+        files[0].primary = true;
+    return withFiles(output, files);
+}

package/dist/cli/eval.d.ts

@@ -0,0 +1,28 @@
+import type { ExperimentCase, LocalFile } from "../evals/types.js";
+import { TargetResult } from "../evals/types.js";
+import { type SimulationTargetContext } from "../evals/simulation.js";
+import type { JsonValue } from "../types.js";
+/** Adapter contract: the module's `default` or `runMyAgent` export. */
+export type EvalAdapter = (inputs: JsonValue, context: SimulationTargetContext) => JsonValue | undefined | Promise<JsonValue | undefined>;
+/**
+ * Context of a direct (file) case: pinned input files in, generated documents out. The same
+ * adapter module serves both modes; `mode` tells it which context it received.
+ */
+export interface DirectTargetContext {
+    mode: "direct";
+    config: JsonValue;
+    item: Pick<ExperimentCase, "id" | "externalKey">;
+    executionId: string;
+    /** Verified copies of the agent-visible pinned files (`source`, templates, originals). */
+    files: LocalFile[];
+    /** Private scratch directory for this case; generated files may be written here. */
+    outputDirectory: string;
+    signal?: AbortSignal;
+}
+export type DirectEvalAdapter = (inputs: JsonValue, context: DirectTargetContext) => JsonValue | TargetResult | undefined | Promise<JsonValue | TargetResult | undefined>;
+/**
+ * `hue eval`: runs a local adapter or command against a Scenario or eval set through
+ * `runSimulation`, or registers it as an outbound worker through `runLocalAgent`.
+ * Returns the process exit code.
+ */
+export declare function runEvalCommand(argv: string[]): Promise<number>;