agent-hitch 0.1.0 → 0.2.0

package/dist/src/eval-tools.js

@@ -0,0 +1,318 @@
+import { constants } from "node:fs";
+import { access, mkdir, rm } from "node:fs/promises";
+import path from "node:path";
+import { spawn } from "node:child_process";
+import { SCHEMA_VERSION, statePaths } from "./config.js";
+import { HitchError, invalidInput } from "./errors.js";
+import { atomicWriteJSON, ensureDir, readJSON } from "./fs.js";
+import { terminateProcess } from "./process.js";
+import { detectVersion, resolveExecutable } from "./registry.js";
+export const DEFAULT_HARBOR_VERSION = "0.21.0";
+export const HARBOR_CREDENTIAL_ENV = [
+    "DEEPSEEK_API_KEY",
+    "DEEPSEEK_BASE_URL",
+    "OPENAI_API_KEY",
+    "OPENAI_BASE_URL",
+    "OPENAI_ORG_ID",
+    "OPENAI_ORGANIZATION",
+    "ANTHROPIC_API_KEY",
+    "ANTHROPIC_AUTH_TOKEN",
+    "ANTHROPIC_BASE_URL",
+    "CLAUDE_CODE_OAUTH_TOKEN",
+    "GOOGLE_API_KEY",
+    "GEMINI_API_KEY",
+    "AZURE_OPENAI_API_KEY",
+    "AZURE_OPENAI_ENDPOINT",
+    "AZURE_OPENAI_API_VERSION",
+    "AWS_ACCESS_KEY_ID",
+    "AWS_SECRET_ACCESS_KEY",
+    "AWS_SESSION_TOKEN",
+    "AWS_REGION",
+    "AWS_DEFAULT_REGION",
+    "GITHUB_TOKEN",
+    "GH_TOKEN",
+];
+const PROVIDER_CREDENTIAL_ENV = [
+    "DEEPSEEK_API_KEY",
+    "OPENAI_API_KEY",
+    "ANTHROPIC_API_KEY",
+    "ANTHROPIC_AUTH_TOKEN",
+    "CLAUDE_CODE_OAUTH_TOKEN",
+    "GOOGLE_API_KEY",
+    "GEMINI_API_KEY",
+    "AZURE_OPENAI_API_KEY",
+    "AWS_ACCESS_KEY_ID",
+];
+export async function setupHarbor({ root, version = DEFAULT_HARBOR_VERSION, python, force = false, env = process.env, onProgress = () => { }, }) {
+    if (!root)
+        throw invalidInput("a Hitch state root is required for Harbor setup");
+    if (!/^\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.-]+)?$/.test(version)) {
+        throw invalidInput("Harbor version must be an exact semantic version");
+    }
+    const pythonInfo = await findPython({ explicit: python, env });
+    if (!pythonInfo.executable) {
+        throw new HitchError("Python 3.12 or newer was not found; install it or pass --python <path>", {
+            code: "python_unavailable",
+            exitCode: 3,
+        });
+    }
+    if (!pythonInfo.supported) {
+        throw new HitchError(`Harbor requires Python 3.12 or newer; found ${pythonInfo.version || "an unknown version"}`, {
+            code: "python_unsupported",
+            exitCode: 3,
+        });
+    }
+    const toolsDirectory = statePaths(root).tools;
+    const installDirectory = path.join(toolsDirectory, `harbor-${version}`);
+    const harborExecutable = venvExecutable(installDirectory, "harbor");
+    const currentPath = path.join(toolsDirectory, "harbor.json");
+    const installedVersion = await installedHarborVersion(harborExecutable);
+    if (installedVersion === version && !force) {
+        const manifest = await writeHarborSelection(currentPath, {
+            version,
+            pythonInfo,
+            installDirectory,
+            harborExecutable,
+            cacheHit: true,
+        });
+        return manifest;
+    }
+    if (installedVersion && !force) {
+        throw new HitchError(`managed Harbor at ${installDirectory} reports ${installedVersion}; rerun with --force to replace it`, {
+            code: "harbor_setup_conflict",
+            exitCode: 3,
+        });
+    }
+    await ensureDir(toolsDirectory);
+    if (force)
+        await rm(installDirectory, { recursive: true, force: true });
+    try {
+        await mkdir(installDirectory);
+    }
+    catch (error) {
+        if (error?.code === "EEXIST") {
+            throw new HitchError(`incomplete Harbor installation already exists at ${installDirectory}; rerun with --force`, {
+                code: "harbor_setup_incomplete",
+                exitCode: 3,
+            });
+        }
+        throw error;
+    }
+    try {
+        onProgress(`Creating an isolated Python environment with ${pythonInfo.version}`);
+        await requireCommand(pythonInfo.executable, ["-m", "venv", installDirectory], {
+            env,
+            code: "harbor_setup_failed",
+            timeoutMs: 5 * 60_000,
+        });
+        const environmentPython = venvExecutable(installDirectory, "python");
+        onProgress(`Installing harbor==${version}`);
+        await requireCommand(environmentPython, [
+            "-m", "pip", "install", "--disable-pip-version-check", "--no-input", `harbor==${version}`,
+        ], {
+            env,
+            code: "harbor_setup_failed",
+            timeoutMs: 20 * 60_000,
+        });
+        const observedVersion = await installedHarborVersion(harborExecutable);
+        if (observedVersion !== version) {
+            throw new HitchError(`installed Harbor reported ${observedVersion || "no version"}; expected ${version}`, {
+                code: "harbor_setup_failed",
+                exitCode: 13,
+            });
+        }
+        return await writeHarborSelection(currentPath, {
+            version,
+            pythonInfo,
+            installDirectory,
+            harborExecutable,
+            cacheHit: false,
+        });
+    }
+    catch (error) {
+        await rm(installDirectory, { recursive: true, force: true });
+        throw error;
+    }
+}
+export async function doctorHarbor({ root, python, harbor, docker, env = process.env }) {
+    if (!root)
+        throw invalidInput("a Hitch state root is required for eval doctor");
+    const pythonInfo = await findPython({ explicit: python, env });
+    const harborInfo = await locateHarbor({ root, explicit: harbor, env });
+    const dockerConfigured = docker || env.HITCH_DOCKER_PATH || "docker";
+    const dockerExecutable = await resolveExecutable(dockerConfigured, env.PATH || "");
+    let dockerVersion = "";
+    let dockerDiagnostic = "";
+    if (dockerExecutable) {
+        const result = await captureCommand(dockerExecutable, ["version", "--format", "{{.Server.Version}}"], {
+            env,
+            timeoutMs: 10_000,
+        });
+        dockerVersion = result.code === 0 ? result.stdout.trim() : "";
+        dockerDiagnostic = result.code === 0 ? "" : (result.stderr.trim() || result.stdout.trim() || "Docker daemon is unavailable");
+    }
+    const credentials = PROVIDER_CREDENTIAL_ENV.filter((name) => typeof env[name] === "string" && env[name].trim());
+    const checks = {
+        python: pythonInfo.executable && pythonInfo.supported
+            ? { status: "ok", executable: pythonInfo.executable, version: pythonInfo.version, required: ">=3.12" }
+            : { status: "error", executable: pythonInfo.executable || null, version: pythonInfo.version || null, required: ">=3.12", message: "Python 3.12 or newer is required to install Harbor" },
+        harbor: harborInfo.executable
+            ? { status: "ok", executable: harborInfo.executable, version: harborInfo.version || null, source: harborInfo.source }
+            : { status: "error", executable: null, version: null, source: harborInfo.source, message: `Harbor executable not found: ${harborInfo.requested}` },
+        docker: dockerExecutable && dockerVersion
+            ? { status: "ok", executable: dockerExecutable, version: dockerVersion, daemon: "running" }
+            : { status: "error", executable: dockerExecutable || null, version: null, daemon: dockerExecutable ? "unavailable" : "not_checked", message: dockerExecutable ? dockerDiagnostic : `Docker executable not found: ${dockerConfigured}` },
+        credentials: credentials.length > 0
+            ? { status: "ok", present: credentials }
+            : { status: "warning", present: [], message: "No common provider credential is set; pass credentials before running a hosted-model eval" },
+    };
+    const ready = [checks.python, checks.harbor, checks.docker].every((check) => check?.status === "ok");
+    return {
+        schema_version: SCHEMA_VERSION,
+        backend: "harbor",
+        status: ready ? (checks.credentials?.status === "ok" ? "ready" : "ready_with_warnings") : "action_required",
+        ready,
+        checks,
+    };
+}
+export async function locateHarbor({ root, explicit, env = process.env } = {}) {
+    const configured = explicit || env.HITCH_HARBOR_PATH;
+    if (configured) {
+        const executable = await resolveExecutable(configured, env.PATH || "");
+        return {
+            executable,
+            version: executable ? await installedHarborVersion(executable) : "",
+            source: explicit ? "option" : "environment",
+            requested: configured,
+        };
+    }
+    const managed = root ? await managedHarborExecutable(root) : null;
+    if (managed) {
+        return {
+            executable: managed,
+            version: await installedHarborVersion(managed),
+            source: "managed",
+            requested: managed,
+        };
+    }
+    const executable = await resolveExecutable("harbor", env.PATH || "");
+    return {
+        executable,
+        version: executable ? await installedHarborVersion(executable) : "",
+        source: "path",
+        requested: "harbor",
+    };
+}
+export async function managedHarborExecutable(root) {
+    const toolsDirectory = statePaths(root).tools;
+    const current = await readJSON(path.join(toolsDirectory, "harbor.json"), null).catch(() => null);
+    if (!/^\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.-]+)?$/.test(current?.version || ""))
+        return null;
+    const version = current?.version;
+    const executable = venvExecutable(path.join(toolsDirectory, `harbor-${version}`), "harbor");
+    try {
+        await access(executable, constants.X_OK);
+        return executable;
+    }
+    catch {
+        return null;
+    }
+}
+async function findPython({ explicit, env }) {
+    const requested = explicit || env.HITCH_PYTHON_PATH;
+    const candidates = requested ? [requested] : ["python3.13", "python3.12", "python3"];
+    let firstFound = null;
+    for (const candidate of candidates) {
+        const executable = await resolveExecutable(candidate, env.PATH || "");
+        if (!executable)
+            continue;
+        const result = await captureCommand(executable, ["-c", "import sys; print('.'.join(map(str, sys.version_info[:3])))"], {
+            env,
+            timeoutMs: 5_000,
+        });
+        const version = result.code === 0 ? result.stdout.trim().split(/\s+/)[0] || "" : "";
+        const supported = pythonVersionSupported(version);
+        const info = { executable, version, supported };
+        firstFound ||= info;
+        if (supported || requested)
+            return info;
+    }
+    return firstFound || { executable: null, version: "", supported: false };
+}
+function pythonVersionSupported(version) {
+    const match = String(version).match(/^(\d+)\.(\d+)(?:\.\d+)?$/);
+    if (!match)
+        return false;
+    const major = Number(match[1]);
+    const minor = Number(match[2]);
+    return major > 3 || (major === 3 && minor >= 12);
+}
+async function installedHarborVersion(executable) {
+    try {
+        await access(executable, constants.X_OK);
+    }
+    catch {
+        return "";
+    }
+    const value = await detectVersion(executable, ["--version"]);
+    return value.match(/\b\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.-]+)?\b/)?.[0] || "";
+}
+async function writeHarborSelection(currentPath, { version, pythonInfo, installDirectory, harborExecutable, cacheHit, }) {
+    const manifest = {
+        schema_version: SCHEMA_VERSION,
+        tool: "harbor",
+        version,
+        executable: harborExecutable,
+        install_directory: installDirectory,
+        python: { executable: pythonInfo.executable || "", version: pythonInfo.version },
+        cache_hit: cacheHit,
+        selected_at: new Date().toISOString(),
+    };
+    await atomicWriteJSON(currentPath, manifest);
+    return manifest;
+}
+function venvExecutable(directory, name) {
+    return process.platform === "win32"
+        ? path.join(directory, "Scripts", `${name}.exe`)
+        : path.join(directory, "bin", name);
+}
+async function requireCommand(executable, args, { env, code, timeoutMs }) {
+    const result = await captureCommand(executable, args, { env, timeoutMs });
+    if (result.code === 0)
+        return result;
+    const detail = result.stderr.trim() || result.stdout.trim();
+    throw new HitchError(`${path.basename(executable)} exited with code ${result.code ?? "null"}${detail ? `: ${detail}` : ""}`, {
+        code,
+        exitCode: 13,
+    });
+}
+function captureCommand(executable, args, { env = process.env, timeoutMs = 10_000 } = {}) {
+    return new Promise((resolve, reject) => {
+        const child = spawn(executable, args, {
+            env,
+            stdio: ["ignore", "pipe", "pipe"],
+            detached: process.platform !== "win32",
+            windowsHide: true,
+        });
+        let stdout = "";
+        let stderr = "";
+        let timedOut = false;
+        const append = (current, chunk) => `${current}${chunk}`.slice(-1_000_000);
+        child.stdout.on("data", (chunk) => { stdout = append(stdout, chunk); });
+        child.stderr.on("data", (chunk) => { stderr = append(stderr, chunk); });
+        const timer = setTimeout(() => {
+            timedOut = true;
+            terminateProcess(child).catch(() => { });
+        }, timeoutMs);
+        timer.unref?.();
+        child.once("error", (error) => {
+            clearTimeout(timer);
+            reject(error);
+        });
+        child.once("close", (code, signal) => {
+            clearTimeout(timer);
+            resolve({ code, signal, stdout, stderr, timed_out: timedOut });
+        });
+    });
+}
+//# sourceMappingURL=eval-tools.js.map

package/dist/src/eval-tools.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-tools.js","sourceRoot":"","sources":["../../src/eval-tools.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzD,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AACvD,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAC/D,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAChD,OAAO,EAAE,aAAa,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAEjE,MAAM,CAAC,MAAM,sBAAsB,GAAG,QAAQ,CAAC;AAC/C,MAAM,CAAC,MAAM,qBAAqB,GAAG;IACnC,kBAAkB;IAClB,mBAAmB;IACnB,gBAAgB;IAChB,iBAAiB;IACjB,eAAe;IACf,qBAAqB;IACrB,mBAAmB;IACnB,sBAAsB;IACtB,oBAAoB;IACpB,yBAAyB;IACzB,gBAAgB;IAChB,gBAAgB;IAChB,sBAAsB;IACtB,uBAAuB;IACvB,0BAA0B;IAC1B,mBAAmB;IACnB,uBAAuB;IACvB,mBAAmB;IACnB,YAAY;IACZ,oBAAoB;IACpB,cAAc;IACd,UAAU;CACX,CAAC;AACF,MAAM,uBAAuB,GAAG;IAC9B,kBAAkB;IAClB,gBAAgB;IAChB,mBAAmB;IACnB,sBAAsB;IACtB,yBAAyB;IACzB,gBAAgB;IAChB,gBAAgB;IAChB,sBAAsB;IACtB,mBAAmB;CACpB,CAAC;AAsBF,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,EAChC,IAAI,EACJ,OAAO,GAAG,sBAAsB,EAChC,MAAM,EACN,KAAK,GAAG,KAAK,EACb,GAAG,GAAG,OAAO,CAAC,GAAG,EACjB,UAAU,GAAG,GAAG,EAAE,GAAE,CAAC,GACF;IACnB,IAAI,CAAC,IAAI;QAAE,MAAM,YAAY,CAAC,iDAAiD,CAAC,CAAC;IACjF,IAAI,CAAC,wCAAwC,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAC5D,MAAM,YAAY,CAAC,kDAAkD,CAAC,CAAC;IACzE,CAAC;IACD,MAAM,UAAU,GAAG,MAAM,UAAU,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;IAC/D,IAAI,CAAC,UAAU,CAAC,UAAU,EAAE,CAAC;QAC3B,MAAM,IAAI,UAAU,CAAC,wEAAwE,EAAE;YAC7F,IAAI,EAAE,oBAAoB;YAC1B,QAAQ,EAAE,CAAC;SACZ,CAAC,CAAC;IACL,CAAC;IACD,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,CAAC;QAC1B,MAAM,IAAI,UAAU,CAAC,+CAA+C,UAAU,CAAC,OAAO,IAAI,oBAAoB,EAAE,EAAE;YAChH,IAAI,EAAE,oBAAoB;YAC1B,QAAQ,EAAE,CAAC;SACZ,CAAC,CAAC;IACL,CAAC;IAED,MAAM,cAAc,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC;IAC9C,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,UAAU,OAAO,EAAE,CAAC,CAAC;IACxE,MAAM,gBAAgB,GAAG,cAAc,CAAC,gBAAgB,EAAE,QAAQ,CAAC,CAAC;IACpE,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,aAAa,CAAC,CAAC;IAC7D,MAAM,gBAAgB,GAAG,MAAM,sBAAsB,CAAC,gBAAgB,CAAC,CAAC;IACxE,IAAI,gBAAgB,KAAK,OAAO,IAAI,CAAC,KAAK,EAAE,CAAC;QAC3C,MAAM,QAAQ,GAAG,MAAM,oBAAoB,CAAC,WAAW,EAAE;YACvD,OAAO;YACP,UAAU;YACV,gBAAgB;YAChB,gBAAgB;YAChB,QAAQ,EAAE,IAAI;SACf,CAAC,CAAC;QACH,OAAO,QAAQ,CAAC;IAClB,CAAC;IACD,IAAI,gBAAgB,IAAI,CAAC,KAAK,EAAE,CAAC;QAC/B,MAAM,IAAI,UAAU,CAAC,qBAAqB,gBAAgB,YAAY,gBAAgB,oCAAoC,EAAE;YAC1H,IAAI,EAAE,uBAAuB;YAC7B,QAAQ,EAAE,CAAC;SACZ,CAAC,CAAC;IACL,CAAC;IAED,MAAM,SAAS,CAAC,cAAc,CAAC,CAAC;IAChC,IAAI,KAAK;QAAE,MAAM,EAAE,CAAC,gBAAgB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACxE,IAAI,CAAC;QACH,MAAM,KAAK,CAAC,gBAAgB,CAAC,CAAC;IAChC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAA+B,EAAE,IAAI,KAAK,QAAQ,EAAE,CAAC;YACxD,MAAM,IAAI,UAAU,CAAC,oDAAoD,gBAAgB,sBAAsB,EAAE;gBAC/G,IAAI,EAAE,yBAAyB;gBAC/B,QAAQ,EAAE,CAAC;aACZ,CAAC,CAAC;QACL,CAAC;QACD,MAAM,KAAK,CAAC;IACd,CAAC;IAED,IAAI,CAAC;QACH,UAAU,CAAC,gDAAgD,UAAU,CAAC,OAAO,EAAE,CAAC,CAAC;QACjF,MAAM,cAAc,CAAC,UAAU,CAAC,UAAU,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,gBAAgB,CAAC,EAAE;YAC5E,GAAG;YACH,IAAI,EAAE,qBAAqB;YAC3B,SAAS,EAAE,CAAC,GAAG,MAAM;SACtB,CAAC,CAAC;QACH,MAAM,iBAAiB,GAAG,cAAc,CAAC,gBAAgB,EAAE,QAAQ,CAAC,CAAC;QACrE,UAAU,CAAC,sBAAsB,OAAO,EAAE,CAAC,CAAC;QAC5C,MAAM,cAAc,CAAC,iBAAiB,EAAE;YACtC,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,6BAA6B,EAAE,YAAY,EAAE,WAAW,OAAO,EAAE;SAC1F,EAAE;YACD,GAAG;YACH,IAAI,EAAE,qBAAqB;YAC3B,SAAS,EAAE,EAAE,GAAG,MAAM;SACvB,CAAC,CAAC;QACH,MAAM,eAAe,GAAG,MAAM,sBAAsB,CAAC,gBAAgB,CAAC,CAAC;QACvE,IAAI,eAAe,KAAK,OAAO,EAAE,CAAC;YAChC,MAAM,IAAI,UAAU,CAAC,6BAA6B,eAAe,IAAI,YAAY,cAAc,OAAO,EAAE,EAAE;gBACxG,IAAI,EAAE,qBAAqB;gBAC3B,QAAQ,EAAE,EAAE;aACb,CAAC,CAAC;QACL,CAAC;QACD,OAAO,MAAM,oBAAoB,CAAC,WAAW,EAAE;YAC7C,OAAO;YACP,UAAU;YACV,gBAAgB;YAChB,gBAAgB;YAChB,QAAQ,EAAE,KAAK;SAChB,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,EAAE,CAAC,gBAAgB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAC7D,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AA6BD,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,GAAG,OAAO,CAAC,GAAG,EAAuB;IACzG,IAAI,CAAC,IAAI;QAAE,MAAM,YAAY,CAAC,gDAAgD,CAAC,CAAC;IAChF,MAAM,UAAU,GAAG,MAAM,UAAU,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;IAC/D,MAAM,UAAU,GAAG,MAAM,YAAY,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;IACvE,MAAM,gBAAgB,GAAG,MAAM,IAAI,GAAG,CAAC,iBAAiB,IAAI,QAAQ,CAAC;IACrE,MAAM,gBAAgB,GAAG,MAAM,iBAAiB,CAAC,gBAAgB,EAAE,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;IAEnF,IAAI,aAAa,GAAG,EAAE,CAAC;IACvB,IAAI,gBAAgB,GAAG,EAAE,CAAC;IAC1B,IAAI,gBAAgB,EAAE,CAAC;QACrB,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,gBAAgB,EAAE,CAAC,SAAS,EAAE,UAAU,EAAE,qBAAqB,CAAC,EAAE;YACpG,GAAG;YACH,SAAS,EAAE,MAAM;SAClB,CAAC,CAAC;QACH,aAAa,GAAG,MAAM,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9D,gBAAgB,GAAG,MAAM,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,8BAA8B,CAAC,CAAC;IAC/H,CAAC;IAED,MAAM,WAAW,GAAG,uBAAuB,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,GAAG,CAAC,IAAI,CAAC,KAAK,QAAQ,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAChH,MAAM,MAAM,GAAgC;QAC1C,MAAM,EAAE,UAAU,CAAC,UAAU,IAAI,UAAU,CAAC,SAAS;YACnD,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,UAAU,CAAC,UAAU,EAAE,OAAO,EAAE,UAAU,CAAC,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE;YACtG,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,UAAU,CAAC,UAAU,IAAI,IAAI,EAAE,OAAO,EAAE,UAAU,CAAC,OAAO,IAAI,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,oDAAoD,EAAE;QAC1L,MAAM,EAAE,UAAU,CAAC,UAAU;YAC3B,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,UAAU,CAAC,UAAU,EAAE,OAAO,EAAE,UAAU,CAAC,OAAO,IAAI,IAAI,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE;YACrH,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,OAAO,EAAE,gCAAgC,UAAU,CAAC,SAAS,EAAE,EAAE;QACpJ,MAAM,EAAE,gBAAgB,IAAI,aAAa;YACvC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,gBAAgB,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,EAAE,SAAS,EAAE;YAC3F,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,gBAAgB,IAAI,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,gBAAgB,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,aAAa,EAAE,OAAO,EAAE,gBAAgB,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,gCAAgC,gBAAgB,EAAE,EAAE;QACzO,WAAW,EAAE,WAAW,CAAC,MAAM,GAAG,CAAC;YACjC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE;YACxC,CAAC,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,EAAE,EAAE,OAAO,EAAE,2FAA2F,EAAE;KAC7I,CAAC;IACF,MAAM,KAAK,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC,CAAC;IACrG,OAAO;QACL,cAAc,EAAE,cAAc;QAC9B,OAAO,EAAE,QAAQ;QACjB,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,EAAE,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,iBAAiB;QAC3G,KAAK;QACL,MAAM;KACP,CAAC;AACJ,CAAC;AAeD,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,GAAG,OAAO,CAAC,GAAG,KAA0B,EAAE;IAChG,MAAM,UAAU,GAAG,QAAQ,IAAI,GAAG,CAAC,iBAAiB,CAAC;IACrD,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,UAAU,GAAG,MAAM,iBAAiB,CAAC,UAAU,EAAE,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;QACvE,OAAO;YACL,UAAU;YACV,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC,MAAM,sBAAsB,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE;YACnE,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,aAAa;YAC3C,SAAS,EAAE,UAAU;SACtB,CAAC;IACJ,CAAC;IACD,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,MAAM,uBAAuB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAClE,IAAI,OAAO,EAAE,CAAC;QACZ,OAAO;YACL,UAAU,EAAE,OAAO;YACnB,OAAO,EAAE,MAAM,sBAAsB,CAAC,OAAO,CAAC;YAC9C,MAAM,EAAE,SAAS;YACjB,SAAS,EAAE,OAAO;SACnB,CAAC;IACJ,CAAC;IACD,MAAM,UAAU,GAAG,MAAM,iBAAiB,CAAC,QAAQ,EAAE,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;IACrE,OAAO;QACL,UAAU;QACV,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC,MAAM,sBAAsB,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE;QACnE,MAAM,EAAE,MAAM;QACd,SAAS,EAAE,QAAQ;KACpB,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,IAAY;IACxD,MAAM,cAAc,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC;IAC9C,MAAM,OAAO,GAAG,MAAM,QAAQ,CAA8B,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,aAAa,CAAC,EAAE,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;IAC9H,IAAI,CAAC,wCAAwC,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,IAAI,EAAE,CAAC;QAAE,OAAO,IAAI,CAAC;IACxF,MAAM,OAAO,GAAG,OAAO,EAAE,OAAiB,CAAC;IAC3C,MAAM,UAAU,GAAG,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,UAAU,OAAO,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAC;IAC5F,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,UAAU,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC;QACzC,OAAO,UAAU,CAAC;IACpB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAaD,KAAK,UAAU,UAAU,CAAC,EAAE,QAAQ,EAAE,GAAG,EAAqB;IAC5D,MAAM,SAAS,GAAG,QAAQ,IAAI,GAAG,CAAC,iBAAiB,CAAC;IACpD,MAAM,UAAU,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,EAAE,YAAY,EAAE,SAAS,CAAC,CAAC;IACrF,IAAI,UAAU,GAAsB,IAAI,CAAC;IACzC,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,MAAM,UAAU,GAAG,MAAM,iBAAiB,CAAC,SAAS,EAAE,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;QACtE,IAAI,CAAC,UAAU;YAAE,SAAS;QAC1B,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,UAAU,EAAE,CAAC,IAAI,EAAE,6DAA6D,CAAC,EAAE;YACrH,GAAG;YACH,SAAS,EAAE,KAAK;SACjB,CAAC,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACpF,MAAM,SAAS,GAAG,sBAAsB,CAAC,OAAO,CAAC,CAAC;QAClD,MAAM,IAAI,GAAe,EAAE,UAAU,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC;QAC5D,UAAU,KAAK,IAAI,CAAC;QACpB,IAAI,SAAS,IAAI,SAAS;YAAE,OAAO,IAAI,CAAC;IAC1C,CAAC;IACD,OAAO,UAAU,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;AAC3E,CAAC;AAED,SAAS,sBAAsB,CAAC,OAAe;IAC7C,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC;IAChE,IAAI,CAAC,KAAK;QAAE,OAAO,KAAK,CAAC;IACzB,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/B,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/B,OAAO,KAAK,GAAG,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,IAAI,KAAK,IAAI,EAAE,CAAC,CAAC;AACnD,CAAC;AAED,KAAK,UAAU,sBAAsB,CAAC,UAAkB;IACtD,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,UAAU,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC;IAC3C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,UAAU,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC;IAC7D,OAAO,KAAK,CAAC,KAAK,CAAC,0CAA0C,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;AAC5E,CAAC;AAED,KAAK,UAAU,oBAAoB,CACjC,WAAmB,EACnB,EACE,OAAO,EACP,UAAU,EACV,gBAAgB,EAChB,gBAAgB,EAChB,QAAQ,GAOT;IAED,MAAM,QAAQ,GAAsB;QAClC,cAAc,EAAE,cAAc;QAC9B,IAAI,EAAE,QAAQ;QACd,OAAO;QACP,UAAU,EAAE,gBAAgB;QAC5B,iBAAiB,EAAE,gBAAgB;QACnC,MAAM,EAAE,EAAE,UAAU,EAAE,UAAU,CAAC,UAAU,IAAI,EAAE,EAAE,OAAO,EAAE,UAAU,CAAC,OAAO,EAAE;QAChF,SAAS,EAAE,QAAQ;QACnB,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KACtC,CAAC;IACF,MAAM,eAAe,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;IAC7C,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAS,cAAc,CAAC,SAAiB,EAAE,IAAY;IACrD,OAAO,OAAO,CAAC,QAAQ,KAAK,OAAO;QACjC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,EAAE,GAAG,IAAI,MAAM,CAAC;QAChD,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC;AACxC,CAAC;AAED,KAAK,UAAU,cAAc,CAAC,UAAkB,EAAE,IAAc,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,SAAS,EAA+D;IACrJ,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,UAAU,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC,CAAC;IAC1E,IAAI,MAAM,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO,MAAM,CAAC;IACrC,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;IAC5D,MAAM,IAAI,UAAU,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,qBAAqB,MAAM,CAAC,IAAI,IAAI,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,KAAK,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE;QAC3H,IAAI;QACJ,QAAQ,EAAE,EAAE;KACb,CAAC,CAAC;AACL,CAAC;AAUD,SAAS,cAAc,CAAC,UAAkB,EAAE,IAAc,EAAE,EAAE,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE,SAAS,GAAG,MAAM,KAAsD,EAAE;IACzJ,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,MAAM,KAAK,GAAG,KAAK,CAAC,UAAU,EAAE,IAAI,EAAE;YACpC,GAAG;YACH,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC;YACjC,QAAQ,EAAE,OAAO,CAAC,QAAQ,KAAK,OAAO;YACtC,WAAW,EAAE,IAAI;SAClB,CAAC,CAAC;QACH,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,MAAM,MAAM,GAAG,CAAC,OAAe,EAAE,KAAsB,EAAE,EAAE,CAAC,GAAG,OAAO,GAAG,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,CAAC;QACnG,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE,GAAG,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAChF,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE,GAAG,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAChF,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC5B,QAAQ,GAAG,IAAI,CAAC;YAChB,gBAAgB,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAC1C,CAAC,EAAE,SAAS,CAAC,CAAC;QACd,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC;QAChB,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;YAC5B,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,MAAM,CAAC,KAAK,CAAC,CAAC;QAChB,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE;YACnC,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,OAAO,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAC;QACjE,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/src/evals.js

@@ -0,0 +1,313 @@
+import { randomUUID } from "node:crypto";
+import { createWriteStream } from "node:fs";
+import { readdir } from "node:fs/promises";
+import path from "node:path";
+import { resolveHarness } from "./artifacts.js";
+import { SCHEMA_VERSION, statePaths } from "./config.js";
+import { HitchError, invalidInput } from "./errors.js";
+import { atomicWriteJSON, ensureDir, readJSON } from "./fs.js";
+import { parseHarnessReference } from "./harness-reference.js";
+import { lockedHarnessRef, runHarborBackend } from "./harbor-backend.js";
+import { ensureControllerRuntime, writeRuntimeReference, inspectEvalRuntimeKind } from "./controller-runtime/store.js";
+export const DEFAULT_EVAL_TIMEOUT_MS = 15 * 60 * 1_000;
+export const DEFAULT_EVAL_SETUP_TIMEOUT_MS = 30 * 60 * 1_000;
+const HARBOR_EVAL_BYPASS_ARG = {
+    codex: "--dangerously-bypass-approvals-and-sandbox",
+    opencode: "--dangerously-skip-permissions",
+};
+export function newEvalId() {
+    return `eval_${randomUUID().replaceAll("-", "")}`;
+}
+export async function validateEvalRequest(input) {
+    if (!input || typeof input !== "object" || Array.isArray(input))
+        throw invalidInput("eval request must be a JSON object");
+    const allowed = new Set([
+        "schema_version", "backend", "dataset", "harness_ref", "model", "attempts",
+        "max_concurrent", "timeout_ms", "setup_timeout_ms", "agent_args", "pass_env",
+    ]);
+    const unexpected = Object.keys(input).find((field) => !allowed.has(field));
+    if (unexpected)
+        throw invalidInput(`unknown eval request field: ${unexpected}`);
+    if (input.schema_version !== undefined && input.schema_version !== SCHEMA_VERSION)
+        throw invalidInput(`unsupported schema_version: ${input.schema_version}`);
+    if ((input.backend || "harbor") !== "harbor")
+        throw invalidInput("only the harbor eval backend is currently supported");
+    if (typeof input.dataset !== "string" || !input.dataset.trim())
+        throw invalidInput("dataset must be a non-empty string");
+    if (typeof input.harness_ref !== "string" || !input.harness_ref.trim())
+        throw invalidInput("harness_ref must be a non-empty string");
+    const reference = parseHarnessReference(input.harness_ref);
+    if (reference.selector.type === "installed") {
+        throw invalidInput("eval requires an immutable harness ref: version:<exact> or commit:<sha>");
+    }
+    if (reference.selector.type === "commit" && reference.selector.source?.explicit) {
+        throw invalidInput("eval does not yet support local git+file harness refs; use a registered remote commit");
+    }
+    const attempts = positiveInteger(input.attempts ?? 1, "attempts");
+    const maxConcurrent = positiveInteger(input.max_concurrent ?? 4, "max_concurrent");
+    const timeout = nonNegativeNumber(input.timeout_ms ?? DEFAULT_EVAL_TIMEOUT_MS, "timeout_ms");
+    const setupTimeout = nonNegativeNumber(input.setup_timeout_ms ?? DEFAULT_EVAL_SETUP_TIMEOUT_MS, "setup_timeout_ms");
+    if (input.model !== undefined && typeof input.model !== "string")
+        throw invalidInput("model must be a string");
+    if (input.agent_args !== undefined && (!Array.isArray(input.agent_args) || input.agent_args.some((value) => typeof value !== "string"))) {
+        throw invalidInput("agent_args must be an array of strings");
+    }
+    if (input.pass_env !== undefined && (!Array.isArray(input.pass_env) || input.pass_env.some((value) => typeof value !== "string"))) {
+        throw invalidInput("pass_env must be an array of strings");
+    }
+    const agentArgs = Array.isArray(input.agent_args) ? [...input.agent_args] : [];
+    const bypassArg = HARBOR_EVAL_BYPASS_ARG[reference.harness_id];
+    if (bypassArg && !agentArgs.includes(bypassArg))
+        agentArgs.unshift(bypassArg);
+    return {
+        schema_version: SCHEMA_VERSION,
+        backend: "harbor",
+        dataset: String(input.dataset).trim(),
+        harness_ref: reference.canonical,
+        model: typeof input.model === "string" ? input.model : "",
+        attempts,
+        max_concurrent: maxConcurrent,
+        timeout_ms: timeout,
+        setup_timeout_ms: setupTimeout,
+        agent_args: agentArgs,
+        pass_env: [...new Set(Array.isArray(input.pass_env) ? input.pass_env : [])],
+    };
+}
+export async function runEval({ evalId = newEvalId(), request, root, env = process.env, harborExecutable, signal, onEvent }) {
+    if (!root)
+        throw invalidInput("a Hitch state root is required for eval");
+    const normalized = await validateEvalRequest(request);
+    const evalDirectory = await ensureDir(path.join(statePaths(root).evals, evalId));
+    const startedAt = new Date();
+    const sink = new EvalEventSink(evalDirectory, evalId, onEvent);
+    await sink.open();
+    await atomicWriteJSON(path.join(evalDirectory, "request.json"), normalized);
+    let result;
+    try {
+        sink.emit({ type: "eval.started", backend: normalized.backend, dataset: normalized.dataset });
+        const resolvedRevision = await resolveHarness(normalized.harness_ref, { root, env });
+        if (resolvedRevision.source.type === "git" && resolvedRevision.source.registered !== true) {
+            throw invalidInput("eval requires a registered remote Git source");
+        }
+        // Phase 2: the shared, read-only, SHA-256-addressed controller runtime
+        // cache replaces the per-eval Hitch runtime copy (spec §4).
+        const controllerRuntime = await ensureControllerRuntime({ root });
+        const runtimeRefFile = await writeRuntimeReference(evalDirectory, controllerRuntime);
+        sink.emit({
+            type: "eval.controller-runtime",
+            runtime_id: controllerRuntime.runtime_id,
+            cache_hit: controllerRuntime.cache_hit,
+            reference: runtimeRefFile,
+        });
+        const plan = {
+            schema_version: SCHEMA_VERSION,
+            eval_id: evalId,
+            backend: "harbor",
+            candidate: {
+                id: "candidate-1",
+                requested_harness_ref: normalized.harness_ref,
+                harness_ref: lockedHarnessRef(resolvedRevision),
+                harness_id: resolvedRevision.harness_id,
+                revision_identity: resolvedRevision.identity,
+                model: normalized.model || null,
+            },
+            dataset: normalized.dataset,
+            attempts: normalized.attempts,
+            max_concurrent: normalized.max_concurrent,
+            controller_runtime: {
+                runtime_id: controllerRuntime.runtime_id,
+                manifest_digest: controllerRuntime.manifest_digest,
+            },
+            created_at: new Date().toISOString(),
+        };
+        await atomicWriteJSON(path.join(evalDirectory, "resolution.json"), resolvedRevision);
+        await atomicWriteJSON(path.join(evalDirectory, "plan.json"), plan);
+        sink.emit({
+            type: "eval.planned",
+            harness: resolvedRevision.harness_id,
+            revision_identity: resolvedRevision.identity,
+            attempts: normalized.attempts,
+        });
+        const backendRun = await runHarborBackend({
+            evalDirectory,
+            request: normalized,
+            root,
+            resolvedRevision,
+            runtimeDirectory: controllerRuntime.directory,
+            runtimeId: controllerRuntime.runtime_id,
+            env,
+            ...(harborExecutable !== undefined ? { harborExecutable } : {}),
+            ...(signal ? { signal } : {}),
+            emit: (event) => sink.emit(event),
+        });
+        const cancelled = signal?.aborted;
+        const succeeded = !cancelled && backendRun.backend.process_exit_code === 0 && backendRun.rawResult !== null;
+        result = {
+            schema_version: SCHEMA_VERSION,
+            eval_id: evalId,
+            status: cancelled ? "cancelled" : succeeded ? "succeeded" : "failed",
+            exit_code: cancelled ? 9 : succeeded ? 0 : 13,
+            backend: backendRun.backend,
+            candidate: plan.candidate,
+            dataset: normalized.dataset,
+            summary: backendRun.summary,
+            ...(succeeded ? {} : {
+                error: {
+                    code: cancelled ? "cancelled" : "harbor_failed",
+                    message: cancelled
+                        ? "eval was cancelled"
+                        : backendRun.rawResult === null
+                            ? `Harbor exited without a result (code ${backendRun.backend.process_exit_code ?? "null"})`
+                            : `Harbor exited with code ${backendRun.backend.process_exit_code ?? "null"}`,
+                },
+            }),
+            started_at: startedAt.toISOString(),
+            completed_at: new Date().toISOString(),
+        };
+    }
+    catch (error) {
+        const typed = error instanceof HitchError;
+        result = {
+            schema_version: SCHEMA_VERSION,
+            eval_id: evalId,
+            status: signal?.aborted ? "cancelled" : "failed",
+            exit_code: signal?.aborted ? 9 : typed ? error.exitCode : 12,
+            error: {
+                code: signal?.aborted ? "cancelled" : typed ? error.code : "internal_error",
+                message: error?.message || String(error),
+            },
+            started_at: startedAt.toISOString(),
+            completed_at: new Date().toISOString(),
+        };
+    }
+    await atomicWriteJSON(path.join(evalDirectory, "result.json"), result);
+    sink.emit({ type: result.status === "succeeded" ? "eval.completed" : "eval.failed", status: result.status, exit_code: result.exit_code, error: result.error });
+    await sink.close();
+    return result;
+}
+export async function listEvals({ root }) {
+    const directory = statePaths(root).evals;
+    let entries;
+    try {
+        entries = await readdir(directory, { withFileTypes: true });
+    }
+    catch (error) {
+        if (error?.code === "ENOENT")
+            return [];
+        throw error;
+    }
+    const evals = [];
+    for (const entry of entries) {
+        if (!entry.isDirectory() || !entry.name.startsWith("eval_"))
+            continue;
+        const result = await readJSON(path.join(directory, entry.name, "result.json"), null).catch(() => null);
+        const request = await readJSON(path.join(directory, entry.name, "request.json"), null).catch(() => null);
+        if (!result && !request)
+            continue;
+        evals.push({
+            eval_id: entry.name,
+            status: result?.status || "running",
+            backend: result?.backend?.name || request?.backend || null,
+            dataset: result?.dataset || request?.dataset || null,
+            harness_ref: result?.candidate?.harness_ref || request?.harness_ref || null,
+            primary_reward: result?.summary?.primary_reward ?? null,
+            started_at: result?.started_at || null,
+            completed_at: result?.completed_at || null,
+        });
+    }
+    return evals.sort((left, right) => String(right.started_at || right.eval_id).localeCompare(String(left.started_at || left.eval_id)));
+}
+export async function inspectEval(evalId, { root }) {
+    if (typeof evalId !== "string" || !/^eval_[a-f0-9]{32}$/.test(evalId))
+        throw invalidInput(`invalid eval ID: ${evalId}`);
+    const directory = path.join(statePaths(root).evals, evalId);
+    const request = await readJSON(path.join(directory, "request.json"), null);
+    if (!request)
+        throw new HitchError(`eval not found: ${evalId}`, { code: "eval_not_found", exitCode: 3 });
+    return {
+        schema_version: SCHEMA_VERSION,
+        eval_id: evalId,
+        directory,
+        request,
+        resolution: await readJSON(path.join(directory, "resolution.json"), null),
+        plan: await readJSON(path.join(directory, "plan.json"), null),
+        result: await readJSON(path.join(directory, "result.json"), null),
+        runtime_storage: await inspectEvalRuntimeKind(directory),
+    };
+}
+class EvalEventSink {
+    path;
+    evalId;
+    onEvent;
+    sequence = 0;
+    pending = Promise.resolve();
+    stream;
+    streamError;
+    constructor(evalDirectory, evalId, onEvent = () => { }) {
+        this.path = path.join(evalDirectory, "events.jsonl");
+        this.evalId = evalId;
+        this.onEvent = onEvent || (() => { });
+    }
+    async open() {
+        await ensureDir(path.dirname(this.path));
+        this.stream = createWriteStream(this.path, { flags: "a", mode: 0o600 });
+        this.stream.on("error", (error) => { this.streamError ||= error; });
+    }
+    emit(event) {
+        const framed = {
+            schema_version: SCHEMA_VERSION,
+            sequence: ++this.sequence,
+            timestamp: new Date().toISOString(),
+            eval_id: this.evalId,
+            ...event,
+        };
+        this.pending = this.pending.then(() => writeChunk(this.stream, `${JSON.stringify(framed)}\n`));
+        try {
+            this.onEvent(framed);
+        }
+        catch { /* Observers cannot break eval persistence. */ }
+        return framed;
+    }
+    async close() {
+        let failure;
+        try {
+            await this.pending;
+        }
+        catch (error) {
+            failure = error;
+        }
+        try {
+            await closeStream(this.stream);
+        }
+        catch (error) {
+            failure ||= error;
+        }
+        failure ||= this.streamError;
+        if (failure)
+            throw failure;
+    }
+}
+function writeChunk(stream, chunk) {
+    return new Promise((resolve, reject) => stream.write(chunk, (error) => error ? reject(error) : resolve()));
+}
+function closeStream(stream) {
+    if (!stream)
+        return Promise.resolve();
+    return new Promise((resolve, reject) => {
+        stream.once("error", reject);
+        stream.end(resolve);
+    });
+}
+function positiveInteger(value, name) {
+    const parsed = Number(value);
+    if (!Number.isInteger(parsed) || parsed <= 0)
+        throw invalidInput(`${name} must be a positive integer`);
+    return parsed;
+}
+function nonNegativeNumber(value, name) {
+    const parsed = Number(value);
+    if (!Number.isFinite(parsed) || parsed < 0)
+        throw invalidInput(`${name} must be a non-negative number`);
+    return parsed;
+}
+//# sourceMappingURL=evals.js.map