agent-hitch 0.2.5 → 0.2.6

package/dist/src/evals/verifier-diagnostics.js

@@ -0,0 +1,235 @@
+import { open, stat } from "node:fs/promises";
+import path from "node:path";
+import { atomicWriteJSON } from "../foundation/index.js";
+const MAX_LOG_BYTES = 256 * 1024;
+const VERIFIER_LOG_FILES = ["test-stdout.txt", "test-stderr.txt", "stdout.txt", "stderr.txt"];
+const INFRASTRUCTURE_PATTERNS = [
+    {
+        signal: "dns_resolution_failed",
+        patterns: [
+            /curl:\s*\(\d+\)\s*Could not resolve host:/i,
+            /Temporary failure in name resolution/i,
+            /Name or service not known/i,
+            /getaddrinfo\s+(?:EAI_AGAIN|ENOTFOUND)/i,
+            /Could not resolve hostname/i,
+        ],
+    },
+    {
+        signal: "network_unreachable",
+        patterns: [
+            /Network is unreachable/i,
+            /Failed to establish a new connection:[^\n]*(?:timed out|connection refused)/i,
+            /Could not connect to (?:host|server)/i,
+        ],
+    },
+    {
+        signal: "package_install_failed",
+        patterns: [
+            /Could not find a version that satisfies the requirement/i,
+            /No matching distribution found for/i,
+            /Failed to (?:download|fetch) [^\n]*(?:package|wheel|index)/i,
+            /error:\s*failed to (?:download|fetch|install)/i,
+        ],
+    },
+    {
+        signal: "test_runner_missing",
+        patterns: [
+            /(?:^|\n)[^\n]*(?:uvx|pytest|pipx|tox|nox): command not found(?:\n|$)/i,
+            /No module named ['"]?(?:pytest|unittest|tox|nox)['"]?/i,
+        ],
+    },
+    {
+        signal: "verifier_environment_missing",
+        patterns: [
+            /\/(?:root|home\/[^/]+)\/\.local\/bin\/env: No such file or directory/i,
+            /(?:^|\n)[^\n]*\/bin\/(?:python|python3|pytest|uv|uvx): No such file or directory(?:\n|$)/i,
+        ],
+    },
+];
+const TEST_EXECUTION_EVIDENCE = [
+    /test session starts/i,
+    /collected\s+\d+\s+items?/i,
+    /(?:^|\n)Ran\s+\d+\s+tests?/i,
+    /(?:^|\n)TAP version\s+\d+/i,
+    /={3,}[^\n]*(?:passed|failed|errors?|skipped)[^\n]*={3,}/i,
+];
+/**
+ * Detect a verifier that wrote a zero reward after its own bootstrap failed.
+ *
+ * Harbor treats reward.txt as authoritative even when a shell verifier masks
+ * an earlier command failure. We fail closed only when reward is exactly zero,
+ * no structured CTRF evidence exists, no test-runner evidence appears in the
+ * bounded logs, and a stable infrastructure signature is present.
+ */
+export async function detectVerifierInfrastructureFailure(trialDirectory, reward) {
+    const verifierDirectory = path.join(trialDirectory, "verifier");
+    const explicit = await readExplicitDiagnostic(path.join(verifierDirectory, "infrastructure-error.json"));
+    if (explicit)
+        return explicit;
+    if (reward !== 0)
+        return null;
+    if (await nonEmptyFile(path.join(verifierDirectory, "ctrf.json")))
+        return null;
+    const logs = [];
+    const sourceFiles = [];
+    for (const name of VERIFIER_LOG_FILES) {
+        const file = path.join(verifierDirectory, name);
+        const value = await readBoundedFile(file, MAX_LOG_BYTES);
+        if (value === null)
+            continue;
+        logs.push(value);
+        sourceFiles.push(`verifier/${name}`);
+    }
+    if (logs.length === 0)
+        return null;
+    const combined = logs.join("\n");
+    if (TEST_EXECUTION_EVIDENCE.some((pattern) => pattern.test(combined)))
+        return null;
+    const signals = INFRASTRUCTURE_PATTERNS
+        .filter(({ patterns }) => patterns.some((pattern) => pattern.test(combined)))
+        .map(({ signal }) => signal);
+    if (signals.length === 0)
+        return null;
+    return {
+        schema_version: "1",
+        code: "verifier_infrastructure_failure",
+        signals,
+        source_files: sourceFiles,
+    };
+}
+async function readExplicitDiagnostic(file) {
+    const raw = await readBoundedFile(file, MAX_LOG_BYTES);
+    if (raw === null)
+        return null;
+    let value;
+    try {
+        value = JSON.parse(raw);
+    }
+    catch {
+        return null;
+    }
+    if (!value || typeof value !== "object" || Array.isArray(value))
+        return null;
+    const record = value;
+    if (record.schema_version !== "1" || record.code !== "verifier_infrastructure_failure")
+        return null;
+    const signals = infrastructureSignals(record.signals);
+    const sourceFiles = stringArray(record.source_files);
+    if (signals.length === 0 || sourceFiles.length === 0)
+        return null;
+    const attempts = Array.isArray(record.attempts)
+        ? record.attempts.flatMap((attempt) => {
+            if (!attempt || typeof attempt !== "object" || Array.isArray(attempt))
+                return [];
+            const item = attempt;
+            const number = item.attempt;
+            const itemSignals = infrastructureSignals(item.signals);
+            const itemSourceFiles = stringArray(item.source_files);
+            if (!Number.isSafeInteger(number) || number < 1 || itemSignals.length === 0 || itemSourceFiles.length === 0)
+                return [];
+            return [{ attempt: number, signals: itemSignals, source_files: itemSourceFiles }];
+        })
+        : [];
+    const maxRetries = record.max_retries;
+    const backoffMs = record.backoff_ms;
+    return {
+        schema_version: "1",
+        code: "verifier_infrastructure_failure",
+        signals,
+        source_files: sourceFiles,
+        ...(attempts.length > 0 ? { attempts } : {}),
+        ...(Number.isSafeInteger(maxRetries) && maxRetries >= 0 ? { max_retries: maxRetries } : {}),
+        ...(Number.isSafeInteger(backoffMs) && backoffMs >= 0 ? { backoff_ms: backoffMs } : {}),
+    };
+}
+function infrastructureSignals(value) {
+    if (!Array.isArray(value))
+        return [];
+    const allowed = new Set(INFRASTRUCTURE_PATTERNS.map(({ signal }) => signal));
+    return [...new Set(value.filter((entry) => typeof entry === "string" && allowed.has(entry)))];
+}
+function stringArray(value) {
+    return Array.isArray(value)
+        ? [...new Set(value.filter((entry) => typeof entry === "string" && entry.length > 0))]
+        : [];
+}
+export function verifierResult(trial) {
+    return trial.verifier_result && typeof trial.verifier_result === "object" && !Array.isArray(trial.verifier_result)
+        ? trial.verifier_result
+        : null;
+}
+export function primaryVerifierReward(trial) {
+    const rewards = (verifierResult(trial)?.rewards || {});
+    const preferred = rewards.reward;
+    if (typeof preferred === "number" && Number.isFinite(preferred))
+        return preferred;
+    return Object.values(rewards).find((value) => typeof value === "number" && Number.isFinite(value));
+}
+export function verifierObservation(input) {
+    const ref = input.verifierRef ? { verifier_result_ref: input.verifierRef } : {};
+    if (input.runStatus === "cancelled")
+        return { status: "invalid", invalid_reason: "cancelled", ...ref };
+    if (input.runStatus !== "succeeded" || input.recordStatus !== "valid")
+        return { status: "invalid", invalid_reason: "infrastructure_failure", ...ref };
+    if (input.trajectoryStatus !== "valid")
+        return { status: "invalid", invalid_reason: "trajectory_missing_or_corrupt", ...ref };
+    if (input.infrastructure)
+        return { status: "invalid", invalid_reason: "verifier_infrastructure_failure", ...ref };
+    // The candidate run and trajectory are already sealed and valid. A later
+    // Harbor exception belongs to verification/teardown and must never trigger
+    // a second candidate execution.
+    if (input.trial.exception_info)
+        return { status: "invalid", invalid_reason: "verifier_infrastructure_failure", ...ref };
+    const reward = primaryVerifierReward(input.trial);
+    if (reward === undefined || !input.verifierRef)
+        return { status: "invalid", invalid_reason: "verifier_result_missing" };
+    return { status: "valid", reward, verifier_result_ref: input.verifierRef };
+}
+export async function writeVerifierInfrastructureDiagnostic(runDirectory, diagnostic) {
+    await atomicWriteJSON(path.join(runDirectory, "verifier", "infrastructure-error.json"), {
+        ...diagnostic,
+        detected_at: new Date().toISOString(),
+    });
+}
+async function nonEmptyFile(file) {
+    try {
+        const info = await stat(file);
+        return info.isFile() && info.size > 0;
+    }
+    catch (error) {
+        if (error.code === "ENOENT")
+            return false;
+        throw error;
+    }
+}
+async function readBoundedFile(file, maxBytes) {
+    let handle;
+    try {
+        handle = await open(file, "r");
+    }
+    catch (error) {
+        if (error.code === "ENOENT")
+            return null;
+        throw error;
+    }
+    try {
+        const size = (await handle.stat()).size;
+        if (size === 0)
+            return "";
+        if (size <= maxBytes) {
+            const buffer = Buffer.alloc(size);
+            await handle.read(buffer, 0, size, 0);
+            return buffer.toString("utf8");
+        }
+        const half = Math.floor(maxBytes / 2);
+        const head = Buffer.alloc(half);
+        const tail = Buffer.alloc(maxBytes - half);
+        await handle.read(head, 0, head.length, 0);
+        await handle.read(tail, 0, tail.length, size - tail.length);
+        return `${head.toString("utf8")}\n[... verifier log truncated ...]\n${tail.toString("utf8")}`;
+    }
+    finally {
+        await handle.close();
+    }
+}
+//# sourceMappingURL=verifier-diagnostics.js.map

package/dist/src/evals/verifier-diagnostics.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"verifier-diagnostics.js","sourceRoot":"","sources":["../../../src/evals/verifier-diagnostics.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAEzD,MAAM,aAAa,GAAG,GAAG,GAAG,IAAI,CAAC;AACjC,MAAM,kBAAkB,GAAG,CAAC,iBAAiB,EAAE,iBAAiB,EAAE,YAAY,EAAE,YAAY,CAAU,CAAC;AAuBvG,MAAM,uBAAuB,GAGxB;IACH;QACE,MAAM,EAAE,uBAAuB;QAC/B,QAAQ,EAAE;YACR,4CAA4C;YAC5C,uCAAuC;YACvC,4BAA4B;YAC5B,wCAAwC;YACxC,6BAA6B;SAC9B;KACF;IACD;QACE,MAAM,EAAE,qBAAqB;QAC7B,QAAQ,EAAE;YACR,yBAAyB;YACzB,8EAA8E;YAC9E,uCAAuC;SACxC;KACF;IACD;QACE,MAAM,EAAE,wBAAwB;QAChC,QAAQ,EAAE;YACR,0DAA0D;YAC1D,qCAAqC;YACrC,6DAA6D;YAC7D,gDAAgD;SACjD;KACF;IACD;QACE,MAAM,EAAE,qBAAqB;QAC7B,QAAQ,EAAE;YACR,uEAAuE;YACvE,wDAAwD;SACzD;KACF;IACD;QACE,MAAM,EAAE,8BAA8B;QACtC,QAAQ,EAAE;YACR,uEAAuE;YACvE,2FAA2F;SAC5F;KACF;CACF,CAAC;AAEF,MAAM,uBAAuB,GAAG;IAC9B,sBAAsB;IACtB,2BAA2B;IAC3B,6BAA6B;IAC7B,4BAA4B;IAC5B,0DAA0D;CAClD,CAAC;AAEX;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,mCAAmC,CACvD,cAAsB,EACtB,MAA0B;IAE1B,MAAM,iBAAiB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,UAAU,CAAC,CAAC;IAChE,MAAM,QAAQ,GAAG,MAAM,sBAAsB,CAAC,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,2BAA2B,CAAC,CAAC,CAAC;IACzG,IAAI,QAAQ;QAAE,OAAO,QAAQ,CAAC;IAC9B,IAAI,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAC9B,IAAI,MAAM,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,WAAW,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IAE/E,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,MAAM,WAAW,GAAa,EAAE,CAAC;IACjC,KAAK,MAAM,IAAI,IAAI,kBAAkB,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,IAAI,CAAC,CAAC;QAChD,MAAM,KAAK,GAAG,MAAM,eAAe,CAAC,IAAI,EAAE,aAAa,CAAC,CAAC;QACzD,IAAI,KAAK,KAAK,IAAI;YAAE,SAAS;QAC7B,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACjB,WAAW,CAAC,IAAI,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC;IACvC,CAAC;IACD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACnC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACjC,IAAI,uBAAuB,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IAEnF,MAAM,OAAO,GAAG,uBAAuB;SACpC,MAAM,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;SAC5E,GAAG,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;IAC/B,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACtC,OAAO;QACL,cAAc,EAAE,GAAG;QACnB,IAAI,EAAE,iCAAiC;QACvC,OAAO;QACP,YAAY,EAAE,WAAW;KAC1B,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,sBAAsB,CAAC,IAAY;IAChD,MAAM,GAAG,GAAG,MAAM,eAAe,CAAC,IAAI,EAAE,aAAa,CAAC,CAAC;IACvD,IAAI,GAAG,KAAK,IAAI;QAAE,OAAO,IAAI,CAAC;IAC9B,IAAI,KAAc,CAAC;IACnB,IAAI,CAAC;QACH,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC1B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAC7E,MAAM,MAAM,GAAG,KAAgC,CAAC;IAChD,IAAI,MAAM,CAAC,cAAc,KAAK,GAAG,IAAI,MAAM,CAAC,IAAI,KAAK,iCAAiC;QAAE,OAAO,IAAI,CAAC;IACpG,MAAM,OAAO,GAAG,qBAAqB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACtD,MAAM,WAAW,GAAG,WAAW,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;IACrD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAClE,MAAM,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC;QAC7C,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YAClC,IAAI,CAAC,OAAO,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;gBAAE,OAAO,EAAE,CAAC;YACjF,MAAM,IAAI,GAAG,OAAkC,CAAC;YAChD,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC;YAC5B,MAAM,WAAW,GAAG,qBAAqB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACxD,MAAM,eAAe,GAAG,WAAW,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YACvD,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,IAAK,MAAiB,GAAG,CAAC,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,EAAE,CAAC;YACnI,OAAO,CAAC,EAAE,OAAO,EAAE,MAAgB,EAAE,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,eAAe,EAAE,CAAC,CAAC;QAC9F,CAAC,CAAC;QACJ,CAAC,CAAC,EAAE,CAAC;IACP,MAAM,UAAU,GAAG,MAAM,CAAC,WAAW,CAAC;IACtC,MAAM,SAAS,GAAG,MAAM,CAAC,UAAU,CAAC;IACpC,OAAO;QACL,cAAc,EAAE,GAAG;QACnB,IAAI,EAAE,iCAAiC;QACvC,OAAO;QACP,YAAY,EAAE,WAAW;QACzB,GAAG,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5C,GAAG,CAAC,MAAM,CAAC,aAAa,CAAC,UAAU,CAAC,IAAK,UAAqB,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,UAAoB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACjH,GAAG,CAAC,MAAM,CAAC,aAAa,CAAC,SAAS,CAAC,IAAK,SAAoB,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,SAAmB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC9G,CAAC;AACJ,CAAC;AAED,SAAS,qBAAqB,CAAC,KAAc;IAC3C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACrC,MAAM,OAAO,GAAG,IAAI,GAAG,CAA+B,uBAAuB,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC;IAC3G,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,EAAyC,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,OAAO,CAAC,GAAG,CAAC,KAAqC,CAAC,CAAC,CAAC,CAAC,CAAC;AACvK,CAAC;AAED,SAAS,WAAW,CAAC,KAAc;IACjC,OAAO,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QACzB,CAAC,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,EAAmB,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;QACvG,CAAC,CAAC,EAAE,CAAC;AACT,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,KAA8B;IAC3D,OAAO,KAAK,CAAC,eAAe,IAAI,OAAO,KAAK,CAAC,eAAe,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,eAAe,CAAC;QAChH,CAAC,CAAC,KAAK,CAAC,eAA0C;QAClD,CAAC,CAAC,IAAI,CAAC;AACX,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,KAA8B;IAClE,MAAM,OAAO,GAAG,CAAC,cAAc,CAAC,KAAK,CAAC,EAAE,OAAO,IAAI,EAAE,CAA4B,CAAC;IAClF,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC;IACjC,IAAI,OAAO,SAAS,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC;QAAE,OAAO,SAAS,CAAC;IAClF,OAAO,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAmB,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;AACtH,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,KAOnC;IACC,MAAM,GAAG,GAAG,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,mBAAmB,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAChF,IAAI,KAAK,CAAC,SAAS,KAAK,WAAW;QAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,cAAc,EAAE,WAAW,EAAE,GAAG,GAAG,EAAE,CAAC;IACvG,IAAI,KAAK,CAAC,SAAS,KAAK,WAAW,IAAI,KAAK,CAAC,YAAY,KAAK,OAAO;QAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,cAAc,EAAE,wBAAwB,EAAE,GAAG,GAAG,EAAE,CAAC;IACtJ,IAAI,KAAK,CAAC,gBAAgB,KAAK,OAAO;QAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,cAAc,EAAE,+BAA+B,EAAE,GAAG,GAAG,EAAE,CAAC;IAC9H,IAAI,KAAK,CAAC,cAAc;QAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,cAAc,EAAE,iCAAiC,EAAE,GAAG,GAAG,EAAE,CAAC;IAClH,yEAAyE;IACzE,2EAA2E;IAC3E,gCAAgC;IAChC,IAAI,KAAK,CAAC,KAAK,CAAC,cAAc;QAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,cAAc,EAAE,iCAAiC,EAAE,GAAG,GAAG,EAAE,CAAC;IACxH,MAAM,MAAM,GAAG,qBAAqB,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAClD,IAAI,MAAM,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,WAAW;QAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,cAAc,EAAE,yBAAyB,EAAE,CAAC;IACxH,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,mBAAmB,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC;AAC7E,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qCAAqC,CACzD,YAAoB,EACpB,UAA4C;IAE5C,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,UAAU,EAAE,2BAA2B,CAAC,EAAE;QACtF,GAAG,UAAU;QACb,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KACtC,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,IAAY;IACtC,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC;QAC9B,OAAO,IAAI,CAAC,MAAM,EAAE,IAAI,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC;IACxC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,KAAK,CAAC;QACrE,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,KAAK,UAAU,eAAe,CAAC,IAAY,EAAE,QAAgB;IAC3D,IAAI,MAAM,CAAC;IACX,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IACjC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,IAAI,CAAC;QACpE,MAAM,KAAK,CAAC;IACd,CAAC;IACD,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,CAAC,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC;QACxC,IAAI,IAAI,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAC1B,IAAI,IAAI,IAAI,QAAQ,EAAE,CAAC;YACrB,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAClC,MAAM,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;YACtC,OAAO,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACjC,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC;QACtC,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAChC,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAC;QAC3C,MAAM,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAC3C,MAAM,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC;QAC5D,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,uCAAuC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;IAChG,CAAC;YAAS,CAAC;QACT,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;AACH,CAAC"}

package/docs/schemas/eval-request.schema.json

@@ -15,6 +15,8 @@
     "model": { "type": "string" },
     "attempts": { "type": "integer", "minimum": 1 },
     "max_concurrent": { "type": "integer", "minimum": 1 },
+    "infrastructure_retries": { "type": "integer", "minimum": 0 },
+    "infrastructure_retry_backoff_ms": { "type": "number", "minimum": 0 },
     "timeout_ms": { "type": "number", "minimum": 0 },
     "setup_timeout_ms": { "type": "number", "minimum": 0 },
     "agent_args": { "type": "array", "items": { "type": "string" } },

package/integrations/harbor/hitch_harbor_agent.py

@@ -60,7 +60,7 @@ class HitchHarborAgent(BaseAgent):
         local_source_transport: dict[str, Any] | None = None,
         hitch_timeout_ms: int = 900_000,
         agent_args: list[str] | None = None,
-        workdir: str = "/app",
+        workdir: str | None = None,
         eval_id: str | None = None,
         benchmark_id: str | None = None,
         benchmark_revision: str | None = None,
@@ -79,7 +79,10 @@ class HitchHarborAgent(BaseAgent):
         self.candidate_id = candidate_id
         self.hitch_timeout_ms = int(hitch_timeout_ms)
         self.agent_args = list(agent_args or [])
-        self.workdir = workdir
+        if workdir is not None and (not isinstance(workdir, str) or not workdir.strip()):
+            raise ValueError("workdir must be a non-empty string when provided")
+        self.workdir = workdir.strip() if isinstance(workdir, str) else None
+        self._workdir_source: str | None = "agent_config" if self.workdir is not None else None
         self.eval_id = eval_id
         self.benchmark_id = benchmark_id
         self.benchmark_revision = benchmark_revision
@@ -145,6 +148,7 @@ class HitchHarborAgent(BaseAgent):
         payload_dir = self.hitch_runtime_dir / "payload"
         if not payload_dir.is_dir():
             raise RuntimeError(f"Hitch runtime bundle has no payload directory: {self.hitch_runtime_dir}")
+        await self._resolve_workdir(environment)
         # Upload the cached bundle's payload (package.json + dist/) as the
         # package root under /opt/hitch; the local cache path is host-side
         # bookkeeping and is not identity (spec §4.2).
@@ -193,6 +197,111 @@ class HitchHarborAgent(BaseAgent):
             if cache_lock is not None:
                 await self._release_artifact_cache_lock(cache_lock)
 
+    async def _resolve_workdir(self, environment: BaseEnvironment) -> str:
+        """Resolve Harbor's effective task directory and prove it is usable.
+
+        Harbor environments already combine task-level ``[environment].workdir``
+        with the container image's ``WORKDIR``. Honor an explicit bridge
+        override, then the task configuration, then ask the running container
+        for its default cwd instead of assuming a global path such as /app.
+        """
+        candidate = self.workdir
+        source = self._workdir_source
+        task_config = getattr(environment, "task_env_config", None)
+        task_workdir = getattr(task_config, "workdir", None)
+        discovery: ExecResult | None = None
+        if candidate is None and isinstance(task_workdir, str) and task_workdir.strip():
+            candidate = task_workdir.strip()
+            source = "task_environment"
+        if candidate is None:
+            discovery = await environment.exec("pwd -P")
+            candidate = (discovery.stdout or "").strip()
+            source = "container_workdir"
+            if discovery.return_code != 0 or not candidate:
+                detail = self._exec_diagnostic(discovery)
+                await self._raise_workdir_error(
+                    environment,
+                    "Could not determine the Harbor task working directory from the container "
+                    f"(exit={discovery.return_code}): {detail}",
+                    source=source,
+                    candidate=candidate or None,
+                    probe=discovery,
+                )
+        if (
+            candidate is None
+            or not PurePosixPath(candidate).is_absolute()
+            or "\x00" in candidate
+            or "\n" in candidate
+            or "\r" in candidate
+        ):
+            await self._raise_workdir_error(
+                environment,
+                f"Harbor task working directory must be an absolute POSIX path; got {candidate!r} "
+                f"from {source or 'unknown'}",
+                source=source,
+                candidate=candidate,
+                probe=discovery,
+            )
+
+        exists = await environment.exec(f"test -d {shlex.quote(candidate)}", cwd="/")
+        if exists.return_code != 0:
+            detail = self._exec_diagnostic(exists)
+            await self._raise_workdir_error(
+                environment,
+                f"Harbor task working directory does not exist or is not a directory: {candidate} "
+                f"(source={source}, exit={exists.return_code}): {detail}",
+                source=source,
+                candidate=candidate,
+                probe=exists,
+            )
+
+        usable = await environment.exec("pwd -P", cwd=candidate)
+        resolved = (usable.stdout or "").strip()
+        if usable.return_code != 0 or not resolved or not PurePosixPath(resolved).is_absolute():
+            detail = self._exec_diagnostic(usable)
+            await self._raise_workdir_error(
+                environment,
+                f"Harbor task working directory exists but cannot be used to start the Hitch agent: {candidate} "
+                f"(source={source}, exit={usable.return_code}): {detail}",
+                source=source,
+                candidate=candidate,
+                probe=usable,
+            )
+        self.workdir = resolved
+        self._workdir_source = source
+        return resolved
+
+    async def _raise_workdir_error(
+        self,
+        environment: BaseEnvironment,
+        message: str,
+        *,
+        source: str | None,
+        candidate: str | None,
+        probe: ExecResult | None,
+    ) -> None:
+        evidence: dict[str, Any] = {
+            "schema_version": "1",
+            "code": "hitch_workdir_invalid",
+            "message": self._bounded_tail(message, 2048),
+            "recorded_at": datetime.now(timezone.utc).isoformat(),
+            "eval_id": self.eval_id,
+            "workdir": {
+                "source": source,
+                "candidate": candidate,
+                "return_code": probe.return_code if probe is not None else None,
+                "stdout_tail": self._bounded_tail(probe.stdout or "") if probe is not None else "",
+                "stderr_tail": self._bounded_tail(probe.stderr or "") if probe is not None else "",
+            },
+        }
+        await self._write_bridge_error(environment, evidence)
+        raise HitchBridgeError("hitch_workdir_invalid", message, evidence)
+
+    def _require_workdir(self) -> str:
+        if self.workdir is None:
+            raise RuntimeError("Hitch agent setup() must resolve the Harbor task working directory before run()")
+        return self.workdir
+
     def _verify_harness_artifact_host(self) -> dict[str, Any]:
         """Pin host artifact metadata before Harbor copies the directory."""
         transport = self.harness_artifact or {}
@@ -834,6 +943,7 @@ git -C {LOCAL_GIT_REMOTE_ROOT}/repo.git update-ref refs/heads/hitch-local {commi
         context: AgentContext,
     ) -> None:
         self.logs_dir.mkdir(parents=True, exist_ok=True)
+        workdir = self._require_workdir()
         assigned_run_id = "run_" + uuid.uuid4().hex
         run_id = assigned_run_id
         trial_id, task_id, attempt = self._trial_identity()
@@ -909,7 +1019,7 @@ git -C {LOCAL_GIT_REMOTE_ROOT}/repo.git update-ref refs/heads/hitch-local {commi
             *self._local_source_cli_args(),
             *self._artifact_cli_args(),
             "--cwd",
-            shlex.quote(self.workdir),
+            shlex.quote(workdir),
             "--workspace-mode",
             "shared",
             "--prompt-file",
@@ -937,7 +1047,7 @@ git -C {LOCAL_GIT_REMOTE_ROOT}/repo.git update-ref refs/heads/hitch-local {commi
             + " 2> >(tee /logs/agent/hitch-stderr.log >&2)"
             + " | tee /logs/agent/hitch-events.jsonl"
         )
-        execution = await environment.exec(command, cwd=self.workdir)
+        execution = await environment.exec(command, cwd=workdir)
         events = self._events(execution.stdout or "")
         observed_run_id = next((
             value
@@ -989,14 +1099,14 @@ mv "$stage_dir" "$target_dir"
         primary_message: str | None = None
         if execution.return_code != 0:
             primary_code = "hitch_process_failed"
-            diagnostic = (execution.stderr or "").strip()
+            diagnostic = self._exec_diagnostic(execution)
             if hitch_result and isinstance(hitch_result.get("error"), dict):
                 result_message = hitch_result["error"].get("message")
                 if isinstance(result_message, str) and result_message.strip():
                     diagnostic = result_message.strip()
             primary_message = (
                 f"Hitch agent run failed with code {execution.return_code} "
-                f"(run_id={run_id}, trial_id={trial_id}): {self._bounded_tail(diagnostic or 'no diagnostic output')}"
+                f"(run_id={run_id}, trial_id={trial_id}): {self._bounded_tail(diagnostic)}"
             )
         elif result_error_code is not None:
             primary_code = result_error_code
@@ -1022,6 +1132,8 @@ mv "$stage_dir" "$target_dir"
             "controller_runtime_id": self.controller_runtime_id,
             "hitch_run_id": run_id,
             "hitch_run_bundle": "hitch-run-bundle",
+            "hitch_workdir": workdir,
+            "hitch_workdir_source": self._workdir_source,
             "eval_id": self.eval_id,
             "trial_id": trial_id,
             "task_id": task_id,
@@ -1226,7 +1338,8 @@ mv "$stage_dir" "$target_dir"
     async def _write_bridge_error(environment: BaseEnvironment, evidence: dict[str, Any]) -> None:
         payload = json.dumps(evidence, ensure_ascii=False, separators=(",", ":"), sort_keys=True)
         await environment.exec(
-            f"umask 077; printf '%s\\n' {shlex.quote(payload)} > {HITCH_BRIDGE_ERROR_LOG}"
+            f"umask 077; printf '%s\\n' {shlex.quote(payload)} > {HITCH_BRIDGE_ERROR_LOG}",
+            cwd="/",
         )
 
     def _trial_identity(self) -> tuple[str, str, int]:
@@ -1276,6 +1389,7 @@ mv "$stage_dir" "$target_dir"
             return Path(handle.name)
 
     async def _workspace_digest(self, environment: BaseEnvironment) -> str:
+        workdir = self._require_workdir()
         script = r"""
 const fs = require('node:fs');
 const path = require('node:path');
@@ -1296,8 +1410,8 @@ walk(process.argv[1]);
 process.stdout.write('sha256:' + hash.digest('hex'));
 """.strip()
         result = await environment.exec(
-            " ".join([self._node_prefix(), "node", "-e", shlex.quote(script), shlex.quote(self.workdir)]),
-            cwd=self.workdir,
+            " ".join([self._node_prefix(), "node", "-e", shlex.quote(script), shlex.quote(workdir)]),
+            cwd=workdir,
         )
         digest = (result.stdout or "").strip()
         if result.return_code == 0 and re.fullmatch(r"sha256:[0-9a-f]{64}", digest):
@@ -1373,6 +1487,13 @@ node -e 'process.exit(Number(process.versions.node.split(".")[0]) >= 22 ? 0 : 1)
             raise RuntimeError(f"container setup command failed ({result.return_code}): {diagnostic}")
         return result
 
+    @staticmethod
+    def _exec_diagnostic(result: ExecResult) -> str:
+        for value in (result.stderr, result.stdout):
+            if value and value.strip():
+                return value.strip()
+        return "no diagnostic output"
+
 
 def canonical_manifest_json(manifest: dict[str, Any]) -> str:
     """Canonically encode the runtime identity `{ schema_version, node_range,