agent-hitch 0.2.6 → 0.2.8

package/dist/src/evals/trial-publication-recovery.js

@@ -0,0 +1,144 @@
+import { readdir, readFile } from "node:fs/promises";
+import path from "node:path";
+import { HitchError, readJSON, sha256Bytes, statePaths } from "../foundation/index.js";
+import { verifyResultBundleIndex } from "../runs/index.js";
+import { evalTrialKey, mergeEvalProgressTrial, replaceInvalidEvalProgressTrial, writeEvalProgress } from "./progress.js";
+import { validateEvalTrialReferences } from "./trial-import.js";
+import { EVAL_TRIAL_PUBLICATION_REF, parseEvalTrialPublication } from "./trial-publication.js";
+const RUN_ID = /^run_[a-f0-9]{32}$/;
+export async function recoverPromotedEvalTrialPublications(input) {
+    const evalId = input.plan.eval_id;
+    const publications = await readEvalTrialPublications(input.root, evalId);
+    const bySlot = new Map();
+    for (const publication of publications) {
+        assertPublicationBelongsToPlan(publication, input.plan);
+        if (!publication.trial.run_group)
+            await verifyResultBundleIndex(path.join(statePaths(input.root).runs, publication.trial.run_id));
+        await validateEvalTrialReferences(input.root, evalId, [publication.trial], {
+            benchmarkId: input.plan.benchmark.id,
+            benchmarkRevision: input.plan.benchmark.revision,
+        });
+        const key = evalTrialKey(publication.trial);
+        const entries = bySlot.get(key) ?? [];
+        entries.push(publication);
+        bySlot.set(key, entries);
+    }
+    let progress = input.progress;
+    const recovered = [];
+    for (const key of [...bySlot.keys()].sort()) {
+        const entries = bySlot.get(key);
+        const settlements = entries.filter((entry) => entry.mode === "settle");
+        const repairs = entries.filter((entry) => entry.mode === "replace-invalid" && entry.trial.observation_status === "valid");
+        if (settlements.length > 1)
+            throw conflict(`multiple promoted authoritative runs exist for ${formatTrial(settlements[0].trial)}`);
+        if (repairs.length > 1)
+            throw conflict(`multiple promoted repair runs exist for ${formatTrial(repairs[0].trial)}`);
+        const selected = repairs[0] ?? settlements[0];
+        if (!selected)
+            continue;
+        const current = progress.trials.find((trial) => evalTrialKey(trial) === key);
+        if (current && JSON.stringify(current) === JSON.stringify(selected.trial))
+            continue;
+        if (current && selected.mode === "settle") {
+            throw conflict(`promoted run conflicts with durable progress for ${formatTrial(selected.trial)}`);
+        }
+        if (current?.observation_status === "valid") {
+            throw conflict(`promoted repair conflicts with an existing valid run for ${formatTrial(selected.trial)}`);
+        }
+        progress = selected.mode === "replace-invalid"
+            ? replaceInvalidEvalProgressTrial(progress, selected.trial)
+            : mergeEvalProgressTrial(progress, selected.trial);
+        recovered.push({ mode: selected.mode, trial: selected.trial });
+    }
+    if (recovered.length > 0)
+        await writeEvalProgress(input.evalDirectory, progress);
+    for (const item of recovered)
+        input.sink?.emit({
+            type: "eval.trial.publication-recovered", mode: item.mode, trial_id: item.trial.trial_id,
+            task_id: item.trial.task_id, attempt: item.trial.attempt, run_id: item.trial.run_id,
+            ...(item.trial.run_group ? { run_group_id: item.trial.run_group.run_group_id } : {}),
+            observation_status: item.trial.observation_status, generation: progress.generation,
+        });
+    return { progress, recovered };
+}
+async function readEvalTrialPublications(root, evalId) {
+    let entries;
+    try {
+        entries = await readdir(statePaths(root).runs, { withFileTypes: true });
+    }
+    catch (error) {
+        if (error.code === "ENOENT")
+            entries = [];
+        else
+            throw error;
+    }
+    const result = [];
+    for (const entry of entries.sort((left, right) => left.name.localeCompare(right.name))) {
+        if (!entry.isDirectory() || !RUN_ID.test(entry.name))
+            continue;
+        const value = await readJSON(path.join(statePaths(root).runs, entry.name, EVAL_TRIAL_PUBLICATION_REF), null);
+        if (value === null || !belongsToEval(value, evalId))
+            continue;
+        result.push(parseEvalTrialPublication(value, entry.name));
+    }
+    const assessments = path.join(statePaths(root).evals, evalId, "assessments");
+    let assessmentEntries;
+    try {
+        assessmentEntries = await readdir(assessments, { withFileTypes: true });
+    }
+    catch (error) {
+        if (error.code === "ENOENT")
+            return result;
+        throw error;
+    }
+    for (const entry of assessmentEntries) {
+        if (!entry.isDirectory() || !/^assessment_[a-f0-9]{32}$/.test(entry.name))
+            continue;
+        let value = await readJSON(path.join(assessments, entry.name, EVAL_TRIAL_PUBLICATION_REF), null);
+        if (value === null) {
+            // assessment.json is the immutable commit. Recover a crash between its
+            // publication and writing the convenience publication receipt.
+            const manifestPath = path.join(assessments, entry.name, "assessment.json");
+            const record = await readJSON(manifestPath, null);
+            if (record?.kind === "native-phase-assessment" && record.eval_id === evalId) {
+                const observation = record.observation;
+                value = { schema_version: "1", eval_id: evalId, mode: record.publication_mode, created_at: record.created_at,
+                    trial: { trial_id: record.trial_id, task_id: record.task_id, attempt: record.attempt, run_group: record.run_group,
+                        assessment: { id: entry.name, digest: sha256Bytes(await readFile(manifestPath)) }, observation_status: observation.status,
+                        ...(observation.reward !== undefined ? { reward: observation.reward } : {}),
+                        ...(record.scores !== undefined ? { scores: record.scores } : {}),
+                        ...(observation.invalid_reason ? { invalid_reason: observation.invalid_reason } : {}),
+                        ...(observation.verifier_result_ref ? { verifier_result_ref: observation.verifier_result_ref } : {}) } };
+            }
+        }
+        if (value === null || !belongsToEval(value, evalId))
+            continue;
+        const publication = parseEvalTrialPublication(value);
+        if (!publication.trial.run_group || publication.trial.assessment.id !== entry.name)
+            throw conflict("native phase publication assessment mismatch");
+        result.push(publication);
+    }
+    return result;
+}
+function belongsToEval(value, evalId) {
+    return Boolean(value && typeof value === "object" && !Array.isArray(value)
+        && value.eval_id === evalId);
+}
+function assertPublicationBelongsToPlan(publication, plan) {
+    if (publication.eval_id !== plan.eval_id)
+        throw conflict("promoted run belongs to another eval plan");
+    const trial = publication.trial;
+    const planned = plan.membership === "known"
+        ? plan.slots.some((slot) => slot.task_id === trial.task_id && slot.attempt === trial.attempt)
+        : plan.work_items.some((work) => work.logical_attempt === trial.attempt
+            && (work.task_ids.length === 0 || work.task_ids.includes(trial.task_id)));
+    if (!planned)
+        throw conflict(`promoted run is outside the frozen plan: ${formatTrial(trial)}`);
+}
+function formatTrial(trial) {
+    return `${trial.task_id} attempt ${trial.attempt}`;
+}
+function conflict(message) {
+    return new HitchError(message, { code: "eval_trial_publication_conflict", exitCode: 12 });
+}
+//# sourceMappingURL=trial-publication-recovery.js.map

package/dist/src/evals/trial-publication-recovery.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"trial-publication-recovery.js","sourceRoot":"","sources":["../../../src/evals/trial-publication-recovery.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAErD,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACvF,OAAO,EAAE,uBAAuB,EAAE,MAAM,kBAAkB,CAAC;AAC3D,OAAO,EAAE,YAAY,EAAE,sBAAsB,EAAE,+BAA+B,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAEzH,OAAO,EAAE,2BAA2B,EAAE,MAAM,mBAAmB,CAAC;AAChE,OAAO,EAAE,0BAA0B,EAAE,yBAAyB,EAAE,MAAM,wBAAwB,CAAC;AAG/F,MAAM,MAAM,GAAG,oBAAoB,CAAC;AAOpC,MAAM,CAAC,KAAK,UAAU,oCAAoC,CAAC,KAM1D;IACC,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC;IAClC,MAAM,YAAY,GAAG,MAAM,yBAAyB,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACzE,MAAM,MAAM,GAAG,IAAI,GAAG,EAAoC,CAAC;IAC3D,KAAK,MAAM,WAAW,IAAI,YAAY,EAAE,CAAC;QACvC,8BAA8B,CAAC,WAAW,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QACxD,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,SAAS;YAAE,MAAM,uBAAuB,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;QAClI,MAAM,2BAA2B,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,WAAW,CAAC,KAAK,CAAC,EAAE;YACzE,WAAW,EAAE,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE;YACpC,iBAAiB,EAAE,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ;SACjD,CAAC,CAAC;QACH,MAAM,GAAG,GAAG,YAAY,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QACtC,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAC1B,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAC3B,CAAC;IAED,IAAI,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC;IAC9B,MAAM,SAAS,GAAoC,EAAE,CAAC;IACtD,KAAK,MAAM,GAAG,IAAI,CAAC,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;QAC5C,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAA6B,CAAC;QAC5D,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC;QACvE,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,iBAAiB,IAAI,KAAK,CAAC,KAAK,CAAC,kBAAkB,KAAK,OAAO,CAAC,CAAC;QAC1H,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC;YAAE,MAAM,QAAQ,CAAC,kDAAkD,WAAW,CAAC,WAAW,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACnI,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;YAAE,MAAM,QAAQ,CAAC,2CAA2C,WAAW,CAAC,OAAO,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACpH,MAAM,QAAQ,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,WAAW,CAAC,CAAC,CAAC,CAAC;QAC9C,IAAI,CAAC,QAAQ;YAAE,SAAS;QACxB,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,YAAY,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC;QAC7E,IAAI,OAAO,IAAI,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC;YAAE,SAAS;QACpF,IAAI,OAAO,IAAI,QAAQ,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC1C,MAAM,QAAQ,CAAC,oDAAoD,WAAW,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACpG,CAAC;QACD,IAAI,OAAO,EAAE,kBAAkB,KAAK,OAAO,EAAE,CAAC;YAC5C,MAAM,QAAQ,CAAC,4DAA4D,WAAW,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC5G,CAAC;QACD,QAAQ,GAAG,QAAQ,CAAC,IAAI,KAAK,iBAAiB;YAC5C,CAAC,CAAC,+BAA+B,CAAC,QAAQ,EAAE,QAAQ,CAAC,KAAK,CAAC;YAC3D,CAAC,CAAC,sBAAsB,CAAC,QAAQ,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC;QACrD,SAAS,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC;IACjE,CAAC;IACD,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC;QAAE,MAAM,iBAAiB,CAAC,KAAK,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC;IACjF,KAAK,MAAM,IAAI,IAAI,SAAS;QAAE,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC;YAC7C,IAAI,EAAE,kCAAkC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ;YACxF,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM;YACnF,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACpF,kBAAkB,EAAE,IAAI,CAAC,KAAK,CAAC,kBAAkB,EAAE,UAAU,EAAE,QAAQ,CAAC,UAAU;SACnF,CAAC,CAAC;IACH,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC;AACjC,CAAC;AAED,KAAK,UAAU,yBAAyB,CAAC,IAAY,EAAE,MAAc;IACnE,IAAI,OAAiB,CAAC;IACtB,IAAI,CAAC;QAAC,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IAAC,CAAC;IAChF,OAAO,KAAK,EAAE,CAAC;QACb,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,GAAG,EAAE,CAAC;;YAChE,MAAM,KAAK,CAAC;IACnB,CAAC;IACD,MAAM,MAAM,GAA6B,EAAE,CAAC;IAC5C,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QACvF,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;YAAE,SAAS;QAC/D,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAiB,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,0BAA0B,CAAC,EAAE,IAAI,CAAC,CAAC;QAC7H,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,MAAM,CAAC;YAAE,SAAS;QAC9D,MAAM,CAAC,IAAI,CAAC,yBAAyB,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;IAC5D,CAAC;IACD,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,aAAa,CAAC,CAAC;IAC7E,IAAI,iBAAiB,CAAC;IACtB,IAAI,CAAC;QAAC,iBAAiB,GAAG,MAAM,OAAO,CAAC,WAAW,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IAAC,CAAC;IAChF,OAAO,KAAK,EAAE,CAAC;QAAC,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,MAAM,CAAC;QAAC,MAAM,KAAK,CAAC;IAAC,CAAC;IACrG,KAAK,MAAM,KAAK,IAAI,iBAAiB,EAAE,CAAC;QACtC,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,IAAI,CAAC,2BAA2B,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;YAAE,SAAS;QACpF,IAAI,KAAK,GAAG,MAAM,QAAQ,CAAiB,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,CAAC,IAAI,EAAE,0BAA0B,CAAC,EAAE,IAAI,CAAC,CAAC;QACjH,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACnB,uEAAuE;YACvE,+DAA+D;YAC/D,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,CAAC,IAAI,EAAE,iBAAiB,CAAC,CAAC;YAC3E,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAiC,YAAY,EAAE,IAAI,CAAC,CAAC;YAClF,IAAI,MAAM,EAAE,IAAI,KAAK,yBAAyB,IAAI,MAAM,CAAC,OAAO,KAAK,MAAM,EAAE,CAAC;gBAC5E,MAAM,WAAW,GAAG,MAAM,CAAC,WAAsC,CAAC;gBAClE,KAAK,GAAG,EAAE,cAAc,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,gBAAgB,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU;oBAC1G,KAAK,EAAE,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS;wBAC/G,UAAU,EAAE,EAAE,EAAE,EAAE,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,WAAW,CAAC,MAAM,QAAQ,CAAC,YAAY,CAAC,CAAC,EAAE,EAAE,kBAAkB,EAAE,WAAW,CAAC,MAAM;wBACzH,GAAG,CAAC,WAAW,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;wBAC3E,GAAG,CAAC,MAAM,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;wBACjE,GAAG,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,WAAW,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;wBACrF,GAAG,CAAC,WAAW,CAAC,mBAAmB,CAAC,CAAC,CAAC,EAAE,mBAAmB,EAAE,WAAW,CAAC,mBAAmB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC/G,CAAC;QACH,CAAC;QACD,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,MAAM,CAAC;YAAE,SAAS;QAC9D,MAAM,WAAW,GAAG,yBAAyB,CAAC,KAAK,CAAC,CAAC;QACrD,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,SAAS,IAAI,WAAW,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,KAAK,KAAK,CAAC,IAAI;YAAE,MAAM,QAAQ,CAAC,8CAA8C,CAAC,CAAC;QACnJ,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IAC3B,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,aAAa,CAAC,KAAc,EAAE,MAAc;IACnD,OAAO,OAAO,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;WACpE,KAAiC,CAAC,OAAO,KAAK,MAAM,CAAC,CAAC;AAC9D,CAAC;AAED,SAAS,8BAA8B,CAAC,WAAmC,EAAE,IAAyB;IACpG,IAAI,WAAW,CAAC,OAAO,KAAK,IAAI,CAAC,OAAO;QAAE,MAAM,QAAQ,CAAC,2CAA2C,CAAC,CAAC;IACtG,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,CAAC;IAChC,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,KAAK,OAAO;QACzC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,KAAK,KAAK,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,KAAK,KAAK,CAAC,OAAO,CAAC;QAC7F,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,eAAe,KAAK,KAAK,CAAC,OAAO;eAClE,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAC9E,IAAI,CAAC,OAAO;QAAE,MAAM,QAAQ,CAAC,4CAA4C,WAAW,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;AACjG,CAAC;AAED,SAAS,WAAW,CAAC,KAAkD;IACrE,OAAO,GAAG,KAAK,CAAC,OAAO,YAAY,KAAK,CAAC,OAAO,EAAE,CAAC;AACrD,CAAC;AAED,SAAS,QAAQ,CAAC,OAAe;IAC/B,OAAO,IAAI,UAAU,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,iCAAiC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;AAC5F,CAAC"}

package/dist/src/evals/trial-publication.js

@@ -0,0 +1,25 @@
+import path from "node:path";
+import { atomicWriteJSON } from "../foundation/index.js";
+import { parseEvalTrialRef } from "./progress.js";
+const EVAL_ID = /^eval_[a-f0-9]{32}$/;
+export const EVAL_TRIAL_PUBLICATION_REF = path.join("eval", "publication.json");
+export async function writeEvalTrialPublication(runDirectory, evalId, mode, trial) {
+    await atomicWriteJSON(path.join(runDirectory, EVAL_TRIAL_PUBLICATION_REF), {
+        schema_version: "1", eval_id: evalId, mode, trial, created_at: new Date().toISOString(),
+    });
+}
+export function parseEvalTrialPublication(value, expectedRunId) {
+    if (!value || typeof value !== "object" || Array.isArray(value))
+        throw new TypeError("eval trial publication must be an object");
+    const record = value;
+    if (record.schema_version !== "1" || typeof record.eval_id !== "string" || !EVAL_ID.test(record.eval_id)
+        || (record.mode !== "settle" && record.mode !== "replace-invalid")
+        || typeof record.created_at !== "string" || !Number.isFinite(Date.parse(record.created_at))) {
+        throw new TypeError("eval trial publication is invalid");
+    }
+    const trial = parseEvalTrialRef(record.trial, "eval trial publication trial");
+    if (expectedRunId !== undefined && trial.run_id !== expectedRunId)
+        throw new TypeError("eval trial publication run identity is invalid");
+    return { schema_version: "1", eval_id: record.eval_id, mode: record.mode, trial, created_at: record.created_at };
+}
+//# sourceMappingURL=trial-publication.js.map

package/dist/src/evals/trial-publication.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"trial-publication.js","sourceRoot":"","sources":["../../../src/evals/trial-publication.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAElD,MAAM,OAAO,GAAG,qBAAqB,CAAC;AAEtC,MAAM,CAAC,MAAM,0BAA0B,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAC;AAWhF,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC7C,YAAoB,EACpB,MAAc,EACd,IAA8B,EAC9B,KAAqB;IAErB,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,0BAA0B,CAAC,EAAE;QACzE,cAAc,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KACxF,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,yBAAyB,CAAC,KAAc,EAAE,aAAsB;IAC9E,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,0CAA0C,CAAC,CAAC;IACjI,MAAM,MAAM,GAAG,KAAgC,CAAC;IAChD,IAAI,MAAM,CAAC,cAAc,KAAK,GAAG,IAAI,OAAO,MAAM,CAAC,OAAO,KAAK,QAAQ,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC;WACnG,CAAC,MAAM,CAAC,IAAI,KAAK,QAAQ,IAAI,MAAM,CAAC,IAAI,KAAK,iBAAiB,CAAC;WAC/D,OAAO,MAAM,CAAC,UAAU,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC;QAC9F,MAAM,IAAI,SAAS,CAAC,mCAAmC,CAAC,CAAC;IAC3D,CAAC;IACD,MAAM,KAAK,GAAG,iBAAiB,CAAC,MAAM,CAAC,KAAK,EAAE,8BAA8B,CAAC,CAAC;IAC9E,IAAI,aAAa,KAAK,SAAS,IAAI,KAAK,CAAC,MAAM,KAAK,aAAa;QAAE,MAAM,IAAI,SAAS,CAAC,gDAAgD,CAAC,CAAC;IACzI,OAAO,EAAE,cAAc,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,CAAC;AACnH,CAAC"}

package/dist/src/evals/trial-reference-validation.js

@@ -0,0 +1,49 @@
+import path from "node:path";
+import { isDeepStrictEqual } from "node:util";
+import { statePaths } from "../foundation/index.js";
+import { benchmarkVerifierIdentity, loadRunRecord } from "../runs/index.js";
+import { parseVerifierScores } from "../domain/index.js";
+import { readJSON } from "../foundation/index.js";
+import { readRegradeObservation } from "./regrade-evidence.js";
+import { readNativePhaseObservation } from "./native-phase-evidence.js";
+export async function validateEvalTrialReferences(root, evalId, trials, expected) {
+    for (const trial of trials) {
+        let observation;
+        if (trial.run_group)
+            observation = await readNativePhaseObservation(root, evalId, trial, expected);
+        else {
+            const loaded = await loadRunRecord(path.join(statePaths(root).runs, trial.run_id), { verifyTrajectory: false });
+            const record = loaded.record;
+            if (record.context.kind !== "benchmark_task")
+                throw new Error(`eval trial ${trial.trial_id} references a non-benchmark run`);
+            if (expected && (record.context.benchmark_id !== expected.benchmarkId
+                || record.context.benchmark_revision !== expected.benchmarkRevision
+                || record.context.verifier_identity !== benchmarkVerifierIdentity(expected.benchmarkId, expected.benchmarkRevision)))
+                throw new Error(`eval trial ${trial.trial_id} benchmark identity mismatch`);
+            if (record.parent?.eval_id !== evalId || record.parent.trial_id !== trial.trial_id || record.parent.attempt !== trial.attempt) {
+                throw new Error(`eval trial ${trial.trial_id} parent mismatch`);
+            }
+            if (record.context.task_id !== trial.task_id)
+                throw new Error(`eval trial ${trial.trial_id} task mismatch`);
+            observation = trial.assessment ? await readRegradeObservation(root, evalId, trial) : record.observation;
+        }
+        if (observation?.status !== trial.observation_status)
+            throw new Error(`eval trial ${trial.trial_id} observation status mismatch`);
+        if (observation?.reward !== trial.reward)
+            throw new Error(`eval trial ${trial.trial_id} reward mismatch`);
+        if (observation?.verifier_result_ref !== trial.verifier_result_ref)
+            throw new Error(`eval trial ${trial.trial_id} verifier ref mismatch`);
+        if (observation?.invalid_reason !== trial.invalid_reason)
+            throw new Error(`eval trial ${trial.trial_id} invalid reason mismatch`);
+        if (trial.scores !== undefined && !trial.run_group) {
+            const directory = trial.assessment
+                ? path.join(statePaths(root).evals, evalId, "assessments", trial.assessment.id)
+                : path.join(statePaths(root).runs, trial.run_id);
+            const result = await readJSON(path.join(directory, observation?.verifier_result_ref ?? "verifier/result.json"));
+            if (!isDeepStrictEqual(parseVerifierScores(result), trial.scores)) {
+                throw new Error(`eval trial ${trial.trial_id} score channels mismatch`);
+            }
+        }
+    }
+}
+//# sourceMappingURL=trial-reference-validation.js.map

package/dist/src/evals/trial-reference-validation.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"trial-reference-validation.js","sourceRoot":"","sources":["../../../src/evals/trial-reference-validation.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAC;AAE9C,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACpD,OAAO,EAAE,yBAAyB,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAC5E,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAClD,OAAO,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AAC/D,OAAO,EAAE,0BAA0B,EAAE,MAAM,4BAA4B,CAAC;AAExE,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAC/C,IAAY,EACZ,MAAc,EACd,MAAwB,EACxB,QAA6D;IAE7D,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,WAAyC,CAAC;QAC9C,IAAI,KAAK,CAAC,SAAS;YAAE,WAAW,GAAG,MAAM,0BAA0B,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC;aAC9F,CAAC;YACJ,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,EAAE,KAAK,EAAE,CAAC,CAAC;YAChH,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;YAC7B,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,KAAK,gBAAgB;gBAAE,MAAM,IAAI,KAAK,CAAC,cAAc,KAAK,CAAC,QAAQ,iCAAiC,CAAC,CAAC;YAC7H,IAAI,QAAQ,IAAI,CACd,MAAM,CAAC,OAAO,CAAC,YAAY,KAAK,QAAQ,CAAC,WAAW;mBACjD,MAAM,CAAC,OAAO,CAAC,kBAAkB,KAAK,QAAQ,CAAC,iBAAiB;mBAChE,MAAM,CAAC,OAAO,CAAC,iBAAiB,KAAK,yBAAyB,CAAC,QAAQ,CAAC,WAAW,EAAE,QAAQ,CAAC,iBAAiB,CAAC,CACpH;gBAAE,MAAM,IAAI,KAAK,CAAC,cAAc,KAAK,CAAC,QAAQ,8BAA8B,CAAC,CAAC;YAC/E,IAAI,MAAM,CAAC,MAAM,EAAE,OAAO,KAAK,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC,QAAQ,KAAK,KAAK,CAAC,QAAQ,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC;gBAC9H,MAAM,IAAI,KAAK,CAAC,cAAc,KAAK,CAAC,QAAQ,kBAAkB,CAAC,CAAC;YAClE,CAAC;YACD,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,KAAK,KAAK,CAAC,OAAO;gBAAE,MAAM,IAAI,KAAK,CAAC,cAAc,KAAK,CAAC,QAAQ,gBAAgB,CAAC,CAAC;YAC5G,WAAW,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM,sBAAsB,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC;QAC1G,CAAC;QACD,IAAI,WAAW,EAAE,MAAM,KAAK,KAAK,CAAC,kBAAkB;YAAE,MAAM,IAAI,KAAK,CAAC,cAAc,KAAK,CAAC,QAAQ,8BAA8B,CAAC,CAAC;QAClI,IAAI,WAAW,EAAE,MAAM,KAAK,KAAK,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,cAAc,KAAK,CAAC,QAAQ,kBAAkB,CAAC,CAAC;QAC1G,IAAI,WAAW,EAAE,mBAAmB,KAAK,KAAK,CAAC,mBAAmB;YAAE,MAAM,IAAI,KAAK,CAAC,cAAc,KAAK,CAAC,QAAQ,wBAAwB,CAAC,CAAC;QAC1I,IAAI,WAAW,EAAE,cAAc,KAAK,KAAK,CAAC,cAAc;YAAE,MAAM,IAAI,KAAK,CAAC,cAAc,KAAK,CAAC,QAAQ,0BAA0B,CAAC,CAAC;QAClI,IAAI,KAAK,CAAC,MAAM,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;YACnD,MAAM,SAAS,GAAG,KAAK,CAAC,UAAU;gBAChC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,aAAa,EAAE,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC;gBAC/E,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;YACnD,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,WAAW,EAAE,mBAAmB,IAAI,sBAAsB,CAAC,CAAC,CAAC;YAChH,IAAI,CAAC,iBAAiB,CAAC,mBAAmB,CAAC,MAAM,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;gBAClE,MAAM,IAAI,KAAK,CAAC,cAAc,KAAK,CAAC,QAAQ,0BAA0B,CAAC,CAAC;YAC1E,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC"}

package/dist/src/evals/verifier-artifacts.js

@@ -0,0 +1,317 @@
+import { createHash } from "node:crypto";
+import { realpath } from "node:fs/promises";
+import { StringDecoder } from "node:string_decoder";
+import path from "node:path";
+import { PROVIDER_ENVIRONMENT_NAMES, atomicWriteJSON, credentialValuesFromEnv, openContainedRegularFile, redactCredentialText, writePrivateFile } from "../foundation/index.js";
+import { captureVerifierScoreEvidence } from "./verifier-score-artifacts.js";
+export const DEFAULT_VERIFIER_DIAGNOSTIC_MAX_BYTES = 64 * 1024;
+export const MAX_VERIFIER_DIAGNOSTIC_MAX_BYTES = 16 * 1024 * 1024;
+export const VERIFIER_DIAGNOSTICS_INDEX_REF = "verifier/diagnostics.json";
+export const VERIFIER_ARTIFACT_NAMES = [
+    "ctrf.json", "test-stdout.txt", "test-stderr.txt", "stdout.txt", "stderr.txt",
+];
+const TRUNCATION_MARKER = Buffer.from("\n[... verifier artifact truncated ...]\n", "utf8");
+const MAX_LOG_LINE_BYTES = 1024 * 1024;
+export async function persistTrialVerifierDiagnostics(input) {
+    const env = input.env ?? process.env;
+    const credentialValues = credentialValuesFromEnv([...PROVIDER_ENVIRONMENT_NAMES, ...(input.passEnv ?? [])], env);
+    throwIfAborted(input.signal);
+    await copyVerifierRetryHistory(input.trialDirectory, input.runDirectory, credentialValues, input.signal);
+    await copyCandidateIneligibleDiagnostic(input.trialDirectory, input.runDirectory, input.signal);
+    const scores = await captureVerifierScoreEvidence({
+        trialDirectory: input.trialDirectory,
+        runDirectory: input.runDirectory,
+        verifierResult: input.verifierResult ?? null,
+        credentialValues,
+        ...(input.dataset === undefined ? {} : { dataset: input.dataset }),
+        ...(input.benchmarkRevision === undefined ? {} : { benchmarkRevision: input.benchmarkRevision }),
+        ...(input.signal ? { signal: input.signal } : {}),
+    });
+    await captureVerifierDiagnostics(input.trialDirectory, input.runDirectory, {
+        ...(input.maxArtifactBytes === undefined ? {} : { maxArtifactBytes: input.maxArtifactBytes }),
+        credentialValues,
+        ...(input.signal ? { signal: input.signal } : {}),
+    });
+    return scores;
+}
+async function copyCandidateIneligibleDiagnostic(trialDirectory, runDirectory, signal) {
+    const root = await realpath(trialDirectory);
+    throwIfAborted(signal);
+    const safe = await safeArtifact(root, "candidate-ineligible.json", 16 * 1024);
+    if (!safe)
+        return;
+    let bytes;
+    try {
+        bytes = await safe.handle.readFile();
+        await safe.assertUnchanged();
+    }
+    finally {
+        await safe.handle.close();
+    }
+    const value = JSON.parse(bytes.toString("utf8"));
+    if (value.schema_version !== "1" || value.code !== "candidate_evidence_unavailable" || value.verifier_executed !== false)
+        return;
+    await atomicWriteJSON(path.join(runDirectory, "verifier", "candidate-ineligible.json"), value);
+}
+/** Persist bounded, redacted verifier artifacts in the immutable run bundle. */
+export async function captureVerifierDiagnostics(trialDirectory, runDirectory, options = {}) {
+    const maxArtifactBytes = options.maxArtifactBytes ?? DEFAULT_VERIFIER_DIAGNOSTIC_MAX_BYTES;
+    if (!Number.isSafeInteger(maxArtifactBytes) || maxArtifactBytes <= TRUNCATION_MARKER.length + 2
+        || maxArtifactBytes > MAX_VERIFIER_DIAGNOSTIC_MAX_BYTES) {
+        throw new TypeError("verifier diagnostic artifact limit is invalid");
+    }
+    throwIfAborted(options.signal);
+    const root = await realpath(trialDirectory);
+    const artifacts = [];
+    const redactions = new Map();
+    for (const name of VERIFIER_ARTIFACT_NAMES) {
+        throwIfAborted(options.signal);
+        const safe = await safeArtifact(root, name);
+        if (!safe)
+            continue;
+        let captured;
+        try {
+            captured = await captureText(safe.handle, maxArtifactBytes, options.credentialValues ?? [], options.signal);
+            await safe.assertUnchanged();
+            if (captured.sourceBytes !== safe.size)
+                throw new TypeError(`verifier artifact changed while being captured: ${name}`);
+        }
+        finally {
+            await safe.handle.close();
+        }
+        mergeCounts(redactions, captured.redactions);
+        const ref = `verifier/${name}`;
+        const destination = path.join(runDirectory, ...ref.split("/"));
+        await writePrivateFile(destination, captured.stored);
+        const stored = captured.stored;
+        artifacts.push({
+            name,
+            ref,
+            media_type: name === "ctrf.json" ? "application/json" : "text/plain",
+            source_bytes: captured.sourceBytes,
+            source_sha256: captured.sourceSha256,
+            bytes: captured.contentBytes,
+            sha256: captured.contentSha256,
+            stored_bytes: stored.length,
+            stored_sha256: sha256(stored),
+            truncated: captured.truncated,
+        });
+    }
+    if (artifacts.length === 0)
+        return null;
+    const index = {
+        schema_version: "1",
+        kind: "verifier-diagnostics",
+        artifacts,
+        redactions: canonicalCounts(redactions),
+    };
+    await atomicWriteJSON(path.join(runDirectory, ...VERIFIER_DIAGNOSTICS_INDEX_REF.split("/")), index);
+    return index;
+}
+async function safeArtifact(root, name, maximumBytes) {
+    try {
+        return await openContainedRegularFile(root, `verifier/${name}`, maximumBytes);
+    }
+    catch (error) {
+        if (error.code === "ENOENT")
+            return null;
+        throw new TypeError(`unsafe verifier artifact: ${name}`, { cause: error });
+    }
+}
+async function captureText(handle, maxBytes, credentialValues, signal) {
+    const sourceHash = createHash("sha256");
+    const contentHash = createHash("sha256");
+    const decoder = new StringDecoder("utf8");
+    const bounded = new BoundedBuffer(maxBytes);
+    const redactions = new Map();
+    let sourceBytes = 0;
+    let pending = "";
+    let discarding = false;
+    const emit = (value) => {
+        const redacted = redactCredentialText(value, credentialValues);
+        mergeCounts(redactions, redacted.redactions);
+        const bytes = Buffer.from(redacted.text, "utf8");
+        contentHash.update(bytes);
+        bounded.append(bytes);
+    };
+    const drain = () => {
+        for (;;) {
+            const newline = pending.indexOf("\n");
+            if (discarding) {
+                if (newline < 0) {
+                    pending = "";
+                    return;
+                }
+                pending = pending.slice(newline + 1);
+                discarding = false;
+                continue;
+            }
+            if (newline >= 0) {
+                emit(pending.slice(0, newline + 1));
+                pending = pending.slice(newline + 1);
+                continue;
+            }
+            if (Buffer.byteLength(pending, "utf8") > MAX_LOG_LINE_BYTES) {
+                emit("[REDACTED OVERSIZED VERIFIER LOG LINE]\n");
+                increment(redactions, "oversized-verifier-log-line-v1");
+                bounded.markOmitted();
+                pending = "";
+                discarding = true;
+            }
+            return;
+        }
+    };
+    const buffer = Buffer.allocUnsafe(64 * 1024);
+    let position = 0;
+    for (;;) {
+        throwIfAborted(signal);
+        const { bytesRead } = await handle.read(buffer, 0, buffer.length, position);
+        if (bytesRead === 0)
+            break;
+        position += bytesRead;
+        const bytes = buffer.subarray(0, bytesRead);
+        sourceHash.update(bytes);
+        sourceBytes += bytes.length;
+        pending += decoder.write(bytes);
+        drain();
+    }
+    pending += decoder.end();
+    drain();
+    if (!discarding && pending)
+        emit(pending);
+    throwIfAborted(signal);
+    return {
+        sourceSha256: `sha256:${sourceHash.digest("hex")}`,
+        sourceBytes,
+        contentSha256: `sha256:${contentHash.digest("hex")}`,
+        contentBytes: bounded.totalBytes,
+        stored: bounded.value(),
+        truncated: bounded.truncated,
+        redactions,
+    };
+}
+class BoundedBuffer {
+    limit;
+    payloadLimit;
+    headLimit;
+    tailLimit;
+    totalBytes = 0;
+    clipped = false;
+    omitted = false;
+    complete = [];
+    head = Buffer.alloc(0);
+    tail = Buffer.alloc(0);
+    constructor(limit) {
+        this.limit = limit;
+        this.payloadLimit = limit - TRUNCATION_MARKER.length;
+        this.headLimit = Math.floor(this.payloadLimit / 2);
+        this.tailLimit = this.payloadLimit - this.headLimit;
+    }
+    append(value) {
+        if (value.length === 0)
+            return;
+        this.totalBytes += value.length;
+        if (!this.clipped) {
+            this.complete.push(value);
+            const combined = Buffer.concat(this.complete);
+            if (combined.length <= this.limit)
+                return;
+            this.clipped = true;
+            this.complete = [];
+            this.head = combined.subarray(0, this.headLimit);
+            this.tail = combined.subarray(Math.max(0, combined.length - this.tailLimit));
+            return;
+        }
+        this.tail = rollingTail(this.tail, value, this.tailLimit);
+    }
+    value() {
+        return this.clipped
+            ? Buffer.concat([
+                validUtf8Prefix(this.head),
+                TRUNCATION_MARKER,
+                validUtf8Suffix(this.tail),
+            ])
+            : Buffer.concat(this.complete);
+    }
+    markOmitted() {
+        this.omitted = true;
+    }
+    get truncated() {
+        return this.clipped || this.omitted;
+    }
+}
+function validUtf8Prefix(value) {
+    let end = value.length;
+    while (end > 0 && (value[end - 1] & 0xc0) === 0x80)
+        end -= 1;
+    const lead = end - 1;
+    return lead >= 0 && utf8SequenceLength(value[lead]) > value.length - lead ? value.subarray(0, lead) : value;
+}
+function validUtf8Suffix(value) {
+    let start = 0;
+    while (start < value.length && (value[start] & 0xc0) === 0x80)
+        start += 1;
+    return value.subarray(start);
+}
+function utf8SequenceLength(byte) {
+    if ((byte & 0x80) === 0)
+        return 1;
+    if ((byte & 0xe0) === 0xc0)
+        return 2;
+    if ((byte & 0xf0) === 0xe0)
+        return 3;
+    if ((byte & 0xf8) === 0xf0)
+        return 4;
+    return 1;
+}
+function rollingTail(current, next, limit) {
+    if (next.length >= limit)
+        return next.subarray(next.length - limit);
+    const keep = Math.max(0, limit - next.length);
+    return Buffer.concat([current.subarray(Math.max(0, current.length - keep)), next]);
+}
+function canonicalCounts(counts) {
+    return [...counts.entries()]
+        .filter(([, count]) => count > 0)
+        .sort(([left], [right]) => left.localeCompare(right))
+        .map(([rule_id, count]) => ({ rule_id, count }));
+}
+function mergeCounts(target, source) {
+    for (const [rule, count] of source)
+        increment(target, rule, count);
+}
+function increment(counts, rule, count = 1) {
+    counts.set(rule, (counts.get(rule) ?? 0) + count);
+}
+function sha256(value) {
+    return `sha256:${createHash("sha256").update(value).digest("hex")}`;
+}
+function throwIfAborted(signal) {
+    if (signal?.aborted)
+        throw signal.reason ?? new DOMException("The operation was aborted", "AbortError");
+}
+async function copyVerifierRetryHistory(trialDirectory, runDirectory, credentialValues, signal) {
+    const root = await realpath(trialDirectory);
+    throwIfAborted(signal);
+    const safe = await safeArtifact(root, "infrastructure-retry-history.json", 1024 * 1024);
+    if (!safe)
+        return;
+    let bytes;
+    try {
+        bytes = await safe.handle.readFile();
+        await safe.assertUnchanged();
+    }
+    finally {
+        await safe.handle.close();
+    }
+    throwIfAborted(signal);
+    const history = JSON.parse(bytes.toString("utf8"));
+    if (history?.schema_version !== "1"
+        || history.code !== "verifier_infrastructure_retry_history"
+        || history.candidate_rerun !== false
+        || !Array.isArray(history.attempts))
+        return;
+    const redacted = redactCredentialText(JSON.stringify(history), credentialValues).text;
+    await atomicWriteJSON(path.join(runDirectory, "verifier", "infrastructure-retry-history.json"), JSON.parse(redacted));
+}
+//# sourceMappingURL=verifier-artifacts.js.map

package/dist/src/evals/verifier-artifacts.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"verifier-artifacts.js","sourceRoot":"","sources":["../../../src/evals/verifier-artifacts.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,QAAQ,EAAmB,MAAM,kBAAkB,CAAC;AAC7D,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,0BAA0B,EAAE,eAAe,EAAE,uBAAuB,EAAE,wBAAwB,EAAE,oBAAoB,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAEhL,OAAO,EAAE,4BAA4B,EAAE,MAAM,+BAA+B,CAAC;AAG7E,MAAM,CAAC,MAAM,qCAAqC,GAAG,EAAE,GAAG,IAAI,CAAC;AAC/D,MAAM,CAAC,MAAM,iCAAiC,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;AAClE,MAAM,CAAC,MAAM,8BAA8B,GAAG,2BAA2B,CAAC;AAC1E,MAAM,CAAC,MAAM,uBAAuB,GAAiD;IACnF,WAAW,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,YAAY,EAAE,YAAY;CAC9E,CAAC;AAEF,MAAM,iBAAiB,GAAG,MAAM,CAAC,IAAI,CAAC,2CAA2C,EAAE,MAAM,CAAC,CAAC;AAC3F,MAAM,kBAAkB,GAAG,IAAI,GAAG,IAAI,CAAC;AA4BvC,MAAM,CAAC,KAAK,UAAU,+BAA+B,CAAC,KAUrD;IACC,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,CAAC;IACrC,MAAM,gBAAgB,GAAG,uBAAuB,CAC9C,CAAC,GAAG,0BAA0B,EAAE,GAAG,CAAC,KAAK,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,EACzD,GAAG,CACJ,CAAC;IACF,cAAc,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAC7B,MAAM,wBAAwB,CAAC,KAAK,CAAC,cAAc,EAAE,KAAK,CAAC,YAAY,EAAE,gBAAgB,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACzG,MAAM,iCAAiC,CAAC,KAAK,CAAC,cAAc,EAAE,KAAK,CAAC,YAAY,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAChG,MAAM,MAAM,GAAG,MAAM,4BAA4B,CAAC;QAChD,cAAc,EAAE,KAAK,CAAC,cAAc;QACpC,YAAY,EAAE,KAAK,CAAC,YAAY;QAChC,cAAc,EAAE,KAAK,CAAC,cAAc,IAAI,IAAI;QAC5C,gBAAgB;QAChB,GAAG,CAAC,KAAK,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC;QAClE,GAAG,CAAC,KAAK,CAAC,iBAAiB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,KAAK,CAAC,iBAAiB,EAAE,CAAC;QAChG,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAClD,CAAC,CAAC;IACH,MAAM,0BAA0B,CAAC,KAAK,CAAC,cAAc,EAAE,KAAK,CAAC,YAAY,EAAE;QACzE,GAAG,CAAC,KAAK,CAAC,gBAAgB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,KAAK,CAAC,gBAAgB,EAAE,CAAC;QAC7F,gBAAgB;QAChB,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAClD,CAAC,CAAC;IACH,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,KAAK,UAAU,iCAAiC,CAAC,cAAsB,EAAE,YAAoB,EAAE,MAAoB;IACjH,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,cAAc,CAAC,CAAC;IAC5C,cAAc,CAAC,MAAM,CAAC,CAAC;IACvB,MAAM,IAAI,GAAG,MAAM,YAAY,CAAC,IAAI,EAAE,2BAA2B,EAAE,EAAE,GAAG,IAAI,CAAC,CAAC;IAC9E,IAAI,CAAC,IAAI;QAAE,OAAO;IAClB,IAAI,KAAa,CAAC;IAClB,IAAI,CAAC;QACH,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;QACrC,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;IAC/B,CAAC;YAAS,CAAC;QACT,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;IAC5B,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAA4B,CAAC;IAC5E,IAAI,KAAK,CAAC,cAAc,KAAK,GAAG,IAAI,KAAK,CAAC,IAAI,KAAK,gCAAgC,IAAI,KAAK,CAAC,iBAAiB,KAAK,KAAK;QAAE,OAAO;IACjI,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,UAAU,EAAE,2BAA2B,CAAC,EAAE,KAAK,CAAC,CAAC;AACjG,CAAC;AAED,gFAAgF;AAChF,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAC9C,cAAsB,EACtB,YAAoB,EACpB,UAA6C,EAAE;IAE/C,MAAM,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,IAAI,qCAAqC,CAAC;IAC3F,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,gBAAgB,CAAC,IAAI,gBAAgB,IAAI,iBAAiB,CAAC,MAAM,GAAG,CAAC;WAC1F,gBAAgB,GAAG,iCAAiC,EAAE,CAAC;QAC1D,MAAM,IAAI,SAAS,CAAC,+CAA+C,CAAC,CAAC;IACvE,CAAC;IACD,cAAc,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAC/B,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,cAAc,CAAC,CAAC;IAC5C,MAAM,SAAS,GAAkC,EAAE,CAAC;IACpD,MAAM,UAAU,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC7C,KAAK,MAAM,IAAI,IAAI,uBAAuB,EAAE,CAAC;QAC3C,cAAc,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC/B,MAAM,IAAI,GAAG,MAAM,YAAY,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAC5C,IAAI,CAAC,IAAI;YAAE,SAAS;QACpB,IAAI,QAAiD,CAAC;QACtD,IAAI,CAAC;YACH,QAAQ,GAAG,MAAM,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,gBAAgB,EAAE,OAAO,CAAC,gBAAgB,IAAI,EAAE,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;YAC5G,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;YAC7B,IAAI,QAAQ,CAAC,WAAW,KAAK,IAAI,CAAC,IAAI;gBAAE,MAAM,IAAI,SAAS,CAAC,mDAAmD,IAAI,EAAE,CAAC,CAAC;QACzH,CAAC;gBAAS,CAAC;YACT,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QAC5B,CAAC;QACD,WAAW,CAAC,UAAU,EAAE,QAAQ,CAAC,UAAU,CAAC,CAAC;QAC7C,MAAM,GAAG,GAAG,YAAY,IAAI,EAAE,CAAC;QAC/B,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;QAC/D,MAAM,gBAAgB,CAAC,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;QACrD,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;QAC/B,SAAS,CAAC,IAAI,CAAC;YACb,IAAI;YACJ,GAAG;YACH,UAAU,EAAE,IAAI,KAAK,WAAW,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,YAAY;YACpE,YAAY,EAAE,QAAQ,CAAC,WAAW;YAClC,aAAa,EAAE,QAAQ,CAAC,YAAY;YACpC,KAAK,EAAE,QAAQ,CAAC,YAAY;YAC5B,MAAM,EAAE,QAAQ,CAAC,aAAa;YAC9B,YAAY,EAAE,MAAM,CAAC,MAAM;YAC3B,aAAa,EAAE,MAAM,CAAC,MAAM,CAAC;YAC7B,SAAS,EAAE,QAAQ,CAAC,SAAS;SAC9B,CAAC,CAAC;IACL,CAAC;IACD,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACxC,MAAM,KAAK,GAA+B;QACxC,cAAc,EAAE,GAAG;QACnB,IAAI,EAAE,sBAAsB;QAC5B,SAAS;QACT,UAAU,EAAE,eAAe,CAAC,UAAU,CAAC;KACxC,CAAC;IACF,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,GAAG,8BAA8B,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IACpG,OAAO,KAAK,CAAC;AACf,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,IAAY,EAAE,IAAY,EAAE,YAAqB;IAC3E,IAAI,CAAC;QACH,OAAO,MAAM,wBAAwB,CAAC,IAAI,EAAE,YAAY,IAAI,EAAE,EAAE,YAAY,CAAC,CAAC;IAChF,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,IAAI,CAAC;QACpE,MAAM,IAAI,SAAS,CAAC,6BAA6B,IAAI,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;IAC7E,CAAC;AACH,CAAC;AAED,KAAK,UAAU,WAAW,CACxB,MAAkB,EAClB,QAAgB,EAChB,gBAAmC,EACnC,MAAgC;IAUhC,MAAM,UAAU,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;IACxC,MAAM,WAAW,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;IACzC,MAAM,OAAO,GAAG,IAAI,aAAa,CAAC,MAAM,CAAC,CAAC;IAC1C,MAAM,OAAO,GAAG,IAAI,aAAa,CAAC,QAAQ,CAAC,CAAC;IAC5C,MAAM,UAAU,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC7C,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,OAAO,GAAG,EAAE,CAAC;IACjB,IAAI,UAAU,GAAG,KAAK,CAAC;IACvB,MAAM,IAAI,GAAG,CAAC,KAAa,EAAQ,EAAE;QACnC,MAAM,QAAQ,GAAG,oBAAoB,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAAC;QAC/D,WAAW,CAAC,UAAU,EAAE,QAAQ,CAAC,UAAU,CAAC,CAAC;QAC7C,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACjD,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC1B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACxB,CAAC,CAAC;IACF,MAAM,KAAK,GAAG,GAAS,EAAE;QACvB,SAAS,CAAC;YACR,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YACtC,IAAI,UAAU,EAAE,CAAC;gBACf,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;oBAAC,OAAO,GAAG,EAAE,CAAC;oBAAC,OAAO;gBAAC,CAAC;gBAC1C,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC;gBACrC,UAAU,GAAG,KAAK,CAAC;gBACnB,SAAS;YACX,CAAC;YACD,IAAI,OAAO,IAAI,CAAC,EAAE,CAAC;gBACjB,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC;gBACpC,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC;gBACrC,SAAS;YACX,CAAC;YACD,IAAI,MAAM,CAAC,UAAU,CAAC,OAAO,EAAE,MAAM,CAAC,GAAG,kBAAkB,EAAE,CAAC;gBAC5D,IAAI,CAAC,0CAA0C,CAAC,CAAC;gBACjD,SAAS,CAAC,UAAU,EAAE,gCAAgC,CAAC,CAAC;gBACxD,OAAO,CAAC,WAAW,EAAE,CAAC;gBACtB,OAAO,GAAG,EAAE,CAAC;gBACb,UAAU,GAAG,IAAI,CAAC;YACpB,CAAC;YACD,OAAO;QACT,CAAC;IACH,CAAC,CAAC;IAEF,MAAM,MAAM,GAAG,MAAM,CAAC,WAAW,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC;IAC7C,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,SAAS,CAAC;QACR,cAAc,CAAC,MAAM,CAAC,CAAC;QACvB,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAC5E,IAAI,SAAS,KAAK,CAAC;YAAE,MAAM;QAC3B,QAAQ,IAAI,SAAS,CAAC;QACtB,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;QAC5C,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACzB,WAAW,IAAI,KAAK,CAAC,MAAM,CAAC;QAC5B,OAAO,IAAI,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAChC,KAAK,EAAE,CAAC;IACV,CAAC;IACD,OAAO,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;IACzB,KAAK,EAAE,CAAC;IACR,IAAI,CAAC,UAAU,IAAI,OAAO;QAAE,IAAI,CAAC,OAAO,CAAC,CAAC;IAC1C,cAAc,CAAC,MAAM,CAAC,CAAC;IACvB,OAAO;QACL,YAAY,EAAE,UAAU,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;QAClD,WAAW;QACX,aAAa,EAAE,UAAU,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;QACpD,YAAY,EAAE,OAAO,CAAC,UAAU;QAChC,MAAM,EAAE,OAAO,CAAC,KAAK,EAAE;QACvB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,UAAU;KACX,CAAC;AACJ,CAAC;AAED,MAAM,aAAa;IAWI;IAVZ,YAAY,CAAS;IACrB,SAAS,CAAS;IAClB,SAAS,CAAS;IAC3B,UAAU,GAAG,CAAC,CAAC;IACP,OAAO,GAAG,KAAK,CAAC;IAChB,OAAO,GAAG,KAAK,CAAC;IAChB,QAAQ,GAAa,EAAE,CAAC;IACxB,IAAI,GAAW,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAC/B,IAAI,GAAW,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAEvC,YAAqB,KAAa;QAAb,UAAK,GAAL,KAAK,CAAQ;QAChC,IAAI,CAAC,YAAY,GAAG,KAAK,GAAG,iBAAiB,CAAC,MAAM,CAAC;QACrD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC;QACnD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,SAAS,CAAC;IACtD,CAAC;IAED,MAAM,CAAC,KAAa;QAClB,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAC/B,IAAI,CAAC,UAAU,IAAI,KAAK,CAAC,MAAM,CAAC;QAChC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC1B,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC9C,IAAI,QAAQ,CAAC,MAAM,IAAI,IAAI,CAAC,KAAK;gBAAE,OAAO;YAC1C,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;YACpB,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC;YACnB,IAAI,CAAC,IAAI,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;YACjD,IAAI,CAAC,IAAI,GAAG,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;YAC7E,OAAO;QACT,CAAC;QACD,IAAI,CAAC,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IAC5D,CAAC;IAED,KAAK;QACH,OAAO,IAAI,CAAC,OAAO;YACjB,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC;gBACd,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC;gBAC1B,iBAAiB;gBACjB,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC;aAC3B,CAAC;YACF,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACnC,CAAC;IAED,WAAW;QACT,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;IACtB,CAAC;IAED,IAAI,SAAS;QACX,OAAO,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,CAAC;IACtC,CAAC;CACF;AAED,SAAS,eAAe,CAAC,KAAa;IACpC,IAAI,GAAG,GAAG,KAAK,CAAC,MAAM,CAAC;IACvB,OAAO,GAAG,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAE,GAAG,IAAI,CAAC,KAAK,IAAI;QAAE,GAAG,IAAI,CAAC,CAAC;IAC9D,MAAM,IAAI,GAAG,GAAG,GAAG,CAAC,CAAC;IACrB,OAAO,IAAI,IAAI,CAAC,IAAI,kBAAkB,CAAC,KAAK,CAAC,IAAI,CAAE,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;AAC/G,CAAC;AAED,SAAS,eAAe,CAAC,KAAa;IACpC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,OAAO,KAAK,GAAG,KAAK,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAE,GAAG,IAAI,CAAC,KAAK,IAAI;QAAE,KAAK,IAAI,CAAC,CAAC;IAC3E,OAAO,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;AAC/B,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAY;IACtC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAClC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,KAAK,IAAI;QAAE,OAAO,CAAC,CAAC;IACrC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,KAAK,IAAI;QAAE,OAAO,CAAC,CAAC;IACrC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,KAAK,IAAI;QAAE,OAAO,CAAC,CAAC;IACrC,OAAO,CAAC,CAAC;AACX,CAAC;AAED,SAAS,WAAW,CAAC,OAAe,EAAE,IAAY,EAAE,KAAa;IAC/D,IAAI,IAAI,CAAC,MAAM,IAAI,KAAK;QAAE,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,CAAC;IACpE,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC;IAC9C,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;AACrF,CAAC;AAED,SAAS,eAAe,CAAC,MAA2B;IAClD,OAAO,CAAC,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC;SACzB,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,KAAK,GAAG,CAAC,CAAC;SAChC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;SACpD,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;AACrD,CAAC;AAED,SAAS,WAAW,CAAC,MAA2B,EAAE,MAA2B;IAC3E,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM;QAAE,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;AACrE,CAAC;AAED,SAAS,SAAS,CAAC,MAA2B,EAAE,IAAY,EAAE,KAAK,GAAG,CAAC;IACrE,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC;AACpD,CAAC;AAED,SAAS,MAAM,CAAC,KAAa;IAC3B,OAAO,UAAU,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;AACtE,CAAC;AAED,SAAS,cAAc,CAAC,MAAoB;IAC1C,IAAI,MAAM,EAAE,OAAO;QAAE,MAAM,MAAM,CAAC,MAAM,IAAI,IAAI,YAAY,CAAC,2BAA2B,EAAE,YAAY,CAAC,CAAC;AAC1G,CAAC;AAED,KAAK,UAAU,wBAAwB,CACrC,cAAsB,EACtB,YAAoB,EACpB,gBAAmC,EACnC,MAAoB;IAEpB,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,cAAc,CAAC,CAAC;IAC5C,cAAc,CAAC,MAAM,CAAC,CAAC;IACvB,MAAM,IAAI,GAAG,MAAM,YAAY,CAAC,IAAI,EAAE,mCAAmC,EAAE,IAAI,GAAG,IAAI,CAAC,CAAC;IACxF,IAAI,CAAC,IAAI;QAAE,OAAO;IAClB,IAAI,KAAa,CAAC;IAClB,IAAI,CAAC;QACH,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;QACrC,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;IAC/B,CAAC;YAAS,CAAC;QACT,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;IAC5B,CAAC;IACD,cAAc,CAAC,MAAM,CAAC,CAAC;IACvB,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAA4B,CAAC;IAC9E,IAAI,OAAO,EAAE,cAAc,KAAK,GAAG;WAC9B,OAAO,CAAC,IAAI,KAAK,uCAAuC;WACxD,OAAO,CAAC,eAAe,KAAK,KAAK;WACjC,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC;QAAE,OAAO;IAC9C,MAAM,QAAQ,GAAG,oBAAoB,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,gBAAgB,CAAC,CAAC,IAAI,CAAC;IACtF,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,UAAU,EAAE,mCAAmC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;AACxH,CAAC"}

package/dist/src/evals/verifier-eligibility.js

@@ -0,0 +1,25 @@
+import { lstat, readFile } from "node:fs/promises";
+import path from "node:path";
+export async function readCandidateIneligibleDiagnostic(trialDirectory) {
+    const source = path.join(trialDirectory, "verifier", "candidate-ineligible.json");
+    try {
+        const info = await lstat(source);
+        if (!info.isFile() || info.size <= 0 || info.size > 16 * 1024)
+            return null;
+        const value = JSON.parse(await readFile(source, "utf8"));
+        if (!value || typeof value !== "object" || Array.isArray(value))
+            return null;
+        const record = value;
+        if (Object.keys(record).some((key) => !["schema_version", "code", "run_id", "candidate_bundle", "reason_code", "verifier_executed"].includes(key))
+            || record.schema_version !== "1" || record.code !== "candidate_evidence_unavailable"
+            || typeof record.run_id !== "string" || !/^run_[a-f0-9]{32}$/.test(record.run_id)
+            || record.candidate_bundle !== "missing" && record.candidate_bundle !== "invalid"
+            || typeof record.reason_code !== "string" || !record.reason_code || record.verifier_executed !== false)
+            return null;
+        return record;
+    }
+    catch {
+        return null;
+    }
+}
+//# sourceMappingURL=verifier-eligibility.js.map

package/dist/src/evals/verifier-eligibility.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"verifier-eligibility.js","sourceRoot":"","sources":["../../../src/evals/verifier-eligibility.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,IAAI,MAAM,WAAW,CAAC;AAW7B,MAAM,CAAC,KAAK,UAAU,iCAAiC,CAAC,cAAsB;IAC5E,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,UAAU,EAAE,2BAA2B,CAAC,CAAC;IAClF,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,CAAC;QACjC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,GAAG,EAAE,GAAG,IAAI;YAAE,OAAO,IAAI,CAAC;QAC3E,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,CAAY,CAAC;QACpE,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QAC7E,MAAM,MAAM,GAAG,KAAgC,CAAC;QAChD,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,gBAAgB,EAAE,MAAM,EAAE,QAAQ,EAAE,kBAAkB,EAAE,aAAa,EAAE,mBAAmB,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;eAC7I,MAAM,CAAC,cAAc,KAAK,GAAG,IAAI,MAAM,CAAC,IAAI,KAAK,gCAAgC;eACjF,OAAO,MAAM,CAAC,MAAM,KAAK,QAAQ,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;eAC9E,MAAM,CAAC,gBAAgB,KAAK,SAAS,IAAI,MAAM,CAAC,gBAAgB,KAAK,SAAS;eAC9E,OAAO,MAAM,CAAC,WAAW,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,WAAW,IAAI,MAAM,CAAC,iBAAiB,KAAK,KAAK;YAAE,OAAO,IAAI,CAAC;QACtH,OAAO,MAAoD,CAAC;IAC9D,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC"}