@tangle-network/agent-eval 1.0.12 → 1.0.14

package/dist/{render-CFrJ7M0o.js → render-DIQ5nqp_.js}

@@ -1,12 +1,13 @@
 import { a as hashCanonical } from "./canonical-CicJlINA.js";
-import { c as unscreenedRewardFields } from "./reward-P0VD_Aft.js";
-import { c as summarizeNumberSeries } from "./descriptive-BTzLL-aj.js";
+import { l as unscreenedRewardFields, o as screenedRewardFields } from "./reward-DqMfaEax.js";
+import { c as summarizeNumberSeries } from "./descriptive-B3Eb7tX0.js";
 import { t as roundDecimal } from "./round-decimal-ZmjAQPo1.js";
 import { S as licenseClassOfName } from "./schema-YUIXT8iW.js";
-import { _ as showMeasured, a as readTerminalRecord, c as parseJsonRecord, d as recordOrNull, f as workerSourceKey, g as isUnavailable, h as SUPERVISOR_RUN_SCHEMA, l as parseJsonl, n as supervisorRunRolloutLinesFromFacts, o as asRecord, p as NO_SOURCE_LIMITS, s as nonEmptyString, u as parseSupervisorTree, v as unavailable, y as expandJournalEventGroup } from "./rollout-nodes-BHR9Q5jP.js";
+import { _ as showMeasured, a as readTerminalRecord, c as parseJsonRecord, d as recordOrNull, f as workerSourceKey, g as isUnavailable, h as SUPERVISOR_RUN_SCHEMA, l as parseJsonl, n as supervisorRunRolloutLinesFromFacts, o as asRecord, p as NO_SOURCE_LIMITS, s as nonEmptyString, u as parseSupervisorTree, v as unavailable, y as expandJournalEventGroup } from "./rollout-nodes-DKZ3vAYY.js";
 import { basename, join } from "node:path";
 import { execFile } from "node:child_process";
 import { readFile, readdir, stat } from "node:fs/promises";
+import { sandboxEventsSession, toChatMessages, toOtlpSpans } from "@tangle-network/harness-sessions";
 import { promisify } from "node:util";
 // src/supervisor-run/analyze-context.ts
 /** The parsed sources and shared lookups every metric group reads. `gap` records an unavailable measurement on the report. */
@@ -1378,16 +1379,19 @@ async function isFile(path) {
 *   - Producer: each node's `materialized` receipt names the model it ran on
 *     (`opencode/zai-coding-plan/glm-5.3` → harness, provider, model), and the
 *     root's `root-segment` names the root's. The license class follows.
-*   - Episode text: the node's task prompt (its `taskDigest` blob) and its
-*     final answer (its `outRef` blob), when the run kept both. The tool calls
-*     between them are not inlined, so the row stays a lossy projection.
+*   - Episode: each execution's input (`execution-input.taskRef`) and result
+*     (`execution-result.outRef`, a `{content, events}` blob), read as Sandbox
+*     sessions in order, so a node steered twice keeps both turns and every tool
+*     call. A node with no event stream stays a lossy projection.
 *   - A node respawned under the same id (a reopened run) is one row.
 *   - Reward: the run's newest known play-check verdict on the root row
 *     (`solved` → 1, a staged `SCORE` → stages passed / stages), and each
 *     agent's own commits on its `workers/<label>` branch of `deliverables.git`.
+*   - Screen: the calibration frozen in the run's receipt, where a fake fixture
+*     failed every stage of the check, and the goal audit, whose `contradiction`
+*     gates the reward. A check nothing calibrated stays unscreened.
 *
-* Every read is of files the run already wrote; nothing in the run dir is
-* changed. Rewards stay unscreened: no realness screen ran on a play-check.
+* Every read is of files the run already wrote; nothing in the run dir is changed.
 */
 const run = promisify(execFile);
 /** A run's pursuit: its directory name without the `-YYYYMMDD<letter>` date suffix. */
@@ -1413,6 +1417,14 @@ function producerOf(id) {
 		model: id
 	};
 }
+/** A row holding the agent's whole stream is no longer lossy; a workspace capture gap still stands. */
+function wholeEpisode({ lossy_projection: _, gap, ...kept }) {
+	const workspace = gap?.match(/workspace capture evidence \w+/u)?.[0];
+	return workspace === void 0 ? kept : {
+		...kept,
+		gap: workspace
+	};
+}
 const record = (v) => v !== null && typeof v === "object" && !Array.isArray(v) ? v : null;
 const text = (v) => typeof v === "string" && v.length > 0 ? v : void 0;
 async function journalFacts(runDir) {
@@ -1423,15 +1435,7 @@ async function journalFacts(runDir) {
 		facts.set(id, fact);
 		return fact;
 	};
-	for (const line of raw.split("\n")) {
-		if (!line.trim()) continue;
-		let row;
-		try {
-			row = record(JSON.parse(line));
-		} catch {
-			continue;
-		}
-		if (row === null) continue;
+	for (const row of parseJsonl(raw)) {
 		const expanded = expandJournalEventGroup(row);
 		if (!expanded.succeeded) continue;
 		for (const entry of expanded.value) {
@@ -1451,19 +1455,49 @@ async function journalFacts(runDir) {
 					model
 				].join("/");
 			} else if (event.kind === "settled" && text(event.outRef)) node(id).outRef = text(event.outRef);
+			else if (event.kind === "execution-input") node(id).inputRef = text(event.taskRef);
+			else if (event.kind === "execution-result" && text(event.outRef)) {
+				const fact = node(id);
+				const taskRef = fact.inputRef ?? fact.taskRef;
+				(fact.executions ??= []).push({
+					taskRef,
+					outRef: event.outRef
+				});
+			}
 		}
 	}
 	return facts;
 }
-async function blobText(runDir, ref) {
+const readJson = (path) => readFile(path, "utf8").then(JSON.parse).catch(() => void 0);
+async function blobValue(runDir, ref) {
 	if (ref === void 0 || !/^sha256:[0-9a-f]{64}$/u.test(ref)) return void 0;
-	const raw = await readFile(join(runDir, "blobs", `${ref.replace(":", "-")}.json`), "utf8").catch(() => void 0);
-	if (raw === void 0) return void 0;
-	try {
-		return text(JSON.parse(raw));
-	} catch {
-		return;
+	return readJson(join(runDir, "blobs", `${ref.replace(":", "-")}.json`));
+}
+const jsonlRows = async (path) => parseJsonl(await readFile(path, "utf8").catch(() => null));
+/** One Sandbox session per execution, oldest first; older runs keep only the settled result. */
+async function nodeSessions(runDir, harness, id, fact) {
+	const executions = fact.executions ?? [{
+		taskRef: fact.taskRef,
+		outRef: fact.outRef
+	}];
+	const sessions = [];
+	for (const [index, { taskRef, outRef }] of executions.entries()) {
+		const [input, result] = await Promise.all([blobValue(runDir, taskRef), blobValue(runDir, outRef)]);
+		const task = text(input) ?? text(record(input)?.prompt) ?? text(record(input)?.instruction);
+		const answer = text(result) ?? text(record(result)?.content);
+		const stream = record(result)?.events;
+		const events = Array.isArray(stream) ? stream : [];
+		if (task === void 0 || answer === void 0 && events.length === 0) continue;
+		const nativeSessionId = `${id}:${index}`;
+		sessions.push(sandboxEventsSession({
+			harness,
+			nativeSessionId,
+			task,
+			answer: answer ?? "",
+			events
+		}));
 	}
+	return sessions;
 }
 /**
 * The newest play-check row whose verdict is known. A staged check
@@ -1472,32 +1506,27 @@ async function blobText(runDir, ref) {
 * Only the SCORE line itself is parsed; the check prints more lines after it.
 */
 async function discoveryPlayCheckReward(runDir) {
-	const rows = [];
-	const lines = await readFile(join(runDir, "play-checks.jsonl"), "utf8").catch(() => "");
-	for (const line of lines.split("\n")) try {
-		const row = record(JSON.parse(line));
-		if (row) rows.push(row);
-	} catch {}
-	if (rows.length === 0) {
-		const single = await readFile(join(runDir, "play-check.json"), "utf8").catch(() => null);
-		try {
-			const row = single === null ? null : record(JSON.parse(single));
-			if (row) rows.push(row);
-		} catch {}
-	}
+	const rows = await jsonlRows(join(runDir, "play-checks.jsonl"));
+	const single = rows.length === 0 ? record(await readJson(join(runDir, "play-check.json"))) : null;
+	if (single) rows.push(single);
 	const last = rows.filter((row) => row.known !== false && typeof row.solved === "boolean").sort((a, b) => String(a.at ?? "").localeCompare(String(b.at ?? ""))).at(-1);
 	if (last === void 0) return null;
 	const detail = typeof last.detail === "string" ? last.detail : "";
 	let reward = last.solved === true ? 1 : 0;
 	const score = detail.split("\n").find((line) => line.startsWith("SCORE {"));
 	if (reward === 0 && score !== void 0) try {
-		const staged = record(JSON.parse(score.slice(6)));
-		const reached = staged?.reached;
-		const of = staged?.of;
+		const { reached, of } = JSON.parse(score.slice(6));
 		if (Number.isSafeInteger(reached) && Number.isSafeInteger(of) && 1 <= reached && reached <= of) reward = (reached - 1) / of;
 	} catch {}
+	const key = last.contentHash ?? last.commit ?? last.at;
+	const ruling = (await jsonlRows(join(runDir, "goal-audits.jsonl"))).filter((a) => a.key === key && a.verdict !== "unavailable").at(-1);
+	const receipt = record(await readJson(join(runDir, "RECEIPT.json")));
+	const calibration = record(record(receipt?.check)?.calibration);
+	const stages = Object.values(record(calibration?.stages) ?? {});
 	return {
 		reward,
+		calibrated: stages.length > 0 && stages.every((state) => state === "calibrated"),
+		contradicted: (ruling?.verdict ?? record(last.goalAudit)?.verdict) === "contradiction",
 		at: String(last.at ?? ""),
 		commit: typeof last.commit === "string" ? last.commit : null,
 		detail: detail.slice(0, 500)
@@ -1559,16 +1588,18 @@ async function discoveryRunRolloutLines(runDir, options = {}) {
 	const check = await discoveryPlayCheckReward(runDir);
 	const commits = await deliverableCommits(runDir);
 	const lines = [];
+	const otlp = /* @__PURE__ */ new Map();
 	for (const node of tree.nodes) {
 		const fact = facts.get(node.rollout_id) ?? {};
 		const producer = fact.model === void 0 ? null : producerOf(fact.model);
 		const license = licenseClassOfName(producer?.model) ?? licenseClassOfName(producer?.harness);
-		const [task, answer] = await Promise.all([blobText(runDir, fact.taskRef), blobText(runDir, fact.outRef)]);
+		const sessions = await nodeSessions(runDir, producer?.harness ?? node.policy.harness ?? "sandbox", node.rollout_id, fact);
+		const messages = sessions.flatMap((session) => toChatMessages(session));
 		const label = typeof node.outcome.metrics.label === "string" ? node.outcome.metrics.label : null;
 		const isRoot = node.parent_rollout_id === null;
 		const outcome = isRoot && check !== null ? {
 			...node.outcome,
-			...unscreenedRewardFields(check.reward),
+			...check.calibrated || check.contradicted ? screenedRewardFields(check.reward, check.contradicted) : unscreenedRewardFields(check.reward),
 			reward_source: "discovery/play-check",
 			verdict: {
 				at: check.at,
@@ -1576,6 +1607,8 @@ async function discoveryRunRolloutLines(runDir, options = {}) {
 				detail: check.detail
 			}
 		} : node.outcome;
+		const spans = sessions.flatMap((session) => toOtlpSpans(session));
+		otlp.set(node.rollout_id, spans.map((span) => `${JSON.stringify(span)}\n`).join(""));
 		lines.push({
 			...node,
 			policy: {
@@ -1587,13 +1620,8 @@ async function discoveryRunRolloutLines(runDir, options = {}) {
 				},
 				...license === null ? {} : { license_class: license }
 			},
-			messages: task !== void 0 && answer !== void 0 ? [{
-				role: "user",
-				content: task
-			}, {
-				role: "assistant",
-				content: answer
-			}] : node.messages,
+			messages: sessions.length === 0 ? node.messages : messages,
+			provenance: messages.some((m) => m.role === "tool") ? wholeEpisode(node.provenance) : node.provenance,
 			outcome: {
 				...outcome,
 				metrics: {
@@ -1609,7 +1637,8 @@ async function discoveryRunRolloutLines(runDir, options = {}) {
 		pursuit,
 		lines,
 		gaps: tree.gaps.filter((gap) => check === null || gap.code !== "root-reward-unavailable"),
-		notes
+		notes,
+		otlp
 	};
 }
 // src/supervisor-run/reader.ts
@@ -1866,4 +1895,4 @@ function renderSupervisorRollupMarkdown(rollup, title = "Round rollup") {
 }
 export { findSupervisorRunDirs as a, discoveryRunRolloutLines as c, analyzeSupervisorRunSources as d, parsePatch as f, analyzeSupervisorRun as i, isRuntimeSupervisorRunDir as l, renderSupervisorRunHeadline as n, discoveryPlayCheckReward as o, renderSupervisorRunMarkdown as r, discoveryPursuitOf as s, renderSupervisorRollupMarkdown as t, readRuntimeSupervisorRun as u };
 
-//# sourceMappingURL=render-CFrJ7M0o.js.map
+//# sourceMappingURL=render-DIQ5nqp_.js.map