@tangle-network/agent-eval 1.0.12 → 1.0.13

package/dist/{render-CFrJ7M0o.js → render-BVPfP7dc.js}

@@ -1,12 +1,13 @@
 import { a as hashCanonical } from "./canonical-CicJlINA.js";
-import { c as unscreenedRewardFields } from "./reward-P0VD_Aft.js";
-import { c as summarizeNumberSeries } from "./descriptive-BTzLL-aj.js";
+import { l as unscreenedRewardFields, o as screenedRewardFields } from "./reward-DqMfaEax.js";
+import { c as summarizeNumberSeries } from "./descriptive-B3Eb7tX0.js";
 import { t as roundDecimal } from "./round-decimal-ZmjAQPo1.js";
 import { S as licenseClassOfName } from "./schema-YUIXT8iW.js";
-import { _ as showMeasured, a as readTerminalRecord, c as parseJsonRecord, d as recordOrNull, f as workerSourceKey, g as isUnavailable, h as SUPERVISOR_RUN_SCHEMA, l as parseJsonl, n as supervisorRunRolloutLinesFromFacts, o as asRecord, p as NO_SOURCE_LIMITS, s as nonEmptyString, u as parseSupervisorTree, v as unavailable, y as expandJournalEventGroup } from "./rollout-nodes-BHR9Q5jP.js";
+import { _ as showMeasured, a as readTerminalRecord, c as parseJsonRecord, d as recordOrNull, f as workerSourceKey, g as isUnavailable, h as SUPERVISOR_RUN_SCHEMA, l as parseJsonl, n as supervisorRunRolloutLinesFromFacts, o as asRecord, p as NO_SOURCE_LIMITS, s as nonEmptyString, u as parseSupervisorTree, v as unavailable, y as expandJournalEventGroup } from "./rollout-nodes-DKZ3vAYY.js";
 import { basename, join } from "node:path";
 import { execFile } from "node:child_process";
 import { readFile, readdir, stat } from "node:fs/promises";
+import { sandboxEventsSession, toChatMessages, toOtlpSpans } from "@tangle-network/harness-sessions";
 import { promisify } from "node:util";
 // src/supervisor-run/analyze-context.ts
 /** The parsed sources and shared lookups every metric group reads. `gap` records an unavailable measurement on the report. */
@@ -1378,16 +1379,18 @@ async function isFile(path) {
 *   - Producer: each node's `materialized` receipt names the model it ran on
 *     (`opencode/zai-coding-plan/glm-5.3` → harness, provider, model), and the
 *     root's `root-segment` names the root's. The license class follows.
-*   - Episode text: the node's task prompt (its `taskDigest` blob) and its
-*     final answer (its `outRef` blob), when the run kept both. The tool calls
-*     between them are not inlined, so the row stays a lossy projection.
+*   - Episode: the node's task prompt (its `taskDigest` blob) and its settled
+*     result (its `outRef` blob, `{content, events}`), read as a Sandbox session
+*     with every tool call. A result without events stays a lossy projection.
 *   - A node respawned under the same id (a reopened run) is one row.
 *   - Reward: the run's newest known play-check verdict on the root row
 *     (`solved` → 1, a staged `SCORE` → stages passed / stages), and each
 *     agent's own commits on its `workers/<label>` branch of `deliverables.git`.
+*   - Screen: the calibration frozen in the run's receipt, where a fake fixture
+*     failed every stage of the check, and the goal audit, whose `contradiction`
+*     gates the reward. A check nothing calibrated stays unscreened.
 *
-* Every read is of files the run already wrote; nothing in the run dir is
-* changed. Rewards stay unscreened: no realness screen ran on a play-check.
+* Every read is of files the run already wrote; nothing in the run dir is changed.
 */
 const run = promisify(execFile);
 /** A run's pursuit: its directory name without the `-YYYYMMDD<letter>` date suffix. */
@@ -1413,6 +1416,14 @@ function producerOf(id) {
 		model: id
 	};
 }
+/** A row holding the agent's whole stream is no longer lossy; a workspace capture gap still stands. */
+function wholeEpisode({ lossy_projection: _, gap, ...kept }) {
+	const workspace = gap?.match(/workspace capture evidence \w+/u)?.[0];
+	return workspace === void 0 ? kept : {
+		...kept,
+		gap: workspace
+	};
+}
 const record = (v) => v !== null && typeof v === "object" && !Array.isArray(v) ? v : null;
 const text = (v) => typeof v === "string" && v.length > 0 ? v : void 0;
 async function journalFacts(runDir) {
@@ -1423,15 +1434,7 @@ async function journalFacts(runDir) {
 		facts.set(id, fact);
 		return fact;
 	};
-	for (const line of raw.split("\n")) {
-		if (!line.trim()) continue;
-		let row;
-		try {
-			row = record(JSON.parse(line));
-		} catch {
-			continue;
-		}
-		if (row === null) continue;
+	for (const row of parseJsonl(raw)) {
 		const expanded = expandJournalEventGroup(row);
 		if (!expanded.succeeded) continue;
 		for (const entry of expanded.value) {
@@ -1455,16 +1458,17 @@ async function journalFacts(runDir) {
 	}
 	return facts;
 }
-async function blobText(runDir, ref) {
+async function blobValue(runDir, ref) {
 	if (ref === void 0 || !/^sha256:[0-9a-f]{64}$/u.test(ref)) return void 0;
 	const raw = await readFile(join(runDir, "blobs", `${ref.replace(":", "-")}.json`), "utf8").catch(() => void 0);
 	if (raw === void 0) return void 0;
 	try {
-		return text(JSON.parse(raw));
+		return JSON.parse(raw);
 	} catch {
 		return;
 	}
 }
+const jsonlRows = async (path) => parseJsonl(await readFile(path, "utf8").catch(() => null));
 /**
 * The newest play-check row whose verdict is known. A staged check
 * (`SCORE {"reached": r, "of": n}`) that is not solved failed at stage r, so it
@@ -1472,12 +1476,7 @@ async function blobText(runDir, ref) {
 * Only the SCORE line itself is parsed; the check prints more lines after it.
 */
 async function discoveryPlayCheckReward(runDir) {
-	const rows = [];
-	const lines = await readFile(join(runDir, "play-checks.jsonl"), "utf8").catch(() => "");
-	for (const line of lines.split("\n")) try {
-		const row = record(JSON.parse(line));
-		if (row) rows.push(row);
-	} catch {}
+	const rows = await jsonlRows(join(runDir, "play-checks.jsonl"));
 	if (rows.length === 0) {
 		const single = await readFile(join(runDir, "play-check.json"), "utf8").catch(() => null);
 		try {
@@ -1496,8 +1495,15 @@ async function discoveryPlayCheckReward(runDir) {
 		const of = staged?.of;
 		if (Number.isSafeInteger(reached) && Number.isSafeInteger(of) && 1 <= reached && reached <= of) reward = (reached - 1) / of;
 	} catch {}
+	const key = last.contentHash ?? last.commit ?? last.at;
+	const ruling = (await jsonlRows(join(runDir, "goal-audits.jsonl"))).filter((a) => a.key === key && a.verdict !== "unavailable").at(-1);
+	const receipt = await readFile(join(runDir, "RECEIPT.json"), "utf8").then(JSON.parse, () => null);
+	const calibration = record(record(record(receipt)?.check)?.calibration);
+	const stages = Object.values(record(calibration?.stages) ?? {});
 	return {
 		reward,
+		calibrated: stages.length > 0 && stages.every((state) => state === "calibrated"),
+		contradicted: (ruling?.verdict ?? record(last.goalAudit)?.verdict) === "contradiction",
 		at: String(last.at ?? ""),
 		commit: typeof last.commit === "string" ? last.commit : null,
 		detail: detail.slice(0, 500)
@@ -1559,16 +1565,28 @@ async function discoveryRunRolloutLines(runDir, options = {}) {
 	const check = await discoveryPlayCheckReward(runDir);
 	const commits = await deliverableCommits(runDir);
 	const lines = [];
+	const otlp = /* @__PURE__ */ new Map();
 	for (const node of tree.nodes) {
 		const fact = facts.get(node.rollout_id) ?? {};
 		const producer = fact.model === void 0 ? null : producerOf(fact.model);
 		const license = licenseClassOfName(producer?.model) ?? licenseClassOfName(producer?.harness);
-		const [task, answer] = await Promise.all([blobText(runDir, fact.taskRef), blobText(runDir, fact.outRef)]);
+		const [taskBlob, result] = await Promise.all([blobValue(runDir, fact.taskRef), blobValue(runDir, fact.outRef)]);
+		const task = text(taskBlob);
+		const answer = text(result) ?? text(record(result)?.content);
+		const events = record(result)?.events;
+		const stream = Array.isArray(events) ? events : [];
+		const session = task === void 0 || answer === void 0 ? null : sandboxEventsSession({
+			harness: producer?.harness ?? node.policy.harness ?? "sandbox",
+			nativeSessionId: node.rollout_id,
+			task,
+			answer,
+			events: stream
+		});
 		const label = typeof node.outcome.metrics.label === "string" ? node.outcome.metrics.label : null;
 		const isRoot = node.parent_rollout_id === null;
 		const outcome = isRoot && check !== null ? {
 			...node.outcome,
-			...unscreenedRewardFields(check.reward),
+			...check.calibrated || check.contradicted ? screenedRewardFields(check.reward, check.contradicted) : unscreenedRewardFields(check.reward),
 			reward_source: "discovery/play-check",
 			verdict: {
 				at: check.at,
@@ -1576,6 +1594,8 @@ async function discoveryRunRolloutLines(runDir, options = {}) {
 				detail: check.detail
 			}
 		} : node.outcome;
+		const spans = session === null ? [] : toOtlpSpans(session);
+		otlp.set(node.rollout_id, spans.map((span) => `${JSON.stringify(span)}\n`).join(""));
 		lines.push({
 			...node,
 			policy: {
@@ -1587,13 +1607,8 @@ async function discoveryRunRolloutLines(runDir, options = {}) {
 				},
 				...license === null ? {} : { license_class: license }
 			},
-			messages: task !== void 0 && answer !== void 0 ? [{
-				role: "user",
-				content: task
-			}, {
-				role: "assistant",
-				content: answer
-			}] : node.messages,
+			messages: session === null ? node.messages : toChatMessages(session),
+			provenance: stream.length === 0 ? node.provenance : wholeEpisode(node.provenance),
 			outcome: {
 				...outcome,
 				metrics: {
@@ -1609,7 +1624,8 @@ async function discoveryRunRolloutLines(runDir, options = {}) {
 		pursuit,
 		lines,
 		gaps: tree.gaps.filter((gap) => check === null || gap.code !== "root-reward-unavailable"),
-		notes
+		notes,
+		otlp
 	};
 }
 // src/supervisor-run/reader.ts
@@ -1866,4 +1882,4 @@ function renderSupervisorRollupMarkdown(rollup, title = "Round rollup") {
 }
 export { findSupervisorRunDirs as a, discoveryRunRolloutLines as c, analyzeSupervisorRunSources as d, parsePatch as f, analyzeSupervisorRun as i, isRuntimeSupervisorRunDir as l, renderSupervisorRunHeadline as n, discoveryPlayCheckReward as o, renderSupervisorRunMarkdown as r, discoveryPursuitOf as s, renderSupervisorRollupMarkdown as t, readRuntimeSupervisorRun as u };
 
-//# sourceMappingURL=render-CFrJ7M0o.js.map
+//# sourceMappingURL=render-BVPfP7dc.js.map