@tangle-network/agent-eval 1.0.13 → 1.0.15

package/dist/openapi.json

@@ -2,7 +2,7 @@
   "openapi": "3.1.0",
   "info": {
     "title": "@tangle-network/agent-eval — wire protocol",
-    "version": "1.0.13",
+    "version": "1.0.15",
     "description": "HTTP and stdio RPC interface to agent-eval. The TypeScript runtime is the source of truth; this spec is the contract that cross-language clients (Python, Rust, Go) generate from.\n\nWire-protocol version: 1.1.0. Bumps on breaking changes to request/response schemas.",
     "contact": {
       "name": "Tangle Network",

package/dist/{render-BVPfP7dc.js → render-CsHLoMUl.js}

@@ -2,8 +2,8 @@ import { a as hashCanonical } from "./canonical-CicJlINA.js";
 import { l as unscreenedRewardFields, o as screenedRewardFields } from "./reward-DqMfaEax.js";
 import { c as summarizeNumberSeries } from "./descriptive-B3Eb7tX0.js";
 import { t as roundDecimal } from "./round-decimal-ZmjAQPo1.js";
-import { S as licenseClassOfName } from "./schema-YUIXT8iW.js";
-import { _ as showMeasured, a as readTerminalRecord, c as parseJsonRecord, d as recordOrNull, f as workerSourceKey, g as isUnavailable, h as SUPERVISOR_RUN_SCHEMA, l as parseJsonl, n as supervisorRunRolloutLinesFromFacts, o as asRecord, p as NO_SOURCE_LIMITS, s as nonEmptyString, u as parseSupervisorTree, v as unavailable, y as expandJournalEventGroup } from "./rollout-nodes-DKZ3vAYY.js";
+import { S as licenseClassOfName } from "./schema-ChIZfnqY.js";
+import { _ as showMeasured, a as readTerminalRecord, c as parseJsonRecord, d as recordOrNull, f as workerSourceKey, g as isUnavailable, h as SUPERVISOR_RUN_SCHEMA, l as parseJsonl, n as supervisorRunRolloutLinesFromFacts, o as asRecord, p as NO_SOURCE_LIMITS, s as nonEmptyString, u as parseSupervisorTree, v as unavailable, y as expandJournalEventGroup } from "./rollout-nodes-DhSPJ2X7.js";
 import { basename, join } from "node:path";
 import { execFile } from "node:child_process";
 import { readFile, readdir, stat } from "node:fs/promises";
@@ -1379,9 +1379,10 @@ async function isFile(path) {
 *   - Producer: each node's `materialized` receipt names the model it ran on
 *     (`opencode/zai-coding-plan/glm-5.3` → harness, provider, model), and the
 *     root's `root-segment` names the root's. The license class follows.
-*   - Episode: the node's task prompt (its `taskDigest` blob) and its settled
-*     result (its `outRef` blob, `{content, events}`), read as a Sandbox session
-*     with every tool call. A result without events stays a lossy projection.
+*   - Episode: each execution's input (`execution-input.taskRef`) and result
+*     (`execution-result.outRef`, a `{content, events}` blob), read as Sandbox
+*     sessions in order, so a node steered twice keeps both turns and every tool
+*     call. A node with no event stream stays a lossy projection.
 *   - A node respawned under the same id (a reopened run) is one row.
 *   - Reward: the run's newest known play-check verdict on the root row
 *     (`solved` → 1, a staged `SCORE` → stages passed / stages), and each
@@ -1424,7 +1425,6 @@ function wholeEpisode({ lossy_projection: _, gap, ...kept }) {
 		gap: workspace
 	};
 }
-const record = (v) => v !== null && typeof v === "object" && !Array.isArray(v) ? v : null;
 const text = (v) => typeof v === "string" && v.length > 0 ? v : void 0;
 async function journalFacts(runDir) {
 	const facts = /* @__PURE__ */ new Map();
@@ -1438,15 +1438,15 @@ async function journalFacts(runDir) {
 		const expanded = expandJournalEventGroup(row);
 		if (!expanded.succeeded) continue;
 		for (const entry of expanded.value) {
-			const event = record(entry.event);
+			const event = recordOrNull(entry.event);
 			const id = text(event?.id);
 			if (event === null || id === void 0) continue;
-			if (event.kind === "spawned") node(id).taskRef ??= text(record(event.identity)?.taskDigest);
+			if (event.kind === "spawned") node(id).taskRef ??= text(recordOrNull(event.identity)?.taskDigest);
 			else if (event.kind === "materialized") {
-				const model = record(record(event.receipt)?.model);
+				const model = recordOrNull(recordOrNull(event.receipt)?.model);
 				if (model?.status === "known" && text(model.id)) node(id).model = text(model.id);
 			} else if (event.kind === "root-segment") {
-				const to = record(event.to);
+				const to = recordOrNull(event.to);
 				const model = text(to?.model);
 				if (model) node(id).model = [
 					text(to?.harness),
@@ -1454,21 +1454,54 @@ async function journalFacts(runDir) {
 					model
 				].join("/");
 			} else if (event.kind === "settled" && text(event.outRef)) node(id).outRef = text(event.outRef);
+			else if (event.kind === "execution-input") {
+				const fact = node(id);
+				fact.inputRef = text(event.taskRef);
+				fact.unansweredSince ??= text(event.at);
+			} else if (event.kind === "execution-result" && text(event.outRef)) {
+				const fact = node(id);
+				fact.unansweredSince = void 0;
+				const taskRef = fact.inputRef ?? fact.taskRef;
+				(fact.executions ??= []).push({
+					taskRef,
+					outRef: event.outRef
+				});
+			}
 		}
 	}
 	return facts;
 }
+const readJson = (path) => readFile(path, "utf8").then(JSON.parse).catch(() => void 0);
 async function blobValue(runDir, ref) {
 	if (ref === void 0 || !/^sha256:[0-9a-f]{64}$/u.test(ref)) return void 0;
-	const raw = await readFile(join(runDir, "blobs", `${ref.replace(":", "-")}.json`), "utf8").catch(() => void 0);
-	if (raw === void 0) return void 0;
-	try {
-		return JSON.parse(raw);
-	} catch {
-		return;
-	}
+	return readJson(join(runDir, "blobs", `${ref.replace(":", "-")}.json`));
 }
 const jsonlRows = async (path) => parseJsonl(await readFile(path, "utf8").catch(() => null));
+/** One Sandbox session per execution, oldest first; older runs keep only the settled result. */
+async function nodeSessions(runDir, harness, id, fact) {
+	const executions = fact.executions ?? [{
+		taskRef: fact.taskRef,
+		outRef: fact.outRef
+	}];
+	const sessions = [];
+	for (const [index, { taskRef, outRef }] of executions.entries()) {
+		const [input, result] = await Promise.all([blobValue(runDir, taskRef), blobValue(runDir, outRef)]);
+		const task = text(input) ?? text(recordOrNull(input)?.prompt) ?? text(recordOrNull(input)?.instruction);
+		const answer = text(result) ?? text(recordOrNull(result)?.content);
+		const stream = recordOrNull(result)?.events;
+		const events = Array.isArray(stream) ? stream : [];
+		if (task === void 0 || answer === void 0 && events.length === 0) continue;
+		const nativeSessionId = `${id}:${index}`;
+		sessions.push(sandboxEventsSession({
+			harness,
+			nativeSessionId,
+			task,
+			answer: answer ?? "",
+			events
+		}));
+	}
+	return sessions;
+}
 /**
 * The newest play-check row whose verdict is known. A staged check
 * (`SCORE {"reached": r, "of": n}`) that is not solved failed at stage r, so it
@@ -1477,38 +1510,34 @@ const jsonlRows = async (path) => parseJsonl(await readFile(path, "utf8").catch(
 */
 async function discoveryPlayCheckReward(runDir) {
 	const rows = await jsonlRows(join(runDir, "play-checks.jsonl"));
-	if (rows.length === 0) {
-		const single = await readFile(join(runDir, "play-check.json"), "utf8").catch(() => null);
-		try {
-			const row = single === null ? null : record(JSON.parse(single));
-			if (row) rows.push(row);
-		} catch {}
-	}
+	const single = rows.length === 0 ? recordOrNull(await readJson(join(runDir, "play-check.json"))) : null;
+	if (single) rows.push(single);
 	const last = rows.filter((row) => row.known !== false && typeof row.solved === "boolean").sort((a, b) => String(a.at ?? "").localeCompare(String(b.at ?? ""))).at(-1);
 	if (last === void 0) return null;
 	const detail = typeof last.detail === "string" ? last.detail : "";
 	let reward = last.solved === true ? 1 : 0;
 	const score = detail.split("\n").find((line) => line.startsWith("SCORE {"));
 	if (reward === 0 && score !== void 0) try {
-		const staged = record(JSON.parse(score.slice(6)));
-		const reached = staged?.reached;
-		const of = staged?.of;
+		const { reached, of } = JSON.parse(score.slice(6));
 		if (Number.isSafeInteger(reached) && Number.isSafeInteger(of) && 1 <= reached && reached <= of) reward = (reached - 1) / of;
 	} catch {}
 	const key = last.contentHash ?? last.commit ?? last.at;
 	const ruling = (await jsonlRows(join(runDir, "goal-audits.jsonl"))).filter((a) => a.key === key && a.verdict !== "unavailable").at(-1);
-	const receipt = await readFile(join(runDir, "RECEIPT.json"), "utf8").then(JSON.parse, () => null);
-	const calibration = record(record(record(receipt)?.check)?.calibration);
-	const stages = Object.values(record(calibration?.stages) ?? {});
+	const receipt = recordOrNull(await readJson(join(runDir, "RECEIPT.json")));
+	const calibration = recordOrNull(recordOrNull(receipt?.check)?.calibration);
+	const stages = Object.values(recordOrNull(calibration?.stages) ?? {});
 	return {
 		reward,
 		calibrated: stages.length > 0 && stages.every((state) => state === "calibrated"),
-		contradicted: (ruling?.verdict ?? record(last.goalAudit)?.verdict) === "contradiction",
+		contradicted: (ruling?.verdict ?? recordOrNull(last.goalAudit)?.verdict) === "contradiction",
 		at: String(last.at ?? ""),
+		taggedAt: text(receipt?.taggedAt),
 		commit: typeof last.commit === "string" ? last.commit : null,
 		detail: detail.slice(0, 500)
 	};
 }
+/** A solved run is stopped, so it reads `cancelled`; it is whole if no turn was open at the tag. */
+const holdsTheSolve = (check, unansweredSince) => check.reward === 1 && !check.contradicted && (!unansweredSince || Date.parse(check.taggedAt ?? "") <= Date.parse(unansweredSince));
 /** Commits each `workers/<label>` branch holds beyond the commit every worker started from. */
 async function deliverableCommits(runDir) {
 	const git = (args) => run("git", [
@@ -1570,23 +1599,17 @@ async function discoveryRunRolloutLines(runDir, options = {}) {
 		const fact = facts.get(node.rollout_id) ?? {};
 		const producer = fact.model === void 0 ? null : producerOf(fact.model);
 		const license = licenseClassOfName(producer?.model) ?? licenseClassOfName(producer?.harness);
-		const [taskBlob, result] = await Promise.all([blobValue(runDir, fact.taskRef), blobValue(runDir, fact.outRef)]);
-		const task = text(taskBlob);
-		const answer = text(result) ?? text(record(result)?.content);
-		const events = record(result)?.events;
-		const stream = Array.isArray(events) ? events : [];
-		const session = task === void 0 || answer === void 0 ? null : sandboxEventsSession({
-			harness: producer?.harness ?? node.policy.harness ?? "sandbox",
-			nativeSessionId: node.rollout_id,
-			task,
-			answer,
-			events: stream
-		});
+		const sessions = await nodeSessions(runDir, producer?.harness ?? node.policy.harness ?? "sandbox", node.rollout_id, fact);
+		const messages = sessions.flatMap((session) => toChatMessages(session));
 		const label = typeof node.outcome.metrics.label === "string" ? node.outcome.metrics.label : null;
 		const isRoot = node.parent_rollout_id === null;
 		const outcome = isRoot && check !== null ? {
 			...node.outcome,
 			...check.calibrated || check.contradicted ? screenedRewardFields(check.reward, check.contradicted) : unscreenedRewardFields(check.reward),
+			...holdsTheSolve(check, fact.unansweredSince) ? {
+				is_completed: true,
+				error: null
+			} : {},
 			reward_source: "discovery/play-check",
 			verdict: {
 				at: check.at,
@@ -1594,7 +1617,7 @@ async function discoveryRunRolloutLines(runDir, options = {}) {
 				detail: check.detail
 			}
 		} : node.outcome;
-		const spans = session === null ? [] : toOtlpSpans(session);
+		const spans = sessions.map((session) => toOtlpSpans(session)).filter((s) => s.length > 1).flat();
 		otlp.set(node.rollout_id, spans.map((span) => `${JSON.stringify(span)}\n`).join(""));
 		lines.push({
 			...node,
@@ -1607,8 +1630,8 @@ async function discoveryRunRolloutLines(runDir, options = {}) {
 				},
 				...license === null ? {} : { license_class: license }
 			},
-			messages: session === null ? node.messages : toChatMessages(session),
-			provenance: stream.length === 0 ? node.provenance : wholeEpisode(node.provenance),
+			messages: sessions.length === 0 ? node.messages : messages,
+			provenance: messages.some((m) => m.role === "tool") ? wholeEpisode(node.provenance) : node.provenance,
 			outcome: {
 				...outcome,
 				metrics: {
@@ -1882,4 +1905,4 @@ function renderSupervisorRollupMarkdown(rollup, title = "Round rollup") {
 }
 export { findSupervisorRunDirs as a, discoveryRunRolloutLines as c, analyzeSupervisorRunSources as d, parsePatch as f, analyzeSupervisorRun as i, isRuntimeSupervisorRunDir as l, renderSupervisorRunHeadline as n, discoveryPlayCheckReward as o, renderSupervisorRunMarkdown as r, discoveryPursuitOf as s, renderSupervisorRollupMarkdown as t, readRuntimeSupervisorRun as u };
 
-//# sourceMappingURL=render-BVPfP7dc.js.map
+//# sourceMappingURL=render-CsHLoMUl.js.map