@tangle-network/agent-eval 0.221.8 → 0.222.1

package/dist/{render-BhJqXoBH.js → render-pfAZ7IR4.js}

@@ -1,10 +1,14 @@
-import { a as hashCanonical } from "./canonical-Dq2hOQ8L.js";
-import { o as summarizeNumberSeries } from "./descriptive-CVNy48rR.js";
-import { t as roundDecimal } from "./round-decimal-COrKglMh.js";
-import { _ as expandJournalEventGroup, a as nonEmptyString, c as parseSupervisorTree, d as NO_SOURCE_LIMITS, g as unavailable, h as showMeasured, i as asRecord, l as recordOrNull, m as isUnavailable, o as parseJsonRecord, p as SUPERVISOR_RUN_SCHEMA, r as readTerminalRecord, s as parseJsonl, u as workerSourceKey } from "./terminal-record-CB9BFtjS.js";
-import { join } from "node:path";
+import { a as hashCanonical } from "./canonical-CicJlINA.js";
+import { c as unscreenedRewardFields } from "./reward-P0VD_Aft.js";
+import { o as summarizeNumberSeries } from "./descriptive-cPwITVfy.js";
+import { t as roundDecimal } from "./round-decimal-ZmjAQPo1.js";
+import { S as licenseClassOfName } from "./schema-YUIXT8iW.js";
+import { _ as showMeasured, a as readTerminalRecord, c as parseJsonRecord, d as recordOrNull, f as workerSourceKey, g as isUnavailable, h as SUPERVISOR_RUN_SCHEMA, l as parseJsonl, n as supervisorRunRolloutLinesFromFacts, o as asRecord, p as NO_SOURCE_LIMITS, s as nonEmptyString, u as parseSupervisorTree, v as unavailable, y as expandJournalEventGroup } from "./rollout-nodes-BHR9Q5jP.js";
+import { basename, join } from "node:path";
+import { execFile } from "node:child_process";
 import { readFile, readdir, stat } from "node:fs/promises";
-//#region src/supervisor-run/analyze-context.ts
+import { promisify } from "node:util";
+// src/supervisor-run/analyze-context.ts
 /** The parsed sources and shared lookups every metric group reads. `gap` records an unavailable measurement on the report. */
 function createAnalysis(src) {
 	const gaps = [];
@@ -64,8 +68,7 @@ function createAnalysis(src) {
 		workersGapReason: src.workersMissingReason ?? "workers/ directory absent"
 	};
 }
-//#endregion
-//#region src/supervisor-run/analyze-decision.ts
+// src/supervisor-run/analyze-decision.ts
 /** Settled verdicts, acceptance and review effort, with a named gap wherever the store never said. */
 function measureDecision(a, m) {
 	const { src, gap, journalMissing, haveJournal, tree, workerCloses, spawnForSource, closeById, workersGapReason } = a;
@@ -112,8 +115,7 @@ function measureDecision(a, m) {
 		workerEvidenceBytes: src.workers === null ? unavailable(workersGapReason) : workerEventsGapReason === null ? evidenceBytes : unavailable(workerEventsGapReason)
 	};
 }
-//#endregion
-//#region src/supervisor-run/analyze-economics.ts
+// src/supervisor-run/analyze-economics.ts
 const NO_CACHE_COUNTERS = "the journal carries no cache-token counters for this role";
 const NO_CACHE_BREAKDOWN = "Runtime recorded cacheBreakdownKnown:false — the provider reported a total without splitting cache reads from writes";
 /** One row per worker log, plus the capture gaps of the root and every worker. */
@@ -212,8 +214,7 @@ function buildEconomics(a, f, t, w, e) {
 		perWorker: src.workers === null ? unavailable(workersGapReason) : perWorker
 	};
 }
-//#endregion
-//#region src/supervisor-run/analyze-orchestration.ts
+// src/supervisor-run/analyze-orchestration.ts
 /** Supervisor wall time and where it ends. Explicit stamps win; the journal span is a lower bound; an inverted stamp pair stays unavailable. */
 function measureWall(a) {
 	const { gap, journalMissing, haveJournal, tree, startedAt, completedAt } = a;
@@ -432,8 +433,7 @@ function registrationMentions(driverLog) {
 	for (const line of driverLog.split("\n")) if (line.includes("registered tools:") && line.includes("supervisor_steer")) n += 1;
 	return n;
 }
-//#endregion
-//#region src/supervisor-run/patch-stats.ts
+// src/supervisor-run/patch-stats.ts
 /** Unified-diff stats. Counts `+++ b/<path>` targets, body +/- lines, and test-file touches. */
 function parsePatch(text) {
 	const files = /* @__PURE__ */ new Set();
@@ -464,8 +464,7 @@ function isTestPath(p) {
 	const base = p.split("/").pop() ?? p;
 	return /(^|\/)(tests?|__tests__|testing|spec)(\/|$)/.test(p) || /\.(test|spec)\.[cm]?[jt]sx?$/.test(base) || /^test_.*\.py$/.test(base) || /_test\.py$/.test(base);
 }
-//#endregion
-//#region src/supervisor-run/analyze-outcome.ts
+// src/supervisor-run/analyze-outcome.ts
 /** Terminal status, verdicts, judge result and the delivered patch. */
 function buildOutcome(a, t) {
 	const { src, gap, state, result, judge, terminal } = a;
@@ -492,8 +491,7 @@ function pickString(rec, key) {
 	const v = rec?.[key];
 	return typeof v === "string" ? v : null;
 }
-//#endregion
-//#region src/supervisor-run/analyze-spend.ts
+// src/supervisor-run/analyze-spend.ts
 /** Spend records of the root and its children, split into reporting and unreporting. */
 function collectSpendFacts(a) {
 	const { tree, rootId, workerSpawns, workerCloses } = a;
@@ -619,8 +617,7 @@ function buildSpend(a, f, t) {
 		}
 	};
 }
-//#endregion
-//#region src/supervisor-run/analyze.ts
+// src/supervisor-run/analyze.ts
 /**
 * The pure analyzer. Takes already-read bytes (`SupervisorRunSources`) and
 * returns the report — every metric derivable from a synthetic journal string
@@ -676,8 +673,7 @@ function analyzeSupervisorRunSources(src, now = Date.now) {
 		traceCommand: src.traceCommand ?? "npx --yes @tangle-network/traces@latest analyze --harness opencode --cwd <worker-clone-cwd>"
 	};
 }
-//#endregion
-//#region src/supervisor-run/coordination-reader.ts
+// src/supervisor-run/coordination-reader.ts
 /**
 * Optional Runtime run-dir files, read as bytes.
 *
@@ -697,8 +693,7 @@ async function readMaybe(path) {
 function readCoordinationLog(runDir) {
 	return readMaybe(join(runDir, COORDINATION_FILE));
 }
-//#endregion
-//#region src/supervisor-run/runtime-source-limits.ts
+// src/supervisor-run/runtime-source-limits.ts
 function spendRecord(value) {
 	const spend = recordOrNull(value);
 	if (spend === null) return null;
@@ -736,8 +731,7 @@ function sourceLimits(root, events, workerIds) {
 		deliverables: workerIds.size === 0 ? null : "Runtime FileRunContext does not retain per-child delivered patches"
 	};
 }
-//#endregion
-//#region src/supervisor-run/runtime-reader.ts
+// src/supervisor-run/runtime-reader.ts
 /**
 * Reader for agent-runtime's file-backed supervision context.
 *
@@ -1376,8 +1370,249 @@ async function isFile(path) {
 		throw error;
 	});
 }
-//#endregion
-//#region src/supervisor-run/reader.ts
+// src/supervisor-run/discovery-rollouts.ts
+/**
+* A Discovery run directory as `tangle.rollout.v1` rows: the supervision tree
+* `supervisorRunRolloutLines` mints, plus what only a Discovery run records.
+*
+*   - Producer: each node's `materialized` receipt names the model it ran on
+*     (`opencode/zai-coding-plan/glm-5.3` → harness, provider, model), and the
+*     root's `root-segment` names the root's. The license class follows.
+*   - Episode text: the node's task prompt (its `taskDigest` blob) and its
+*     final answer (its `outRef` blob), when the run kept both. The tool calls
+*     between them are not inlined, so the row stays a lossy projection.
+*   - A node respawned under the same id (a reopened run) is one row.
+*   - Reward: the run's newest known play-check verdict on the root row
+*     (`solved` → 1, a staged `SCORE` → stages passed / stages), and each
+*     agent's own commits on its `workers/<label>` branch of `deliverables.git`.
+*
+* Every read is of files the run already wrote; nothing in the run dir is
+* changed. Rewards stay unscreened: no realness screen ran on a play-check.
+*/
+const run = promisify(execFile);
+/** A run's pursuit: its directory name without the `-YYYYMMDD<letter>` date suffix. */
+function discoveryPursuitOf(runDir) {
+	return basename(runDir).replace(/-20\d{6}[a-z0-9]*$/u, "");
+}
+/** `opencode/zai-coding-plan/glm-5.3` → harness, provider, model. */
+function producerOf(id) {
+	const parts = id.split("/");
+	if (parts.length >= 3) return {
+		harness: parts[0],
+		provider: parts[1],
+		model: parts.slice(2).join("/")
+	};
+	if (parts.length === 2) return {
+		harness: null,
+		provider: parts[0],
+		model: parts[1]
+	};
+	return {
+		harness: null,
+		provider: null,
+		model: id
+	};
+}
+const record = (v) => v !== null && typeof v === "object" && !Array.isArray(v) ? v : null;
+const text = (v) => typeof v === "string" && v.length > 0 ? v : void 0;
+async function journalFacts(runDir) {
+	const facts = /* @__PURE__ */ new Map();
+	const raw = await readFile(join(runDir, "spawn-journal.jsonl"), "utf8").catch(() => "");
+	const node = (id) => {
+		const fact = facts.get(id) ?? {};
+		facts.set(id, fact);
+		return fact;
+	};
+	for (const line of raw.split("\n")) {
+		if (!line.trim()) continue;
+		let row;
+		try {
+			row = record(JSON.parse(line));
+		} catch {
+			continue;
+		}
+		if (row === null) continue;
+		const expanded = expandJournalEventGroup(row);
+		if (!expanded.succeeded) continue;
+		for (const entry of expanded.value) {
+			const event = record(entry.event);
+			const id = text(event?.id);
+			if (event === null || id === void 0) continue;
+			if (event.kind === "spawned") node(id).taskRef ??= text(record(event.identity)?.taskDigest);
+			else if (event.kind === "materialized") {
+				const model = record(record(event.receipt)?.model);
+				if (model?.status === "known" && text(model.id)) node(id).model = text(model.id);
+			} else if (event.kind === "root-segment") {
+				const to = record(event.to);
+				const model = text(to?.model);
+				if (model) node(id).model = [
+					text(to?.harness),
+					text(to?.provider),
+					model
+				].join("/");
+			} else if (event.kind === "settled" && text(event.outRef)) node(id).outRef = text(event.outRef);
+		}
+	}
+	return facts;
+}
+async function blobText(runDir, ref) {
+	if (ref === void 0 || !/^sha256:[0-9a-f]{64}$/u.test(ref)) return void 0;
+	const raw = await readFile(join(runDir, "blobs", `${ref.replace(":", "-")}.json`), "utf8").catch(() => void 0);
+	if (raw === void 0) return void 0;
+	try {
+		return text(JSON.parse(raw));
+	} catch {
+		return;
+	}
+}
+/**
+* The newest play-check row whose verdict is known. A staged check
+* (`SCORE {"reached": r, "of": n}`) that is not solved failed at stage r, so it
+* passed r − 1 of n: Discovery's own `stagesPassed` (runner/calibration.mjs).
+* Only the SCORE line itself is parsed; the check prints more lines after it.
+*/
+async function discoveryPlayCheckReward(runDir) {
+	const rows = [];
+	const lines = await readFile(join(runDir, "play-checks.jsonl"), "utf8").catch(() => "");
+	for (const line of lines.split("\n")) try {
+		const row = record(JSON.parse(line));
+		if (row) rows.push(row);
+	} catch {}
+	if (rows.length === 0) {
+		const single = await readFile(join(runDir, "play-check.json"), "utf8").catch(() => null);
+		try {
+			const row = single === null ? null : record(JSON.parse(single));
+			if (row) rows.push(row);
+		} catch {}
+	}
+	const last = rows.filter((row) => row.known !== false && typeof row.solved === "boolean").sort((a, b) => String(a.at ?? "").localeCompare(String(b.at ?? ""))).at(-1);
+	if (last === void 0) return null;
+	const detail = typeof last.detail === "string" ? last.detail : "";
+	let reward = last.solved === true ? 1 : 0;
+	const score = detail.split("\n").find((line) => line.startsWith("SCORE {"));
+	if (reward === 0 && score !== void 0) try {
+		const staged = record(JSON.parse(score.slice(6)));
+		const reached = staged?.reached;
+		const of = staged?.of;
+		if (Number.isSafeInteger(reached) && Number.isSafeInteger(of) && 1 <= reached && reached <= of) reward = (reached - 1) / of;
+	} catch {}
+	return {
+		reward,
+		at: String(last.at ?? ""),
+		commit: typeof last.commit === "string" ? last.commit : null,
+		detail: detail.slice(0, 500)
+	};
+}
+/** Commits each `workers/<label>` branch holds beyond the commit every worker started from. */
+async function deliverableCommits(runDir) {
+	const git = (args) => run("git", [
+		"-C",
+		join(runDir, "deliverables.git"),
+		...args
+	], { timeout: 3e4 });
+	let refs;
+	try {
+		const { stdout } = await git([
+			"for-each-ref",
+			"--format=%(refname:short)",
+			"refs/heads/workers"
+		]);
+		refs = stdout.split("\n").filter(Boolean);
+	} catch {
+		return null;
+	}
+	if (refs.length === 0) return /* @__PURE__ */ new Map();
+	const base = await git([
+		"merge-base",
+		"--octopus",
+		...refs
+	]).then(({ stdout }) => stdout.trim()).catch(() => "");
+	const counts = /* @__PURE__ */ new Map();
+	for (const ref of refs) {
+		const counted = await git([
+			"rev-list",
+			"--count",
+			ref,
+			...base ? [`^${base}`] : []
+		]).catch(() => null);
+		if (counted === null) return null;
+		counts.set(ref.slice(8), Number(counted.stdout.trim()));
+	}
+	return counts;
+}
+/** Read one Discovery run directory into rollout rows. */
+async function discoveryRunRolloutLines(runDir, options = {}) {
+	const pursuit = discoveryPursuitOf(runDir);
+	const src = await readRuntimeSupervisorRun(runDir);
+	const facts0 = parseSupervisorTree(src);
+	const seen = /* @__PURE__ */ new Set();
+	const workerSpawns = facts0.workerSpawns.filter((spawn) => !seen.has(spawn.id) && seen.add(spawn.id));
+	const tree = supervisorRunRolloutLinesFromFacts(src, {
+		...facts0,
+		workerSpawns
+	}, {
+		suite: `discovery/${pursuit}`,
+		runId: basename(runDir),
+		...options.capturedAt === void 0 ? {} : { capturedAt: options.capturedAt }
+	});
+	const facts = await journalFacts(runDir);
+	const check = await discoveryPlayCheckReward(runDir);
+	const commits = await deliverableCommits(runDir);
+	const lines = [];
+	for (const node of tree.nodes) {
+		const fact = facts.get(node.rollout_id) ?? {};
+		const producer = fact.model === void 0 ? null : producerOf(fact.model);
+		const license = licenseClassOfName(producer?.model) ?? licenseClassOfName(producer?.harness);
+		const [task, answer] = await Promise.all([blobText(runDir, fact.taskRef), blobText(runDir, fact.outRef)]);
+		const label = typeof node.outcome.metrics.label === "string" ? node.outcome.metrics.label : null;
+		const isRoot = node.parent_rollout_id === null;
+		const outcome = isRoot && check !== null ? {
+			...node.outcome,
+			...unscreenedRewardFields(check.reward),
+			reward_source: "discovery/play-check",
+			verdict: {
+				at: check.at,
+				commit: check.commit,
+				detail: check.detail
+			}
+		} : node.outcome;
+		lines.push({
+			...node,
+			policy: {
+				...node.policy,
+				...producer === null ? {} : {
+					harness: producer.harness ?? node.policy.harness,
+					provider: producer.provider,
+					model: producer.model
+				},
+				...license === null ? {} : { license_class: license }
+			},
+			messages: task !== void 0 && answer !== void 0 ? [{
+				role: "user",
+				content: task
+			}, {
+				role: "assistant",
+				content: answer
+			}] : node.messages,
+			outcome: {
+				...outcome,
+				metrics: {
+					...outcome.metrics,
+					deliverable_commits: commits === null ? null : commits.get(isRoot ? "root" : label ?? "") ?? 0
+				}
+			}
+		});
+	}
+	const notes = commits === null ? ["deliverables.git unreadable: deliverable_commits is null"] : [];
+	return {
+		runDir,
+		pursuit,
+		lines,
+		gaps: tree.gaps.filter((gap) => check === null || gap.code !== "root-reward-unavailable"),
+		notes
+	};
+}
+// src/supervisor-run/reader.ts
 /**
 * Runtime supervisor-run reader and report writers.
 *
@@ -1424,8 +1659,7 @@ async function findSupervisorRunDirs(root) {
 	await walk(root, 0);
 	return found.sort();
 }
-//#endregion
-//#region src/supervisor-run/render.ts
+// src/supervisor-run/render.ts
 /**
 * Human-readable renderings of a supervisor-run report. Zero and unavailable
 * render differently on purpose (`0` vs `unavailable — <reason>`), because the
@@ -1630,7 +1864,6 @@ function renderSupervisorRollupMarkdown(rollup, title = "Round rollup") {
 	for (const cell of rollup.perCell) if (cell.resourceRecords !== void 0) out.push(`## Resource receipts — ${cell.instanceId ?? "?"} [${cell.arm ?? "?"}]`, "", ...resourceRows(cell.resourceRecords));
 	return out.join("\n");
 }
-//#endregion
-export { findSupervisorRunDirs as a, analyzeSupervisorRunSources as c, analyzeSupervisorRun as i, parsePatch as l, renderSupervisorRunHeadline as n, isRuntimeSupervisorRunDir as o, renderSupervisorRunMarkdown as r, readRuntimeSupervisorRun as s, renderSupervisorRollupMarkdown as t };
+export { findSupervisorRunDirs as a, discoveryRunRolloutLines as c, analyzeSupervisorRunSources as d, parsePatch as f, analyzeSupervisorRun as i, isRuntimeSupervisorRunDir as l, renderSupervisorRunHeadline as n, discoveryPlayCheckReward as o, renderSupervisorRunMarkdown as r, discoveryPursuitOf as s, renderSupervisorRollupMarkdown as t, readRuntimeSupervisorRun as u };
 
-//# sourceMappingURL=render-BhJqXoBH.js.map
+//# sourceMappingURL=render-pfAZ7IR4.js.map