@tangle-network/agent-eval 0.218.1 → 0.219.0

package/dist/trajectory-replay/index.js

@@ -1025,9 +1025,9 @@ async function runFixPhase(run, selected, rows, verdicts) {
 function pct(value) {
 	return value === null ? "—" : `${(value * 100).toFixed(1)}%`;
 }
-function renderBatchReport(report) {
-	const lines = [];
-	const { headline, totals } = report;
+/** The title and the headline rates, prefix divergence and fix-flip numbers. */
+function pushHeadline(report, lines) {
+	const { headline } = report;
 	lines.push("# Replay-verify batch report");
 	lines.push("");
 	lines.push(`Generated ${report.generatedAt}.`);
@@ -1045,6 +1045,10 @@ function renderBatchReport(report) {
 		const parts = Object.entries(headline.flipsByAttempt).sort((a, b) => Number(a[0]) - Number(b[0])).map(([attempt, count]) => `attempt ${attempt}: ${count}`);
 		lines.push(`- Flips by attempt: ${parts.join(", ")}.`);
 	}
+}
+/** How many label entries were replayable, executed and excluded, and why. */
+function pushEnumeration(report, lines) {
+	const { totals } = report;
 	lines.push("");
 	lines.push("## Enumeration");
 	lines.push("");
@@ -1062,6 +1066,9 @@ function renderBatchReport(report) {
 		lines.push("| --- | --- | --- |");
 		for (const [corpus, stats] of submitEntries.sort((a, b) => a[0].localeCompare(b[0]))) lines.push(`| ${corpus} | ${stats.submitOnlyCases} | ${stats.goldsSkippedWithinReplayable} |`);
 	}
+}
+/** The images that could not be pulled or built. */
+function pushPullFailures(report, lines) {
 	if (report.pullFailures.length > 0) {
 		lines.push("");
 		lines.push("## Image pull/build failures");
@@ -1070,6 +1077,9 @@ function renderBatchReport(report) {
 		lines.push("| --- | --- | --- | --- |");
 		for (const failure of report.pullFailures) lines.push(`| ${failure.corpus} | ${failure.trajId} | \`${failure.image}\` | ${failure.error.replaceAll("|", "\\|")} |`);
 	}
+}
+/** One row per replayed case. */
+function pushCaseResults(report, lines) {
 	lines.push("");
 	lines.push("## Per-case results");
 	lines.push("");
@@ -1100,12 +1110,23 @@ function renderBatchReport(report) {
 			(row.wallMs / 1e3).toFixed(1)
 		].join(" | "));
 	}
+}
+/** The model, calls and tokens spent generating fixes. */
+function pushLlmUsage(report, lines) {
 	if (report.llm) {
 		lines.push("");
 		lines.push("## LLM fix generation");
 		lines.push("");
 		lines.push(`Model ${report.llm.model}: ${report.llm.calls} calls (${report.llm.failures} failed), ${report.llm.promptTokens} prompt + ${report.llm.completionTokens} completion tokens.`);
 	}
+}
+function renderBatchReport(report) {
+	const lines = [];
+	pushHeadline(report, lines);
+	pushEnumeration(report, lines);
+	pushPullFailures(report, lines);
+	pushCaseResults(report, lines);
+	pushLlmUsage(report, lines);
 	lines.push("");
 	return lines.join("\n");
 }