@ls-stack/agent-eval 0.58.1 → 0.58.2

package/dist/{app-DhMIbjlE.mjs → app-BxD6aHbp.mjs}

@@ -1,7 +1,7 @@
-import { et as createRunRequestSchema, nt as extractCacheEntries, tt as updateManualScoreRequestSchema, ut as getEvalTitle } from "./runExecution-d42Lm0i5.mjs";
-import { o as stageManualInputFile } from "./cli-_g2qOMK6.mjs";
-import "./src-CdZsOn6y.mjs";
-import { t as getRunnerInstance } from "./runner-MSr8sAWm.mjs";
+import { Et as getCaseRowCaseKey, Ot as caseRowSchema, dt as getEvalTitle, nt as updateManualScoreRequestSchema, rt as extractCacheEntries, tt as createRunRequestSchema } from "./runExecution-pHJ0_TzH.mjs";
+import { o as stageManualInputFile } from "./cli-HBwXIJsg.mjs";
+import "./src-AeXGBJ26.mjs";
+import { t as getRunnerInstance } from "./runner-D_pz2NON.mjs";
 import { z } from "zod/v4";
 import { readFile } from "node:fs/promises";
 import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
@@ -231,9 +231,24 @@ function logStartedAppRunEvals(params) {
 	const targetEvals = getRunTargetEvalSummaries(params.evals, params.target);
 	if (targetEvals.length === 0) return;
 	const label = targetEvals.length === 1 ? "eval" : "evals";
-	console.info(`[agent-evals] Starting app run ${params.shortId} (${params.runId}) with ${String(targetEvals.length)} ${label}:`);
+	console.info(`[agent-evals] Queued app run ${params.shortId} (${params.runId}) with ${String(targetEvals.length)} ${label}; concurrency ${String(params.concurrency)}:`);
 	for (const ev of targetEvals) console.info(`  - ${getEvalTitle(ev)} (${ev.filePath}#${ev.id})`);
 }
+function getEvalSummaryLabel(evalsByKey, evalsById, evalKey, evalId) {
+	const summary = (evalKey === void 0 ? void 0 : evalsByKey.get(evalKey)) ?? evalsById.get(evalId);
+	if (summary === void 0) return evalId;
+	return `${getEvalTitle(summary)} (${summary.filePath}#${summary.id})`;
+}
+function getRunCaseLabel(caseId, caseKey) {
+	return caseKey === void 0 || caseKey === caseId ? caseId : `${caseId} [${caseKey}]`;
+}
+function formatCaseStartedLog(params) {
+	return [
+		`[agent-evals] Run ${params.shortId} started `,
+		`${String(params.activeCount)}/${String(params.concurrency)}: `,
+		`${params.evalLabel} / ${params.caseLabel}`
+	].join("");
+}
 function formatDurationMs(durationMs) {
 	if (durationMs === null) return "";
 	if (durationMs < 1e3) return ` in ${String(durationMs)}ms`;
@@ -247,8 +262,34 @@ function isTerminalRunEvent(eventType) {
 	return eventType === "run.finished" || eventType === "run.error" || eventType === "run.cancelled";
 }
 function subscribeToAppRunResultLog(params) {
+	const evalsByKey = new Map(params.evals.map((ev) => [ev.key, ev]));
+	const evalsById = new Map(params.evals.map((ev) => [ev.id, ev]));
+	const activeCases = /* @__PURE__ */ new Set();
+	const loggedStarts = /* @__PURE__ */ new Set();
 	let unsubscribe;
 	unsubscribe = params.runner.subscribe(params.runId, (event) => {
+		if (event.type === "case.started") {
+			const parsed = caseRowSchema.safeParse(event.payload);
+			if (!parsed.success) return;
+			const caseRow = parsed.data;
+			const caseKey = `${getCaseRowCaseKey(caseRow)}:${String(caseRow.trial)}`;
+			activeCases.add(caseKey);
+			if (loggedStarts.has(caseKey)) return;
+			loggedStarts.add(caseKey);
+			console.info(formatCaseStartedLog({
+				shortId: params.shortId,
+				activeCount: activeCases.size,
+				concurrency: params.concurrency,
+				evalLabel: getEvalSummaryLabel(evalsByKey, evalsById, caseRow.evalKey, caseRow.evalId),
+				caseLabel: getRunCaseLabel(caseRow.caseId, caseRow.caseKey)
+			}));
+			return;
+		}
+		if (event.type === "case.finished") {
+			const parsed = caseRowSchema.safeParse(event.payload);
+			if (parsed.success) activeCases.delete(`${getCaseRowCaseKey(parsed.data)}:${String(parsed.data.trial)}`);
+			return;
+		}
 		if (!isTerminalRunEvent(event.type)) return;
 		unsubscribe?.();
 		unsubscribe = void 0;
@@ -301,6 +342,7 @@ const runsRoutes = new Hono().get("/", (c) => {
 		failures: validation.failures
 	}, 400);
 	const evalsForTerminalLog = runner.getEvals();
+	const concurrency = runner.getConfiguredConcurrency();
 	const runResult = await resultify(() => runner.startRun(body));
 	if (runResult.error) return c.json({
 		error: "Failed to start run",
@@ -310,12 +352,15 @@ const runsRoutes = new Hono().get("/", (c) => {
 		runId: runResult.value.manifest.id,
 		shortId: runResult.value.manifest.shortId,
 		evals: evalsForTerminalLog,
-		target: body.target
+		target: body.target,
+		concurrency
 	});
 	subscribeToAppRunResultLog({
 		runner,
 		runId: runResult.value.manifest.id,
-		shortId: runResult.value.manifest.shortId
+		shortId: runResult.value.manifest.shortId,
+		evals: evalsForTerminalLog,
+		concurrency
 	});
 	return c.json(runResult.value, 201);
 }).post("/actions/open-location", zValidator("json", openRunLocationRequestSchema), (c) => {