@ls-stack/agent-eval 0.55.0 → 0.55.2

package/dist/{app-CunZ8Dku.mjs → app-NI4to6lp.mjs}

@@ -1,7 +1,7 @@
-import { ft as createRunRequestSchema, mt as extractCacheEntries, pt as updateManualScoreRequestSchema } from "./runOrchestration-BWyE5lRX.mjs";
-import { o as stageManualInputFile } from "./cli-rvPrUj6S.mjs";
-import "./src-DEENkbkn.mjs";
-import { t as getRunnerInstance } from "./runner-CFQ8LZmY.mjs";
+import { et as createRunRequestSchema, nt as extractCacheEntries, tt as updateManualScoreRequestSchema, ut as getEvalTitle } from "./runExecution-C31dpemR.mjs";
+import { o as stageManualInputFile } from "./cli-Bu9347r1.mjs";
+import "./src-FR60ZR_4.mjs";
+import { t as getRunnerInstance } from "./runner-B4EfMn1d.mjs";
 import { z } from "zod/v4";
 import { readFile } from "node:fs/promises";
 import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
@@ -189,6 +189,77 @@ const openRunLocationRequestSchema = z.object({
 	column: z.number().int().min(1)
 });
 const importQuerySeparatorRegex = /[?#]/;
+function escapeRegex(value) {
+	return value.replace(/[|\\{}()[\]^$+?.]/g, "\\$&");
+}
+function globToRegex(pattern) {
+	const normalized = pattern.replaceAll("\\", "/");
+	let regex = "^";
+	for (let i = 0; i < normalized.length; i++) {
+		const char = normalized[i];
+		const next = normalized[i + 1];
+		if (char === "*" && next === "*") {
+			regex += ".*";
+			i++;
+		} else if (char === "*") regex += "[^/]*";
+		else if (char === "?") regex += "[^/]";
+		else regex += escapeRegex(char ?? "");
+	}
+	return new RegExp(`${regex}$`);
+}
+function fileMatches(pattern, filePath) {
+	const normalizedPattern = pattern.replaceAll("\\", "/");
+	if (normalizedPattern === filePath) return true;
+	return globToRegex(normalizedPattern).test(filePath);
+}
+function matchesRunTarget(ev, target) {
+	if (target.evalKeys !== void 0 && target.evalKeys.length > 0) {
+		if (!target.evalKeys.includes(ev.key)) return false;
+	}
+	if (target.evalIds !== void 0 && target.evalIds.length > 0) {
+		if (!target.evalIds.includes(ev.id)) return false;
+	}
+	if (target.files !== void 0 && target.files.length > 0) {
+		if (!target.files.some((file) => fileMatches(file, ev.filePath))) return false;
+	}
+	return true;
+}
+function getRunTargetEvalSummaries(evals, target) {
+	return evals.filter((ev) => matchesRunTarget(ev, target)).toSorted((left, right) => left.filePath.localeCompare(right.filePath) || left.id.localeCompare(right.id));
+}
+function logStartedAppRunEvals(params) {
+	const targetEvals = getRunTargetEvalSummaries(params.evals, params.target);
+	if (targetEvals.length === 0) return;
+	const label = targetEvals.length === 1 ? "eval" : "evals";
+	console.info(`[agent-evals] Starting app run ${params.shortId} (${params.runId}) with ${String(targetEvals.length)} ${label}:`);
+	for (const ev of targetEvals) console.info(`  - ${getEvalTitle(ev)} (${ev.filePath}#${ev.id})`);
+}
+function formatDurationMs(durationMs) {
+	if (durationMs === null) return "";
+	if (durationMs < 1e3) return ` in ${String(durationMs)}ms`;
+	return ` in ${(durationMs / 1e3).toFixed(1)}s`;
+}
+function formatRunResultSummary(summary) {
+	const cancelled = summary.cancelledCases > 0 ? `, ${String(summary.cancelledCases)} cancelled` : "";
+	return `${summary.status}: ${String(summary.totalCases)} total, ${String(summary.passedCases)} passed, ${String(summary.failedCases)} failed, ${String(summary.errorCases)} errors${cancelled}${formatDurationMs(summary.totalDurationMs)}`;
+}
+function isTerminalRunEvent(eventType) {
+	return eventType === "run.finished" || eventType === "run.error" || eventType === "run.cancelled";
+}
+function subscribeToAppRunResultLog(params) {
+	let unsubscribe;
+	unsubscribe = params.runner.subscribe(params.runId, (event) => {
+		if (!isTerminalRunEvent(event.type)) return;
+		unsubscribe?.();
+		unsubscribe = void 0;
+		const run = params.runner.getRun(params.runId);
+		if (run === void 0) {
+			console.info(`[agent-evals] Run ${params.shortId} (${params.runId}) finished.`);
+			return;
+		}
+		console.info(`[agent-evals] Run ${params.shortId} (${params.runId}) ${formatRunResultSummary(run.summary)}`);
+	});
+}
 function isInsideWorkspace(path, workspaceRoot) {
 	return path === workspaceRoot || path.startsWith(workspaceRoot + sep);
 }
@@ -229,11 +300,23 @@ const runsRoutes = new Hono().get("/", (c) => {
 		error: "Manual input validation failed",
 		failures: validation.failures
 	}, 400);
+	const evalsForTerminalLog = runner.getEvals();
 	const runResult = await resultify(() => runner.startRun(body));
 	if (runResult.error) return c.json({
 		error: "Failed to start run",
 		message: formatUnknownErrorDetails(runResult.error)
 	}, 500);
+	logStartedAppRunEvals({
+		runId: runResult.value.manifest.id,
+		shortId: runResult.value.manifest.shortId,
+		evals: evalsForTerminalLog,
+		target: body.target
+	});
+	subscribeToAppRunResultLog({
+		runner,
+		runId: runResult.value.manifest.id,
+		shortId: runResult.value.manifest.shortId
+	});
 	return c.json(runResult.value, 201);
 }).post("/actions/open-location", zValidator("json", openRunLocationRequestSchema), (c) => {
 	const body = c.req.valid("json");