@ls-stack/agent-eval 0.55.0 → 0.55.1

package/dist/{app-CunZ8Dku.mjs → app-BD0D9-7k.mjs}

@@ -1,7 +1,7 @@
-import { ft as createRunRequestSchema, mt as extractCacheEntries, pt as updateManualScoreRequestSchema } from "./runOrchestration-BWyE5lRX.mjs";
-import { o as stageManualInputFile } from "./cli-rvPrUj6S.mjs";
-import "./src-DEENkbkn.mjs";
-import { t as getRunnerInstance } from "./runner-CFQ8LZmY.mjs";
+import { et as createRunRequestSchema, nt as extractCacheEntries, tt as updateManualScoreRequestSchema, ut as getEvalTitle } from "./runExecution-Sw38bCaq.mjs";
+import { o as stageManualInputFile } from "./cli-BR3wMZMx.mjs";
+import "./src-hBGtzWuA.mjs";
+import { t as getRunnerInstance } from "./runner-72rsqJRq.mjs";
 import { z } from "zod/v4";
 import { readFile } from "node:fs/promises";
 import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
@@ -189,6 +189,77 @@ const openRunLocationRequestSchema = z.object({
 	column: z.number().int().min(1)
 });
 const importQuerySeparatorRegex = /[?#]/;
+function escapeRegex(value) {
+	return value.replace(/[|\\{}()[\]^$+?.]/g, "\\$&");
+}
+function globToRegex(pattern) {
+	const normalized = pattern.replaceAll("\\", "/");
+	let regex = "^";
+	for (let i = 0; i < normalized.length; i++) {
+		const char = normalized[i];
+		const next = normalized[i + 1];
+		if (char === "*" && next === "*") {
+			regex += ".*";
+			i++;
+		} else if (char === "*") regex += "[^/]*";
+		else if (char === "?") regex += "[^/]";
+		else regex += escapeRegex(char ?? "");
+	}
+	return new RegExp(`${regex}$`);
+}
+function fileMatches(pattern, filePath) {
+	const normalizedPattern = pattern.replaceAll("\\", "/");
+	if (normalizedPattern === filePath) return true;
+	return globToRegex(normalizedPattern).test(filePath);
+}
+function matchesRunTarget(ev, target) {
+	if (target.evalKeys !== void 0 && target.evalKeys.length > 0) {
+		if (!target.evalKeys.includes(ev.key)) return false;
+	}
+	if (target.evalIds !== void 0 && target.evalIds.length > 0) {
+		if (!target.evalIds.includes(ev.id)) return false;
+	}
+	if (target.files !== void 0 && target.files.length > 0) {
+		if (!target.files.some((file) => fileMatches(file, ev.filePath))) return false;
+	}
+	return true;
+}
+function getRunTargetEvalSummaries(evals, target) {
+	return evals.filter((ev) => matchesRunTarget(ev, target)).toSorted((left, right) => left.filePath.localeCompare(right.filePath) || left.id.localeCompare(right.id));
+}
+function logStartedAppRunEvals(params) {
+	const targetEvals = getRunTargetEvalSummaries(params.evals, params.target);
+	if (targetEvals.length === 0) return;
+	const label = targetEvals.length === 1 ? "eval" : "evals";
+	console.info(`[agent-evals] Starting app run ${params.shortId} (${params.runId}) with ${String(targetEvals.length)} ${label}:`);
+	for (const ev of targetEvals) console.info(`  - ${getEvalTitle(ev)} (${ev.filePath}#${ev.id})`);
+}
+function formatDurationMs(durationMs) {
+	if (durationMs === null) return "";
+	if (durationMs < 1e3) return ` in ${String(durationMs)}ms`;
+	return ` in ${(durationMs / 1e3).toFixed(1)}s`;
+}
+function formatRunResultSummary(summary) {
+	const cancelled = summary.cancelledCases > 0 ? `, ${String(summary.cancelledCases)} cancelled` : "";
+	return `${summary.status}: ${String(summary.totalCases)} total, ${String(summary.passedCases)} passed, ${String(summary.failedCases)} failed, ${String(summary.errorCases)} errors${cancelled}${formatDurationMs(summary.totalDurationMs)}`;
+}
+function isTerminalRunEvent(eventType) {
+	return eventType === "run.finished" || eventType === "run.error" || eventType === "run.cancelled";
+}
+function subscribeToAppRunResultLog(params) {
+	let unsubscribe;
+	unsubscribe = params.runner.subscribe(params.runId, (event) => {
+		if (!isTerminalRunEvent(event.type)) return;
+		unsubscribe?.();
+		unsubscribe = void 0;
+		const run = params.runner.getRun(params.runId);
+		if (run === void 0) {
+			console.info(`[agent-evals] Run ${params.shortId} (${params.runId}) finished.`);
+			return;
+		}
+		console.info(`[agent-evals] Run ${params.shortId} (${params.runId}) ${formatRunResultSummary(run.summary)}`);
+	});
+}
 function isInsideWorkspace(path, workspaceRoot) {
 	return path === workspaceRoot || path.startsWith(workspaceRoot + sep);
 }
@@ -229,11 +300,23 @@ const runsRoutes = new Hono().get("/", (c) => {
 		error: "Manual input validation failed",
 		failures: validation.failures
 	}, 400);
+	const evalsForTerminalLog = runner.getEvals();
 	const runResult = await resultify(() => runner.startRun(body));
 	if (runResult.error) return c.json({
 		error: "Failed to start run",
 		message: formatUnknownErrorDetails(runResult.error)
 	}, 500);
+	logStartedAppRunEvals({
+		runId: runResult.value.manifest.id,
+		shortId: runResult.value.manifest.shortId,
+		evals: evalsForTerminalLog,
+		target: body.target
+	});
+	subscribeToAppRunResultLog({
+		runner,
+		runId: runResult.value.manifest.id,
+		shortId: runResult.value.manifest.shortId
+	});
 	return c.json(runResult.value, 201);
 }).post("/actions/open-location", zValidator("json", openRunLocationRequestSchema), (c) => {
 	const body = c.req.valid("json");

package/dist/bin.mjs

@@ -1,5 +1,5 @@
 #!/usr/bin/env node
-import { t as runCli } from "./cli-rvPrUj6S.mjs";
+import { t as runCli } from "./cli-BR3wMZMx.mjs";
 import { spawn } from "node:child_process";
 //#region src/bin.ts
 const moduleMocksFlag = "--experimental-test-module-mocks";

package/dist/caseChild.mjs

@@ -0,0 +1,167 @@
+import { I as configureEvalRunLogs, Pt as runWithEvalRegistry, St as resolveLlmCallsConfig, _ as createBufferedCacheStore, a as isCaseChildParentMessage, d as loadEvalModule, l as registerAgentEvalsPackageResolutionHooks, p as loadConfig, q as runInEvalRuntimeScope, r as runCase, v as createFsCacheStore, xt as resolveApiCallsConfig } from "./runExecution-Sw38bCaq.mjs";
+//#region ../runner/src/caseChild.ts
+let fatalErrorReported = false;
+let disconnectExpected = false;
+let runStarted = false;
+const pendingMessageSends = /* @__PURE__ */ new Set();
+function sendMessage(message) {
+	if (process.send === void 0) return;
+	const sendPromise = new Promise((resolvePromise) => {
+		try {
+			process.send?.(message, (error) => {
+				if (error) {
+					console.error("Failed to send case child message:");
+					console.error(formatUnknownErrorDetails(error));
+				}
+				resolvePromise();
+			});
+		} catch (error) {
+			console.error("Failed to send case child message:");
+			console.error(formatUnknownErrorDetails(error));
+			resolvePromise();
+		}
+	});
+	pendingMessageSends.add(sendPromise);
+	sendPromise.finally(() => {
+		pendingMessageSends.delete(sendPromise);
+	});
+}
+async function flushMessageSends() {
+	while (pendingMessageSends.size > 0) await Promise.allSettled([...pendingMessageSends]);
+}
+function installFatalCaseChildErrorHandlers() {
+	process.once("uncaughtException", (error) => {
+		reportFatalCaseChildErrorAndExit(error);
+	});
+	process.once("unhandledRejection", (reason) => {
+		reportFatalCaseChildErrorAndExit(toUnhandledRejectionError(reason));
+	});
+}
+async function useEvalDefinition(params) {
+	const entry = (await runWithEvalRegistry(async (activeRegistry) => {
+		await runInEvalRuntimeScope("env", async () => {
+			await loadEvalModule(params.evalFilePath, params.sourceFingerprint);
+		});
+		return activeRegistry;
+	})).get(params.evalId);
+	if (entry === void 0) throw new Error(`Eval "${params.evalId}" was not registered after importing ${params.evalFilePath}`);
+	return await entry.use(async (evalDef) => await params.use(evalDef));
+}
+async function executeCaseChild(context) {
+	process.chdir(context.workspaceRoot);
+	registerAgentEvalsPackageResolutionHooks();
+	const config = await loadConfig();
+	configureEvalRunLogs({ captureConsole: config.runLogs?.captureConsole !== false });
+	const cacheStore = createFsCacheStore({
+		workspaceRoot: context.workspaceRoot,
+		dir: config.cache?.dir,
+		maxEntriesPerNamespace: config.cache?.maxEntriesPerNamespace ?? config.cache?.maxEntriesPerEval,
+		maxEntriesByNamespace: config.cache?.maxEntriesByNamespace
+	});
+	const bufferedCacheStore = context.cacheEnabled && context.cacheMode !== "bypass" ? createBufferedCacheStore(cacheStore) : null;
+	const llmCallsConfig = resolveLlmCallsConfig(config.llmCalls);
+	const apiCallsConfig = resolveApiCallsConfig(config.apiCalls);
+	const { caseDetail, caseRowUpdate } = await useEvalDefinition({
+		evalId: context.evalId,
+		evalFilePath: context.evalFilePath,
+		sourceFingerprint: context.sourceFingerprint,
+		use: async (evalDef) => await runCase({
+			evalDef,
+			evalId: context.evalId,
+			evalKey: context.evalKey,
+			evalCase: context.evalCase,
+			globalTraceDisplay: context.globalTraceDisplay,
+			globalColumns: config.columns,
+			globalDeriveFromTracing: config.deriveFromTracing,
+			llmCallsConfig,
+			apiCallsConfig,
+			globalRemoveDefaultConfig: config.removeDefaultConfig,
+			trial: context.trial,
+			startTime: context.startTime,
+			cacheAdapter: bufferedCacheStore ?? (context.cacheEnabled ? cacheStore : null),
+			cacheMode: context.cacheMode,
+			moduleIsolation: void 0,
+			evalFilePath: context.evalFilePath,
+			evalFileRelativePath: context.evalFileRelativePath,
+			workspaceRoot: context.workspaceRoot,
+			artifactDir: context.artifactDir,
+			runId: context.runId
+		})
+	});
+	return {
+		caseDetail,
+		caseRow: {
+			caseId: context.evalCase.id,
+			evalId: context.evalId,
+			evalKey: context.evalKey,
+			caseKey: caseDetail.caseKey,
+			tags: caseDetail.tags,
+			status: caseRowUpdate.status ?? "pending",
+			durationMs: caseRowUpdate.durationMs ?? null,
+			cacheHits: caseRowUpdate.cacheHits ?? 0,
+			cacheOperations: caseRowUpdate.cacheOperations ?? 0,
+			columns: caseRowUpdate.columns ?? {},
+			trial: context.trial
+		},
+		pendingCacheWrites: bufferedCacheStore?.getPendingWrites() ?? []
+	};
+}
+async function handleFatalCaseChildError(error) {
+	if (fatalErrorReported) return;
+	fatalErrorReported = true;
+	const message = formatUnknownErrorDetails(error);
+	process.exitCode = 1;
+	console.error(message);
+	sendMessage({
+		type: "error",
+		message
+	});
+	await flushMessageSends();
+}
+function formatUnknownErrorDetails(error) {
+	if (error instanceof Error) return error.stack ?? error.message;
+	if (typeof error === "string") return error;
+	return String(error);
+}
+function toUnhandledRejectionError(reason) {
+	if (reason instanceof Error) return reason;
+	return /* @__PURE__ */ new Error(`Unhandled rejection: ${formatUnknownErrorDetails(reason)}`);
+}
+async function reportFatalCaseChildErrorAndExit(error) {
+	try {
+		await handleFatalCaseChildError(error);
+	} catch (reportError) {
+		console.error("Failed to report fatal case child error:");
+		console.error(formatUnknownErrorDetails(reportError));
+	} finally {
+		process.exit(1);
+	}
+}
+installFatalCaseChildErrorHandlers();
+process.on("disconnect", () => {
+	if (disconnectExpected) return;
+	process.exit(1);
+});
+process.on("message", (message) => {
+	if (runStarted) return;
+	runStarted = true;
+	if (!isCaseChildParentMessage(message)) {
+		reportFatalCaseChildErrorAndExit(/* @__PURE__ */ new Error("Case child received an invalid start message"));
+		return;
+	}
+	executeCaseChild(message.context).then(async (result) => {
+		sendMessage({
+			type: "done",
+			result
+		});
+		await flushMessageSends();
+		disconnectExpected = true;
+		process.disconnect();
+	}).catch(async (error) => {
+		await handleFatalCaseChildError(error);
+		disconnectExpected = true;
+		process.disconnect();
+	});
+});
+//#endregion
+export {};

package/dist/{cli-rvPrUj6S.mjs → cli-BR3wMZMx.mjs}

@@ -1,4 +1,5 @@
-import { A as validateCharts, At as buildEvalKey, C as deriveEvalFreshness, Ct as getEvalDisplayStatus, D as loadConfig, Dt as runSummarySchema, E as resolveEvalDefaultConfig, O as buildDeclaredColumnDefs, Ot as resolveApiCallsConfig, Rt as getEvalRegistry, S as parseManualInputValues, St as getEvalTitle, T as parseEvalDiscovery, Tt as matchesTagsFilter, _ as recomputePersistedCaseStatus, a as validateTagsFilters, b as resolveArtifactPath, bt as applyDerivedCallAttributes, c as getLastRunStatuses, d as loadPersistedRunSnapshots, f as nextShortIdFromSnapshots, g as recomputeEvalStatusesInRuns, h as persistRunState, i as resolveEvalTags, j as createFsCacheStore, jt as getCaseRowCaseKey, k as normalizeScoreDef, kt as resolveLlmCallsConfig, l as getLatestRunInfos, m as deleteTemporaryRuns, n as getTargetEvalKeys, o as stripTerminalControlCodes, p as persistCaseDetail, s as generateRunId, u as loadPersistedRunSnapshot, v as runTouchesEval, w as loadIsolatedEvalRegistry, wt as deriveScopedSummaryFromCases, x as buildManualInputDescriptor, y as resolveTracePresentation } from "./runOrchestration-BWyE5lRX.mjs";
+import { Ct as buildEvalKey, Nt as getEvalRegistry, St as resolveLlmCallsConfig, bt as runSummarySchema, c as resolveArtifactPath, ct as applyDerivedCallAttributes, dt as getEvalDisplayStatus, f as resolveEvalDefaultConfig, ft as deriveScopedSummaryFromCases, gt as matchesTagsFilter, h as normalizeScoreDef, m as buildDeclaredColumnDefs, o as stripTerminalControlCodes, p as loadConfig, s as resolveTracePresentation, ut as getEvalTitle, v as createFsCacheStore, wt as getCaseRowCaseKey, xt as resolveApiCallsConfig } from "./runExecution-Sw38bCaq.mjs";
+import { C as validateCharts, S as parseEvalDiscovery, _ as runTouchesEval, a as validateTagsFilters, b as deriveEvalFreshness, c as getLatestRunInfos, d as nextShortIdFromSnapshots, f as persistCaseDetail, g as recomputePersistedCaseStatus, h as recomputeEvalStatusesInRuns, i as resolveEvalTags, l as loadPersistedRunSnapshot, m as persistRunState, n as getTargetEvalKeys, o as generateRunId, p as deleteTemporaryRuns, s as getLastRunStatuses, u as loadPersistedRunSnapshots, v as buildManualInputDescriptor, x as loadIsolatedEvalRegistry, y as parseManualInputValues } from "./runOrchestration-DJsdLYeZ.mjs";
 import { copyFile, mkdir, readFile, rm, writeFile } from "node:fs/promises";
 import { basename, dirname, extname, isAbsolute, join, relative, resolve, sep } from "node:path";
 import { createHash, randomUUID } from "node:crypto";
@@ -2135,8 +2136,8 @@ async function commandApp(args) {
 	const { serve } = await import("@hono/node-server");
 	const bundledWebDist = resolve(currentDir, "apps/web/dist");
 	if (existsSync(bundledWebDist)) process.env.AGENT_EVALS_WEB_DIST = bundledWebDist;
-	const appModule = await import("./app-CunZ8Dku.mjs");
-	const runnerModule = await import("./runner-C2fvjKZP.mjs");
+	const appModule = await import("./app-BD0D9-7k.mjs");
+	const runnerModule = await import("./runner-dB69WsnM.mjs");
 	if (!isHonoAppModule(appModule)) throw new Error("Server app module is invalid");
 	if (!isServerRunnerModule(runnerModule)) throw new Error("Server runner module is invalid");
 	await runnerModule.initRunner();