npm - @ls-stack/agent-eval - Versions diffs - 0.36.0 → 0.38.0 - Mend

@ls-stack/agent-eval 0.36.0 → 0.38.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (18) hide show

package/dist/{app-BlNzXWDM.mjs → app-DD-8kx5H.mjs} +50 -6
package/dist/apps/web/dist/assets/index-C6PgBOfP.css +1 -0
package/dist/apps/web/dist/assets/index-CO86PsY-.js +140 -0
package/dist/apps/web/dist/index.html +2 -2
package/dist/bin.mjs +14 -3
package/dist/{cli-Dg3abrOv.mjs → cli-BUX6tr9J.mjs} +106 -25
package/dist/index.d.mts +159 -150
package/dist/index.mjs +3 -3
package/dist/runChild.mjs +73 -2
package/dist/{runOrchestration-V1TxX8es.mjs → runOrchestration-BhUFWvq9.mjs} +293 -121
package/dist/{runner-BCs5rzej.mjs → runner-B1wfPKNH.mjs} +2 -2
package/dist/{runner-znY6PY1M.mjs → runner-CoRf7Vef.mjs} +1 -1
package/dist/src-BwJ5tod2.mjs +3 -0
package/package.json +3 -3
package/skills/agent-eval/SKILL.md +18 -6
package/dist/apps/web/dist/assets/index-BYtcGddU.js +0 -140
package/dist/apps/web/dist/assets/index-D0rC5MSS.css +0 -1
package/dist/src-DBypR4TV.mjs +0 -3

package/dist/apps/web/dist/index.html CHANGED Viewed

@@ -25,8 +25,8 @@
       href="https://fonts.googleapis.com/css2?family=Geist:wght@400;500;600&family=JetBrains+Mono:wght@400;500&display=swap"
       rel="stylesheet"
     />
-    <script type="module" crossorigin src="/assets/index-BYtcGddU.js"></script>
-    <link rel="stylesheet" crossorigin href="/assets/index-D0rC5MSS.css">
+    <script type="module" crossorigin src="/assets/index-CO86PsY-.js"></script>
+    <link rel="stylesheet" crossorigin href="/assets/index-C6PgBOfP.css">
   </head>
   <body>
     <div id="root"></div>

package/dist/bin.mjs CHANGED Viewed

@@ -1,5 +1,5 @@
 #!/usr/bin/env node
-import { t as runCli } from "./cli-Dg3abrOv.mjs";
+import { t as runCli } from "./cli-BUX6tr9J.mjs";
 import { spawn } from "node:child_process";
 //#region src/bin.ts
 const moduleMocksFlag = "--experimental-test-module-mocks";
@@ -70,10 +70,21 @@ async function reexecWithNodeArgs(argv, execArgv) {
 		});
 	});
 }
+function formatUnknownErrorDetails(error) {
+	if (error instanceof Error) return error.stack ?? error.message;
+	if (typeof error === "string") return error;
+	return String(error);
+}
 const { argv, inspectArg } = parseDebugFlags(process.argv.slice(2));
 setRunChildInspectArg(inspectArg);
 const execArgv = buildExecArgv(inspectArg);
-if (needsModuleMocksFlag() || !execArgvMatches(execArgv)) await reexecWithNodeArgs(argv, execArgv);
-else await runCli(argv);
+if (needsModuleMocksFlag() || !execArgvMatches(execArgv)) await reexecWithNodeArgs(argv, execArgv).catch((error) => {
+	console.error(formatUnknownErrorDetails(error));
+	process.exitCode = 1;
+});
+else await runCli(argv).catch((error) => {
+	console.error(formatUnknownErrorDetails(error));
+	process.exitCode = 1;
+});
 //#endregion
 export {};

package/dist/{cli-Dg3abrOv.mjs → cli-BUX6tr9J.mjs} RENAMED Viewed

@@ -1,4 +1,4 @@
-import { C as loadConfig, D as createFsCacheStore, E as validateCharts, H as getEvalDisplayStatus, S as resolveEvalDefaultConfig, T as normalizeScoreDef, U as deriveScopedSummaryFromCases, V as getEvalTitle, _ as buildManualInputDescriptor, a as getLastRunStatuses, b as loadEvalModule, bt as getCaseRowEvalKey, c as loadPersistedRunSnapshots, d as persistRunState, f as recomputeEvalStatusesInRuns, g as resolveArtifactPath, h as resolveTracePresentation, i as generateRunId, l as nextShortIdFromSnapshots, m as runTouchesEval, mt as resolveLlmCallsConfig, n as getTargetEvalKeys, o as getLatestRunInfos, p as recomputePersistedCaseStatus, pr as getEvalRegistry, pt as resolveApiCallsConfig, q as runSummarySchema, s as loadPersistedRunSnapshot, u as persistCaseDetail, v as parseManualInputValues, vt as buildEvalKey, w as buildDeclaredColumnDefs, x as parseEvalDiscovery, y as deriveEvalFreshness, yt as getCaseRowCaseKey, z as applyDerivedCallAttributes } from "./runOrchestration-V1TxX8es.mjs";
+import { C as parseEvalDiscovery, D as normalizeScoreDef, E as buildDeclaredColumnDefs, G as deriveScopedSummaryFromCases, O as validateCharts, S as loadEvalModule, St as getCaseRowEvalKey, T as loadConfig, U as getEvalTitle, V as applyDerivedCallAttributes, W as getEvalDisplayStatus, Y as runSummarySchema, _ as resolveTracePresentation, a as generateRunId, b as parseManualInputValues, bt as buildEvalKey, c as loadPersistedRunSnapshot, d as persistCaseDetail, f as deleteTemporaryRuns, g as runTouchesEval, gt as resolveLlmCallsConfig, h as recomputePersistedCaseStatus, hr as getEvalRegistry, ht as resolveApiCallsConfig, i as stripTerminalControlCodes, k as createFsCacheStore, l as loadPersistedRunSnapshots, m as recomputeEvalStatusesInRuns, n as getTargetEvalKeys, o as getLastRunStatuses, p as persistRunState, s as getLatestRunInfos, u as nextShortIdFromSnapshots, v as resolveArtifactPath, w as resolveEvalDefaultConfig, x as deriveEvalFreshness, xt as getCaseRowCaseKey, y as buildManualInputDescriptor } from "./runOrchestration-BhUFWvq9.mjs";
 import { createHash, randomUUID } from "node:crypto";
 import { copyFile, mkdir, readFile, rm, writeFile } from "node:fs/promises";
 import { basename, dirname, extname, isAbsolute, join, relative, resolve, sep } from "node:path";
@@ -536,6 +536,8 @@ function isRunChildMessage(value) {
 const runChildInspectArgEnv = "AGENT_EVALS_RUN_CHILD_INSPECT_ARG";
 const inspectFlagPrefix = "--inspect";
 const inspectBrkFlagPrefix = "--inspect-brk";
+const childOutputTailMaxLength = 12e3;
+const outputHeadlineMaxLength = 240;
 function startRunChild(params) {
 	const child = spawn(process.execPath, [
 		...getRunChildExecArgv(),
@@ -546,12 +548,18 @@ function startRunChild(params) {
 		env: process.env,
 		stdio: [
 			"ignore",
-			"inherit",
-			"inherit",
+			"pipe",
+			"pipe",
 			"ipc"
 		]
 	});
+	const outputTail = createRunChildOutputTail(child);
 	params.runState.childProcess = child;
+	child.once("error", (error) => {
+		if (params.runState.childProcess === child) params.runState.childProcess = void 0;
+		if (params.runState.manifest.status !== "running" || params.runState.childTerminalReceived) return;
+		markRunErrored(params.runState, `Failed to start run child: ${error.message}`, params.managerContext);
+	});
 	child.on("message", (message) => {
 		if (!isRunChildMessage(message)) return;
 		handleRunChildMessage({
@@ -560,12 +568,69 @@ function startRunChild(params) {
 			managerContext: params.managerContext
 		});
 	});
-	child.once("exit", (code, signal) => {
+	child.once("close", (code, signal) => {
 		if (params.runState.childProcess === child) params.runState.childProcess = void 0;
 		if (params.runState.manifest.status !== "running" || params.runState.childTerminalReceived) return;
-		const reason = signal !== null ? `Run child exited with signal ${signal}` : `Run child exited with code ${String(code)}`;
-		markRunErrored(params.runState, reason, params.managerContext);
+		const reason = formatChildExitReason(code, signal);
+		markRunErrored(params.runState, formatUnexpectedRunChildExit(reason, outputTail), params.managerContext);
+	});
+}
+function createRunChildOutputTail(child) {
+	const tail = {
+		stdout: "",
+		stderr: "",
+		stdoutTruncated: false,
+		stderrTruncated: false
+	};
+	child.stdout?.on("data", (chunk) => {
+		process.stdout.write(chunk);
+		const nextTail = appendOutputTail(tail.stdout, chunkToText(chunk));
+		tail.stdout = nextTail.text;
+		tail.stdoutTruncated = tail.stdoutTruncated || nextTail.truncated;
+	});
+	child.stderr?.on("data", (chunk) => {
+		process.stderr.write(chunk);
+		const nextTail = appendOutputTail(tail.stderr, chunkToText(chunk));
+		tail.stderr = nextTail.text;
+		tail.stderrTruncated = tail.stderrTruncated || nextTail.truncated;
 	});
+	return tail;
+}
+function chunkToText(chunk) {
+	return typeof chunk === "string" ? chunk : chunk.toString("utf-8");
+}
+function appendOutputTail(current, next) {
+	const combined = current + next;
+	if (combined.length <= childOutputTailMaxLength) return {
+		text: combined,
+		truncated: false
+	};
+	return {
+		text: combined.slice(combined.length - childOutputTailMaxLength),
+		truncated: true
+	};
+}
+function formatUnexpectedRunChildExit(reason, outputTail) {
+	const stderr = stripTerminalControlCodes(outputTail.stderr).trim();
+	const stdout = stripTerminalControlCodes(outputTail.stdout).trim();
+	const headline = getChildStderrHeadline(stderr);
+	const sections = [headline === null ? `${reason} before sending a structured run error.` : `Run child exited before sending a structured run error: ${headline}`, reason];
+	if (stderr.length > 0) sections.push(formatOutputSection("stderr", stderr, outputTail.stderrTruncated));
+	if (stdout.length > 0) sections.push(formatOutputSection("stdout", stdout, outputTail.stdoutTruncated));
+	return sections.join("\n\n");
+}
+function formatChildExitReason(code, signal) {
+	if (signal !== null) return `Run child exited with signal ${signal}`;
+	return `Run child exited with code ${String(code)}`;
+}
+function getChildStderrHeadline(stderr) {
+	const line = stderr.split("\n").map((entry) => entry.trim()).find((entry) => entry.length > 0);
+	if (line === void 0) return null;
+	if (line.length <= outputHeadlineMaxLength) return line;
+	return `${line.slice(0, outputHeadlineMaxLength)}...`;
+}
+function formatOutputSection(streamName, output, truncated) {
+	return `${truncated ? `Child ${streamName} (last ${String(output.length)} chars)` : `Child ${streamName}`}:\n${output}`;
 }
 function getRunChildExecArgv() {
 	const execArgv = [];
@@ -647,6 +712,7 @@ function applyChildEvalMetas(evals, childMetas) {
 		}
 		evalMeta.columnDefs = childMeta.columnDefs;
 		evalMeta.caseCount = childMeta.caseCount;
+		evalMeta.caseIds = childMeta.caseIds;
 		evalMeta.stats = childMeta.stats;
 		evalMeta.charts = childMeta.charts;
 		evalMeta.sourceFingerprint = childMeta.sourceFingerprint;
@@ -1064,6 +1130,7 @@ function createRunner({ watchForChanges = true } = {}) {
 						sourceFingerprint,
 						columnDefs,
 						caseCount: null,
+						caseIds: void 0,
 						stats,
 						charts,
 						manualInputDescriptor,
@@ -1074,18 +1141,24 @@ function createRunner({ watchForChanges = true } = {}) {
 			emitDiscoveryEvent();
 		},
 		async startRun(request) {
+			const deletedTemporaryRuns = await deleteTemporaryRuns({
+				runs,
+				cancelRunningRun: killRunChild
+			});
 			const runId = generateRunId();
 			const shortId = `r${String(nextShortIdNum++)}`;
 			const now = (/* @__PURE__ */ new Date()).toISOString();
 			const cacheMode = request.cache?.mode ?? "use";
 			const runDir = join(localStateDir, "runs", runId);
+			const gitState = readGitWorktreeState(workspaceRoot);
 			const manifest = {
 				id: runId,
 				shortId,
 				status: "running",
+				temporary: request.temporary === true,
 				startedAt: now,
 				endedAt: null,
-				commitSha: readGitWorktreeState(workspaceRoot).commitSha,
+				commitSha: gitState.commitSha,
 				evalSourceFingerprints: {},
 				target: request.target,
 				trials: request.trials,
@@ -1162,6 +1235,7 @@ function createRunner({ watchForChanges = true } = {}) {
 					emitDiscoveryEvent
 				}
 			});
+			if (deletedTemporaryRuns > 0) emitDiscoveryEvent();
 			return {
 				manifest,
 				summary,
@@ -1445,6 +1519,7 @@ Flags:
   --no-cache                 Shortcut for --cache bypass
   --refresh-cache            Shortcut for --cache refresh
   --clear-cache              Clear the cache before starting the run
+  --temporary                Persist until the next run starts, then delete
   --input <json>             Manual input value for a single targeted eval
                              that declares manualInput
   --input-file <path>        JSON object keyed by eval key (or eval id) with
@@ -1515,6 +1590,7 @@ Options:
   --no-cache                 Shortcut for --cache bypass
   --refresh-cache            Shortcut for --cache refresh
   --clear-cache              Clear the cache before starting the run
+  --temporary                Persist until the next run starts, then delete
   --no-env                   Disable automatic .env loading
   --help, -h                 Show help
   `);
@@ -1753,6 +1829,7 @@ function parseArgs(argv) {
 		port: 4100,
 		cacheMode: "use",
 		clearCache: false,
+		temporary: false,
 		all: false,
 		loadEnv: normalizedArgv.length === argv.length,
 		inputJson: void 0,
@@ -1803,6 +1880,7 @@ function parseArgs(argv) {
 		} else if (arg === "--no-cache") args.cacheMode = "bypass";
 		else if (arg === "--refresh-cache") args.cacheMode = "refresh";
 		else if (arg === "--clear-cache") args.clearCache = true;
+		else if (arg === "--temporary") args.temporary = true;
 		else if (arg === "--input" && next !== void 0) {
 			args.inputJson = next;
 			i++;
@@ -1821,15 +1899,11 @@ function parseArgs(argv) {
 */
 async function runCli(argv) {
 	const args = parseArgs(argv);
-	if (args.loadEnv && !loadWorkspaceEnv()) {
-		process.exit(1);
-		return;
-	}
+	if (args.loadEnv && !loadWorkspaceEnv()) process.exit(1);
 	if (args.showHelp) {
 		if (args.unknownHelpTarget !== void 0) {
 			console.error(`No help found for "${args.unknownHelpTarget}".`);
 			process.exit(1);
-			return;
 		}
 		printHelp(args.helpTopic);
 		return;
@@ -1892,6 +1966,11 @@ function loadWorkspaceEnv() {
 	}
 	return true;
 }
+function formatUnknownErrorDetails(error) {
+	if (error instanceof Error) return error.stack ?? error.message;
+	if (typeof error === "string") return error;
+	return String(error);
+}
 const currentDir = dirname(fileURLToPath(import.meta.url));
 const repoRoot = resolve(currentDir, "../../..");
 const pnpmCommand = process.platform === "win32" ? "pnpm.cmd" : "pnpm";
@@ -1940,8 +2019,8 @@ async function commandApp(args) {
 	const { serve } = await import("@hono/node-server");
 	const bundledWebDist = resolve(currentDir, "apps/web/dist");
 	if (existsSync(bundledWebDist)) process.env.AGENT_EVALS_WEB_DIST = bundledWebDist;
-	const appModule = await import("./app-BlNzXWDM.mjs");
-	const runnerModule = await import("./runner-znY6PY1M.mjs");
+	const appModule = await import("./app-DD-8kx5H.mjs");
+	const runnerModule = await import("./runner-CoRf7Vef.mjs");
 	if (!isHonoAppModule(appModule)) throw new Error("Server app module is invalid");
 	if (!isServerRunnerModule(runnerModule)) throw new Error("Server runner module is invalid");
 	await runnerModule.initRunner();
@@ -1990,7 +2069,6 @@ async function commandRun(args) {
 	if (args.evalIds.length === 0 && args.caseIds.length === 0 && args.files.length === 0 && !runner.getAllowCliRunAll()) {
 		console.error("This workspace disables running all evals from the CLI. Pass --eval <id>, --file <path|glob>, or --case <id> to run a targeted subset.");
 		process.exit(1);
-		return;
 	}
 	if (args.clearCache) {
 		await runner.clearCache();
@@ -2025,26 +2103,30 @@ async function commandRun(args) {
 	if (manualInputsResult.error !== null) {
 		console.error(manualInputsResult.error);
 		process.exit(1);
-		return;
 	}
-	const run = await runner.startRun({
+	const runResult = await resultify(() => runner.startRun({
 		target,
 		trials: args.trials,
+		temporary: args.temporary,
 		cache: { mode: args.cacheMode },
 		manualInputs: manualInputsResult.value
-	});
+	}));
+	if (runResult.error) {
+		console.error("Failed to start run:");
+		console.error(formatUnknownErrorDetails(runResult.error));
+		process.exit(1);
+	}
+	const run = runResult.value;
 	if (!args.json) {
 		console.info(`Run started: ${run.manifest.id}`);
 		console.info(`Trials: ${String(args.trials)}`);
 		if (args.cacheMode !== "use") console.info(`Cache mode: ${args.cacheMode}`);
+		if (args.temporary) console.info("Temporary: yes");
 		console.info("");
 	}
 	await waitForRunCompletion(runner, run.manifest.id);
 	const finalRun = runner.getRun(run.manifest.id);
-	if (!finalRun) {
-		process.exit(1);
-		return;
-	}
+	if (!finalRun) process.exit(1);
 	const { summary } = finalRun;
 	if (args.json) console.info(JSON.stringify(summary, null, 2));
 	else {
@@ -2072,7 +2154,6 @@ async function commandShowRuns(args) {
 		if (!run) {
 			printMissingRun(runRef);
 			process.exit(1);
-			return;
 		}
 		const index = buildRunFileIndex(runner.getWorkspaceRoot(), run);
 		if (args.json) {
@@ -2136,7 +2217,6 @@ async function commandCache(args) {
 		}
 		console.info("Refusing to clear cache without --eval <id> or --all. Use one of these flags to confirm.");
 		process.exit(1);
-		return;
 	}
 	printHelp(args.helpTopic);
 }
@@ -2152,6 +2232,7 @@ function buildRunFileIndex(workspaceRoot, run) {
 		id: run.manifest.id,
 		shortId: run.manifest.shortId,
 		status: run.manifest.status,
+		temporary: run.manifest.temporary,
 		startedAt: run.manifest.startedAt,
 		endedAt: run.manifest.endedAt,
 		target: run.manifest.target,
@@ -2212,7 +2293,7 @@ function printRunFileIndexes(indexes) {
 	}
 }
 function printRunFileIndex(index) {
-	console.info(`${index.shortId} (${index.id})  ${index.status}  ${formatCaseCounts(index.summary)}`);
+	console.info(`${index.shortId} (${index.id})  ${index.status}${index.temporary ? "  temporary" : ""}  ${formatCaseCounts(index.summary)}`);
 	console.info(`  dir: ${index.files.dir}`);
 	console.info(`  run: ${index.files.run}`);
 	console.info(`  summary: ${index.files.summary}`);