npm - @ls-stack/agent-eval - Versions diffs - 0.37.0 → 0.39.0 - Mend

@ls-stack/agent-eval 0.37.0 → 0.39.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (17) hide show

package/dist/{app-C7ON9Wdh.mjs → app-DD-8kx5H.mjs} +15 -6
package/dist/apps/web/dist/assets/index-C6PgBOfP.css +1 -0
package/dist/apps/web/dist/assets/{index-BiwYbMem.js → index-CO86PsY-.js} +43 -43
package/dist/apps/web/dist/index.html +2 -2
package/dist/bin.mjs +14 -3
package/dist/{cli-CwGcJYWe.mjs → cli-BUX6tr9J.mjs} +60 -28
package/dist/index.d.mts +124 -121
package/dist/index.mjs +3 -3
package/dist/runChild.mjs +56 -2
package/dist/{runOrchestration-C4o5TcIu.mjs → runOrchestration-BhUFWvq9.mjs} +251 -114
package/dist/{runner-BTH8m_Er.mjs → runner-B1wfPKNH.mjs} +2 -2
package/dist/{runner-LqeHPID6.mjs → runner-CoRf7Vef.mjs} +1 -1
package/dist/src-BwJ5tod2.mjs +3 -0
package/package.json +1 -1
package/skills/agent-eval/SKILL.md +9 -5
package/dist/apps/web/dist/assets/index-CKdoOah2.css +0 -1
package/dist/src--13_4uDG.mjs +0 -3

package/dist/apps/web/dist/index.html CHANGED Viewed

@@ -25,8 +25,8 @@
       href="https://fonts.googleapis.com/css2?family=Geist:wght@400;500;600&family=JetBrains+Mono:wght@400;500&display=swap"
       rel="stylesheet"
     />
-    <script type="module" crossorigin src="/assets/index-BiwYbMem.js"></script>
-    <link rel="stylesheet" crossorigin href="/assets/index-CKdoOah2.css">
+    <script type="module" crossorigin src="/assets/index-CO86PsY-.js"></script>
+    <link rel="stylesheet" crossorigin href="/assets/index-C6PgBOfP.css">
   </head>
   <body>
     <div id="root"></div>

package/dist/bin.mjs CHANGED Viewed

@@ -1,5 +1,5 @@
 #!/usr/bin/env node
-import { t as runCli } from "./cli-CwGcJYWe.mjs";
+import { t as runCli } from "./cli-BUX6tr9J.mjs";
 import { spawn } from "node:child_process";
 //#region src/bin.ts
 const moduleMocksFlag = "--experimental-test-module-mocks";
@@ -70,10 +70,21 @@ async function reexecWithNodeArgs(argv, execArgv) {
 		});
 	});
 }
+function formatUnknownErrorDetails(error) {
+	if (error instanceof Error) return error.stack ?? error.message;
+	if (typeof error === "string") return error;
+	return String(error);
+}
 const { argv, inspectArg } = parseDebugFlags(process.argv.slice(2));
 setRunChildInspectArg(inspectArg);
 const execArgv = buildExecArgv(inspectArg);
-if (needsModuleMocksFlag() || !execArgvMatches(execArgv)) await reexecWithNodeArgs(argv, execArgv);
-else await runCli(argv);
+if (needsModuleMocksFlag() || !execArgvMatches(execArgv)) await reexecWithNodeArgs(argv, execArgv).catch((error) => {
+	console.error(formatUnknownErrorDetails(error));
+	process.exitCode = 1;
+});
+else await runCli(argv).catch((error) => {
+	console.error(formatUnknownErrorDetails(error));
+	process.exitCode = 1;
+});
 //#endregion
 export {};

package/dist/{cli-CwGcJYWe.mjs → cli-BUX6tr9J.mjs} RENAMED Viewed

@@ -1,4 +1,4 @@
-import { C as parseEvalDiscovery, D as normalizeScoreDef, E as buildDeclaredColumnDefs, G as deriveScopedSummaryFromCases, O as validateCharts, S as loadEvalModule, St as getCaseRowEvalKey, T as loadConfig, U as getEvalTitle, V as applyDerivedCallAttributes, W as getEvalDisplayStatus, Y as runSummarySchema, _ as resolveTracePresentation, a as generateRunId, b as parseManualInputValues, bt as buildEvalKey, c as loadPersistedRunSnapshot, d as persistCaseDetail, f as deleteTemporaryRuns, g as runTouchesEval, gt as resolveLlmCallsConfig, h as recomputePersistedCaseStatus, hr as getEvalRegistry, ht as resolveApiCallsConfig, i as stripTerminalControlCodes, k as createFsCacheStore, l as loadPersistedRunSnapshots, m as recomputeEvalStatusesInRuns, n as getTargetEvalKeys, o as getLastRunStatuses, p as persistRunState, s as getLatestRunInfos, u as nextShortIdFromSnapshots, v as resolveArtifactPath, w as resolveEvalDefaultConfig, x as deriveEvalFreshness, xt as getCaseRowCaseKey, y as buildManualInputDescriptor } from "./runOrchestration-C4o5TcIu.mjs";
+import { C as parseEvalDiscovery, D as normalizeScoreDef, E as buildDeclaredColumnDefs, G as deriveScopedSummaryFromCases, O as validateCharts, S as loadEvalModule, St as getCaseRowEvalKey, T as loadConfig, U as getEvalTitle, V as applyDerivedCallAttributes, W as getEvalDisplayStatus, Y as runSummarySchema, _ as resolveTracePresentation, a as generateRunId, b as parseManualInputValues, bt as buildEvalKey, c as loadPersistedRunSnapshot, d as persistCaseDetail, f as deleteTemporaryRuns, g as runTouchesEval, gt as resolveLlmCallsConfig, h as recomputePersistedCaseStatus, hr as getEvalRegistry, ht as resolveApiCallsConfig, i as stripTerminalControlCodes, k as createFsCacheStore, l as loadPersistedRunSnapshots, m as recomputeEvalStatusesInRuns, n as getTargetEvalKeys, o as getLastRunStatuses, p as persistRunState, s as getLatestRunInfos, u as nextShortIdFromSnapshots, v as resolveArtifactPath, w as resolveEvalDefaultConfig, x as deriveEvalFreshness, xt as getCaseRowCaseKey, y as buildManualInputDescriptor } from "./runOrchestration-BhUFWvq9.mjs";
 import { createHash, randomUUID } from "node:crypto";
 import { copyFile, mkdir, readFile, rm, writeFile } from "node:fs/promises";
 import { basename, dirname, extname, isAbsolute, join, relative, resolve, sep } from "node:path";
@@ -537,6 +537,7 @@ const runChildInspectArgEnv = "AGENT_EVALS_RUN_CHILD_INSPECT_ARG";
 const inspectFlagPrefix = "--inspect";
 const inspectBrkFlagPrefix = "--inspect-brk";
 const childOutputTailMaxLength = 12e3;
+const outputHeadlineMaxLength = 240;
 function startRunChild(params) {
 	const child = spawn(process.execPath, [
 		...getRunChildExecArgv(),
@@ -554,6 +555,11 @@ function startRunChild(params) {
 	});
 	const outputTail = createRunChildOutputTail(child);
 	params.runState.childProcess = child;
+	child.once("error", (error) => {
+		if (params.runState.childProcess === child) params.runState.childProcess = void 0;
+		if (params.runState.manifest.status !== "running" || params.runState.childTerminalReceived) return;
+		markRunErrored(params.runState, `Failed to start run child: ${error.message}`, params.managerContext);
+	});
 	child.on("message", (message) => {
 		if (!isRunChildMessage(message)) return;
 		handleRunChildMessage({
@@ -562,25 +568,31 @@ function startRunChild(params) {
 			managerContext: params.managerContext
 		});
 	});
-	child.once("exit", (code, signal) => {
+	child.once("close", (code, signal) => {
 		if (params.runState.childProcess === child) params.runState.childProcess = void 0;
 		if (params.runState.manifest.status !== "running" || params.runState.childTerminalReceived) return;
-		const reason = signal !== null ? `Run child exited with signal ${signal}` : `Run child exited with code ${String(code)}`;
+		const reason = formatChildExitReason(code, signal);
 		markRunErrored(params.runState, formatUnexpectedRunChildExit(reason, outputTail), params.managerContext);
 	});
 }
 function createRunChildOutputTail(child) {
 	const tail = {
 		stdout: "",
-		stderr: ""
+		stderr: "",
+		stdoutTruncated: false,
+		stderrTruncated: false
 	};
 	child.stdout?.on("data", (chunk) => {
 		process.stdout.write(chunk);
-		tail.stdout = appendOutputTail(tail.stdout, chunkToText(chunk));
+		const nextTail = appendOutputTail(tail.stdout, chunkToText(chunk));
+		tail.stdout = nextTail.text;
+		tail.stdoutTruncated = tail.stdoutTruncated || nextTail.truncated;
 	});
 	child.stderr?.on("data", (chunk) => {
 		process.stderr.write(chunk);
-		tail.stderr = appendOutputTail(tail.stderr, chunkToText(chunk));
+		const nextTail = appendOutputTail(tail.stderr, chunkToText(chunk));
+		tail.stderr = nextTail.text;
+		tail.stderrTruncated = tail.stderrTruncated || nextTail.truncated;
 	});
 	return tail;
 }
@@ -589,17 +601,37 @@ function chunkToText(chunk) {
 }
 function appendOutputTail(current, next) {
 	const combined = current + next;
-	if (combined.length <= childOutputTailMaxLength) return combined;
-	return combined.slice(combined.length - childOutputTailMaxLength);
+	if (combined.length <= childOutputTailMaxLength) return {
+		text: combined,
+		truncated: false
+	};
+	return {
+		text: combined.slice(combined.length - childOutputTailMaxLength),
+		truncated: true
+	};
 }
 function formatUnexpectedRunChildExit(reason, outputTail) {
-	const sections = [reason];
 	const stderr = stripTerminalControlCodes(outputTail.stderr).trim();
 	const stdout = stripTerminalControlCodes(outputTail.stdout).trim();
-	if (stderr.length > 0) sections.push(`Child stderr (last ${String(stderr.length)} chars):\n${stderr}`);
-	if (stdout.length > 0) sections.push(`Child stdout (last ${String(stdout.length)} chars):\n${stdout}`);
+	const headline = getChildStderrHeadline(stderr);
+	const sections = [headline === null ? `${reason} before sending a structured run error.` : `Run child exited before sending a structured run error: ${headline}`, reason];
+	if (stderr.length > 0) sections.push(formatOutputSection("stderr", stderr, outputTail.stderrTruncated));
+	if (stdout.length > 0) sections.push(formatOutputSection("stdout", stdout, outputTail.stdoutTruncated));
 	return sections.join("\n\n");
 }
+function formatChildExitReason(code, signal) {
+	if (signal !== null) return `Run child exited with signal ${signal}`;
+	return `Run child exited with code ${String(code)}`;
+}
+function getChildStderrHeadline(stderr) {
+	const line = stderr.split("\n").map((entry) => entry.trim()).find((entry) => entry.length > 0);
+	if (line === void 0) return null;
+	if (line.length <= outputHeadlineMaxLength) return line;
+	return `${line.slice(0, outputHeadlineMaxLength)}...`;
+}
+function formatOutputSection(streamName, output, truncated) {
+	return `${truncated ? `Child ${streamName} (last ${String(output.length)} chars)` : `Child ${streamName}`}:\n${output}`;
+}
 function getRunChildExecArgv() {
 	const execArgv = [];
 	let skipNext = false;
@@ -1867,15 +1899,11 @@ function parseArgs(argv) {
 */
 async function runCli(argv) {
 	const args = parseArgs(argv);
-	if (args.loadEnv && !loadWorkspaceEnv()) {
-		process.exit(1);
-		return;
-	}
+	if (args.loadEnv && !loadWorkspaceEnv()) process.exit(1);
 	if (args.showHelp) {
 		if (args.unknownHelpTarget !== void 0) {
 			console.error(`No help found for "${args.unknownHelpTarget}".`);
 			process.exit(1);
-			return;
 		}
 		printHelp(args.helpTopic);
 		return;
@@ -1938,6 +1966,11 @@ function loadWorkspaceEnv() {
 	}
 	return true;
 }
+function formatUnknownErrorDetails(error) {
+	if (error instanceof Error) return error.stack ?? error.message;
+	if (typeof error === "string") return error;
+	return String(error);
+}
 const currentDir = dirname(fileURLToPath(import.meta.url));
 const repoRoot = resolve(currentDir, "../../..");
 const pnpmCommand = process.platform === "win32" ? "pnpm.cmd" : "pnpm";
@@ -1986,8 +2019,8 @@ async function commandApp(args) {
 	const { serve } = await import("@hono/node-server");
 	const bundledWebDist = resolve(currentDir, "apps/web/dist");
 	if (existsSync(bundledWebDist)) process.env.AGENT_EVALS_WEB_DIST = bundledWebDist;
-	const appModule = await import("./app-C7ON9Wdh.mjs");
-	const runnerModule = await import("./runner-LqeHPID6.mjs");
+	const appModule = await import("./app-DD-8kx5H.mjs");
+	const runnerModule = await import("./runner-CoRf7Vef.mjs");
 	if (!isHonoAppModule(appModule)) throw new Error("Server app module is invalid");
 	if (!isServerRunnerModule(runnerModule)) throw new Error("Server runner module is invalid");
 	await runnerModule.initRunner();
@@ -2036,7 +2069,6 @@ async function commandRun(args) {
 	if (args.evalIds.length === 0 && args.caseIds.length === 0 && args.files.length === 0 && !runner.getAllowCliRunAll()) {
 		console.error("This workspace disables running all evals from the CLI. Pass --eval <id>, --file <path|glob>, or --case <id> to run a targeted subset.");
 		process.exit(1);
-		return;
 	}
 	if (args.clearCache) {
 		await runner.clearCache();
@@ -2071,15 +2103,20 @@ async function commandRun(args) {
 	if (manualInputsResult.error !== null) {
 		console.error(manualInputsResult.error);
 		process.exit(1);
-		return;
 	}
-	const run = await runner.startRun({
+	const runResult = await resultify(() => runner.startRun({
 		target,
 		trials: args.trials,
 		temporary: args.temporary,
 		cache: { mode: args.cacheMode },
 		manualInputs: manualInputsResult.value
-	});
+	}));
+	if (runResult.error) {
+		console.error("Failed to start run:");
+		console.error(formatUnknownErrorDetails(runResult.error));
+		process.exit(1);
+	}
+	const run = runResult.value;
 	if (!args.json) {
 		console.info(`Run started: ${run.manifest.id}`);
 		console.info(`Trials: ${String(args.trials)}`);
@@ -2089,10 +2126,7 @@ async function commandRun(args) {
 	}
 	await waitForRunCompletion(runner, run.manifest.id);
 	const finalRun = runner.getRun(run.manifest.id);
-	if (!finalRun) {
-		process.exit(1);
-		return;
-	}
+	if (!finalRun) process.exit(1);
 	const { summary } = finalRun;
 	if (args.json) console.info(JSON.stringify(summary, null, 2));
 	else {
@@ -2120,7 +2154,6 @@ async function commandShowRuns(args) {
 		if (!run) {
 			printMissingRun(runRef);
 			process.exit(1);
-			return;
 		}
 		const index = buildRunFileIndex(runner.getWorkspaceRoot(), run);
 		if (args.json) {
@@ -2184,7 +2217,6 @@ async function commandCache(args) {
 		}
 		console.info("Refusing to clear cache without --eval <id> or --all. Use one of these flags to confirm.");
 		process.exit(1);
-		return;
 	}
 	printHelp(args.helpTopic);
 }