npm - @ls-stack/agent-eval - Versions diffs - 0.58.1 → 0.58.3 - Mend

@ls-stack/agent-eval 0.58.1 → 0.58.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (19) hide show

package/dist/{app-DhMIbjlE.mjs → app-ROCEce9X.mjs} +52 -7
package/dist/apps/web/dist/assets/index-CHH7m5Cv.css +1 -0
package/dist/apps/web/dist/assets/index-PTikBbhf.js +377 -0
package/dist/apps/web/dist/index.html +2 -2
package/dist/bin.mjs +1 -1
package/dist/caseChild.mjs +2 -1
package/dist/{cli-_g2qOMK6.mjs → cli-SP4kEtYL.mjs} +31 -5
package/dist/index.d.mts +184 -129
package/dist/index.mjs +4 -4
package/dist/runChild.mjs +2 -2
package/dist/{runExecution-d42Lm0i5.mjs → runExecution-CFw0MQFs.mjs} +114 -21
package/dist/{runOrchestration-CvmFeOmT.mjs → runOrchestration-CxjiQmof.mjs} +73 -6
package/dist/{runner-BKogjiYd.mjs → runner-BlFQyvN2.mjs} +1 -1
package/dist/{runner-MSr8sAWm.mjs → runner-CY3bgsjU.mjs} +2 -2
package/dist/{src-CdZsOn6y.mjs → src-7GbQj1sb.mjs} +2 -2
package/package.json +3 -3
package/skills/agent-eval/SKILL.md +19 -3
package/dist/apps/web/dist/assets/index-Cz9p4l-t.js +0 -377
package/dist/apps/web/dist/assets/index-DtARRwsS.css +0 -1

package/dist/apps/web/dist/index.html CHANGED Viewed

@@ -25,8 +25,8 @@
       href="https://fonts.googleapis.com/css2?family=Geist:wght@400;500;600&family=JetBrains+Mono:wght@400;500&display=swap"
       rel="stylesheet"
     />
-    <script type="module" crossorigin src="/assets/index-Cz9p4l-t.js"></script>
-    <link rel="stylesheet" crossorigin href="/assets/index-DtARRwsS.css">
+    <script type="module" crossorigin src="/assets/index-PTikBbhf.js"></script>
+    <link rel="stylesheet" crossorigin href="/assets/index-CHH7m5Cv.css">
   </head>
   <body>
     <div id="root"></div>

package/dist/bin.mjs CHANGED Viewed

@@ -1,5 +1,5 @@
 #!/usr/bin/env node
-import { t as runCli } from "./cli-_g2qOMK6.mjs";
+import { t as runCli } from "./cli-SP4kEtYL.mjs";
 import { spawn } from "node:child_process";
 //#region src/bin.ts
 const moduleMocksFlag = "--experimental-test-module-mocks";

package/dist/caseChild.mjs CHANGED Viewed

@@ -1,4 +1,4 @@
-import { I as configureEvalRunLogs, Pt as runWithEvalRegistry, St as resolveLlmCallsConfig, _ as createBufferedCacheStore, a as isCaseChildParentMessage, d as loadEvalModule, l as registerAgentEvalsPackageResolutionHooks, p as loadConfig, q as runInEvalRuntimeScope, r as runCase, v as createFsCacheStore, xt as resolveApiCallsConfig } from "./runExecution-d42Lm0i5.mjs";
+import { Ct as resolveLlmCallsConfig, It as runWithEvalRegistry, J as runInEvalRuntimeScope, L as configureEvalRunLogs, St as resolveApiCallsConfig, _ as createBufferedCacheStore, a as isCaseChildParentMessage, d as loadEvalModule, l as registerAgentEvalsPackageResolutionHooks, p as loadConfig, r as runCase, v as createFsCacheStore } from "./runExecution-CFw0MQFs.mjs";
 //#region ../runner/src/caseChild.ts
 let fatalErrorReported = false;
 let disconnectExpected = false;
@@ -74,6 +74,7 @@ async function executeCaseChild(context) {
 			globalTraceDisplay: context.globalTraceDisplay,
 			globalColumns: config.columns,
 			globalDeriveFromTracing: config.deriveFromTracing,
+			globalTracingAssertions: config.tracingAssertions,
 			llmCallsConfig,
 			apiCallsConfig,
 			globalRemoveDefaultConfig: config.removeDefaultConfig,

package/dist/{cli-_g2qOMK6.mjs → cli-SP4kEtYL.mjs} RENAMED Viewed

@@ -1,5 +1,5 @@
-import { Ct as buildEvalKey, Nt as getEvalRegistry, St as resolveLlmCallsConfig, bt as runSummarySchema, c as resolveArtifactPath, ct as applyDerivedCallAttributes, dt as getEvalDisplayStatus, f as resolveEvalDefaultConfig, ft as deriveScopedSummaryFromCases, gt as matchesTagsFilter, h as normalizeScoreDef, m as buildDeclaredColumnDefs, o as stripTerminalControlCodes, p as loadConfig, s as resolveTracePresentation, ut as getEvalTitle, v as createFsCacheStore, wt as getCaseRowCaseKey, xt as resolveApiCallsConfig } from "./runExecution-d42Lm0i5.mjs";
-import { C as validateCharts, S as parseEvalDiscovery, _ as runTouchesEval, a as validateTagsFilters, b as deriveEvalFreshness, c as getLatestRunInfos, d as nextShortIdFromSnapshots, f as persistCaseDetail, g as recomputePersistedCaseStatus, h as recomputeEvalStatusesInRuns, i as resolveEvalTags, l as loadPersistedRunSnapshot, m as persistRunState, n as getTargetEvalKeys, o as generateRunId, p as deleteTemporaryRuns, s as getLastRunStatuses, u as loadPersistedRunSnapshots, v as buildManualInputDescriptor, x as loadIsolatedEvalRegistry, y as parseManualInputValues } from "./runOrchestration-CvmFeOmT.mjs";
+import { Ct as resolveLlmCallsConfig, Et as getCaseRowCaseKey, Ft as getEvalRegistry, Ot as caseRowSchema, St as resolveApiCallsConfig, Tt as buildEvalKey, _t as matchesTagsFilter, c as resolveArtifactPath, dt as getEvalTitle, f as resolveEvalDefaultConfig, ft as getEvalDisplayStatus, h as normalizeScoreDef, lt as applyDerivedCallAttributes, m as buildDeclaredColumnDefs, o as stripTerminalControlCodes, p as loadConfig, pt as deriveScopedSummaryFromCases, s as resolveTracePresentation, v as createFsCacheStore, xt as runSummarySchema } from "./runExecution-CFw0MQFs.mjs";
+import { C as validateCharts, S as parseEvalDiscovery, _ as runTouchesEval, a as validateTagsFilters, b as deriveEvalFreshness, c as getLatestRunInfos, d as nextShortIdFromSnapshots, f as persistCaseDetail, g as recomputePersistedCaseStatus, h as recomputeEvalStatusesInRuns, i as resolveEvalTags, l as loadPersistedRunSnapshot, m as persistRunState, n as getTargetEvalKeys, o as generateRunId, p as deleteTemporaryRuns, s as getLastRunStatuses, u as loadPersistedRunSnapshots, v as buildManualInputDescriptor, x as loadIsolatedEvalRegistry, y as parseManualInputValues } from "./runOrchestration-CxjiQmof.mjs";
 import { copyFile, mkdir, readFile, rm, writeFile } from "node:fs/promises";
 import { basename, dirname, extname, isAbsolute, join, relative, resolve, sep } from "node:path";
 import { createHash, randomUUID } from "node:crypto";
@@ -733,10 +733,24 @@ function handleRunChildMessage(params) {
 	handleRunChildEvent(runState, message.event, managerContext);
 }
 function upsertFinishedCase(runState, caseDetail, caseRow) {
+	removeLiveCaseRows(runState.cases, caseRow);
+	upsertCaseRow(runState, caseRow);
+	runState.caseDetails.set(caseDetail.caseKey ?? caseDetail.caseId, caseDetail);
+}
+function upsertCaseRow(runState, caseRow) {
 	const existingIndex = runState.cases.findIndex((row) => getCaseRowCaseKey(row) === getCaseRowCaseKey(caseRow) && row.trial === caseRow.trial);
 	if (existingIndex === -1) runState.cases.push(caseRow);
 	else runState.cases[existingIndex] = caseRow;
-	runState.caseDetails.set(caseDetail.caseKey ?? caseDetail.caseId, caseDetail);
+}
+function removeLiveCaseRows(caseRows, nextCaseRow) {
+	const caseKey = getCaseRowCaseKey(nextCaseRow);
+	for (let i = caseRows.length - 1; i >= 0; i--) {
+		const caseRow = caseRows[i];
+		if (caseRow === void 0) continue;
+		if (getCaseRowCaseKey(caseRow) !== caseKey) continue;
+		if (caseRow.status !== "pending" && caseRow.status !== "running") continue;
+		caseRows.splice(i, 1);
+	}
 }
 function applyChildEvalMetas(evals, childMetas) {
 	for (const childMeta of childMetas) {
@@ -763,6 +777,12 @@ function handleRunChildEvent(runState, event, managerContext) {
 		managerContext.emitEvent(runState, event);
 		return;
 	}
+	if (event.type === "case.started" || event.type === "case.updated") {
+		const parsed = caseRowSchema.safeParse(event.payload);
+		if (parsed.success) upsertCaseRow(runState, parsed.data);
+		managerContext.emitEvent(runState, event);
+		return;
+	}
 	if (event.type === "run.finished") {
 		runState.childTerminalReceived = true;
 		runState.childProcess = void 0;
@@ -929,6 +949,10 @@ function createRunner({ watchForChanges = true } = {}) {
 	function getSourceFingerprint(source) {
 		return createHash("sha256").update(source).digest("hex");
 	}
+	function getConfiguredConcurrency() {
+		if (typeof config.concurrency !== "number" || !Number.isFinite(config.concurrency)) return 1;
+		return Math.max(1, Math.floor(config.concurrency));
+	}
 	function nextRegistryLoadIsolationKey(prefix, filePath) {
 		registryLoadCounter++;
 		return `${prefix}:${String(registryLoadCounter)}:${filePath}`;
@@ -1142,6 +1166,7 @@ function createRunner({ watchForChanges = true } = {}) {
 		getConfigReloadState() {
 			return configReload.currentState();
 		},
+		getConfiguredConcurrency,
 		async refreshDiscovery() {
 			const patterns = config.include;
 			const discovered = [];
@@ -1385,6 +1410,7 @@ function createRunner({ watchForChanges = true } = {}) {
 			if (!run) return;
 			if (run.manifest.status !== "running") return;
 			const endedAt = /* @__PURE__ */ new Date();
+			run.cases = run.cases.filter((caseRow) => caseRow.status !== "pending" && caseRow.status !== "running");
 			run.manifest.status = "cancelled";
 			run.manifest.endedAt = endedAt.toISOString();
 			run.summary.status = "cancelled";
@@ -2172,8 +2198,8 @@ async function commandApp(args) {
 	const { serve } = await import("@hono/node-server");
 	const bundledWebDist = resolve(currentDir, "apps/web/dist");
 	if (existsSync(bundledWebDist)) process.env.AGENT_EVALS_WEB_DIST = bundledWebDist;
-	const appModule = await import("./app-DhMIbjlE.mjs");
-	const runnerModule = await import("./runner-BKogjiYd.mjs");
+	const appModule = await import("./app-ROCEce9X.mjs");
+	const runnerModule = await import("./runner-BlFQyvN2.mjs");
 	if (!isHonoAppModule(appModule)) throw new Error("Server app module is invalid");
 	if (!isServerRunnerModule(runnerModule)) throw new Error("Server runner module is invalid");
 	await runnerModule.initRunner();