@ls-stack/agent-eval 0.6.0 → 0.7.0

package/dist/index.d.mts

@@ -1887,7 +1887,6 @@ type EvalTraceTree = {
 /** Context passed to an eval's `execute` function for a single case run. */
 type EvalExecuteContext<TInput> = {
   input: TInput;
-  signal: AbortSignal;
 };
 /** Context passed to `deriveFromTracing` after execution has completed. */
 type EvalDeriveContext<TInput> = {
@@ -2383,8 +2382,8 @@ type EvalRunner = {
     manifest: RunManifest;
     summary: RunSummary;
     cases: CaseRow[];
-  } | undefined; /** Request cancellation for an in-flight run. */
-  cancelRun(id: string): void; /** Return full details for a single case in a run, when available. */
+  } | undefined; /** Request cancellation for an in-flight run and persist its cancelled state. */
+  cancelRun(id: string): Promise<void>; /** Return full details for a single case in a run, when available. */
   getCaseDetail(runId: string, caseId: string): CaseDetail | undefined; /** Subscribe to streamed events for a specific run. */
   subscribe(runId: string, listener: (event: SseEnvelope) => void): () => void; /** Subscribe to discovery updates triggered by file changes or manual refresh. */
   subscribeDiscovery(listener: (event: SseEnvelope) => void): () => void; /** Stop background filesystem watchers owned by this runner instance. */

package/dist/index.mjs

@@ -1,3 +1,4 @@
-import { $ as columnDefSchema, A as evalStatsConfigSchema, B as evalChartTypeSchema, C as spanCacheOptionsSchema, Ct as setEvalOutput, D as evalFreshnessStatusSchema, Dt as getEvalRegistry, E as caseRowSchema, Et as defineEval, F as evalChartBuiltinMetricSchema, G as traceAttributeDisplaySchema, H as traceAttributeDisplayFormatSchema, I as evalChartColorSchema, J as traceSpanErrorSchema, K as traceDisplayConfigSchema, L as evalChartConfigSchema, M as scoreTraceSchema, N as evalChartAggregateSchema, O as evalStatAggregateSchema, P as evalChartAxisSchema, Q as cellValueSchema, R as evalChartMetricSchema, S as serializedCacheSpanSchema, St as runInEvalScope, T as caseDetailSchema, Tt as repoFile, U as traceAttributeDisplayInputSchema, V as evalChartsConfigSchema, W as traceAttributeDisplayPlacementSchema, X as traceSpanSchema, Y as traceSpanKindSchema, Z as traceSpanWarningSchema, _ as cacheListItemSchema, _t as getCurrentScope, a as sseEnvelopeSchema, at as repoFileRefSchema, b as cacheRecordingOpSchema, bt as isInEvalScope, c as deriveScopedSummaryFromCases, ct as buildTraceTree, d as runManifestSchema, dt as evalTracer, et as columnFormatSchema, f as runSummarySchema, ft as hashCacheKey, g as cacheFileSchema, gt as evalAssert, h as cacheEntrySchema, ht as appendToEvalOutput, i as updateManualScoreRequestSchema, it as numberDisplayOptionsSchema, j as evalSummarySchema, k as evalStatItemSchema, l as deriveStatusFromCaseRows, lt as captureEvalSpanError, m as trialSelectionModeSchema, mt as EvalAssertionError, n as createRunner, nt as fileRefSchema, o as getEvalTitle, ot as runArtifactRefSchema, p as agentEvalsConfigSchema, pt as hashCacheKeySync, q as traceDisplayInputConfigSchema, r as createRunRequestSchema, rt as jsonCellSchema, s as getEvalDisplayStatus, st as z, t as runCli, tt as columnKindSchema, u as deriveStatusFromChildStatuses, ut as evalSpan, v as cacheModeSchema, vt as getEvalCaseInput, w as assertionFailureSchema, wt as setScopeCacheContext, x as cacheRecordingSchema, xt as mergeEvalOutput, y as cacheOperationTypeSchema, yt as incrementEvalOutput, z as evalChartTooltipExtraSchema } from "./cli-C-n-Fd4o.mjs";
-import "./src-Dy31CPXH.mjs";
+import { $ as evalChartAxisSchema, A as runManifestSchema, At as evalTracer, B as cacheRecordingSchema, Bt as mergeEvalOutput, C as updateManualScoreRequestSchema, Ct as numberDisplayOptionsSchema, D as deriveScopedSummaryFromCases, Dt as buildTraceTree, E as getEvalDisplayStatus, Et as z, F as cacheFileSchema, Ft as evalAssert, G as caseRowSchema, Gt as defineEval, H as spanCacheOptionsSchema, Ht as setEvalOutput, I as cacheListItemSchema, It as getCurrentScope, J as evalStatItemSchema, K as evalFreshnessStatusSchema, Kt as getEvalRegistry, L as cacheModeSchema, Lt as getEvalCaseInput, M as agentEvalsConfigSchema, Mt as hashCacheKeySync, N as trialSelectionModeSchema, Nt as EvalAssertionError, O as deriveStatusFromCaseRows, Ot as captureEvalSpanError, P as cacheEntrySchema, Pt as appendToEvalOutput, Q as evalChartAggregateSchema, R as cacheOperationTypeSchema, Rt as incrementEvalOutput, S as createRunRequestSchema, St as jsonCellSchema, T as getEvalTitle, Tt as runArtifactRefSchema, U as assertionFailureSchema, Ut as setScopeCacheContext, V as serializedCacheSpanSchema, Vt as runInEvalScope, W as caseDetailSchema, Wt as repoFile, X as evalSummarySchema, Y as evalStatsConfigSchema, Z as scoreTraceSchema, _t as cellValueSchema, at as evalChartTypeSchema, bt as columnKindSchema, ct as traceAttributeDisplayInputSchema, dt as traceDisplayConfigSchema, et as evalChartBuiltinMetricSchema, ft as traceDisplayInputConfigSchema, gt as traceSpanWarningSchema, ht as traceSpanSchema, it as evalChartTooltipExtraSchema, j as runSummarySchema, jt as hashCacheKey, k as deriveStatusFromChildStatuses, kt as evalSpan, lt as traceAttributeDisplayPlacementSchema, mt as traceSpanKindSchema, nt as evalChartConfigSchema, ot as evalChartsConfigSchema, pt as traceSpanErrorSchema, q as evalStatAggregateSchema, rt as evalChartMetricSchema, st as traceAttributeDisplayFormatSchema, tt as evalChartColorSchema, ut as traceAttributeDisplaySchema, vt as columnDefSchema, w as sseEnvelopeSchema, wt as repoFileRefSchema, xt as fileRefSchema, yt as columnFormatSchema, z as cacheRecordingOpSchema, zt as isInEvalScope } from "./runOrchestration-HaMahl6b.mjs";
+import { n as createRunner, t as runCli } from "./cli-BTtgQLjB.mjs";
+import "./src-Bt5Fz9HS.mjs";
 export { EvalAssertionError, agentEvalsConfigSchema, appendToEvalOutput, assertionFailureSchema, buildTraceTree, cacheEntrySchema, cacheFileSchema, cacheListItemSchema, cacheModeSchema, cacheOperationTypeSchema, cacheRecordingOpSchema, cacheRecordingSchema, captureEvalSpanError, caseDetailSchema, caseRowSchema, cellValueSchema, columnDefSchema, columnFormatSchema, columnKindSchema, createRunRequestSchema, createRunner, defineEval, deriveScopedSummaryFromCases, deriveStatusFromCaseRows, deriveStatusFromChildStatuses, evalAssert, evalChartAggregateSchema, evalChartAxisSchema, evalChartBuiltinMetricSchema, evalChartColorSchema, evalChartConfigSchema, evalChartMetricSchema, evalChartTooltipExtraSchema, evalChartTypeSchema, evalChartsConfigSchema, evalFreshnessStatusSchema, evalSpan, evalStatAggregateSchema, evalStatItemSchema, evalStatsConfigSchema, evalSummarySchema, evalTracer, fileRefSchema, getCurrentScope, getEvalCaseInput, getEvalDisplayStatus, getEvalRegistry, getEvalTitle, hashCacheKey, hashCacheKeySync, incrementEvalOutput, isInEvalScope, jsonCellSchema, mergeEvalOutput, numberDisplayOptionsSchema, repoFile, repoFileRefSchema, runArtifactRefSchema, runCli, runInEvalScope, runManifestSchema, runSummarySchema, scoreTraceSchema, serializedCacheSpanSchema, setEvalOutput, setScopeCacheContext, spanCacheOptionsSchema, sseEnvelopeSchema, traceAttributeDisplayFormatSchema, traceAttributeDisplayInputSchema, traceAttributeDisplayPlacementSchema, traceAttributeDisplaySchema, traceDisplayConfigSchema, traceDisplayInputConfigSchema, traceSpanErrorSchema, traceSpanKindSchema, traceSpanSchema, traceSpanWarningSchema, trialSelectionModeSchema, updateManualScoreRequestSchema, z };

package/dist/runChild.d.mts

@@ -0,0 +1 @@
+export { };

package/dist/runChild.mjs

@@ -0,0 +1,107 @@
+import { A as runManifestSchema, S as createRunRequestSchema, Y as evalStatsConfigSchema, j as runSummarySchema, ot as evalChartsConfigSchema, t as executeRun, v as loadConfig, vt as columnDefSchema, x as createFsCacheStore } from "./runOrchestration-HaMahl6b.mjs";
+import { createHash } from "node:crypto";
+import { readFile } from "node:fs/promises";
+import { z } from "zod/v4";
+//#region ../runner/src/runChild.ts
+const evalMetaSchema = z.object({
+	id: z.string(),
+	title: z.string().optional(),
+	filePath: z.string(),
+	sourceFilePath: z.string(),
+	sourceFingerprint: z.string().nullable(),
+	columnDefs: z.array(columnDefSchema),
+	caseCount: z.number().nullable(),
+	stats: evalStatsConfigSchema.optional(),
+	charts: evalChartsConfigSchema.optional()
+});
+const runChildContextSchema = z.object({
+	request: createRunRequestSchema,
+	workspaceRoot: z.string(),
+	runDir: z.string(),
+	manifest: runManifestSchema,
+	summary: runSummarySchema,
+	evals: z.array(evalMetaSchema)
+});
+function sendMessage(message) {
+	if (process.send === void 0) return;
+	process.send(message);
+}
+function getSourceFingerprint(source) {
+	return createHash("sha256").update(source).digest("hex");
+}
+function getConfiguredConcurrency(configConcurrency) {
+	if (typeof configConcurrency !== "number" || !Number.isFinite(configConcurrency)) return 1;
+	return Math.max(1, Math.floor(configConcurrency));
+}
+function getTargetEvals(params) {
+	if (params.request.target.evalIds && params.request.target.evalIds.length > 0) return params.request.target.evalIds.map((id) => params.evals.get(id)).filter((entry) => entry !== void 0);
+	return [...params.evals.values()].toSorted((a, b) => a.filePath.localeCompare(b.filePath));
+}
+async function readContext(contextPath) {
+	if (contextPath === void 0) throw new Error("Missing run child context path");
+	return runChildContextSchema.parse(JSON.parse(await readFile(contextPath, "utf-8")));
+}
+async function main() {
+	process.on("disconnect", () => {
+		process.exit(1);
+	});
+	const context = await readContext(process.argv[2]);
+	process.chdir(context.workspaceRoot);
+	const config = await loadConfig();
+	const cacheStore = createFsCacheStore({
+		workspaceRoot: context.workspaceRoot,
+		dir: config.cache?.dir,
+		maxEntriesPerEval: config.cache?.maxEntriesPerEval
+	});
+	const evals = new Map(context.evals.map((evalMeta) => [evalMeta.id, evalMeta]));
+	const lastRunStatusMap = /* @__PURE__ */ new Map();
+	const latestRunInfoMap = /* @__PURE__ */ new Map();
+	await executeRun({
+		runState: {
+			runDir: context.runDir,
+			manifest: context.manifest,
+			summary: context.summary,
+			cases: [],
+			caseDetails: /* @__PURE__ */ new Map(),
+			listeners: /* @__PURE__ */ new Set()
+		},
+		request: context.request,
+		runDir: context.runDir,
+		config,
+		evals,
+		cacheStore,
+		lastRunStatusMap,
+		latestRunInfoMap,
+		emitEvent(_runState, event) {
+			if (event.type === "case.finished") return;
+			sendMessage({
+				type: "event",
+				event
+			});
+		},
+		emitDiscoveryEvent() {},
+		workspaceRoot: context.workspaceRoot,
+		getSourceFingerprint,
+		getConfiguredConcurrency: () => getConfiguredConcurrency(config.concurrency),
+		getSortedEvalMetas: () => [...evals.values()].toSorted((a, b) => a.filePath.localeCompare(b.filePath)),
+		getTargetEvals: (request) => getTargetEvals({
+			evals,
+			request
+		}),
+		onCaseFinished(caseDetail, caseRow) {
+			sendMessage({
+				type: "case.finished",
+				caseDetail,
+				caseRow
+			});
+		}
+	});
+	sendMessage({
+		type: "done",
+		evals: [...evals.values()]
+	});
+}
+await main();
+process.disconnect();
+//#endregion
+export {};