@ls-stack/agent-eval 0.5.0 → 0.7.0

package/dist/index.mjs

@@ -1,3 +1,4 @@
-import { $ as columnKindSchema, A as evalSummarySchema, B as evalChartsConfigSchema, C as assertionFailureSchema, D as evalStatAggregateSchema, E as evalFreshnessStatusSchema, F as evalChartColorSchema, G as traceDisplayConfigSchema, H as traceAttributeDisplayInputSchema, I as evalChartConfigSchema, J as traceSpanKindSchema, K as traceDisplayInputConfigSchema, L as evalChartMetricSchema, M as evalChartAggregateSchema, N as evalChartAxisSchema, O as evalStatItemSchema, P as evalChartBuiltinMetricSchema, Q as columnFormatSchema, R as evalChartTooltipExtraSchema, S as spanCacheOptionsSchema, T as caseRowSchema, U as traceAttributeDisplayPlacementSchema, V as traceAttributeDisplayFormatSchema, W as traceAttributeDisplaySchema, X as cellValueSchema, Y as traceSpanSchema, Z as columnDefSchema, _ as cacheListItemSchema, _t as setEvalOutput, a as sseEnvelopeSchema, at as buildTraceTree, b as cacheRecordingSchema, bt as defineEval, c as deriveScopedSummaryFromCases, ct as evalTracer, d as runManifestSchema, dt as EvalAssertionError, et as fileRefSchema, f as runSummarySchema, ft as evalAssert, g as cacheFileSchema, gt as runInEvalScope, h as cacheEntrySchema, ht as isInEvalScope, i as updateManualScoreRequestSchema, it as runArtifactRefSchema, j as scoreTraceSchema, k as evalStatsConfigSchema, l as deriveStatusFromCaseRows, lt as hashCacheKey, m as trialSelectionModeSchema, mt as incrementEvalOutput, n as createRunner, nt as numberDisplayOptionsSchema, o as getEvalTitle, ot as captureEvalSpanError, p as agentEvalsConfigSchema, pt as getCurrentScope, q as traceSpanErrorSchema, r as createRunRequestSchema, rt as repoFileRefSchema, s as getEvalDisplayStatus, st as evalSpan, t as runCli, tt as jsonCellSchema, u as deriveStatusFromChildStatuses, ut as hashCacheKeySync, v as cacheModeSchema, vt as setScopeCacheContext, w as caseDetailSchema, x as serializedCacheSpanSchema, xt as getEvalRegistry, y as cacheRecordingOpSchema, yt as repoFile, z as evalChartTypeSchema } from "./cli-C5FL7C4G.mjs";
-import "./src-gqm1z1Nu.mjs";
-export { EvalAssertionError, agentEvalsConfigSchema, assertionFailureSchema, buildTraceTree, cacheEntrySchema, cacheFileSchema, cacheListItemSchema, cacheModeSchema, cacheRecordingOpSchema, cacheRecordingSchema, captureEvalSpanError, caseDetailSchema, caseRowSchema, cellValueSchema, columnDefSchema, columnFormatSchema, columnKindSchema, createRunRequestSchema, createRunner, defineEval, deriveScopedSummaryFromCases, deriveStatusFromCaseRows, deriveStatusFromChildStatuses, evalAssert, evalChartAggregateSchema, evalChartAxisSchema, evalChartBuiltinMetricSchema, evalChartColorSchema, evalChartConfigSchema, evalChartMetricSchema, evalChartTooltipExtraSchema, evalChartTypeSchema, evalChartsConfigSchema, evalFreshnessStatusSchema, evalSpan, evalStatAggregateSchema, evalStatItemSchema, evalStatsConfigSchema, evalSummarySchema, evalTracer, fileRefSchema, getCurrentScope, getEvalDisplayStatus, getEvalRegistry, getEvalTitle, hashCacheKey, hashCacheKeySync, incrementEvalOutput, isInEvalScope, jsonCellSchema, numberDisplayOptionsSchema, repoFile, repoFileRefSchema, runArtifactRefSchema, runCli, runInEvalScope, runManifestSchema, runSummarySchema, scoreTraceSchema, serializedCacheSpanSchema, setEvalOutput, setScopeCacheContext, spanCacheOptionsSchema, sseEnvelopeSchema, traceAttributeDisplayFormatSchema, traceAttributeDisplayInputSchema, traceAttributeDisplayPlacementSchema, traceAttributeDisplaySchema, traceDisplayConfigSchema, traceDisplayInputConfigSchema, traceSpanErrorSchema, traceSpanKindSchema, traceSpanSchema, trialSelectionModeSchema, updateManualScoreRequestSchema };
+import { $ as evalChartAxisSchema, A as runManifestSchema, At as evalTracer, B as cacheRecordingSchema, Bt as mergeEvalOutput, C as updateManualScoreRequestSchema, Ct as numberDisplayOptionsSchema, D as deriveScopedSummaryFromCases, Dt as buildTraceTree, E as getEvalDisplayStatus, Et as z, F as cacheFileSchema, Ft as evalAssert, G as caseRowSchema, Gt as defineEval, H as spanCacheOptionsSchema, Ht as setEvalOutput, I as cacheListItemSchema, It as getCurrentScope, J as evalStatItemSchema, K as evalFreshnessStatusSchema, Kt as getEvalRegistry, L as cacheModeSchema, Lt as getEvalCaseInput, M as agentEvalsConfigSchema, Mt as hashCacheKeySync, N as trialSelectionModeSchema, Nt as EvalAssertionError, O as deriveStatusFromCaseRows, Ot as captureEvalSpanError, P as cacheEntrySchema, Pt as appendToEvalOutput, Q as evalChartAggregateSchema, R as cacheOperationTypeSchema, Rt as incrementEvalOutput, S as createRunRequestSchema, St as jsonCellSchema, T as getEvalTitle, Tt as runArtifactRefSchema, U as assertionFailureSchema, Ut as setScopeCacheContext, V as serializedCacheSpanSchema, Vt as runInEvalScope, W as caseDetailSchema, Wt as repoFile, X as evalSummarySchema, Y as evalStatsConfigSchema, Z as scoreTraceSchema, _t as cellValueSchema, at as evalChartTypeSchema, bt as columnKindSchema, ct as traceAttributeDisplayInputSchema, dt as traceDisplayConfigSchema, et as evalChartBuiltinMetricSchema, ft as traceDisplayInputConfigSchema, gt as traceSpanWarningSchema, ht as traceSpanSchema, it as evalChartTooltipExtraSchema, j as runSummarySchema, jt as hashCacheKey, k as deriveStatusFromChildStatuses, kt as evalSpan, lt as traceAttributeDisplayPlacementSchema, mt as traceSpanKindSchema, nt as evalChartConfigSchema, ot as evalChartsConfigSchema, pt as traceSpanErrorSchema, q as evalStatAggregateSchema, rt as evalChartMetricSchema, st as traceAttributeDisplayFormatSchema, tt as evalChartColorSchema, ut as traceAttributeDisplaySchema, vt as columnDefSchema, w as sseEnvelopeSchema, wt as repoFileRefSchema, xt as fileRefSchema, yt as columnFormatSchema, z as cacheRecordingOpSchema, zt as isInEvalScope } from "./runOrchestration-HaMahl6b.mjs";
+import { n as createRunner, t as runCli } from "./cli-BTtgQLjB.mjs";
+import "./src-Bt5Fz9HS.mjs";
+export { EvalAssertionError, agentEvalsConfigSchema, appendToEvalOutput, assertionFailureSchema, buildTraceTree, cacheEntrySchema, cacheFileSchema, cacheListItemSchema, cacheModeSchema, cacheOperationTypeSchema, cacheRecordingOpSchema, cacheRecordingSchema, captureEvalSpanError, caseDetailSchema, caseRowSchema, cellValueSchema, columnDefSchema, columnFormatSchema, columnKindSchema, createRunRequestSchema, createRunner, defineEval, deriveScopedSummaryFromCases, deriveStatusFromCaseRows, deriveStatusFromChildStatuses, evalAssert, evalChartAggregateSchema, evalChartAxisSchema, evalChartBuiltinMetricSchema, evalChartColorSchema, evalChartConfigSchema, evalChartMetricSchema, evalChartTooltipExtraSchema, evalChartTypeSchema, evalChartsConfigSchema, evalFreshnessStatusSchema, evalSpan, evalStatAggregateSchema, evalStatItemSchema, evalStatsConfigSchema, evalSummarySchema, evalTracer, fileRefSchema, getCurrentScope, getEvalCaseInput, getEvalDisplayStatus, getEvalRegistry, getEvalTitle, hashCacheKey, hashCacheKeySync, incrementEvalOutput, isInEvalScope, jsonCellSchema, mergeEvalOutput, numberDisplayOptionsSchema, repoFile, repoFileRefSchema, runArtifactRefSchema, runCli, runInEvalScope, runManifestSchema, runSummarySchema, scoreTraceSchema, serializedCacheSpanSchema, setEvalOutput, setScopeCacheContext, spanCacheOptionsSchema, sseEnvelopeSchema, traceAttributeDisplayFormatSchema, traceAttributeDisplayInputSchema, traceAttributeDisplayPlacementSchema, traceAttributeDisplaySchema, traceDisplayConfigSchema, traceDisplayInputConfigSchema, traceSpanErrorSchema, traceSpanKindSchema, traceSpanSchema, traceSpanWarningSchema, trialSelectionModeSchema, updateManualScoreRequestSchema, z };

package/dist/runChild.d.mts

@@ -0,0 +1 @@
+export { };

package/dist/runChild.mjs

@@ -0,0 +1,107 @@
+import { A as runManifestSchema, S as createRunRequestSchema, Y as evalStatsConfigSchema, j as runSummarySchema, ot as evalChartsConfigSchema, t as executeRun, v as loadConfig, vt as columnDefSchema, x as createFsCacheStore } from "./runOrchestration-HaMahl6b.mjs";
+import { createHash } from "node:crypto";
+import { readFile } from "node:fs/promises";
+import { z } from "zod/v4";
+//#region ../runner/src/runChild.ts
+const evalMetaSchema = z.object({
+	id: z.string(),
+	title: z.string().optional(),
+	filePath: z.string(),
+	sourceFilePath: z.string(),
+	sourceFingerprint: z.string().nullable(),
+	columnDefs: z.array(columnDefSchema),
+	caseCount: z.number().nullable(),
+	stats: evalStatsConfigSchema.optional(),
+	charts: evalChartsConfigSchema.optional()
+});
+const runChildContextSchema = z.object({
+	request: createRunRequestSchema,
+	workspaceRoot: z.string(),
+	runDir: z.string(),
+	manifest: runManifestSchema,
+	summary: runSummarySchema,
+	evals: z.array(evalMetaSchema)
+});
+function sendMessage(message) {
+	if (process.send === void 0) return;
+	process.send(message);
+}
+function getSourceFingerprint(source) {
+	return createHash("sha256").update(source).digest("hex");
+}
+function getConfiguredConcurrency(configConcurrency) {
+	if (typeof configConcurrency !== "number" || !Number.isFinite(configConcurrency)) return 1;
+	return Math.max(1, Math.floor(configConcurrency));
+}
+function getTargetEvals(params) {
+	if (params.request.target.evalIds && params.request.target.evalIds.length > 0) return params.request.target.evalIds.map((id) => params.evals.get(id)).filter((entry) => entry !== void 0);
+	return [...params.evals.values()].toSorted((a, b) => a.filePath.localeCompare(b.filePath));
+}
+async function readContext(contextPath) {
+	if (contextPath === void 0) throw new Error("Missing run child context path");
+	return runChildContextSchema.parse(JSON.parse(await readFile(contextPath, "utf-8")));
+}
+async function main() {
+	process.on("disconnect", () => {
+		process.exit(1);
+	});
+	const context = await readContext(process.argv[2]);
+	process.chdir(context.workspaceRoot);
+	const config = await loadConfig();
+	const cacheStore = createFsCacheStore({
+		workspaceRoot: context.workspaceRoot,
+		dir: config.cache?.dir,
+		maxEntriesPerEval: config.cache?.maxEntriesPerEval
+	});
+	const evals = new Map(context.evals.map((evalMeta) => [evalMeta.id, evalMeta]));
+	const lastRunStatusMap = /* @__PURE__ */ new Map();
+	const latestRunInfoMap = /* @__PURE__ */ new Map();
+	await executeRun({
+		runState: {
+			runDir: context.runDir,
+			manifest: context.manifest,
+			summary: context.summary,
+			cases: [],
+			caseDetails: /* @__PURE__ */ new Map(),
+			listeners: /* @__PURE__ */ new Set()
+		},
+		request: context.request,
+		runDir: context.runDir,
+		config,
+		evals,
+		cacheStore,
+		lastRunStatusMap,
+		latestRunInfoMap,
+		emitEvent(_runState, event) {
+			if (event.type === "case.finished") return;
+			sendMessage({
+				type: "event",
+				event
+			});
+		},
+		emitDiscoveryEvent() {},
+		workspaceRoot: context.workspaceRoot,
+		getSourceFingerprint,
+		getConfiguredConcurrency: () => getConfiguredConcurrency(config.concurrency),
+		getSortedEvalMetas: () => [...evals.values()].toSorted((a, b) => a.filePath.localeCompare(b.filePath)),
+		getTargetEvals: (request) => getTargetEvals({
+			evals,
+			request
+		}),
+		onCaseFinished(caseDetail, caseRow) {
+			sendMessage({
+				type: "case.finished",
+				caseDetail,
+				caseRow
+			});
+		}
+	});
+	sendMessage({
+		type: "done",
+		evals: [...evals.values()]
+	});
+}
+await main();
+process.disconnect();
+//#endregion
+export {};