@leostera/evalkit 0.6.1 → 0.8.0

package/dist/{src-CZPufWN_.mjs → src-LeGSyDD2.mjs}

@@ -283,10 +283,10 @@ function catalogScorer(rule, at) {
 }
 function catalogStep(step, at) {
 	switch (step.kind) {
-		case "user": return {
+		case "user": return typeof step.message === "string" ? {
 			kind: "user",
 			message: step.message
-		};
+		} : { kind: "user" };
 		case "agent": return {
 			kind: "agent",
 			expectation: step.expectation
@@ -316,6 +316,22 @@ function shapeOf(value) {
 	return { type: typeof json };
 }
 //#endregion
+//#region ../core/src/config.ts
+/** Left out of candidate snapshots unless `artifacts.exclude` says otherwise. */
+const DEFAULT_ARTIFACT_EXCLUDE = [
+	".git",
+	"node_modules",
+	".venv",
+	"__pycache__",
+	"_build",
+	"_opam",
+	"target"
+];
+/** Pure, typed project configuration. Loading/discovery belongs to the CLI. */
+function defineConfig(configuration) {
+	return configuration;
+}
+//#endregion
 //#region ../core/src/canonical-json.ts
 /** Canonical JSON: keys sorted recursively, array order preserved. Rejects non-JSON inputs. */
 function canonicalParameters(value) {
@@ -603,6 +619,8 @@ const TrialMetadataSchema = Schema.Struct({
 	matrix: Schema.optional(MatrixCellSchema),
 	aut: Schema.optional(AutIdentitySchema),
 	startedAt: Schema.String,
+	/** The effective snapshot ignore patterns: the project's, then the scenario's. */
+	workspaceIgnore: Schema.optional(Schema.mutable(Schema.Array(Schema.String))),
 	datasets: Schema.optional(Schema.mutable(Schema.Array(Schema.Struct({
 		id: Schema.String,
 		visibility: Schema.Literal("candidate", "evaluator"),
@@ -663,8 +681,17 @@ const CheckpointResultSchema = Schema.Struct({
 		id: Schema.String
 	}))
 });
+const MetricResultSchema = Schema.Struct({
+	name: Schema.String,
+	value: Schema.optional(Schema.Number),
+	unit: Schema.optional(Schema.String),
+	better: Schema.optional(Schema.Literal("lower", "higher")),
+	builtIn: Schema.optional(Schema.Literal(true)),
+	error: Schema.optional(RecordedErrorSchema)
+});
 const TrialScoringSchema = Schema.Struct({
 	results: Schema.mutable(Schema.Array(ScoreResultSchema)),
+	metrics: Schema.optional(Schema.mutable(Schema.Array(MetricResultSchema))),
 	checkpoints: Schema.optional(Schema.mutable(Schema.Array(CheckpointResultSchema))),
 	skippedScorers: Schema.optional(Schema.mutable(Schema.Array(Schema.String))),
 	overall: Schema.optional(Schema.Number),
@@ -874,6 +901,10 @@ function expectToolCall(expected) {
 		expected
 	};
 }
+/**
+* A user message. Pass a function to compute it for each trial, e.g. from a
+* matrix axis: `user(({ parameters }) => `Set up OCaml with ${parameters.toolchain}.`)`.
+*/
 function user(message) {
 	return {
 		kind: "user",
@@ -907,6 +938,15 @@ function predicate(name, run, options = {}) {
 		...options
 	});
 }
+function metric(name, run, options = {}) {
+	if (!name.trim()) throw new Error("Metric name must not be empty");
+	return markDefinitionSite({
+		kind: "metric",
+		name,
+		run,
+		...options
+	});
+}
 function judge(name, options) {
 	if (!name.trim() || !options.rubric.trim()) throw new Error("Judge name and rubric must not be empty");
 	return markDefinitionSite({
@@ -919,12 +959,19 @@ function defineEval(definition) {
 	authoringId(definition);
 	if ("fixtures" in definition) throw new Error(`Eval ${definition.id} uses fixtures, which were replaced by datasets: [defineDataset(...)]`);
 	if (definition.metadata !== void 0) toJsonObject(definition.metadata, `${definition.id}.metadata`);
+	for (const [index, pattern] of (definition.workspace?.ignore ?? []).entries()) if (typeof pattern !== "string" || pattern.replace(/^!/, "").trim() === "") throw new Error(`Eval ${definition.id} workspace.ignore[${index}] must be a non-empty glob, got ${JSON.stringify(pattern)}`);
 	const datasetIds = /* @__PURE__ */ new Set();
 	for (const [index, dataset] of (definition.datasets ?? []).entries()) {
 		if (!isDataset(dataset)) throw new Error(`Eval ${definition.id} datasets[${index}] is not a dataset; create it with defineDataset(...)`);
 		if (datasetIds.has(dataset.id)) throw new Error(`Eval ${definition.id} uses dataset ${dataset.id} more than once`);
 		datasetIds.add(dataset.id);
 	}
+	const metricNames = new Set(BUILT_IN_METRICS);
+	for (const [index, entry] of (definition.metrics ?? []).entries()) {
+		if (entry?.kind !== "metric") throw new Error(`Eval ${definition.id} metrics[${index}] is not a metric; create it with metric(...)`);
+		if (metricNames.has(entry.name)) throw new Error(`Eval ${definition.id} has more than one metric named ${entry.name} (built-in: ${BUILT_IN_METRICS.join(", ")})`);
+		metricNames.add(entry.name);
+	}
 	let hasUser = false;
 	for (const step of definition.transcript) if (step.kind === "user") hasUser = true;
 	else if (step.kind !== "agent" && !hasUser) throw new Error("Checkpoint requires a preceding user step");
@@ -1001,6 +1048,12 @@ function registerEvals(registrations) {
 		catalog: (options) => evals.map((evaluation) => catalogEntry(evaluation, suiteIdByEvalId.get(authoringId(evaluation)), options))
 	};
 }
+/** Metrics EvalKit records for every trial; scenario metrics may not reuse these names. */
+const BUILT_IN_METRICS = [
+	"agent time",
+	"tokens",
+	"tool calls"
+];
 function recordError(error) {
 	if (error instanceof Error) return {
 		name: error.name,
@@ -1013,4 +1066,4 @@ function recordError(error) {
 	};
 }
 //#endregion
-export { resourceUriSchema as $, TrialManifestSchema as A, NotJsonError as B, RunManifestSchema as C, RunSummarySchema as D, RunStatusSchema as E, normalizeAutEvent as F, relativeDatasetPath as G, toJsonObject as H, turnResultEvents as I, ResourceUriSchema as J, definitionSite as K, defineEvalMatrix as L, TrialScoringSchema as M, TrialSummarySchema as N, ScoreResultSchema as O, UsageSchema as P, resourceUri as Q, canonicalParameters as R, RecordedErrorSchema as S, RunSourceSchema as T, defineDataset as U, toJson as V, isDataset as W, authorId as X, UuidSchema as Y, parseResourceUri as Z, ArtifactEntrySchema as _, defineAgent as a, JsonValueSchema as b, expectToolCall as c, recordError as d, resourceUuid as et, registerEvals as f, ApiErrorSchema as g, parseTrajectoryJsonl as h, authoringId as i, TrialMetadataSchema as j, TrajectoryEventSchema as k, judge as l, user as m, agent as n, defineEval as o, skip as p, ResourceKindSchema as q, agentRuntimes as r, defineSuite as s, DEFAULT_AGENT_RUNTIMES as t, predicate as u, AutIdentitySchema as v, RunMetadataSchema as w, JudgeRunInfoSchema as x, CheckpointResultSchema as y, catalogEntry as z };
+export { ResourceUriSchema as $, RunSummarySchema as A, defineEvalMatrix as B, JudgeRunInfoSchema as C, RunMetadataSchema as D, RunManifestSchema as E, TrialScoringSchema as F, NotJsonError as G, DEFAULT_ARTIFACT_EXCLUDE as H, TrialSummarySchema as I, defineDataset as J, toJson as K, UsageSchema as L, TrajectoryEventSchema as M, TrialManifestSchema as N, RunSourceSchema as O, TrialMetadataSchema as P, ResourceKindSchema as Q, normalizeAutEvent as R, JsonValueSchema as S, RecordedErrorSchema as T, defineConfig as U, canonicalParameters as V, catalogEntry as W, relativeDatasetPath as X, isDataset as Y, definitionSite as Z, parseTrajectoryJsonl as _, authoringId as a, resourceUuid as at, AutIdentitySchema as b, defineSuite as c, metric as d, UuidSchema as et, predicate as f, user as g, skip as h, agentRuntimes as i, resourceUriSchema as it, ScoreResultSchema as j, RunStatusSchema as k, expectToolCall as l, registerEvals as m, DEFAULT_AGENT_RUNTIMES as n, parseResourceUri as nt, defineAgent as o, recordError as p, toJsonObject as q, agent as r, resourceUri as rt, defineEval as s, BUILT_IN_METRICS as t, authorId as tt, judge as u, ApiErrorSchema as v, MetricResultSchema as w, CheckpointResultSchema as x, ArtifactEntrySchema as y, turnResultEvents as z };

package/dist/{src-CKqOzeBi.mjs → src-mm8L_IxM.mjs}

@@ -1,4 +1,4 @@
-import { A as TrialManifestSchema, C as RunManifestSchema, D as RunSummarySchema, F as normalizeAutEvent, G as relativeDatasetPath, H as toJsonObject, I as turnResultEvents, M as TrialScoringSchema, N as TrialSummarySchema, O as ScoreResultSchema, Q as resourceUri, R as canonicalParameters, V as toJson, b as JsonValueSchema, d as recordError, h as parseTrajectoryJsonl, i as authoringId, j as TrialMetadataSchema, k as TrajectoryEventSchema, r as agentRuntimes, w as RunMetadataSchema, y as CheckpointResultSchema } from "./src-CZPufWN_.mjs";
+import { A as RunSummarySchema, D as RunMetadataSchema, E as RunManifestSchema, F as TrialScoringSchema, H as DEFAULT_ARTIFACT_EXCLUDE, I as TrialSummarySchema, K as toJson, M as TrajectoryEventSchema, N as TrialManifestSchema, P as TrialMetadataSchema, R as normalizeAutEvent, S as JsonValueSchema, V as canonicalParameters, X as relativeDatasetPath, _ as parseTrajectoryJsonl, a as authoringId, i as agentRuntimes, j as ScoreResultSchema, p as recordError, q as toJsonObject, rt as resourceUri, x as CheckpointResultSchema, z as turnResultEvents } from "./src-LeGSyDD2.mjs";
 import * as Schema from "effect/Schema";
 import { Data, Effect, Either, Exit, Scope } from "effect";
 import os from "node:os";
@@ -32,30 +32,113 @@ async function unlessAborted(work, signal) {
 	}
 }
 //#endregion
+//#region ../core/src/glob.ts
+/**
+* A glob matcher for discovery: `*` and `?` stay within a path segment, `**`
+* spans segments, and `{a,b}` is an alternation.
+*/
+function globMatcher(pattern) {
+	let source = "";
+	for (let index = 0; index < pattern.length; index++) {
+		const char = pattern[index];
+		if (char === "*" && pattern[index + 1] === "*") {
+			index++;
+			if (pattern[index + 1] === "/") {
+				index++;
+				source += "(?:.*/)?";
+			} else source += ".*";
+		} else if (char === "*") source += "[^/]*";
+		else if (char === "?") source += "[^/]";
+		else if (char === "{") {
+			const end = pattern.indexOf("}", index);
+			if (end === -1) source += "\\{";
+			else {
+				source += `(?:${pattern.slice(index + 1, end).split(",").map((part) => part.replace(/[.+^$()|[\]\\]/g, "\\$&").replace(/\*/g, "[^/]*")).join("|")})`;
+				index = end;
+			}
+		} else source += char.replace(/[.+^$()|[\]\\]/g, "\\$&");
+	}
+	const expression = new RegExp(`^${source}$`);
+	return (path) => expression.test(path);
+}
+//#endregion
 //#region ../runner/src/snapshot.ts
+const GLOB_CHARS = /[*?{]/;
+function ignoreRule(pattern) {
+	const negated = pattern.startsWith("!");
+	const trimmed = (negated ? pattern.slice(1) : pattern).replace(/^\.\//, "").replace(/\/$/, "");
+	const match = globMatcher(trimmed);
+	if (!trimmed.includes("/")) return {
+		negated,
+		match: (relativePath) => match(path.posix.basename(relativePath)),
+		mayMatchBelow: () => true
+	};
+	const segments = trimmed.split("/");
+	const firstGlob = segments.findIndex((segment) => GLOB_CHARS.test(segment));
+	const prefix = segments.slice(0, firstGlob === -1 ? segments.length : firstGlob).join("/");
+	return {
+		negated,
+		match,
+		mayMatchBelow: (directory) => prefix === "" || prefix.startsWith(`${directory}/`) || prefix === directory || directory.startsWith(`${prefix}/`)
+	};
+}
 /**
-* Copies every regular file in the candidate workspace into the trial report.
-* Symbolic links are rejected so snapshots cannot follow paths outside the trial.
+* Ignore rules for candidate snapshots, read like `.gitignore`: globs relative
+* to the workspace root, where a pattern without `/` matches that name at any
+* depth, and later patterns win. `!pattern` re-includes a path; unlike Git, it
+* also works below an ignored directory (`_build` then `!_build/doc/**`).
 */
-async function snapshotCandidateWorkspace(candidateRoot, writer) {
+function ignoreRules(patterns) {
+	const rules = patterns.map(ignoreRule);
+	return {
+		/** Whether a path is ignored, given whether its directory was. */
+		ignored(relativePath, parentIgnored = false) {
+			let ignored = parentIgnored;
+			for (const rule of rules) if (rule.match(relativePath)) ignored = !rule.negated;
+			return ignored;
+		},
+		/** Whether an ignored directory must still be walked for re-included paths. */
+		mayReinclude(directory) {
+			return rules.some((rule) => rule.negated && rule.mayMatchBelow(directory));
+		}
+	};
+}
+/**
+* Copies the candidate workspace's regular files into the trial report,
+* leaving out ignored paths. Symbolic links and special files are skipped,
+* never followed, so a snapshot cannot read outside the trial. An ignored
+* directory is walked only when a `!` pattern could re-include something in
+* it, and is listed only if it ends up containing a snapshotted file.
+*/
+async function snapshotCandidateWorkspace(candidateRoot, writer, ignore = DEFAULT_ARTIFACT_EXCLUDE) {
 	const artifacts = [];
-	async function visit(directory, relativeDirectory) {
+	const rules = ignoreRules(ignore);
+	/** Returns whether anything below `directory` was snapshotted. */
+	async function visit(directory, relativeDirectory, directoryIgnored) {
 		const entries = await readdir(directory, { withFileTypes: true });
 		entries.sort((left, right) => left.name.localeCompare(right.name));
+		let kept = false;
 		for (const entry of entries) {
 			const absolutePath = path.join(directory, entry.name);
 			const relativePath = path.join(relativeDirectory, entry.name);
+			const posixPath = relativePath.split(path.sep).join("/");
+			const ignored = rules.ignored(posixPath, directoryIgnored);
 			const stat = await lstat(absolutePath);
-			if (stat.isSymbolicLink()) throw new Error(`Candidate workspace contains a symbolic link: ${relativePath}`);
+			if (stat.isSymbolicLink()) continue;
 			if (stat.isDirectory()) {
-				artifacts.push({
-					path: path.join("candidate", relativePath),
-					kind: "directory"
-				});
-				await visit(absolutePath, relativePath);
+				if (ignored && !rules.mayReinclude(posixPath)) continue;
+				const entryIndex = artifacts.length;
+				const keptBelow = await visit(absolutePath, relativePath, ignored);
+				if (!ignored || keptBelow) {
+					artifacts.splice(entryIndex, 0, {
+						path: path.join("candidate", relativePath),
+						kind: "directory"
+					});
+					kept = true;
+				}
 				continue;
 			}
-			if (!stat.isFile()) throw new Error(`Candidate workspace contains an unsupported entry: ${relativePath}`);
+			if (ignored || !stat.isFile()) continue;
 			const data = await readFile(absolutePath);
 			const artifactPath = path.join("candidate", relativePath);
 			await writer.writeArtifact(artifactPath, data);
@@ -64,9 +147,11 @@ async function snapshotCandidateWorkspace(candidateRoot, writer) {
 				kind: "file",
 				size: data.byteLength
 			});
+			kept = true;
 		}
+		return kept;
 	}
-	await visit(candidateRoot, "");
+	await visit(candidateRoot, "", false);
 	return artifacts;
 }
 //#endregion
@@ -247,8 +332,9 @@ async function staticDatasetFile(dataset, root, file) {
 }
 //#endregion
 //#region ../runner/src/workspace.ts
-function trialDirectoryName(trialId) {
-	return trialId.replace(/[^a-zA-Z0-9._-]/g, "_");
+/** Where a trial's persistent workspace lives below `root`. */
+function trialWorkspacePath(root, trialId) {
+	return path.join(path.resolve(root), trialId.replace(/[^a-zA-Z0-9._-]/g, "_"));
 }
 /**
 * Creates isolated candidate and evaluator directories and writes each
@@ -256,7 +342,7 @@ function trialDirectoryName(trialId) {
 * Two datasets writing the same file is an error, never an overwrite.
 */
 async function createTrialWorkspace(trialId, datasets = [], persistentRoot) {
-	const root = persistentRoot ? path.join(path.resolve(persistentRoot), trialDirectoryName(trialId)) : await mkdtemp(path.join(os.tmpdir(), "evalkit-trial-"));
+	const root = persistentRoot ? trialWorkspacePath(persistentRoot, trialId) : await mkdtemp(path.join(os.tmpdir(), "evalkit-trial-"));
 	const persistent = persistentRoot !== void 0;
 	const artifacts = {
 		candidate: { root: path.join(root, "candidate") },
@@ -270,10 +356,13 @@ async function createTrialWorkspace(trialId, datasets = [], persistentRoot) {
 		}
 		return {
 			artifacts,
-			cleanup: persistent ? async () => {} : () => rm(root, {
-				recursive: true,
-				force: true
-			})
+			cleanup: async (keep = false) => {
+				if (persistent && keep) return;
+				await rm(root, {
+					recursive: true,
+					force: true
+				});
+			}
 		};
 	} catch (error) {
 		await rm(root, {
@@ -545,7 +634,7 @@ async function executeTranscript(definition, session, input) {
 	let turn = 0;
 	for (const [index, step] of definition.transcript.entries()) {
 		if (input.signal.aborted) throw cancelled();
-		const message = step.kind === "user" ? step.message.replaceAll("{{randomSeed}}", String(input.randomSeed)) : void 0;
+		const message = step.kind === "user" ? userMessage(step, index, input) : void 0;
 		await emitRunner({
 			kind: "transcript-step-started",
 			step: index,
@@ -586,6 +675,18 @@ async function executeTranscript(definition, session, input) {
 		});
 	}
 }
+/** A computed message sees the parameters and candidate data the agent also sees. */
+function userMessage(step, index, { context, randomSeed }) {
+	if (typeof step.message === "string") return step.message;
+	const message = step.message({
+		parameters: context.parameters ?? {},
+		data: context.data,
+		seed: randomSeed,
+		trialIndex: context.trialIndex
+	});
+	if (typeof message !== "string" || !message.trim()) throw new Error(`The user message at step ${index + 1} must compute a non-empty string`);
+	return message;
+}
 async function sendUserTurn(session, message, signal) {
 	return runBoundary(Effect.tryPromise({
 		try: () => unlessAborted(() => session.send(message), signal),
@@ -611,7 +712,8 @@ async function runCheckpoint(definition, step, index, input) {
 		trajectory: { events: input.events },
 		turn: progress.lastTurn,
 		...turnOutput(progress.lastTurn),
-		data: input.data
+		data: input.data,
+		checkpoints: [...progress.checkpoints]
 	}, now, definition.judge)).catch(async (error) => {
 		const recorded = recordError(error);
 		progress.checkpoints.push({
@@ -656,20 +758,33 @@ function turnOutput(turn) {
 }
 //#endregion
 //#region ../runner/src/final-scoring.ts
-function scoreSummary(results, checkpoints, skippedScorers) {
+function scoreSummary(results, checkpoints, skippedScorers, metrics = []) {
 	const valid = results.filter((result) => result.value !== void 0 && !result.skipped);
 	const hasError = results.some((result) => result.error !== void 0);
 	return {
 		results,
+		...metrics.length ? { metrics } : {},
 		...checkpoints.length ? { checkpoints } : {},
 		...skippedScorers.length ? { skippedScorers } : {},
 		...valid.length === 0 ? {} : { overall: valid.reduce((sum, result) => sum + (result.value ?? 0), 0) / valid.length },
 		passed: !hasError && results.every((result) => result.skipped || result.passed === true) && checkpoints.every((result) => result.status === "passed" || result.status === "skipped") && !skippedScorers.length
 	};
 }
+/** What final scorers and metrics receive. */
+function scoringContext(input) {
+	return {
+		context: input.context,
+		artifacts: input.workspace.artifacts,
+		trajectory: { events: input.events },
+		...input.lastTurn ? { turn: input.lastTurn } : {},
+		...turnOutput(input.lastTurn),
+		data: input.data,
+		checkpoints: input.checkpoints
+	};
+}
 /** Append completed scores to the supplied arrays, even when a later scorer or report write fails. */
 async function evaluateFinalScorers(definition, input) {
-	const { context, workspace, events, lastTurn, results, skippedScorers, now, emitRunner } = input;
+	const { context, workspace, results, skippedScorers, now, emitRunner } = input;
 	for (const scorer of definition.scoring) {
 		if (input.cancelled || input.reportingFailed || !context || !workspace || (input.primaryError || input.stoppedEarly) && (!("supportsPartial" in scorer) || !scorer.supportsPartial)) {
 			skippedScorers.push(scorer.name);
@@ -683,14 +798,11 @@ async function evaluateFinalScorers(definition, input) {
 		});
 		let result;
 		try {
-			const score = await runBoundary(evaluateRule(scorer, {
+			const score = await runBoundary(evaluateRule(scorer, scoringContext({
+				...input,
 				context,
-				artifacts: workspace.artifacts,
-				trajectory: { events },
-				...lastTurn ? { turn: lastTurn } : {},
-				...turnOutput(lastTurn),
-				data: input.data
-			}, definition.judge, "scoring").pipe(Effect.mapError((cause) => new ScoringError({
+				workspace
+			}), definition.judge, "scoring").pipe(Effect.mapError((cause) => new ScoringError({
 				cause,
 				message: cause.message
 			}))));
@@ -727,6 +839,68 @@ async function evaluateFinalScorers(definition, input) {
 	}
 }
 //#endregion
+//#region ../runner/src/metrics.ts
+/**
+* What EvalKit measures for every trial from its trajectory: the agent's time
+* (first event to the last one before final scoring), tokens when the agent
+* reported usage, and tool calls.
+*/
+function builtInMetrics(events) {
+	const times = events.filter((event) => !event.kind.startsWith("scorer-")).map((event) => Date.parse(event.timestamp)).filter((time) => !Number.isNaN(time));
+	const metrics = [];
+	if (times.length) metrics.push({
+		name: "agent time",
+		value: Math.max(...times) - Math.min(...times),
+		unit: "ms",
+		better: "lower",
+		builtIn: true
+	});
+	let tokens;
+	for (const event of events) if (event.source === "aut" && event.kind === "turn-completed" && event.usage) {
+		const { totalTokens, inputTokens, outputTokens } = event.usage;
+		const turn = totalTokens ?? (inputTokens === void 0 && outputTokens === void 0 ? void 0 : (inputTokens ?? 0) + (outputTokens ?? 0));
+		if (turn !== void 0) tokens = (tokens ?? 0) + turn;
+	}
+	if (tokens !== void 0) metrics.push({
+		name: "tokens",
+		value: tokens,
+		unit: "tokens",
+		better: "lower",
+		builtIn: true
+	});
+	metrics.push({
+		name: "tool calls",
+		value: events.filter((event) => event.source === "aut" && event.kind === "tool-call").length,
+		builtIn: true
+	});
+	return metrics;
+}
+/** Runs the scenario's metrics; a failing metric records its error and the rest still run. */
+async function evaluateMetrics(definition, context) {
+	const results = [];
+	for (const entry of definition.metrics ?? []) {
+		const shape = {
+			name: entry.name,
+			...entry.unit ? { unit: entry.unit } : {},
+			...entry.better ? { better: entry.better } : {}
+		};
+		try {
+			const value = await entry.run(context);
+			if (value !== void 0 && !Number.isFinite(value)) throw new Error(`Metric ${entry.name} must return a finite number or undefined; got ${String(value)}`);
+			results.push({
+				...shape,
+				...value === void 0 ? {} : { value }
+			});
+		} catch (error) {
+			results.push({
+				...shape,
+				error: recordError(error)
+			});
+		}
+	}
+	return results;
+}
+//#endregion
 //#region ../runner/src/local-report-store.ts
 const MANIFEST_FILE = "manifest.json";
 /** The project's evals and suites when the run started (schema version 4+). */
@@ -1071,6 +1245,7 @@ async function runTrial(definition, options) {
 	}
 	const candidateData = datasetValues(prepared, "candidate");
 	const scoringData = datasetValues(prepared, "all");
+	const workspaceIgnore = [...options.artifactExclude ?? DEFAULT_ARTIFACT_EXCLUDE, ...definition.workspace?.ignore ?? []];
 	const events = [];
 	const runWriter = options.runWriter ?? await options.report.startRun(runMetadata(definition, options, runId, startedAt), { catalog: options.catalog });
 	const trialWriter = await runWriter.startTrial({
@@ -1085,6 +1260,7 @@ async function runTrial(definition, options) {
 		parameters: options.parameters,
 		matrix: options.matrix,
 		startedAt,
+		workspaceIgnore,
 		...prepared.length ? { datasets: trialDatasets(prepared) } : {}
 	});
 	let writeChain = Promise.resolve();
@@ -1155,7 +1331,7 @@ async function runTrial(definition, options) {
 				message: cause instanceof Error ? cause.message : String(cause)
 			})
 		}), (created) => Effect.tryPromise({
-			try: () => created.cleanup(),
+			try: () => created.cleanup(keepWorkspace(options.keepWorkspaces, scoring?.passed === true)),
 			catch: (cause) => new DatasetError({
 				cause,
 				message: cause instanceof Error ? cause.message : String(cause)
@@ -1220,6 +1396,7 @@ async function runTrial(definition, options) {
 	const { checkpoints, stoppedEarly, lastTurn } = progress;
 	const scoreResults = [];
 	const skippedScorers = [];
+	const metrics = [];
 	try {
 		await evaluateFinalScorers(definition, {
 			context,
@@ -1233,13 +1410,23 @@ async function runTrial(definition, options) {
 			results: scoreResults,
 			skippedScorers,
 			data: scoringData,
+			checkpoints,
 			now,
 			emitRunner
 		});
-		scoring = scoreSummary(scoreResults, checkpoints, skippedScorers);
+		metrics.push(...builtInMetrics(events));
+		if (context && workspace && !wasCancelled) metrics.push(...await evaluateMetrics(definition, scoringContext({
+			context,
+			workspace,
+			events,
+			lastTurn,
+			data: scoringData,
+			checkpoints
+		})));
+		scoring = scoreSummary(scoreResults, checkpoints, skippedScorers, metrics);
 		await trialWriter.writeScores(scoring);
 		if (workspace) try {
-			artifacts = await snapshotCandidateWorkspace(workspace.artifacts.candidate.root, trialWriter);
+			artifacts = await snapshotCandidateWorkspace(workspace.artifacts.candidate.root, trialWriter, workspaceIgnore);
 		} catch (captureError) {
 			if (!primaryError) primaryError = captureError;
 			await tryEmitError(captureError, emitRunner, now);
@@ -1254,7 +1441,7 @@ async function runTrial(definition, options) {
 			await tryEmitError(cleanupError, emitRunner, now);
 		}
 	}
-	scoring ??= scoreSummary(scoreResults, checkpoints, skippedScorers);
+	scoring ??= scoreSummary(scoreResults, checkpoints, skippedScorers, metrics);
 	if (!primaryError) try {
 		await emitRunner({
 			kind: "trial-completed",
@@ -1327,5 +1514,9 @@ function trialCounts(trials) {
 		cancelled: cancelledTrials
 	};
 }
+/** Persistent workspaces of trials that did not pass are kept for debugging by default. */
+function keepWorkspace(policy = "failed", passed) {
+	return policy === "all" || policy === "failed" && !passed;
+}
 //#endregion
-export { ReportError as _, readRunSummary as a, readTrialScoring as c, staticDatasetFile as d, staticDatasetFiles as f, DatasetError as g, CheckpointExecutionError as h, readRunManifest as i, readTrialSummary as l, CancelledError as m, runMatrix as n, readTrialEvents as o, AutExecutionError as p, ownerIsGone as r, readTrialManifest as s, runEval as t, localReportStore as u, ScoringError as v };
+export { CheckpointExecutionError as _, readRunSummary as a, ScoringError as b, readTrialScoring as c, trialWorkspacePath as d, staticDatasetFile as f, CancelledError as g, AutExecutionError as h, readRunManifest as i, readTrialSummary as l, globMatcher as m, runMatrix as n, readTrialEvents as o, staticDatasetFiles as p, ownerIsGone as r, readTrialManifest as s, runEval as t, localReportStore as u, DatasetError as v, ReportError as y };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leostera/evalkit",
-  "version": "0.6.1",
+  "version": "0.8.0",
   "description": "Author and run agent evals as code",
   "type": "module",
   "license": "MIT",