@leostera/evalkit 0.9.0 → 0.10.0

package/dist/{src-CcdLZqFk.mjs → src-BfSQKOJP.mjs}

@@ -1,5 +1,5 @@
-import { $ as relativeDatasetPath, A as RunManifestSchema, B as turnResultEvents, C as CheckpointResultSchema, F as RunSummarySchema, G as DEFAULT_ATTACHMENT_LIMITS, H as canonicalParameters, I as ScoreResultSchema, L as TrajectoryEventSchema, M as RunScoringSchema, U as turnTokens, W as DEFAULT_ARTIFACT_EXCLUDE, X as toJsonObject, Y as toJson, _ as parseTrajectoryJsonl, a as authoringId, i as agentRuntimes, j as RunMetadataSchema, ot as resourceUri, p as recordError, w as JsonValueSchema, z as normalizeAutEvent } from "./src-BrfIXfO2.mjs";
-import { a as upgradeManifest, n as assertNoRemovedOptions, o as upgradeStatus, r as upgradeCatalog, s as upgradeTrajectory, t as UPGRADABLE_SCHEMA_VERSIONS } from "./upgrade-60SIRius.mjs";
+import { $ as judgeInputNames, B as normalizeAutEvent, G as DEFAULT_ARTIFACT_EXCLUDE, I as RunSummarySchema, K as DEFAULT_ATTACHMENT_LIMITS, L as ScoreResultSchema, M as RunMetadataSchema, N as RunScoringSchema, R as TrajectoryEventSchema, T as JsonValueSchema, U as canonicalParameters, V as turnResultEvents, W as turnTokens, X as answerInstruction, Y as UnreadableAnswer, a as assertJudgesIndependent, at as toJson, et as judgePrompt, gt as resourceUri, i as agentRuntimes, j as RunManifestSchema, lt as relativeDatasetPath, m as recordError, nt as readJudgeAnswer, o as authoringId, ot as toJsonObject, rt as weightedMean, v as parseTrajectoryJsonl, w as CheckpointResultSchema } from "./src-B14fXAGp.mjs";
+import { a as upgradeManifest, n as assertNoRemovedOptions, o as upgradeStatus, r as upgradeCatalog, s as upgradeTrajectory, t as UPGRADABLE_SCHEMA_VERSIONS } from "./upgrade-G-uFPJKg.mjs";
 import * as Schema from "effect/Schema";
 import { Data, Effect, Either, Exit, Scope } from "effect";
 import os from "node:os";
@@ -105,6 +105,22 @@ function attachmentStore(writer, limits = {}) {
 		}
 	};
 }
+/**
+* An `attach` that remembers what it saved, so a score or checkpoint can
+* link the files its scorer attached, even when the scorer then failed.
+*/
+function trackAttachments(attach) {
+	const attached = [];
+	return {
+		attached,
+		async attach(name, content) {
+			await attach(name, content);
+			attached.push(name);
+		}
+	};
+}
+/** `{ attachments }` when a scorer attached any. */
+const attachedFields = (attached) => attached.length ? { attachments: [...attached] } : {};
 //#endregion
 //#region ../core/src/glob.ts
 /**
@@ -483,28 +499,85 @@ function checkpointLookup(transcript, results) {
 }
 //#endregion
 //#region ../runner/src/score.ts
+const inUnitInterval = (value) => typeof value === "number" && Number.isFinite(value) && value >= 0 && value <= 1;
+/**
+* Criteria must be well formed before they can stand for the score. With
+* declared criteria, each takes its declared weight unless it sets one, a
+* declared one left out is not reached, and an undeclared one is an error,
+* so the checks shown before a run are the checks recorded after it.
+*/
+function checkedCriteria(criteria, declared) {
+	if (!Array.isArray(criteria) || !criteria.length && !declared?.length) throw new Error("criteria must be a non-empty array");
+	const seen = /* @__PURE__ */ new Set();
+	const results = criteria.map((entry, index) => {
+		if (typeof entry?.name !== "string" || !entry.name.trim()) throw new Error(`criteria[${index}] needs a name`);
+		const { name } = entry;
+		if (seen.has(name)) throw new Error(`Criterion ${name} is listed twice`);
+		seen.add(name);
+		const declaration = declared?.find((criterion) => criterion.name === name);
+		if (declared && !declaration) throw new Error(`Criterion ${name} is not declared; declared: ${declared.map((criterion) => criterion.name).join(", ")}`);
+		const weight = entry.weight ?? declaration?.weight;
+		if (weight !== void 0 && !(Number.isFinite(weight) && weight > 0)) throw new Error(`Criterion ${name} weight must be a positive number; received ${String(weight)}`);
+		const explanation = entry.explanation;
+		if (explanation !== void 0 && typeof explanation !== "string") throw new Error(`Criterion ${name} explanation must be a string`);
+		const common = {
+			name,
+			...weight !== void 0 ? { weight } : {},
+			...explanation?.trim() ? { explanation: explanation.trim() } : {}
+		};
+		if ("reached" in entry) {
+			if (entry.reached !== false) throw new Error(`Criterion ${name}: reached can only be false; leave it out for a check that ran`);
+			return {
+				...common,
+				reached: false
+			};
+		}
+		const value = entry.value;
+		if (!inUnitInterval(value)) throw new Error(`Criterion ${name} value must be a number from 0 to 1; received ${String(value)}`);
+		const evidence = entry.evidence;
+		return {
+			...common,
+			value,
+			...evidence !== void 0 ? { evidence: toJson(evidence, `criteria.${name}.evidence`) } : {}
+		};
+	});
+	const missing = (declared ?? []).filter((criterion) => !seen.has(criterion.name)).map((criterion) => ({
+		name: criterion.name,
+		weight: criterion.weight,
+		reached: false
+	}));
+	return [...results, ...missing];
+}
 /**
 * Shared normalization for inline and final predicate and judge rules.
-* `passAt` turns a numeric value into a verdict; an explicit `passed` wins.
+* Without a `value`, the weighted mean of `criteria` is the value. `passAt`
+* turns the value into a verdict, and is recorded with it; an explicit
+* `passed` wins.
 */
-function normalizeScore(value, passAt) {
+function normalizeScore(value, passAt, declared) {
 	if (typeof value === "object" && value !== null && "skipped" in value && value.skipped === true) return {
 		skipped: true,
 		explanation: value.reason
 	};
 	const normalized = typeof value === "boolean" ? { value: Number(value) } : typeof value === "number" ? { value } : value;
-	if (!normalized || !Number.isFinite(normalized.value) || normalized.value < 0 || normalized.value > 1) throw new Error(`Score value must be a finite number from 0 to 1; received ${normalized?.value}`);
-	if ("judge" in normalized && normalized.judge) {
+	if (!normalized || typeof normalized !== "object") throw new Error(`A scorer must return a boolean, a number from 0 to 1, or a score object; received ${String(value)}`);
+	const criteria = normalized.criteria === void 0 ? void 0 : checkedCriteria(normalized.criteria, declared);
+	const score = normalized.value ?? (criteria ? weightedMean(criteria) : void 0);
+	if (!inUnitInterval(score)) throw new Error(`Score value must be a finite number from 0 to 1; received ${String(score)}`);
+	if (normalized.judge?.usage) {
 		const { usage } = normalized.judge;
-		if (usage && Object.values(usage).some((tokens) => !Number.isSafeInteger(tokens) || tokens < 0)) throw new Error("Judge token usage must contain nonnegative integers");
+		if (Object.values(usage).some((tokens) => !Number.isSafeInteger(tokens) || tokens < 0)) throw new Error("Judge token usage must contain nonnegative integers");
 	}
-	const passed = "passed" in normalized && normalized.passed !== void 0 ? normalized.passed : normalized.value >= (passAt ?? 1);
+	const mark = passAt ?? 1;
 	return {
-		value: normalized.value,
-		passed,
-		..."explanation" in normalized && normalized.explanation ? { explanation: normalized.explanation } : {},
-		..."evidence" in normalized && normalized.evidence !== void 0 ? { evidence: toJson(normalized.evidence, "evidence") } : {},
-		..."judge" in normalized && normalized.judge !== void 0 ? { judge: normalized.judge } : {}
+		value: score,
+		passed: normalized.passed ?? score >= mark,
+		passAt: mark,
+		...normalized.explanation ? { explanation: normalized.explanation } : {},
+		...normalized.evidence !== void 0 ? { evidence: toJson(normalized.evidence, "evidence") } : {},
+		...normalized.choice ? { choice: normalized.choice } : {},
+		...criteria ? { criteria } : {},
+		...normalized.judge !== void 0 ? { judge: normalized.judge } : {}
 	};
 }
 //#endregion
@@ -514,8 +587,28 @@ const failure$2 = (name, cause) => new RuleExecutionError({
 	cause,
 	message: cause instanceof Error ? cause.message : String(cause)
 });
-/** Drive a distinct agent session for one assessment; do not add its events to AUT trajectory. */
+/** A long tool result or reply is cut to this many characters in a judge's conversation. */
+const MAX_PART = 4e3;
+/**
+* Drives one judge agent session for one assessment: builds the prompt from
+* the rule, sends it, reads the answer in the rule's format, and asks once
+* more if it cannot. Its events stay out of the AUT trajectory.
+*/
 function runJudgeAgent(rule, agent, input, placement) {
+	return Effect.tryPromise({
+		try: () => judgeOnce(rule, agent, input, placement),
+		catch: (cause) => cause instanceof RuleExecutionError ? cause : failure$2(rule.name, cause)
+	});
+}
+async function judgeOnce(rule, agent, input, placement) {
+	const inputs = judgeInputNames(rule);
+	const values = await inputValues(rule, input, placement);
+	const prompt = judgePrompt({
+		rubric: rule.rubric,
+		answer: rule.answer,
+		inputs,
+		values
+	});
 	const events = [];
 	const context = {
 		...input.context,
@@ -523,64 +616,117 @@ function runJudgeAgent(rule, agent, input, placement) {
 		workspace: input.artifacts.evaluator,
 		evaluatorWorkspace: input.artifacts.evaluator
 	};
-	const prompt = JSON.stringify({
-		name: rule.name,
-		rubric: rule.rubric,
-		placement,
-		evidence: placement === "transcript" ? { turn: input.turn?.events ?? [] } : { trajectory: input.trajectory.events },
-		instruction: "Return only a JSON verdict with value (a number from 0 to 1), optional passed, explanation, and evidence."
+	const session = await agent.start({
+		context,
+		attach: input.attach,
+		onEvent: (event) => {
+			const observed = Schema.decodeUnknownSync(TrajectoryEventSchema)({
+				...normalizeAutEvent(event, () => /* @__PURE__ */ new Date()),
+				source: "aut"
+			});
+			if (observed.source !== "aut") throw new Error("Invalid judge event source");
+			const { source: _, ...validated } = observed;
+			events.push(validated);
+		}
 	});
-	let closeError;
-	return Effect.gen(function* () {
-		yield* Effect.acquireUseRelease(Effect.tryPromise({
-			try: () => agent.start({
-				context,
-				attach: input.attach,
-				onEvent: (event) => {
-					const observed = Schema.decodeUnknownSync(TrajectoryEventSchema)({
-						...normalizeAutEvent(event, () => /* @__PURE__ */ new Date()),
-						source: "aut"
-					});
-					if (observed.source !== "aut") throw new Error("Invalid judge event source");
-					const { source: _, ...validated } = observed;
-					events.push(validated);
-				}
-			}),
-			catch: (cause) => failure$2(rule.name, cause)
-		}), (session) => Effect.tryPromise({
-			try: async () => {
-				const result = await session.send(prompt);
-				if (result !== void 0) events.push(...turnResultEvents(result, 1, () => /* @__PURE__ */ new Date()));
-			},
-			catch: (cause) => failure$2(rule.name, cause)
-		}), (session) => Effect.tryPromise({
-			try: async () => session.close?.(),
-			catch: (cause) => failure$2(rule.name, cause)
-		}).pipe(Effect.catchAll((error) => Effect.sync(() => {
-			closeError = error;
-		}))));
-		if (closeError) return yield* Effect.fail(closeError);
-		const completed = events.slice().reverse().find((event) => event.kind === "completed" && event.output !== void 0);
-		const assistant = events.slice().reverse().find((event) => event.kind === "message" && event.role === "assistant");
-		const output = completed?.kind === "completed" ? completed.output : assistant?.kind === "message" ? assistant.content : void 0;
-		if (output === void 0) return yield* Effect.fail(failure$2(rule.name, /* @__PURE__ */ new Error("Judge agent did not emit a verdict")));
-		const value = yield* Effect.try({
-			try: () => typeof output === "string" ? JSON.parse(output) : output,
-			catch: (cause) => failure$2(rule.name, cause)
-		});
-		const encodedEvents = yield* Schema.decodeUnknown(JsonValueSchema)(events).pipe(Effect.mapError((cause) => failure$2(rule.name, cause)));
-		if (!Array.isArray(encodedEvents)) return yield* Effect.fail(failure$2(rule.name, /* @__PURE__ */ new Error("Judge events must be an array")));
-		const usage = events.slice().reverse().find((event) => event.kind === "turn-completed" && event.usage);
-		return {
-			value,
-			judge: {
-				...agent.identity ? { agent: agent.identity } : {},
-				...usage?.kind === "turn-completed" && usage.usage ? { usage: usage.usage } : {},
-				events: encodedEvents
+	let reading;
+	let reply;
+	let attempts = 0;
+	try {
+		let message = prompt;
+		let problem = "";
+		while (attempts < 2 && !reading) {
+			attempts += 1;
+			const from = events.length;
+			const result = await session.send(message);
+			if (result !== void 0) events.push(...turnResultEvents(result, attempts, () => /* @__PURE__ */ new Date()));
+			reply = replyIn(events.slice(from));
+			if (reply === void 0) problem = "it did not reply";
+			else try {
+				reading = readJudgeAnswer(rule.answer, reply);
+			} catch (error) {
+				if (!(error instanceof UnreadableAnswer)) throw error;
+				problem = error.message;
 			}
-		};
+			message = `Your answer could not be read: ${problem}. Answer again, only in this format.\n\n${answerInstruction(rule.answer)}`;
+		}
+		if (!reading) throw new Error(`Judge ${rule.name}'s answer could not be read after ${attempts} tries: ${problem}${reply === void 0 ? "" : `. Its last reply: ${preview(reply)}`}`);
+	} finally {
+		await session.close?.();
+	}
+	const recorded = Schema.decodeUnknownSync(JsonValueSchema)(events);
+	const model = lastOf(events, (event) => event.kind === "turn-completed" && event.model);
+	const usage = totalUsage(events);
+	const judge = {
+		...agent.identity ? { agent: agent.identity } : {},
+		...model?.kind === "turn-completed" && model.model ? { model: model.model } : {},
+		...usage ? { usage } : {},
+		prompt,
+		...reply !== void 0 ? { reply } : {},
+		...attempts > 1 ? { attempts } : {},
+		events: recorded
+	};
+	return { value: {
+		...reading,
+		judge
+	} };
+}
+/** Each input's text: what the rule's function returned, or the defaults. */
+async function inputValues(rule, input, placement) {
+	if (!rule.inputs) return {
+		conversation: conversationText(input.trajectory.events, placement),
+		reply: text(input.output)
+	};
+	const values = {};
+	for (const [name, read] of Object.entries(rule.inputs)) values[name] = text(await read(input));
+	return values;
+}
+/** The run's conversation as plain lines, for a judge to read: this turn at a checkpoint, else all of it. */
+function conversationText(events, placement) {
+	let start = 0;
+	if (placement === "transcript") events.forEach((event, index) => {
+		if (event.kind === "transcript-step-started" && "message" in event) start = index;
 	});
+	const lines = [];
+	for (const event of events.slice(start)) if (event.kind === "transcript-step-started" && "message" in event) lines.push(`User: ${cut(text(event.message))}`);
+	else if (event.source !== "aut") continue;
+	else if (event.kind === "message") lines.push(`${capitalize(event.role)}: ${cut(text(event.content))}`);
+	else if (event.kind === "tool-call") lines.push(`Tool call ${event.name}: ${cut(text(event.arguments))}`);
+	else if (event.kind === "tool-result") lines.push(`Tool result ${event.name ?? event.id}: ${cut(text(event.result))}`);
+	return lines.join("\n") || "(nothing happened)";
+}
+/** The judge's answer in one attempt's events: its final output, else its last reply. */
+function replyIn(events) {
+	const completed = lastOf(events, (event) => event.kind === "completed" && event.output !== void 0);
+	if (completed?.kind === "completed") return completed.output;
+	const assistant = lastOf(events, (event) => event.kind === "message" && event.role === "assistant");
+	return assistant?.kind === "message" ? assistant.content : void 0;
+}
+function totalUsage(events) {
+	let total;
+	for (const event of events) {
+		if (event.kind !== "turn-completed" || !event.usage) continue;
+		total ??= {};
+		for (const [key, tokens] of Object.entries(event.usage)) if (tokens !== void 0) total[key] = (total[key] ?? 0) + tokens;
+	}
+	return total;
+}
+function text(value) {
+	if (value === void 0) return "(none)";
+	return typeof value === "string" ? value : JSON.stringify(value, null, 2);
 }
+function cut(value) {
+	return value.length > MAX_PART ? `${value.slice(0, MAX_PART)}… (${value.length - MAX_PART} more characters)` : value;
+}
+function preview(value) {
+	const shown = text(value).replace(/\s+/g, " ");
+	return shown.length > 300 ? `${shown.slice(0, 300)}…` : shown;
+}
+/** The last element matching, scanning from the end (no ES2023 lib here). */
+function lastOf(items, match) {
+	for (let index = items.length - 1; index >= 0; index -= 1) if (match(items[index])) return items[index];
+}
+const capitalize = (value) => value.charAt(0).toUpperCase() + value.slice(1);
 //#endregion
 //#region ../runner/src/evaluate-rule.ts
 const failure$1 = (name, cause) => new RuleExecutionError({
@@ -589,38 +735,30 @@ const failure$1 = (name, cause) => new RuleExecutionError({
 	message: cause instanceof Error ? cause.message : String(cause)
 });
 /** Both placements call the same rule executor; only input timing and persistence differ. */
-function evaluateRule(rule, input, judgeAgent, placement) {
+function evaluateRule(rule, input, defaultJudge, placement) {
 	return Effect.gen(function* () {
-		if (rule.kind === "judge" && !judgeAgent) return yield* Effect.fail(failure$1(rule.name, /* @__PURE__ */ new Error("Judge rules require an eval judge agent")));
-		const evaluated = rule.kind === "predicate" ? { value: yield* Effect.tryPromise({
+		const judgeAgent = rule.kind === "judge" ? rule.agent ?? defaultJudge : void 0;
+		if (rule.kind === "judge" && !judgeAgent) return yield* Effect.fail(failure$1(rule.name, /* @__PURE__ */ new Error("This judge has no agent: pass judge(name, { agent }) or set judge in evalkit.config.ts")));
+		const evaluated = rule.kind === "predicate" ? yield* Effect.tryPromise({
 			try: async () => rule.run(input),
 			catch: (cause) => failure$1(rule.name, cause)
-		}) } : yield* runJudgeAgent(rule, judgeAgent, input, placement);
+		}) : (yield* runJudgeAgent(rule, judgeAgent, input, placement)).value;
 		const score = yield* Effect.try({
-			try: () => normalizeScore(evaluated.value, rule.kind === "predicate" ? rule.passAt : void 0),
+			try: () => normalizeScore(evaluated, rule.passAt, rule.kind === "predicate" ? rule.criteria : void 0),
 			catch: (cause) => failure$1(rule.name, cause)
 		});
-		const withJudge = rule.kind === "judge" && "judge" in evaluated ? {
-			...score,
-			judge: evaluated.judge
-		} : score;
 		const validated = yield* Schema.decodeUnknown(ScoreResultSchema)({
 			name: rule.name,
 			kind: rule.kind,
 			durationMs: 0,
-			...withJudge
+			...score
 		}).pipe(Effect.mapError((cause) => failure$1(rule.name, cause)));
 		if (validated.skipped) return {
 			skipped: true,
 			...validated.explanation === void 0 ? {} : { explanation: validated.explanation }
 		};
-		return {
-			value: validated.value,
-			passed: validated.passed,
-			...validated.explanation === void 0 ? {} : { explanation: validated.explanation },
-			...validated.evidence === void 0 ? {} : { evidence: validated.evidence },
-			...validated.judge === void 0 ? {} : { judge: validated.judge }
-		};
+		const { name: _name, kind: _kind, durationMs: _duration, ...result } = validated;
+		return result;
 	});
 }
 //#endregion
@@ -794,6 +932,7 @@ async function runCheckpoint(definition, step, index, input) {
 		name: step.name,
 		timestamp: started.toISOString()
 	});
+	const tracked = trackAttachments(input.attach);
 	return runBoundary(evaluateCheckpoint(step, index, {
 		context: input.context,
 		artifacts: input.artifacts,
@@ -803,8 +942,11 @@ async function runCheckpoint(definition, step, index, input) {
 		data: input.data,
 		checkpoints: [...progress.checkpoints],
 		checkpoint: checkpointLookup(definition.transcript, [...progress.checkpoints]),
-		attach: input.attach
-	}, now, definition.judge)).catch(async (error) => {
+		attach: tracked.attach
+	}, now, input.defaultJudge)).then((result) => ({
+		...result,
+		...attachedFields(tracked.attached)
+	})).catch(async (error) => {
 		const recorded = recordError(error);
 		progress.checkpoints.push({
 			step: index,
@@ -812,7 +954,8 @@ async function runCheckpoint(definition, step, index, input) {
 			name: step.name,
 			status: "error",
 			durationMs: now().getTime() - started.getTime(),
-			error: recorded
+			error: recorded,
+			...attachedFields(tracked.attached)
 		});
 		try {
 			await emitRunner({
@@ -889,13 +1032,15 @@ async function evaluateFinalScorers(definition, input) {
 			timestamp: started.toISOString()
 		});
 		let result;
+		const tracked = trackAttachments(input.attach);
 		try {
 			const score = await runBoundary(evaluateRule(scorer, scoringContext({
 				...input,
+				attach: tracked.attach,
 				context,
 				workspace,
 				transcript: definition.transcript
-			}), definition.judge, "scoring").pipe(Effect.mapError((cause) => new ScoringError({
+			}), input.defaultJudge, "scoring").pipe(Effect.mapError((cause) => new ScoringError({
 				cause,
 				message: cause.message
 			}))));
@@ -903,7 +1048,8 @@ async function evaluateFinalScorers(definition, input) {
 				name: scorer.name,
 				kind: scorer.kind,
 				durationMs: now().getTime() - started.getTime(),
-				...score
+				...score,
+				...attachedFields(tracked.attached)
 			};
 			results.push(result);
 		} catch (error) {
@@ -913,7 +1059,8 @@ async function evaluateFinalScorers(definition, input) {
 				kind: scorer.kind,
 				durationMs: now().getTime() - started.getTime(),
 				error: recorded,
-				passed: false
+				passed: false,
+				...attachedFields(tracked.attached)
 			});
 			await emitRunner({
 				kind: "scorer-failed",
@@ -934,11 +1081,12 @@ async function evaluateFinalScorers(definition, input) {
 //#endregion
 //#region ../runner/src/metrics.ts
 /**
-* What EvalKit measures for every run from its trajectory: the agent's time
+* What EvalKit measures for every run: from its trajectory, the agent's time
 * (first event to the last one before final scoring), tokens when the agent
-* reported usage, and tool calls.
+* reported usage, and tool calls; from its scores and checkpoints, the
+* tokens its judges used, kept apart from the agent's.
 */
-function builtInMetrics(events) {
+function builtInMetrics(events, verdicts = []) {
 	const times = events.filter((event) => !event.kind.startsWith("scorer-")).map((event) => Date.parse(event.timestamp)).filter((time) => !Number.isNaN(time));
 	const metrics = [];
 	if (times.length) metrics.push({
@@ -965,6 +1113,18 @@ function builtInMetrics(events) {
 		value: events.filter((event) => event.source === "aut" && event.kind === "tool-call").length,
 		builtIn: true
 	});
+	let judgeTokens;
+	for (const verdict of verdicts) {
+		const used = verdict.judge?.usage && turnTokens(verdict.judge.usage);
+		if (used !== void 0) judgeTokens = (judgeTokens ?? 0) + used;
+	}
+	if (judgeTokens !== void 0) metrics.push({
+		name: "judge tokens",
+		value: judgeTokens,
+		unit: "tokens",
+		better: "lower",
+		builtIn: true
+	});
 	return metrics;
 }
 /** Runs the scenario's metrics; a failing metric records its error and the rest still run. */
@@ -1410,8 +1570,9 @@ async function executeBatch(definition, options) {
 	let hasUser = false;
 	for (const step of definition.transcript) if (step.kind === "user") hasUser = true;
 	else if (step.kind !== "agent" && !hasUser) throw new Error("Checkpoint requires a preceding user step");
-	if (definition.judge && definition.judge === definition.agent) throw new Error("Judge agent must be distinct from the agent under test");
 	assertNoRemovedOptions(definition);
+	assertJudgesIndependent(definition, options.judge);
+	for (const rule of [...definition.transcript, ...definition.scoring]) if (rule.kind === "judge" && !rule.agent && !options.judge) throw new Error(`Judge ${rule.name} in ${definition.id} has no agent: pass judge(name, { agent }) or set judge in evalkit.config.ts`);
 	if (options.runId) assertUuid(options.runId, "runId");
 	const repeat = options.repeat ?? definition.repeat ?? 1;
 	if (!Number.isInteger(repeat) || repeat < 1) throw new Error(`repeat must be a positive integer; received ${repeat}`);
@@ -1587,7 +1748,8 @@ async function runOnce(definition, options) {
 				data: scoringData,
 				signal,
 				progress,
-				attach: attachments.attach
+				attach: attachments.attach,
+				defaultJudge: options.judge
 			}),
 			catch: (cause) => cause
 		}), (opened) => Effect.tryPromise({
@@ -1630,10 +1792,11 @@ async function runOnce(definition, options) {
 			data: scoringData,
 			checkpoints,
 			attach: attachments.attach,
+			defaultJudge: options.judge,
 			now,
 			emitRunner
 		});
-		metrics.push(...builtInMetrics(events));
+		metrics.push(...builtInMetrics(events, [...scoreResults, ...checkpoints]));
 		if (context && workspace && !wasCancelled) metrics.push(...await evaluateMetrics(definition, scoringContext({
 			context,
 			workspace,

package/dist/{upgrade-60SIRius.mjs → upgrade-G-uFPJKg.mjs}

@@ -104,6 +104,7 @@ function upgradeManifest(input) {
 */
 function assertNoRemovedOptions(definition) {
 	if ("fixtures" in definition) throw new Error(`Eval ${String(definition.id)} uses fixtures, which were replaced by datasets: [defineDataset(...)]`);
+	if ("judge" in definition) throw new Error(`Eval ${String(definition.id)} sets judge, which moved to each rule: judge(name, { agent, rubric }), or judge in evalkit.config.ts for every judge (EvalKit 0.10)`);
 	if (isRecord(definition.policy) && "trials" in definition.policy) throw new Error(`Eval ${String(definition.id)} sets policy.trials, which was replaced by repeat: ${JSON.stringify(definition.policy.trials)} (EvalKit 0.9)`);
 }
 //#endregion

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leostera/evalkit",
-  "version": "0.9.0",
+  "version": "0.10.0",
   "description": "Author and run agent evals as code",
   "type": "module",
   "license": "MIT",