@leostera/evalkit 0.8.0 → 0.10.0

package/dist/{src-mm8L_IxM.mjs → src-BfSQKOJP.mjs}

@@ -1,8 +1,9 @@
-import { A as RunSummarySchema, D as RunMetadataSchema, E as RunManifestSchema, F as TrialScoringSchema, H as DEFAULT_ARTIFACT_EXCLUDE, I as TrialSummarySchema, K as toJson, M as TrajectoryEventSchema, N as TrialManifestSchema, P as TrialMetadataSchema, R as normalizeAutEvent, S as JsonValueSchema, V as canonicalParameters, X as relativeDatasetPath, _ as parseTrajectoryJsonl, a as authoringId, i as agentRuntimes, j as ScoreResultSchema, p as recordError, q as toJsonObject, rt as resourceUri, x as CheckpointResultSchema, z as turnResultEvents } from "./src-LeGSyDD2.mjs";
+import { $ as judgeInputNames, B as normalizeAutEvent, G as DEFAULT_ARTIFACT_EXCLUDE, I as RunSummarySchema, K as DEFAULT_ATTACHMENT_LIMITS, L as ScoreResultSchema, M as RunMetadataSchema, N as RunScoringSchema, R as TrajectoryEventSchema, T as JsonValueSchema, U as canonicalParameters, V as turnResultEvents, W as turnTokens, X as answerInstruction, Y as UnreadableAnswer, a as assertJudgesIndependent, at as toJson, et as judgePrompt, gt as resourceUri, i as agentRuntimes, j as RunManifestSchema, lt as relativeDatasetPath, m as recordError, nt as readJudgeAnswer, o as authoringId, ot as toJsonObject, rt as weightedMean, v as parseTrajectoryJsonl, w as CheckpointResultSchema } from "./src-B14fXAGp.mjs";
+import { a as upgradeManifest, n as assertNoRemovedOptions, o as upgradeStatus, r as upgradeCatalog, s as upgradeTrajectory, t as UPGRADABLE_SCHEMA_VERSIONS } from "./upgrade-G-uFPJKg.mjs";
 import * as Schema from "effect/Schema";
 import { Data, Effect, Either, Exit, Scope } from "effect";
 import os from "node:os";
-import { appendFile, copyFile, lstat, mkdir, mkdtemp, readFile, readdir, rename, rm, stat, writeFile } from "node:fs/promises";
+import { appendFile, copyFile, cp, lstat, mkdir, mkdtemp, readFile, readdir, rename, rm, stat, writeFile } from "node:fs/promises";
 import path, { join } from "node:path";
 import { createHash, randomUUID } from "node:crypto";
 //#region ../runner/src/errors.ts
@@ -12,7 +13,7 @@ var ReportError = class extends Data.TaggedError("ReportError") {};
 var ScoringError = class extends Data.TaggedError("ScoringError") {};
 var RuleExecutionError = class extends Data.TaggedError("RuleExecutionError") {};
 var CheckpointExecutionError = class extends Data.TaggedError("CheckpointExecutionError") {};
-/** The run was cancelled (Ctrl+C); the trial stops and is finalized as cancelled. */
+/** The run was cancelled (Ctrl+C); the run stops and is finalized as cancelled. */
 var CancelledError = class extends Data.TaggedError("CancelledError") {};
 function cancelled() {
 	return new CancelledError({ message: "Cancelled" });
@@ -32,6 +33,95 @@ async function unlessAborted(work, signal) {
 	}
 }
 //#endregion
+//#region ../runner/src/attachments.ts
+const SEGMENT = /^[A-Za-z0-9_-][A-Za-z0-9._-]{0,99}$/;
+const MAX_NAME = 200;
+/**
+* Checks an attachment name: a relative path whose segments are letters,
+* digits, `.`, `_` and `-`, not starting with a dot. That rules out
+* absolute paths, `..`, hidden files, separators other than `/`, and
+* anything a filesystem or URL would read differently.
+*/
+function attachmentName(name) {
+	if (typeof name !== "string" || !name || name.length > MAX_NAME || !name.split("/").every((segment) => SEGMENT.test(segment))) throw new Error(`Attachment name ${JSON.stringify(name)} is not allowed: use a relative path of letters, digits, ".", "_" and "-", like "logs/agent.log"`);
+	return name;
+}
+/** The last `max` bytes, starting on a whole UTF-8 character when the content is text. */
+function lastBytes(bytes, max, text) {
+	let start = bytes.byteLength - max;
+	if (text) while (start < bytes.byteLength && (bytes[start] & 192) === 128) start++;
+	return bytes.subarray(start);
+}
+/**
+* Saves a run's attachments as `attachments/<name>` below its artifacts.
+* Content is stored as given and never interpreted. A file over
+* `maxFileBytes` keeps its end (where a log's errors are); one that would
+* take the run past `maxRunBytes` is rejected.
+*/
+function attachmentStore(writer, limits = {}) {
+	const { maxFileBytes, maxRunBytes } = {
+		...DEFAULT_ATTACHMENT_LIMITS,
+		...limits
+	};
+	const entries = [];
+	const names = /* @__PURE__ */ new Set();
+	const pending = /* @__PURE__ */ new Set();
+	let used = 0;
+	let closed = false;
+	async function attach(name, content) {
+		if (closed) throw new Error(`Cannot attach ${JSON.stringify(name)}: the run has finished`);
+		attachmentName(name);
+		if (typeof content !== "string" && !(content instanceof Uint8Array)) throw new Error(`Attachment ${name} must be a string or a Uint8Array, not ${content === null ? "null" : typeof content}`);
+		if (names.has(name)) throw new Error(`Attachment ${name} was already attached`);
+		const text = typeof content === "string";
+		const original = text ? new TextEncoder().encode(content) : content;
+		const bytes = original.byteLength > maxFileBytes ? lastBytes(original, maxFileBytes, text) : original;
+		if (used + bytes.byteLength > maxRunBytes) throw new Error(`Attachment ${name} (${bytes.byteLength} bytes) would take this run past its ${maxRunBytes}-byte attachment limit (${used} used)`);
+		names.add(name);
+		used += bytes.byteLength;
+		const write = writer.writeArtifact(`attachments/${name}`, bytes);
+		pending.add(write);
+		try {
+			await write;
+			entries.push({
+				name,
+				size: bytes.byteLength,
+				...bytes.byteLength < original.byteLength ? { truncatedFrom: original.byteLength } : {}
+			});
+		} catch (error) {
+			names.delete(name);
+			used -= bytes.byteLength;
+			throw error;
+		} finally {
+			pending.delete(write);
+		}
+	}
+	return {
+		attach,
+		async close() {
+			closed = true;
+			await Promise.allSettled(pending);
+			return [...entries].sort((a, b) => a.name.localeCompare(b.name));
+		}
+	};
+}
+/**
+* An `attach` that remembers what it saved, so a score or checkpoint can
+* link the files its scorer attached, even when the scorer then failed.
+*/
+function trackAttachments(attach) {
+	const attached = [];
+	return {
+		attached,
+		async attach(name, content) {
+			await attach(name, content);
+			attached.push(name);
+		}
+	};
+}
+/** `{ attachments }` when a scorer attached any. */
+const attachedFields = (attached) => attached.length ? { attachments: [...attached] } : {};
+//#endregion
 //#region ../core/src/glob.ts
 /**
 * A glob matcher for discovery: `*` and `?` stay within a path segment, `**`
@@ -104,9 +194,9 @@ function ignoreRules(patterns) {
 	};
 }
 /**
-* Copies the candidate workspace's regular files into the trial report,
+* Copies the candidate workspace's regular files into the run report,
 * leaving out ignored paths. Symbolic links and special files are skipped,
-* never followed, so a snapshot cannot read outside the trial. An ignored
+* never followed, so a snapshot cannot read outside the run. An ignored
 * directory is walked only when a `!` pattern could re-include something in
 * it, and is listed only if it ends up containing a snapshotted file.
 */
@@ -159,7 +249,7 @@ async function snapshotCandidateWorkspace(candidateRoot, writer, ignore = DEFAUL
 const loads = /* @__PURE__ */ new WeakMap();
 /**
 * Loads a dataset at most once per process and project root. A failed load is
-* forgotten, so the next trial that needs the dataset tries again.
+* forgotten, so the next run that needs the dataset tries again.
 */
 function loadDataset(dataset, root, signal) {
 	let byRoot = loads.get(dataset);
@@ -210,12 +300,12 @@ async function readValueFile(id, file) {
 	}
 	throw new Error(`Dataset ${id} valueFile must be a .json or .jsonl file: ${file}`);
 }
-/** Loads (cached), derives this trial's content with `onRun`, and resolves it. */
-async function prepareDatasets(datasets, root, trial) {
+/** Loads (cached), derives this run's content with `onRun`, and resolves it. */
+async function prepareDatasets(datasets, root, run) {
 	const prepared = [];
 	for (const dataset of datasets) {
-		const loaded = await loadDataset(dataset, root, trial.signal);
-		const content = dataset.source.kind === "computed" && dataset.source.onRun ? await dataset.source.onRun(loaded, trial) : loaded;
+		const loaded = await loadDataset(dataset, root, run.signal);
+		const content = dataset.source.kind === "computed" && dataset.source.onRun ? await dataset.source.onRun(loaded, run) : loaded;
 		if (typeof content !== "object" || content === null) throw new Error(`Dataset ${dataset.id} onRun must return { files?, value? }`);
 		prepared.push(await resolveContent(dataset, content, root));
 	}
@@ -283,7 +373,7 @@ async function digest(files, value) {
 	if (value !== void 0) hash.update(`value ${canonicalParameters(value)}\n`);
 	return `sha256:${hash.digest("hex")}`;
 }
-function trialDatasets(prepared) {
+function runDatasets(prepared) {
 	return prepared.map(({ dataset, digest }) => ({
 		id: dataset.id,
 		visibility: dataset.visibility,
@@ -332,17 +422,17 @@ async function staticDatasetFile(dataset, root, file) {
 }
 //#endregion
 //#region ../runner/src/workspace.ts
-/** Where a trial's persistent workspace lives below `root`. */
-function trialWorkspacePath(root, trialId) {
-	return path.join(path.resolve(root), trialId.replace(/[^a-zA-Z0-9._-]/g, "_"));
+/** Where a run's persistent workspace lives below `root`. */
+function runWorkspacePath(root, runId) {
+	return path.join(path.resolve(root), runId.replace(/[^a-zA-Z0-9._-]/g, "_"));
 }
 /**
 * Creates isolated candidate and evaluator directories and writes each
 * dataset's files below its mount, on the side its visibility allows.
 * Two datasets writing the same file is an error, never an overwrite.
 */
-async function createTrialWorkspace(trialId, datasets = [], persistentRoot) {
-	const root = persistentRoot ? trialWorkspacePath(persistentRoot, trialId) : await mkdtemp(path.join(os.tmpdir(), "evalkit-trial-"));
+async function createRunWorkspace(runId, datasets = [], persistentRoot) {
+	const root = persistentRoot ? runWorkspacePath(persistentRoot, runId) : await mkdtemp(path.join(os.tmpdir(), "evalkit-run-"));
 	const persistent = persistentRoot !== void 0;
 	const artifacts = {
 		candidate: { root: path.join(root, "candidate") },
@@ -394,29 +484,100 @@ function collision(datasetId, file, cause) {
 	return cause.code === "EEXIST" ? /* @__PURE__ */ new Error(`Dataset ${datasetId} would overwrite ${file}, which another dataset already wrote`) : cause instanceof Error ? cause : new Error(String(cause));
 }
 //#endregion
+//#region ../runner/src/checkpoint-lookup.ts
+/**
+* `checkpoint(rule)` for a scoring context: the latest result of the steps
+* that are this very rule. A rule outside the transcript is a mistake in the
+* scenario, so it throws rather than reading as "not run".
+*/
+function checkpointLookup(transcript, results) {
+	return ((rule) => {
+		const steps = new Set(transcript.flatMap((step, index) => step === rule ? [index] : []));
+		if (!steps.size) throw new Error(`checkpoint(${JSON.stringify(rule.name)}) is not a checkpoint in this scenario's transcript; pass the same rule object the transcript uses`);
+		for (let index = results.length - 1; index >= 0; index--) if (steps.has(results[index].step)) return results[index];
+	});
+}
+//#endregion
 //#region ../runner/src/score.ts
+const inUnitInterval = (value) => typeof value === "number" && Number.isFinite(value) && value >= 0 && value <= 1;
+/**
+* Criteria must be well formed before they can stand for the score. With
+* declared criteria, each takes its declared weight unless it sets one, a
+* declared one left out is not reached, and an undeclared one is an error,
+* so the checks shown before a run are the checks recorded after it.
+*/
+function checkedCriteria(criteria, declared) {
+	if (!Array.isArray(criteria) || !criteria.length && !declared?.length) throw new Error("criteria must be a non-empty array");
+	const seen = /* @__PURE__ */ new Set();
+	const results = criteria.map((entry, index) => {
+		if (typeof entry?.name !== "string" || !entry.name.trim()) throw new Error(`criteria[${index}] needs a name`);
+		const { name } = entry;
+		if (seen.has(name)) throw new Error(`Criterion ${name} is listed twice`);
+		seen.add(name);
+		const declaration = declared?.find((criterion) => criterion.name === name);
+		if (declared && !declaration) throw new Error(`Criterion ${name} is not declared; declared: ${declared.map((criterion) => criterion.name).join(", ")}`);
+		const weight = entry.weight ?? declaration?.weight;
+		if (weight !== void 0 && !(Number.isFinite(weight) && weight > 0)) throw new Error(`Criterion ${name} weight must be a positive number; received ${String(weight)}`);
+		const explanation = entry.explanation;
+		if (explanation !== void 0 && typeof explanation !== "string") throw new Error(`Criterion ${name} explanation must be a string`);
+		const common = {
+			name,
+			...weight !== void 0 ? { weight } : {},
+			...explanation?.trim() ? { explanation: explanation.trim() } : {}
+		};
+		if ("reached" in entry) {
+			if (entry.reached !== false) throw new Error(`Criterion ${name}: reached can only be false; leave it out for a check that ran`);
+			return {
+				...common,
+				reached: false
+			};
+		}
+		const value = entry.value;
+		if (!inUnitInterval(value)) throw new Error(`Criterion ${name} value must be a number from 0 to 1; received ${String(value)}`);
+		const evidence = entry.evidence;
+		return {
+			...common,
+			value,
+			...evidence !== void 0 ? { evidence: toJson(evidence, `criteria.${name}.evidence`) } : {}
+		};
+	});
+	const missing = (declared ?? []).filter((criterion) => !seen.has(criterion.name)).map((criterion) => ({
+		name: criterion.name,
+		weight: criterion.weight,
+		reached: false
+	}));
+	return [...results, ...missing];
+}
 /**
 * Shared normalization for inline and final predicate and judge rules.
-* `passAt` turns a numeric value into a verdict; an explicit `passed` wins.
+* Without a `value`, the weighted mean of `criteria` is the value. `passAt`
+* turns the value into a verdict, and is recorded with it; an explicit
+* `passed` wins.
 */
-function normalizeScore(value, passAt) {
+function normalizeScore(value, passAt, declared) {
 	if (typeof value === "object" && value !== null && "skipped" in value && value.skipped === true) return {
 		skipped: true,
 		explanation: value.reason
 	};
 	const normalized = typeof value === "boolean" ? { value: Number(value) } : typeof value === "number" ? { value } : value;
-	if (!normalized || !Number.isFinite(normalized.value) || normalized.value < 0 || normalized.value > 1) throw new Error(`Score value must be a finite number from 0 to 1; received ${normalized?.value}`);
-	if ("judge" in normalized && normalized.judge) {
+	if (!normalized || typeof normalized !== "object") throw new Error(`A scorer must return a boolean, a number from 0 to 1, or a score object; received ${String(value)}`);
+	const criteria = normalized.criteria === void 0 ? void 0 : checkedCriteria(normalized.criteria, declared);
+	const score = normalized.value ?? (criteria ? weightedMean(criteria) : void 0);
+	if (!inUnitInterval(score)) throw new Error(`Score value must be a finite number from 0 to 1; received ${String(score)}`);
+	if (normalized.judge?.usage) {
 		const { usage } = normalized.judge;
-		if (usage && Object.values(usage).some((tokens) => !Number.isSafeInteger(tokens) || tokens < 0)) throw new Error("Judge token usage must contain nonnegative integers");
+		if (Object.values(usage).some((tokens) => !Number.isSafeInteger(tokens) || tokens < 0)) throw new Error("Judge token usage must contain nonnegative integers");
 	}
-	const passed = "passed" in normalized && normalized.passed !== void 0 ? normalized.passed : normalized.value >= (passAt ?? 1);
+	const mark = passAt ?? 1;
 	return {
-		value: normalized.value,
-		passed,
-		..."explanation" in normalized && normalized.explanation ? { explanation: normalized.explanation } : {},
-		..."evidence" in normalized && normalized.evidence !== void 0 ? { evidence: toJson(normalized.evidence, "evidence") } : {},
-		..."judge" in normalized && normalized.judge !== void 0 ? { judge: normalized.judge } : {}
+		value: score,
+		passed: normalized.passed ?? score >= mark,
+		passAt: mark,
+		...normalized.explanation ? { explanation: normalized.explanation } : {},
+		...normalized.evidence !== void 0 ? { evidence: toJson(normalized.evidence, "evidence") } : {},
+		...normalized.choice ? { choice: normalized.choice } : {},
+		...criteria ? { criteria } : {},
+		...normalized.judge !== void 0 ? { judge: normalized.judge } : {}
 	};
 }
 //#endregion
@@ -426,8 +587,28 @@ const failure$2 = (name, cause) => new RuleExecutionError({
 	cause,
 	message: cause instanceof Error ? cause.message : String(cause)
 });
-/** Drive a distinct agent session for one assessment; do not add its events to AUT trajectory. */
+/** A long tool result or reply is cut to this many characters in a judge's conversation. */
+const MAX_PART = 4e3;
+/**
+* Drives one judge agent session for one assessment: builds the prompt from
+* the rule, sends it, reads the answer in the rule's format, and asks once
+* more if it cannot. Its events stay out of the AUT trajectory.
+*/
 function runJudgeAgent(rule, agent, input, placement) {
+	return Effect.tryPromise({
+		try: () => judgeOnce(rule, agent, input, placement),
+		catch: (cause) => cause instanceof RuleExecutionError ? cause : failure$2(rule.name, cause)
+	});
+}
+async function judgeOnce(rule, agent, input, placement) {
+	const inputs = judgeInputNames(rule);
+	const values = await inputValues(rule, input, placement);
+	const prompt = judgePrompt({
+		rubric: rule.rubric,
+		answer: rule.answer,
+		inputs,
+		values
+	});
 	const events = [];
 	const context = {
 		...input.context,
@@ -435,63 +616,117 @@ function runJudgeAgent(rule, agent, input, placement) {
 		workspace: input.artifacts.evaluator,
 		evaluatorWorkspace: input.artifacts.evaluator
 	};
-	const prompt = JSON.stringify({
-		name: rule.name,
-		rubric: rule.rubric,
-		placement,
-		evidence: placement === "transcript" ? { turn: input.turn?.events ?? [] } : { trajectory: input.trajectory.events },
-		instruction: "Return only a JSON verdict with value (a number from 0 to 1), optional passed, explanation, and evidence."
+	const session = await agent.start({
+		context,
+		attach: input.attach,
+		onEvent: (event) => {
+			const observed = Schema.decodeUnknownSync(TrajectoryEventSchema)({
+				...normalizeAutEvent(event, () => /* @__PURE__ */ new Date()),
+				source: "aut"
+			});
+			if (observed.source !== "aut") throw new Error("Invalid judge event source");
+			const { source: _, ...validated } = observed;
+			events.push(validated);
+		}
 	});
-	let closeError;
-	return Effect.gen(function* () {
-		yield* Effect.acquireUseRelease(Effect.tryPromise({
-			try: () => agent.start({
-				context,
-				onEvent: (event) => {
-					const observed = Schema.decodeUnknownSync(TrajectoryEventSchema)({
-						...normalizeAutEvent(event, () => /* @__PURE__ */ new Date()),
-						source: "aut"
-					});
-					if (observed.source !== "aut") throw new Error("Invalid judge event source");
-					const { source: _, ...validated } = observed;
-					events.push(validated);
-				}
-			}),
-			catch: (cause) => failure$2(rule.name, cause)
-		}), (session) => Effect.tryPromise({
-			try: async () => {
-				const result = await session.send(prompt);
-				if (result !== void 0) events.push(...turnResultEvents(result, 1, () => /* @__PURE__ */ new Date()));
-			},
-			catch: (cause) => failure$2(rule.name, cause)
-		}), (session) => Effect.tryPromise({
-			try: async () => session.close?.(),
-			catch: (cause) => failure$2(rule.name, cause)
-		}).pipe(Effect.catchAll((error) => Effect.sync(() => {
-			closeError = error;
-		}))));
-		if (closeError) return yield* Effect.fail(closeError);
-		const completed = events.slice().reverse().find((event) => event.kind === "completed" && event.output !== void 0);
-		const assistant = events.slice().reverse().find((event) => event.kind === "message" && event.role === "assistant");
-		const output = completed?.kind === "completed" ? completed.output : assistant?.kind === "message" ? assistant.content : void 0;
-		if (output === void 0) return yield* Effect.fail(failure$2(rule.name, /* @__PURE__ */ new Error("Judge agent did not emit a verdict")));
-		const value = yield* Effect.try({
-			try: () => typeof output === "string" ? JSON.parse(output) : output,
-			catch: (cause) => failure$2(rule.name, cause)
-		});
-		const encodedEvents = yield* Schema.decodeUnknown(JsonValueSchema)(events).pipe(Effect.mapError((cause) => failure$2(rule.name, cause)));
-		if (!Array.isArray(encodedEvents)) return yield* Effect.fail(failure$2(rule.name, /* @__PURE__ */ new Error("Judge events must be an array")));
-		const usage = events.slice().reverse().find((event) => event.kind === "turn-completed" && event.usage);
-		return {
-			value,
-			judge: {
-				...agent.identity ? { agent: agent.identity } : {},
-				...usage?.kind === "turn-completed" && usage.usage ? { usage: usage.usage } : {},
-				events: encodedEvents
+	let reading;
+	let reply;
+	let attempts = 0;
+	try {
+		let message = prompt;
+		let problem = "";
+		while (attempts < 2 && !reading) {
+			attempts += 1;
+			const from = events.length;
+			const result = await session.send(message);
+			if (result !== void 0) events.push(...turnResultEvents(result, attempts, () => /* @__PURE__ */ new Date()));
+			reply = replyIn(events.slice(from));
+			if (reply === void 0) problem = "it did not reply";
+			else try {
+				reading = readJudgeAnswer(rule.answer, reply);
+			} catch (error) {
+				if (!(error instanceof UnreadableAnswer)) throw error;
+				problem = error.message;
 			}
-		};
+			message = `Your answer could not be read: ${problem}. Answer again, only in this format.\n\n${answerInstruction(rule.answer)}`;
+		}
+		if (!reading) throw new Error(`Judge ${rule.name}'s answer could not be read after ${attempts} tries: ${problem}${reply === void 0 ? "" : `. Its last reply: ${preview(reply)}`}`);
+	} finally {
+		await session.close?.();
+	}
+	const recorded = Schema.decodeUnknownSync(JsonValueSchema)(events);
+	const model = lastOf(events, (event) => event.kind === "turn-completed" && event.model);
+	const usage = totalUsage(events);
+	const judge = {
+		...agent.identity ? { agent: agent.identity } : {},
+		...model?.kind === "turn-completed" && model.model ? { model: model.model } : {},
+		...usage ? { usage } : {},
+		prompt,
+		...reply !== void 0 ? { reply } : {},
+		...attempts > 1 ? { attempts } : {},
+		events: recorded
+	};
+	return { value: {
+		...reading,
+		judge
+	} };
+}
+/** Each input's text: what the rule's function returned, or the defaults. */
+async function inputValues(rule, input, placement) {
+	if (!rule.inputs) return {
+		conversation: conversationText(input.trajectory.events, placement),
+		reply: text(input.output)
+	};
+	const values = {};
+	for (const [name, read] of Object.entries(rule.inputs)) values[name] = text(await read(input));
+	return values;
+}
+/** The run's conversation as plain lines, for a judge to read: this turn at a checkpoint, else all of it. */
+function conversationText(events, placement) {
+	let start = 0;
+	if (placement === "transcript") events.forEach((event, index) => {
+		if (event.kind === "transcript-step-started" && "message" in event) start = index;
 	});
+	const lines = [];
+	for (const event of events.slice(start)) if (event.kind === "transcript-step-started" && "message" in event) lines.push(`User: ${cut(text(event.message))}`);
+	else if (event.source !== "aut") continue;
+	else if (event.kind === "message") lines.push(`${capitalize(event.role)}: ${cut(text(event.content))}`);
+	else if (event.kind === "tool-call") lines.push(`Tool call ${event.name}: ${cut(text(event.arguments))}`);
+	else if (event.kind === "tool-result") lines.push(`Tool result ${event.name ?? event.id}: ${cut(text(event.result))}`);
+	return lines.join("\n") || "(nothing happened)";
+}
+/** The judge's answer in one attempt's events: its final output, else its last reply. */
+function replyIn(events) {
+	const completed = lastOf(events, (event) => event.kind === "completed" && event.output !== void 0);
+	if (completed?.kind === "completed") return completed.output;
+	const assistant = lastOf(events, (event) => event.kind === "message" && event.role === "assistant");
+	return assistant?.kind === "message" ? assistant.content : void 0;
+}
+function totalUsage(events) {
+	let total;
+	for (const event of events) {
+		if (event.kind !== "turn-completed" || !event.usage) continue;
+		total ??= {};
+		for (const [key, tokens] of Object.entries(event.usage)) if (tokens !== void 0) total[key] = (total[key] ?? 0) + tokens;
+	}
+	return total;
+}
+function text(value) {
+	if (value === void 0) return "(none)";
+	return typeof value === "string" ? value : JSON.stringify(value, null, 2);
+}
+function cut(value) {
+	return value.length > MAX_PART ? `${value.slice(0, MAX_PART)}… (${value.length - MAX_PART} more characters)` : value;
+}
+function preview(value) {
+	const shown = text(value).replace(/\s+/g, " ");
+	return shown.length > 300 ? `${shown.slice(0, 300)}…` : shown;
 }
+/** The last element matching, scanning from the end (no ES2023 lib here). */
+function lastOf(items, match) {
+	for (let index = items.length - 1; index >= 0; index -= 1) if (match(items[index])) return items[index];
+}
+const capitalize = (value) => value.charAt(0).toUpperCase() + value.slice(1);
 //#endregion
 //#region ../runner/src/evaluate-rule.ts
 const failure$1 = (name, cause) => new RuleExecutionError({
@@ -500,38 +735,30 @@ const failure$1 = (name, cause) => new RuleExecutionError({
 	message: cause instanceof Error ? cause.message : String(cause)
 });
 /** Both placements call the same rule executor; only input timing and persistence differ. */
-function evaluateRule(rule, input, judgeAgent, placement) {
+function evaluateRule(rule, input, defaultJudge, placement) {
 	return Effect.gen(function* () {
-		if (rule.kind === "judge" && !judgeAgent) return yield* Effect.fail(failure$1(rule.name, /* @__PURE__ */ new Error("Judge rules require an eval judge agent")));
-		const evaluated = rule.kind === "predicate" ? { value: yield* Effect.tryPromise({
+		const judgeAgent = rule.kind === "judge" ? rule.agent ?? defaultJudge : void 0;
+		if (rule.kind === "judge" && !judgeAgent) return yield* Effect.fail(failure$1(rule.name, /* @__PURE__ */ new Error("This judge has no agent: pass judge(name, { agent }) or set judge in evalkit.config.ts")));
+		const evaluated = rule.kind === "predicate" ? yield* Effect.tryPromise({
 			try: async () => rule.run(input),
 			catch: (cause) => failure$1(rule.name, cause)
-		}) } : yield* runJudgeAgent(rule, judgeAgent, input, placement);
+		}) : (yield* runJudgeAgent(rule, judgeAgent, input, placement)).value;
 		const score = yield* Effect.try({
-			try: () => normalizeScore(evaluated.value, rule.kind === "predicate" ? rule.passAt : void 0),
+			try: () => normalizeScore(evaluated, rule.passAt, rule.kind === "predicate" ? rule.criteria : void 0),
 			catch: (cause) => failure$1(rule.name, cause)
 		});
-		const withJudge = rule.kind === "judge" && "judge" in evaluated ? {
-			...score,
-			judge: evaluated.judge
-		} : score;
 		const validated = yield* Schema.decodeUnknown(ScoreResultSchema)({
 			name: rule.name,
 			kind: rule.kind,
 			durationMs: 0,
-			...withJudge
+			...score
 		}).pipe(Effect.mapError((cause) => failure$1(rule.name, cause)));
 		if (validated.skipped) return {
 			skipped: true,
 			...validated.explanation === void 0 ? {} : { explanation: validated.explanation }
 		};
-		return {
-			value: validated.value,
-			passed: validated.passed,
-			...validated.explanation === void 0 ? {} : { explanation: validated.explanation },
-			...validated.evidence === void 0 ? {} : { evidence: validated.evidence },
-			...validated.judge === void 0 ? {} : { judge: validated.judge }
-		};
+		const { name: _name, kind: _kind, durationMs: _duration, ...result } = validated;
+		return result;
 	});
 }
 //#endregion
@@ -583,7 +810,6 @@ function evaluateCheckpoint(step, index, input, now, judgeAgent) {
 	const annotations = {
 		evalId: input.context.evalId,
 		runId: input.context.runId,
-		trialId: input.context.trialId,
 		step: index,
 		checkpoint: step.name
 	};
@@ -682,7 +908,7 @@ function userMessage(step, index, { context, randomSeed }) {
 		parameters: context.parameters ?? {},
 		data: context.data,
 		seed: randomSeed,
-		trialIndex: context.trialIndex
+		...context.batch ? { batch: context.batch } : {}
 	});
 	if (typeof message !== "string" || !message.trim()) throw new Error(`The user message at step ${index + 1} must compute a non-empty string`);
 	return message;
@@ -706,6 +932,7 @@ async function runCheckpoint(definition, step, index, input) {
 		name: step.name,
 		timestamp: started.toISOString()
 	});
+	const tracked = trackAttachments(input.attach);
 	return runBoundary(evaluateCheckpoint(step, index, {
 		context: input.context,
 		artifacts: input.artifacts,
@@ -713,8 +940,13 @@ async function runCheckpoint(definition, step, index, input) {
 		turn: progress.lastTurn,
 		...turnOutput(progress.lastTurn),
 		data: input.data,
-		checkpoints: [...progress.checkpoints]
-	}, now, definition.judge)).catch(async (error) => {
+		checkpoints: [...progress.checkpoints],
+		checkpoint: checkpointLookup(definition.transcript, [...progress.checkpoints]),
+		attach: tracked.attach
+	}, now, input.defaultJudge)).then((result) => ({
+		...result,
+		...attachedFields(tracked.attached)
+	})).catch(async (error) => {
 		const recorded = recordError(error);
 		progress.checkpoints.push({
 			step: index,
@@ -722,7 +954,8 @@ async function runCheckpoint(definition, step, index, input) {
 			name: step.name,
 			status: "error",
 			durationMs: now().getTime() - started.getTime(),
-			error: recorded
+			error: recorded,
+			...attachedFields(tracked.attached)
 		});
 		try {
 			await emitRunner({
@@ -779,7 +1012,9 @@ function scoringContext(input) {
 		...input.lastTurn ? { turn: input.lastTurn } : {},
 		...turnOutput(input.lastTurn),
 		data: input.data,
-		checkpoints: input.checkpoints
+		checkpoints: input.checkpoints,
+		checkpoint: checkpointLookup(input.transcript, input.checkpoints),
+		attach: input.attach
 	};
 }
 /** Append completed scores to the supplied arrays, even when a later scorer or report write fails. */
@@ -797,12 +1032,15 @@ async function evaluateFinalScorers(definition, input) {
 			timestamp: started.toISOString()
 		});
 		let result;
+		const tracked = trackAttachments(input.attach);
 		try {
 			const score = await runBoundary(evaluateRule(scorer, scoringContext({
 				...input,
+				attach: tracked.attach,
 				context,
-				workspace
-			}), definition.judge, "scoring").pipe(Effect.mapError((cause) => new ScoringError({
+				workspace,
+				transcript: definition.transcript
+			}), input.defaultJudge, "scoring").pipe(Effect.mapError((cause) => new ScoringError({
 				cause,
 				message: cause.message
 			}))));
@@ -810,7 +1048,8 @@ async function evaluateFinalScorers(definition, input) {
 				name: scorer.name,
 				kind: scorer.kind,
 				durationMs: now().getTime() - started.getTime(),
-				...score
+				...score,
+				...attachedFields(tracked.attached)
 			};
 			results.push(result);
 		} catch (error) {
@@ -820,7 +1059,8 @@ async function evaluateFinalScorers(definition, input) {
 				kind: scorer.kind,
 				durationMs: now().getTime() - started.getTime(),
 				error: recorded,
-				passed: false
+				passed: false,
+				...attachedFields(tracked.attached)
 			});
 			await emitRunner({
 				kind: "scorer-failed",
@@ -841,11 +1081,12 @@ async function evaluateFinalScorers(definition, input) {
 //#endregion
 //#region ../runner/src/metrics.ts
 /**
-* What EvalKit measures for every trial from its trajectory: the agent's time
+* What EvalKit measures for every run: from its trajectory, the agent's time
 * (first event to the last one before final scoring), tokens when the agent
-* reported usage, and tool calls.
+* reported usage, and tool calls; from its scores and checkpoints, the
+* tokens its judges used, kept apart from the agent's.
 */
-function builtInMetrics(events) {
+function builtInMetrics(events, verdicts = []) {
 	const times = events.filter((event) => !event.kind.startsWith("scorer-")).map((event) => Date.parse(event.timestamp)).filter((time) => !Number.isNaN(time));
 	const metrics = [];
 	if (times.length) metrics.push({
@@ -857,8 +1098,7 @@ function builtInMetrics(events) {
 	});
 	let tokens;
 	for (const event of events) if (event.source === "aut" && event.kind === "turn-completed" && event.usage) {
-		const { totalTokens, inputTokens, outputTokens } = event.usage;
-		const turn = totalTokens ?? (inputTokens === void 0 && outputTokens === void 0 ? void 0 : (inputTokens ?? 0) + (outputTokens ?? 0));
+		const turn = turnTokens(event.usage);
 		if (turn !== void 0) tokens = (tokens ?? 0) + turn;
 	}
 	if (tokens !== void 0) metrics.push({
@@ -873,6 +1113,18 @@ function builtInMetrics(events) {
 		value: events.filter((event) => event.source === "aut" && event.kind === "tool-call").length,
 		builtIn: true
 	});
+	let judgeTokens;
+	for (const verdict of verdicts) {
+		const used = verdict.judge?.usage && turnTokens(verdict.judge.usage);
+		if (used !== void 0) judgeTokens = (judgeTokens ?? 0) + used;
+	}
+	if (judgeTokens !== void 0) metrics.push({
+		name: "judge tokens",
+		value: judgeTokens,
+		unit: "tokens",
+		better: "lower",
+		builtIn: true
+	});
 	return metrics;
 }
 /** Runs the scenario's metrics; a failing metric records its error and the rest still run. */
@@ -903,7 +1155,7 @@ async function evaluateMetrics(definition, context) {
 //#endregion
 //#region ../runner/src/local-report-store.ts
 const MANIFEST_FILE = "manifest.json";
-/** The project's evals and suites when the run started (schema version 4+). */
+/** The project's evals and suites when the run started. */
 const CATALOG_FILE = "catalog.json";
 async function writeJson(filePath, value) {
 	await writeFile(filePath, `${JSON.stringify(value, null, 2)}\n`);
@@ -922,10 +1174,12 @@ function artifactPath(root, value) {
 	if (normalized === ".." || normalized.startsWith(`..${path.sep}`)) throw new Error(`Artifact path escapes its root: ${value}`);
 	return path.join(root, normalized);
 }
-var LocalTrialWriter = class {
+var LocalRunWriter = class {
+	location;
 	directory;
 	metadata;
-	constructor(directory, metadata) {
+	constructor(location, directory, metadata) {
+		this.location = location;
 		this.directory = directory;
 		this.metadata = metadata;
 	}
@@ -933,40 +1187,13 @@ var LocalTrialWriter = class {
 		return appendFile(path.join(this.directory, "trajectory.jsonl"), `${JSON.stringify(Schema.encodeSync(TrajectoryEventSchema)(event))}\n`);
 	}
 	writeScores(scoring) {
-		return writeJson(path.join(this.directory, "scoring.json"), Schema.encodeSync(TrialScoringSchema)(scoring));
+		return writeJson(path.join(this.directory, "scoring.json"), Schema.encodeSync(RunScoringSchema)(scoring));
 	}
 	async writeArtifact(relativePath, data) {
 		const target = artifactPath(path.join(this.directory, "artifacts"), relativePath);
 		await mkdir(path.dirname(target), { recursive: true });
 		await writeFile(target, data);
 	}
-	async finalize(summary) {
-		await writeJson(path.join(this.directory, "summary.json"), Schema.encodeSync(TrialSummarySchema)(summary));
-		await replaceJson(path.join(this.directory, MANIFEST_FILE), {
-			...Schema.encodeSync(TrialMetadataSchema)(this.metadata),
-			status: summary.status
-		});
-	}
-};
-var LocalRunWriter = class {
-	location;
-	directory;
-	metadata;
-	constructor(location, directory, metadata) {
-		this.location = location;
-		this.directory = directory;
-		this.metadata = metadata;
-	}
-	async startTrial(metadata) {
-		assertPathSegment(metadata.trialId, "trialId");
-		const directory = path.join(this.directory, "trials", metadata.trialId);
-		await mkdir(directory, { recursive: true });
-		await writeJson(path.join(directory, MANIFEST_FILE), {
-			...Schema.encodeSync(TrialMetadataSchema)(metadata),
-			status: "running"
-		});
-		return new LocalTrialWriter(directory, metadata);
-	}
 	async finalize(summary) {
 		await writeJson(path.join(this.directory, "summary.json"), Schema.encodeSync(RunSummarySchema)(summary));
 		await replaceJson(path.join(this.directory, MANIFEST_FILE), {
@@ -991,17 +1218,15 @@ function localReportStore(rootDirectory) {
 }
 //#endregion
 //#region ../runner/src/local-report-reader.ts
-/** Shared v2/v3 local report boundary for CLI, dashboard, and CI consumers. */
+/** The local report boundary for CLI, dashboard, and CI consumers (schema version 6). */
 async function decode(file, schema) {
 	return Schema.decodeUnknownSync(schema)(JSON.parse(await readFile(file, "utf8")));
 }
 const readRunManifest = (root, runId) => decode(join(root, runId, "manifest.json"), RunManifestSchema);
 const readRunSummary = (root, runId) => decode(join(root, runId, "summary.json"), RunSummarySchema);
-const readTrialManifest = (root, runId, trialId) => decode(join(root, runId, "trials", trialId, "manifest.json"), TrialManifestSchema);
-const readTrialSummary = (root, runId, trialId) => decode(join(root, runId, "trials", trialId, "summary.json"), TrialSummarySchema);
-const readTrialScoring = (root, runId, trialId) => decode(join(root, runId, "trials", trialId, "scoring.json"), TrialScoringSchema);
-async function readTrialEvents(root, runId, trialId) {
-	return parseTrajectoryJsonl(await readFile(join(root, runId, "trials", trialId, "trajectory.jsonl"), "utf8"));
+const readRunScoring = (root, runId) => decode(join(root, runId, "scoring.json"), RunScoringSchema);
+async function readRunEvents(root, runId) {
+	return parseTrajectoryJsonl(await readFile(join(root, runId, "trajectory.jsonl"), "utf8"));
 }
 /**
 * True when a run's owning process is known to be gone: it ran on this host
@@ -1028,6 +1253,7 @@ function runMatrix(matrix, options) {
 		const iterator = matrix.cells(options.selection);
 		const summary = {
 			cells: 0,
+			runs: 0,
 			passed: 0,
 			failed: 0,
 			cancelled: 0
@@ -1043,7 +1269,7 @@ function runMatrix(matrix, options) {
 					"sandbox"
 				].find((name) => agentRuntimes(cell.eval.agent)[name]);
 				const suiteId = options.suiteId ?? options.suiteFor?.(cell);
-				const result = yield* runEval(cell.eval, {
+				const runs = yield* runEval(cell.eval, {
 					...options,
 					...suiteId ? { suiteId } : {},
 					concurrency: 1,
@@ -1055,18 +1281,232 @@ function runMatrix(matrix, options) {
 					}
 				});
 				summary.cells++;
-				if (result.status === "cancelled") summary.cancelled++;
-				else if (result.status === "completed" && (result.aggregateScoring?.passRate === 1 || result.scoring?.passed)) summary.passed++;
-				else summary.failed++;
-				if (options.onResult) yield* Effect.tryPromise(async () => options.onResult(cell, result));
+				for (const run of runs) {
+					summary.runs++;
+					if (run.status === "cancelled") summary.cancelled++;
+					else if (run.status === "passed") summary.passed++;
+					else summary.failed++;
+				}
+				if (options.onResult) yield* Effect.tryPromise(async () => options.onResult(cell, runs));
 			}
 		})), { concurrency });
 		return summary;
 	});
 }
 //#endregion
+//#region ../runner/src/upgrade.ts
+/** A new run being written. */
+const TEMPORARY = ".upgrade-";
+/** An old run set aside while its replacement, with the same ID, moves in. */
+const RETIRED = ".upgraded-";
+const PER_RUN_FILES = ["archived.json"];
+/**
+* Rewrites results from report schema versions 2 to 5 into version 6, in
+* place (RFD0008). An old run of one trial (most of them) becomes a run with
+* the same ID, so links to it keep working. An old run of several trials
+* becomes runs with the trials' IDs, in a batch with the old run's ID.
+*
+* Each new run is written under a temporary name and renamed into place;
+* artifacts are moved, not copied, as the last step before that rename. An
+* old run is removed only after its new runs exist, and one replaced under
+* the same ID is set aside first, so a crash at any point leaves the old run
+* or complete new ones, and re-running finishes the job. Version 6 runs are
+* left alone.
+*/
+async function upgradeReports(root, options = {}) {
+	const report = {
+		upgraded: 0,
+		runs: 0,
+		batches: 0,
+		running: []
+	};
+	let entries;
+	try {
+		entries = await readdir(root);
+	} catch {
+		return report;
+	}
+	if (!options.dryRun) await finishReplacements(root, entries);
+	for (const entry of entries) {
+		if (entry.startsWith(".")) continue;
+		const directory = join(root, entry);
+		const manifest = await readJson(join(directory, "manifest.json"));
+		if (!manifest || !UPGRADABLE_SCHEMA_VERSIONS.includes(manifest.schemaVersion)) continue;
+		const summary = await readJson(join(directory, "summary.json"));
+		if (!summary && !ownerIsGone(manifest.process)) {
+			report.running.push(entry);
+			continue;
+		}
+		const trials = await trialIds(directory);
+		report.upgraded += 1;
+		report.runs += Math.max(trials.length, 1);
+		if (trials.length > 1) report.batches += 1;
+		if (options.dryRun) continue;
+		await upgradeRun(root, entry, manifest, summary, trials);
+	}
+	if (!options.dryRun) await removeLeftovers(root);
+	return report;
+}
+/**
+* Finishes same-ID replacements a crash interrupted: the old run was set
+* aside, and its replacement may or may not have moved in.
+*/
+async function finishReplacements(root, entries) {
+	for (const entry of entries) {
+		if (!entry.startsWith(RETIRED)) continue;
+		const id = entry.slice(10);
+		const retired = join(root, entry);
+		const directory = join(root, id);
+		const replacement = join(root, `${TEMPORARY}${id}`);
+		if (!await exists(directory)) await rename(await exists(replacement) ? replacement : retired, directory);
+		await rm(retired, {
+			recursive: true,
+			force: true
+		});
+	}
+}
+/**
+* Temporary folders nothing will finish. One holding artifacts is kept: it
+* could be the only copy, and an old run would have moved them back.
+*/
+async function removeLeftovers(root) {
+	for (const entry of await readdir(root)) if (entry.startsWith(TEMPORARY) && !await exists(join(root, entry, "artifacts"))) await rm(join(root, entry), {
+		recursive: true,
+		force: true
+	});
+}
+/** Swaps a fully written replacement in for the old run of the same ID. */
+async function replaceInPlace(root, id, replacement) {
+	const retired = join(root, `${RETIRED}${id}`);
+	await rename(join(root, id), retired);
+	await rename(replacement, join(root, id));
+	await rm(retired, {
+		recursive: true,
+		force: true
+	});
+}
+async function upgradeRun(root, oldRunId, runManifest, runSummary, trials) {
+	const directory = join(root, oldRunId);
+	const catalog = await readJson(join(directory, "catalog.json"));
+	const extras = await presentFiles(directory, PER_RUN_FILES);
+	if (!trials.length) {
+		const target = join(root, `${TEMPORARY}${oldRunId}`);
+		await mkdir(target, { recursive: true });
+		const status = runSummary ? "cancelled" : "running";
+		await writeJsonFile(join(target, "manifest.json"), upgradeManifest({
+			run: runManifest,
+			runId: oldRunId,
+			status
+		}));
+		if (runSummary) await writeJsonFile(join(target, "summary.json"), {
+			status,
+			endedAt: runSummary.endedAt,
+			...runSummary.durationMs === void 0 ? {} : { durationMs: runSummary.durationMs },
+			...runSummary.error ? { error: runSummary.error } : {}
+		});
+		await copyShared(directory, target, catalog, extras);
+		await replaceInPlace(root, oldRunId, target);
+		return;
+	}
+	const inPlace = trials.length === 1;
+	for (const trialId of trials) {
+		const runId = inPlace ? oldRunId : trialId;
+		const destination = join(root, runId);
+		if (!inPlace && await exists(join(destination, "manifest.json"))) continue;
+		const source = join(directory, "trials", trialId);
+		const trialManifest = await readJson(join(source, "manifest.json")) ?? {};
+		const trialSummary = await readJson(join(source, "summary.json"));
+		const passed = (trialSummary?.scoring)?.passed;
+		const status = upgradeStatus(trialSummary?.status ?? trialManifest.status, passed);
+		const index = typeof trialManifest.trialIndex === "number" ? trialManifest.trialIndex : trials.indexOf(trialId);
+		const target = join(root, `${TEMPORARY}${runId}`);
+		if (!await exists(join(source, "artifacts")) && await exists(join(target, "artifacts"))) await rename(join(target, "artifacts"), join(source, "artifacts"));
+		await rm(target, {
+			recursive: true,
+			force: true
+		});
+		await mkdir(target, { recursive: true });
+		await writeJsonFile(join(target, "manifest.json"), upgradeManifest({
+			run: runManifest,
+			trial: trialManifest,
+			runId,
+			...!inPlace ? { batch: {
+				id: oldRunId,
+				index,
+				size: trials.length
+			} } : {},
+			status
+		}));
+		if (trialSummary) await writeJsonFile(join(target, "summary.json"), {
+			...trialSummary,
+			status
+		});
+		const scoring = await readText(join(source, "scoring.json"));
+		if (scoring !== void 0) await writeFile(join(target, "scoring.json"), scoring);
+		const trajectory = await readText(join(source, "trajectory.jsonl"));
+		if (trajectory !== void 0) await writeFile(join(target, "trajectory.jsonl"), upgradeTrajectory(trajectory));
+		await copyShared(directory, target, catalog, extras);
+		if (await exists(join(source, "artifacts"))) await rename(join(source, "artifacts"), join(target, "artifacts"));
+		if (inPlace) await replaceInPlace(root, oldRunId, target);
+		else await rename(target, destination);
+	}
+	if (!inPlace) await rm(directory, {
+		recursive: true,
+		force: true
+	});
+}
+async function copyShared(from, to, catalog, extras) {
+	if (catalog) await writeJsonFile(join(to, "catalog.json"), upgradeCatalog(catalog));
+	for (const file of extras) await cp(join(from, file), join(to, file));
+}
+async function trialIds(directory) {
+	try {
+		return (await readdir(join(directory, "trials"), { withFileTypes: true })).filter((entry) => entry.isDirectory()).map((entry) => entry.name).sort();
+	} catch {
+		return [];
+	}
+}
+async function presentFiles(directory, names) {
+	const present = [];
+	for (const name of names) if (await exists(join(directory, name))) present.push(name);
+	return present;
+}
+async function exists(path) {
+	try {
+		await stat(path);
+		return true;
+	} catch {
+		return false;
+	}
+}
+async function readText(path) {
+	try {
+		return await readFile(path, "utf8");
+	} catch {
+		return;
+	}
+}
+async function readJson(path) {
+	const text = await readText(path);
+	if (text === void 0) return void 0;
+	let value;
+	try {
+		value = JSON.parse(text);
+	} catch (error) {
+		throw new Error(`${path} is not valid JSON: ${error instanceof Error ? error.message : String(error)}`);
+	}
+	return typeof value === "object" && value !== null && !Array.isArray(value) ? value : void 0;
+}
+async function writeJsonFile(path, value) {
+	await writeFile(path, `${JSON.stringify(value, null, 2)}\n`);
+}
+//#endregion
 //#region ../runner/src/index.ts
-/** Effect-native aggregate eval execution entry point. */
+/**
+* Runs a scenario: once, or `repeat` times as one batch. Resolves to one
+* result per run started; runs never started because of cancellation are
+* left out.
+*/
 function runEval(definition, options) {
 	return Effect.gen(function* () {
 		yield* Effect.logInfo("eval run started").pipe(Effect.annotateLogs({
@@ -1074,19 +1514,19 @@ function runEval(definition, options) {
 			...options.runId ? { runId: options.runId } : {},
 			...options.suiteId ? { suiteId: options.suiteId } : {}
 		}));
-		const result = yield* Effect.tryPromise({
-			try: () => executeRun(definition, options),
+		const runs = yield* Effect.tryPromise({
+			try: () => executeBatch(definition, options),
 			catch: (error) => error
 		});
 		yield* Effect.logInfo("eval run finished").pipe(Effect.annotateLogs({
 			evalId: authoringId(definition),
-			runId: result.runId,
-			status: result.status
+			runs: runs.length,
+			passed: runs.filter((run) => run.status === "passed").length
 		}));
-		return result;
+		return runs;
 	});
 }
-function createId(_kind) {
+function createId() {
 	return crypto.randomUUID();
 }
 function canonicalId(kind, id) {
@@ -1095,9 +1535,9 @@ function canonicalId(kind, id) {
 function assertUuid(value, label) {
 	if (!/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(value)) throw new Error(`${label} must be a UUID`);
 }
-function runMetadata(definition, options, runId, startedAt) {
+function runMetadata(definition, options, runId, startedAt, extra) {
 	return {
-		schemaVersion: 5,
+		schemaVersion: 6,
 		runId,
 		runUri: canonicalId("run", runId),
 		evalId: definition.id,
@@ -1110,7 +1550,9 @@ function runMetadata(definition, options, runId, startedAt) {
 		...definition.agent.identity ? { aut: definition.agent.identity } : {},
 		parameters: options.parameters,
 		matrix: options.matrix,
-		startedAt
+		startedAt,
+		...options.batch ? { batch: options.batch } : {},
+		...extra
 	};
 }
 /** Recording a secondary error must never mask the original failure or block cleanup. */
@@ -1123,104 +1565,53 @@ async function tryEmitError(error, emitRunner, now) {
 		});
 	} catch {}
 }
-/** Executes one aggregate eval run and its requested isolated trials. */
-async function executeRun(definition, options) {
+/** Starts `repeat` runs of a scenario, as one batch when there are several. */
+async function executeBatch(definition, options) {
 	let hasUser = false;
 	for (const step of definition.transcript) if (step.kind === "user") hasUser = true;
 	else if (step.kind !== "agent" && !hasUser) throw new Error("Checkpoint requires a preceding user step");
-	if (definition.judge && definition.judge === definition.agent) throw new Error("Judge agent must be distinct from the agent under test");
+	assertNoRemovedOptions(definition);
+	assertJudgesIndependent(definition, options.judge);
+	for (const rule of [...definition.transcript, ...definition.scoring]) if (rule.kind === "judge" && !rule.agent && !options.judge) throw new Error(`Judge ${rule.name} in ${definition.id} has no agent: pass judge(name, { agent }) or set judge in evalkit.config.ts`);
 	if (options.runId) assertUuid(options.runId, "runId");
-	if (options.trialId) assertUuid(options.trialId, "trialId");
-	const requestedTrials = options.trials ?? definition.policy?.trials ?? 1;
-	if (!Number.isInteger(requestedTrials) || requestedTrials < 1) throw new Error(`Trial count must be a positive integer; received ${requestedTrials}`);
-	if (requestedTrials === 1) return runTrial(definition, options);
-	const now = options.now ?? (() => /* @__PURE__ */ new Date());
-	const aggregateStartedAt = now();
-	const runId = options.runId ?? createId("run");
-	const runWriter = await options.report.startRun(runMetadata(definition, options, runId, aggregateStartedAt.toISOString()), { catalog: options.catalog });
+	const repeat = options.repeat ?? definition.repeat ?? 1;
+	if (!Number.isInteger(repeat) || repeat < 1) throw new Error(`repeat must be a positive integer; received ${repeat}`);
+	if (repeat === 1) return [await runOnce(definition, options)];
+	if (options.runId) throw new Error("runId names one run; it cannot be used with repeat");
+	const batchId = createId();
 	const signal = options.signal;
-	const trialEffects = Array.from({ length: requestedTrials }, (_, trialIndex) => {
-		const trial = Effect.tryPromise({
-			try: async () => signal?.aborted ? void 0 : runTrial(definition, {
+	const runs = Array.from({ length: repeat }, (_, index) => {
+		const run = Effect.tryPromise({
+			try: async () => signal?.aborted ? void 0 : runOnce(definition, {
 				...options,
-				runId,
-				trialId: createId("trial"),
-				trials: 1,
-				runWriter,
-				trialIndex
+				batch: {
+					id: batchId,
+					index,
+					size: repeat
+				}
 			}),
 			catch: (error) => error
 		});
-		return options.semaphore ? options.semaphore.withPermits(1)(trial) : trial;
-	});
-	const trials = (await Effect.runPromise(Effect.all(trialEffects, { concurrency: options.concurrency ?? 32 }))).filter((trial) => trial !== void 0);
-	const counts = trialCounts(trials);
-	const status = counts.cancelled || signal?.aborted ? "cancelled" : trials.some((trial) => trial.status === "failed") ? "failed" : "completed";
-	const endedAt = now();
-	await runWriter.finalize({
-		status,
-		endedAt: endedAt.toISOString(),
-		durationMs: endedAt.getTime() - aggregateStartedAt.getTime(),
-		trialCount: trials.length,
-		passed: counts.passed,
-		failed: counts.failed,
-		...counts.cancelled ? { cancelled: counts.cancelled } : {}
+		return options.semaphore ? options.semaphore.withPermits(1)(run) : run;
 	});
-	if (!trials.length) return {
-		runId,
-		runUri: canonicalId("run", runId),
-		trialId: "",
-		trialUri: canonicalId("trial", runId),
-		trialIndex: 0,
-		evalId: authoringId(definition),
-		status,
-		reportLocation: runWriter.location,
-		trialCount: 0,
-		passed: 0,
-		failed: 0,
-		aggregateScoring: {
-			passed: 0,
-			failed: 0,
-			passRate: 0
-		},
-		trials: []
-	};
-	const overallScores = trials.map((trial) => trial.scoring?.overall).filter((score) => score !== void 0);
-	return {
-		...trials[0],
-		runId,
-		evalId: authoringId(definition),
-		status,
-		trialCount: trials.length,
-		passed: counts.passed,
-		failed: counts.failed,
-		aggregateScoring: {
-			passed: counts.passed,
-			failed: counts.failed,
-			...counts.cancelled ? { cancelled: counts.cancelled } : {},
-			passRate: counts.passed / trials.length,
-			...overallScores.length ? { overall: overallScores.reduce((sum, score) => sum + score, 0) / overallScores.length } : {}
-		},
-		trials
-	};
+	return (await Effect.runPromise(Effect.all(runs, { concurrency: options.concurrency ?? 32 }))).filter((run) => run !== void 0);
 }
-async function runTrial(definition, options) {
+/** One execution: a session in its own sandbox, scored and reported. */
+async function runOnce(definition, options) {
 	const now = options.now ?? (() => /* @__PURE__ */ new Date());
-	const runId = options.runId ?? createId("run");
-	const trialId = options.trialId ?? createId("trial");
+	const runId = options.runId ?? createId();
 	const startedAt = now().toISOString();
 	const selectedRuntime = options.runtime;
 	const runtime = selectedRuntime ? agentRuntimes(definition.agent)[selectedRuntime] : void 0;
 	if (selectedRuntime && !runtime) throw new Error(`AUT does not declare the requested ${selectedRuntime} runtime`);
 	const randomSeed = crypto.getRandomValues(/* @__PURE__ */ new Uint32Array(1))[0];
-	const trialIndex = options.trialIndex ?? 0;
+	const batch = options.batch;
 	const signal = options.signal ?? new AbortController().signal;
 	const root = options.root ?? process.cwd();
 	const baseContext = {
 		runId,
 		evalId: authoringId(definition),
-		trialId,
-		trialIndex,
+		...batch ? { batch } : {},
 		metadata: {
 			...toJsonObject(definition.metadata ?? {}, `${definition.id}.metadata`),
 			randomSeed
@@ -1234,8 +1625,7 @@ async function runTrial(definition, options) {
 		prepared = await prepareDatasets(definition.datasets ?? [], root, {
 			evalId: baseContext.evalId,
 			runId,
-			trialId,
-			trialIndex,
+			...batch ? { batch } : {},
 			seed: randomSeed,
 			parameters: options.parameters ?? {},
 			signal
@@ -1247,40 +1637,28 @@ async function runTrial(definition, options) {
 	const scoringData = datasetValues(prepared, "all");
 	const workspaceIgnore = [...options.artifactExclude ?? DEFAULT_ARTIFACT_EXCLUDE, ...definition.workspace?.ignore ?? []];
 	const events = [];
-	const runWriter = options.runWriter ?? await options.report.startRun(runMetadata(definition, options, runId, startedAt), { catalog: options.catalog });
-	const trialWriter = await runWriter.startTrial({
-		schemaVersion: 5,
-		runId,
-		runUri: canonicalId("run", runId),
-		trialId,
-		trialUri: canonicalId("trial", trialId),
-		trialIndex: options.trialIndex ?? 0,
-		evalId: definition.id,
-		...definition.agent.identity ? { aut: definition.agent.identity } : {},
-		parameters: options.parameters,
-		matrix: options.matrix,
-		startedAt,
+	const runWriter = await options.report.startRun(runMetadata(definition, options, runId, startedAt, {
 		workspaceIgnore,
-		...prepared.length ? { datasets: trialDatasets(prepared) } : {}
-	});
+		...prepared.length ? { datasets: runDatasets(prepared) } : {}
+	}), { catalog: options.catalog });
+	const attachments = attachmentStore(runWriter, options.attachmentLimits);
 	let writeChain = Promise.resolve();
 	let reportingFailed = false;
-	const trialRef = {
+	const runRef = {
 		runId,
-		trialId,
-		trialIndex,
+		...batch ? { batch } : {},
 		evalId: definition.id,
 		...options.suiteId ? { suiteId: options.suiteId } : {},
 		...options.parameters ? { parameters: options.parameters } : {}
 	};
 	const observe = (event) => {
 		try {
-			options.onEvent?.(trialRef, event);
+			options.onEvent?.(runRef, event);
 		} catch {}
 	};
 	const append = async (event) => {
 		events.push(event);
-		writeChain = writeChain.then(() => trialWriter.appendEvent(event));
+		writeChain = writeChain.then(() => runWriter.appendEvent(event));
 		try {
 			await writeChain;
 			observe(event);
@@ -1316,7 +1694,7 @@ async function runTrial(definition, options) {
 	const workspaceScope = await Effect.runPromise(Scope.make());
 	try {
 		await emitRunner({
-			kind: "trial-started",
+			kind: "run-started",
 			timestamp: now().toISOString()
 		});
 		if (signal.aborted) throw cancelled();
@@ -1325,7 +1703,7 @@ async function runTrial(definition, options) {
 			message: datasetsError instanceof Error ? datasetsError.message : String(datasetsError)
 		});
 		workspace = await runBoundary(Effect.acquireRelease(Effect.tryPromise({
-			try: () => createTrialWorkspace(trialId, prepared, options.workspaceRoot),
+			try: () => createRunWorkspace(runId, prepared, options.workspaceRoot),
 			catch: (cause) => new DatasetError({
 				cause,
 				message: cause instanceof Error ? cause.message : String(cause)
@@ -1338,20 +1716,21 @@ async function runTrial(definition, options) {
 			})
 		}).pipe(Effect.catchAll((error) => Effect.sync(() => {
 			cleanupError = error;
-		}).pipe(Effect.tap(() => Effect.logError("trial workspace cleanup failed")))))).pipe(Effect.provideService(Scope.Scope, workspaceScope)));
-		const trialWorkspace = workspace;
-		const trialContext = {
+		}).pipe(Effect.tap(() => Effect.logError("run workspace cleanup failed")))))).pipe(Effect.provideService(Scope.Scope, workspaceScope)));
+		const runWorkspace = workspace;
+		const runContext = {
 			...baseContext,
 			data: candidateData,
-			workspace: trialWorkspace.artifacts.candidate,
-			evaluatorWorkspace: trialWorkspace.artifacts.evaluator
+			workspace: runWorkspace.artifacts.candidate,
+			evaluatorWorkspace: runWorkspace.artifacts.evaluator
 		};
-		context = trialContext;
+		context = runContext;
 		await runBoundary(Effect.acquireUseRelease(Effect.tryPromise({
 			try: () => definition.agent.start({
-				context: trialContext,
+				context: runContext,
 				...runtime ? { runtime } : {},
-				onEvent: onAutEvent
+				onEvent: onAutEvent,
+				attach: attachments.attach
 			}),
 			catch: (cause) => new AutExecutionError({
 				cause,
@@ -1359,8 +1738,8 @@ async function runTrial(definition, options) {
 			})
 		}), (session) => Effect.tryPromise({
 			try: () => executeTranscript(definition, session, {
-				context: trialContext,
-				artifacts: trialWorkspace.artifacts,
+				context: runContext,
+				artifacts: runWorkspace.artifacts,
 				events,
 				now,
 				randomSeed,
@@ -1368,7 +1747,9 @@ async function runTrial(definition, options) {
 				emitAut,
 				data: scoringData,
 				signal,
-				progress
+				progress,
+				attach: attachments.attach,
+				defaultJudge: options.judge
 			}),
 			catch: (cause) => cause
 		}), (opened) => Effect.tryPromise({
@@ -1381,7 +1762,6 @@ async function runTrial(definition, options) {
 			closeError = error;
 		}).pipe(Effect.tap(() => Effect.logError("AUT session close failed")))))).pipe(Effect.annotateLogs({
 			runId,
-			trialId,
 			evalId: definition.id
 		})));
 	} catch (error) {
@@ -1411,22 +1791,26 @@ async function runTrial(definition, options) {
 			skippedScorers,
 			data: scoringData,
 			checkpoints,
+			attach: attachments.attach,
+			defaultJudge: options.judge,
 			now,
 			emitRunner
 		});
-		metrics.push(...builtInMetrics(events));
+		metrics.push(...builtInMetrics(events, [...scoreResults, ...checkpoints]));
 		if (context && workspace && !wasCancelled) metrics.push(...await evaluateMetrics(definition, scoringContext({
 			context,
 			workspace,
 			events,
 			lastTurn,
 			data: scoringData,
-			checkpoints
+			checkpoints,
+			transcript: definition.transcript,
+			attach: attachments.attach
 		})));
 		scoring = scoreSummary(scoreResults, checkpoints, skippedScorers, metrics);
-		await trialWriter.writeScores(scoring);
+		await runWriter.writeScores(scoring);
 		if (workspace) try {
-			artifacts = await snapshotCandidateWorkspace(workspace.artifacts.candidate.root, trialWriter, workspaceIgnore);
+			artifacts = await snapshotCandidateWorkspace(workspace.artifacts.candidate.root, runWriter, workspaceIgnore);
 		} catch (captureError) {
 			if (!primaryError) primaryError = captureError;
 			await tryEmitError(captureError, emitRunner, now);
@@ -1444,49 +1828,34 @@ async function runTrial(definition, options) {
 	scoring ??= scoreSummary(scoreResults, checkpoints, skippedScorers, metrics);
 	if (!primaryError) try {
 		await emitRunner({
-			kind: "trial-completed",
+			kind: "run-completed",
 			timestamp: now().toISOString()
 		});
 	} catch (error) {
 		primaryError = error;
 	}
-	status = wasCancelled ? "cancelled" : primaryError ? "failed" : "completed";
+	status = wasCancelled ? "cancelled" : primaryError ? "errored" : scoring.passed ? "passed" : "failed";
 	const error = primaryError ? recordError(primaryError) : void 0;
+	const attached = await attachments.close();
 	const endedAt = now().toISOString();
-	await trialWriter.finalize({
+	const durationMs = new Date(endedAt).getTime() - new Date(startedAt).getTime();
+	await runWriter.finalize({
 		status,
 		endedAt,
-		durationMs: new Date(endedAt).getTime() - new Date(startedAt).getTime(),
+		durationMs,
 		scoring,
 		...artifacts ? { artifacts } : {},
+		...attached.length ? { attachments: attached } : {},
 		...error ? { error } : {}
 	});
-	if (!options.runWriter) {
-		const counts = trialCounts([{
-			status,
-			scoring
-		}]);
-		await runWriter.finalize({
-			status,
-			endedAt,
-			durationMs: new Date(endedAt).getTime() - new Date(startedAt).getTime(),
-			trialCount: 1,
-			passed: counts.passed,
-			failed: counts.failed,
-			...counts.cancelled ? { cancelled: counts.cancelled } : {},
-			...error ? { error } : {}
-		});
-	}
 	return {
 		runId,
 		runUri: canonicalId("run", runId),
-		trialId,
-		trialUri: canonicalId("trial", trialId),
-		trialIndex: options.trialIndex ?? 0,
 		evalId: definition.id,
 		status,
+		...batch ? { batch } : {},
 		reportLocation: runWriter.location,
-		durationMs: new Date(endedAt).getTime() - new Date(startedAt).getTime(),
+		durationMs,
 		scoring,
 		...error ? { error } : {}
 	};
@@ -1500,23 +1869,9 @@ function datasetValues(prepared, audience) {
 	}
 	return values;
 }
-/** A cancelled trial is neither passed nor failed. */
-function trialCounts(trials) {
-	let passed = 0;
-	let failed = 0;
-	let cancelledTrials = 0;
-	for (const trial of trials) if (trial.status === "cancelled") cancelledTrials++;
-	else if (trial.status === "completed" && trial.scoring?.passed) passed++;
-	else failed++;
-	return {
-		passed,
-		failed,
-		cancelled: cancelledTrials
-	};
-}
-/** Persistent workspaces of trials that did not pass are kept for debugging by default. */
+/** Persistent workspaces of runs that did not pass are kept for debugging by default. */
 function keepWorkspace(policy = "failed", passed) {
 	return policy === "all" || policy === "failed" && !passed;
 }
 //#endregion
-export { CheckpointExecutionError as _, readRunSummary as a, ScoringError as b, readTrialScoring as c, trialWorkspacePath as d, staticDatasetFile as f, CancelledError as g, AutExecutionError as h, readRunManifest as i, readTrialSummary as l, globMatcher as m, runMatrix as n, readTrialEvents as o, staticDatasetFiles as p, ownerIsGone as r, readTrialManifest as s, runEval as t, localReportStore as u, DatasetError as v, ReportError as y };
+export { DatasetError as _, readRunEvents as a, readRunSummary as c, staticDatasetFile as d, staticDatasetFiles as f, CheckpointExecutionError as g, CancelledError as h, ownerIsGone as i, localReportStore as l, AutExecutionError as m, upgradeReports as n, readRunManifest as o, globMatcher as p, runMatrix as r, readRunScoring as s, runEval as t, runWorkspacePath as u, ReportError as v, ScoringError as y };