@leostera/evalkit 0.5.0 → 0.6.0

package/dist/{src-DoPs9Yug.mjs → src-CVSCn3bZ.mjs}

@@ -1,17 +1,36 @@
-import { A as TrajectoryEventSchema, L as canonicalParameters, M as TrialMetadataSchema, N as TrialScoringSchema, O as RunSummarySchema, P as TrialSummarySchema, T as RunMetadataSchema, U as resourceUri, b as CheckpointResultSchema, g as parseTrajectoryJsonl, j as TrialManifestSchema, k as ScoreResultSchema, n as authoringId, p as recordError, w as RunManifestSchema, x as JsonValueSchema } from "./src-ButXPY8P.mjs";
+import { A as TrialManifestSchema, C as RunManifestSchema, D as RunSummarySchema, F as normalizeAutEvent, G as relativeDatasetPath, H as toJsonObject, I as turnResultEvents, M as TrialScoringSchema, N as TrialSummarySchema, O as ScoreResultSchema, Q as resourceUri, R as canonicalParameters, V as toJson, b as JsonValueSchema, d as recordError, h as parseTrajectoryJsonl, i as authoringId, j as TrialMetadataSchema, k as TrajectoryEventSchema, r as agentRuntimes, w as RunMetadataSchema, y as CheckpointResultSchema } from "./src-D4tCwxNb.mjs";
 import * as Schema from "effect/Schema";
 import { Data, Effect, Either, Exit, Scope } from "effect";
-import { appendFile, copyFile, cp, lstat, mkdir, mkdtemp, readFile, readdir, rename, rm, writeFile } from "node:fs/promises";
-import path, { join } from "node:path";
 import os from "node:os";
-import { randomUUID } from "node:crypto";
+import { appendFile, copyFile, lstat, mkdir, mkdtemp, readFile, readdir, rename, rm, stat, writeFile } from "node:fs/promises";
+import path, { join } from "node:path";
+import { createHash, randomUUID } from "node:crypto";
 //#region ../runner/src/errors.ts
 var AutExecutionError = class extends Data.TaggedError("AutExecutionError") {};
-var FixtureError = class extends Data.TaggedError("FixtureError") {};
+var DatasetError = class extends Data.TaggedError("DatasetError") {};
 var ReportError = class extends Data.TaggedError("ReportError") {};
 var ScoringError = class extends Data.TaggedError("ScoringError") {};
 var RuleExecutionError = class extends Data.TaggedError("RuleExecutionError") {};
 var CheckpointExecutionError = class extends Data.TaggedError("CheckpointExecutionError") {};
+/** The run was cancelled (Ctrl+C); the trial stops and is finalized as cancelled. */
+var CancelledError = class extends Data.TaggedError("CancelledError") {};
+function cancelled() {
+	return new CancelledError({ message: "Cancelled" });
+}
+/** Settles like `work`, or rejects with `CancelledError` as soon as `signal` aborts. */
+async function unlessAborted(work, signal) {
+	if (signal.aborted) throw cancelled();
+	let onAbort;
+	const aborted = new Promise((_, reject) => {
+		onAbort = () => reject(cancelled());
+		signal.addEventListener("abort", onAbort, { once: true });
+	});
+	try {
+		return await Promise.race([work(), aborted]);
+	} finally {
+		signal.removeEventListener("abort", onAbort);
+	}
+}
 //#endregion
 //#region ../runner/src/snapshot.ts
 /**
@@ -51,46 +70,193 @@ async function snapshotCandidateWorkspace(candidateRoot, writer) {
 	return artifacts;
 }
 //#endregion
-//#region ../runner/src/workspace.ts
-const MAX_DYNAMIC_FIXTURE_DEPTH = 16;
-function workspaceRoot(visibility, artifacts) {
-	return visibility === "candidate" ? artifacts.candidate.root : artifacts.evaluator.root;
-}
-function destinationPath(root, destination) {
-	if (!destination || path.isAbsolute(destination)) throw new Error(`Fixture destination must be a non-empty relative path: ${destination}`);
-	const normalized = path.normalize(destination);
-	if (normalized === ".." || normalized.startsWith(`..${path.sep}`)) throw new Error(`Fixture destination escapes its workspace: ${destination}`);
-	return path.join(root, normalized);
+//#region ../runner/src/datasets.ts
+const loads = /* @__PURE__ */ new WeakMap();
+/**
+* Loads a dataset at most once per process and project root. A failed load is
+* forgotten, so the next trial that needs the dataset tries again.
+*/
+function loadDataset(dataset, root, signal) {
+	let byRoot = loads.get(dataset);
+	if (!byRoot) loads.set(dataset, byRoot = /* @__PURE__ */ new Map());
+	const cached = byRoot.get(root);
+	if (cached) return cached;
+	const loading = loadUncached(dataset, root, signal);
+	byRoot.set(root, loading);
+	loading.catch(() => {
+		if (byRoot.get(root) === loading) byRoot.delete(root);
+	});
+	return loading;
 }
-async function resolveFixtures(fixtures, context, depth = 0) {
-	if (depth > MAX_DYNAMIC_FIXTURE_DEPTH) throw new Error("Dynamic fixture nesting exceeds the supported depth");
-	const resolved = [];
-	for (const fixture of fixtures) {
-		if (fixture.kind !== "dynamic") {
-			resolved.push(fixture);
-			continue;
+async function loadUncached(dataset, root, signal) {
+	const { source } = dataset;
+	if (source.kind === "computed") {
+		const loaded = await source.onLoad({
+			root,
+			signal
+		});
+		if (typeof loaded !== "object" || loaded === null) throw new Error(`Dataset ${dataset.id} onLoad must return { files?, value? }`);
+		return loaded;
+	}
+	return {
+		...source.files !== void 0 ? { files: source.files } : {},
+		...source.valueFile !== void 0 ? { value: await readValueFile(dataset.id, path.resolve(root, source.valueFile)) } : source.value !== void 0 ? { value: source.value } : {}
+	};
+}
+/** `.json` is one value; `.jsonl` is an array of one JSON object per non-blank line. */
+async function readValueFile(id, file) {
+	const text = await readFile(file, "utf8").catch((cause) => {
+		throw new Error(`Dataset ${id} cannot read ${file}: ${String(cause)}`);
+	});
+	if (file.endsWith(".jsonl")) return text.split("\n").flatMap((line, index) => {
+		if (!line.trim()) return [];
+		try {
+			const row = JSON.parse(line);
+			if (typeof row !== "object" || row === null || Array.isArray(row)) throw new Error("expected a JSON object");
+			return [row];
+		} catch (cause) {
+			throw new Error(`Dataset ${id} ${file}:${index + 1}: ${cause instanceof Error ? cause.message : String(cause)}`);
 		}
-		const created = await fixture.create(context);
-		const nested = Array.isArray(created) ? created : [created];
-		resolved.push(...await resolveFixtures(nested, context, depth + 1));
+	});
+	if (file.endsWith(".json")) try {
+		return JSON.parse(text);
+	} catch (cause) {
+		throw new Error(`Dataset ${id} ${file}: ${cause instanceof Error ? cause.message : String(cause)}`);
+	}
+	throw new Error(`Dataset ${id} valueFile must be a .json or .jsonl file: ${file}`);
+}
+/** Loads (cached), derives this trial's content with `onRun`, and resolves it. */
+async function prepareDatasets(datasets, root, trial) {
+	const prepared = [];
+	for (const dataset of datasets) {
+		const loaded = await loadDataset(dataset, root, trial.signal);
+		const content = dataset.source.kind === "computed" && dataset.source.onRun ? await dataset.source.onRun(loaded, trial) : loaded;
+		if (typeof content !== "object" || content === null) throw new Error(`Dataset ${dataset.id} onRun must return { files?, value? }`);
+		prepared.push(await resolveContent(dataset, content, root));
+	}
+	return prepared;
+}
+async function resolveContent(dataset, content, root) {
+	const label = `Dataset ${dataset.id}`;
+	const value = content.value === void 0 ? void 0 : toJson(content.value, `datasets.${dataset.id}.value`);
+	let files = { kind: "none" };
+	if (typeof content.files === "string") {
+		const source = path.resolve(root, content.files);
+		if (!(await stat(source).catch(() => void 0))?.isDirectory()) throw new Error(`${label} files directory does not exist: ${content.files}`);
+		files = {
+			kind: "directory",
+			source
+		};
+	} else if (content.files !== void 0) files = {
+		kind: "entries",
+		entries: Object.entries(content.files).map(([file, source]) => preparedFile(relativeDatasetPath(file, `${label} file`), source, root, label))
+	};
+	return {
+		dataset,
+		files,
+		...value === void 0 ? {} : { value },
+		digest: await digest(files, value)
+	};
+}
+function preparedFile(file, source, root, label) {
+	if (typeof source === "string") return {
+		path: file,
+		kind: "bytes",
+		data: new TextEncoder().encode(source)
+	};
+	if (source instanceof Uint8Array) return {
+		path: file,
+		kind: "bytes",
+		data: source
+	};
+	if (typeof source === "object" && source !== null && typeof source.file === "string") return {
+		path: file,
+		kind: "file",
+		source: path.resolve(root, source.file)
+	};
+	throw new Error(`${label} file ${file} must be a string, bytes, or { file: 'path' }`);
+}
+/** Lists a directory's files as sorted relative POSIX paths. */
+async function listFiles(directory, prefix = "") {
+	const entries = await readdir(path.join(directory, prefix), { withFileTypes: true });
+	const files = [];
+	for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) {
+		const relative = prefix ? `${prefix}/${entry.name}` : entry.name;
+		if (entry.isDirectory()) files.push(...await listFiles(directory, relative));
+		else if (entry.isFile()) files.push(relative);
 	}
-	return resolved;
-}
-function assertUniqueDestinations(fixtures) {
-	const destinations = /* @__PURE__ */ new Set();
-	for (const fixture of fixtures) {
-		const destination = fixture.kind === "inline" ? fixture.file : fixture.dst;
-		const key = `${fixture.visibility}:${destination}`;
-		if (destinations.has(key)) throw new Error(`Duplicate ${fixture.visibility} fixture destination: ${destination}`);
-		destinations.add(key);
+	return files;
+}
+async function digest(files, value) {
+	const hash = createHash("sha256");
+	const addFile = (file, data) => {
+		hash.update(`file ${file} ${data.byteLength}\n`);
+		hash.update(data);
+	};
+	if (files.kind === "directory") for (const file of await listFiles(files.source)) addFile(file, await readFile(path.join(files.source, file)));
+	else if (files.kind === "entries") for (const entry of [...files.entries].sort((a, b) => a.path.localeCompare(b.path))) addFile(entry.path, entry.kind === "bytes" ? entry.data : await readFile(entry.source));
+	if (value !== void 0) hash.update(`value ${canonicalParameters(value)}\n`);
+	return `sha256:${hash.digest("hex")}`;
+}
+function trialDatasets(prepared) {
+	return prepared.map(({ dataset, digest }) => ({
+		id: dataset.id,
+		visibility: dataset.visibility,
+		source: dataset.source.kind,
+		digest
+	}));
+}
+/**
+* The files a static dataset places in every workspace, relative to its
+* mount. Computed datasets return `undefined`: their files only exist once
+* `onLoad` runs, and browsing must never run project code.
+*/
+async function staticDatasetFiles(dataset, root) {
+	const { source } = dataset;
+	if (source.kind === "computed") return void 0;
+	if (source.files === void 0) return [];
+	if (typeof source.files === "string") {
+		const directory = path.resolve(root, source.files);
+		return Promise.all((await listFiles(directory)).map(async (file) => ({
+			path: file,
+			size: (await stat(path.join(directory, file))).size
+		})));
 	}
+	return Promise.all(Object.entries(source.files).sort(([a], [b]) => a.localeCompare(b)).map(async ([file, content]) => ({
+		path: file,
+		size: typeof content === "string" ? new TextEncoder().encode(content).byteLength : content instanceof Uint8Array ? content.byteLength : (await stat(path.resolve(root, content.file))).size
+	})));
 }
-/** Creates isolated candidate and evaluator directories and materializes fixtures into each. */
+/** One file of a static dataset, by its path below the mount. */
+async function staticDatasetFile(dataset, root, file) {
+	const { source } = dataset;
+	const missing = () => /* @__PURE__ */ new Error(`Dataset ${dataset.id} has no file ${JSON.stringify(file)}`);
+	if (source.kind === "computed" || source.files === void 0) throw missing();
+	if (typeof source.files === "string") {
+		const directory = path.resolve(root, source.files);
+		const resolved = path.resolve(directory, file);
+		if (!resolved.startsWith(`${directory}${path.sep}`)) throw missing();
+		if (!(await stat(resolved).catch(() => void 0))?.isFile()) throw missing();
+		return readFile(resolved);
+	}
+	if (!Object.hasOwn(source.files, file)) throw missing();
+	const content = source.files[file];
+	if (typeof content === "string") return new TextEncoder().encode(content);
+	if (content instanceof Uint8Array) return new Uint8Array(content);
+	return readFile(path.resolve(root, content.file));
+}
+//#endregion
+//#region ../runner/src/workspace.ts
 function trialDirectoryName(trialId) {
 	return trialId.replace(/[^a-zA-Z0-9._-]/g, "_");
 }
-async function createTrialWorkspace(context, fixtures = [], persistentRoot) {
-	const root = persistentRoot ? path.join(path.resolve(persistentRoot), trialDirectoryName(context.trialId)) : await mkdtemp(path.join(os.tmpdir(), "evalkit-trial-"));
+/**
+* Creates isolated candidate and evaluator directories and writes each
+* dataset's files below its mount, on the side its visibility allows.
+* Two datasets writing the same file is an error, never an overwrite.
+*/
+async function createTrialWorkspace(trialId, datasets = [], persistentRoot) {
+	const root = persistentRoot ? path.join(path.resolve(persistentRoot), trialDirectoryName(trialId)) : await mkdtemp(path.join(os.tmpdir(), "evalkit-trial-"));
 	const persistent = persistentRoot !== void 0;
 	const artifacts = {
 		candidate: { root: path.join(root, "candidate") },
@@ -98,18 +264,9 @@ async function createTrialWorkspace(context, fixtures = [], persistentRoot) {
 	};
 	try {
 		await Promise.all([mkdir(artifacts.candidate.root, { recursive: true }), mkdir(artifacts.evaluator.root, { recursive: true })]);
-		const resolved = await resolveFixtures(fixtures, context);
-		assertUniqueDestinations(resolved);
-		for (const fixture of resolved) {
-			const target = destinationPath(workspaceRoot(fixture.visibility, artifacts), fixture.kind === "inline" ? fixture.file : fixture.dst);
-			await mkdir(path.dirname(target), { recursive: true });
-			if (fixture.kind === "directory") await cp(fixture.src, target, {
-				recursive: true,
-				force: false,
-				errorOnExist: true
-			});
-			else if (fixture.kind === "file") await copyFile(fixture.src, target);
-			else await writeFile(target, fixture.data);
+		for (const prepared of datasets) {
+			const side = prepared.dataset.visibility === "candidate" ? artifacts.candidate.root : artifacts.evaluator.root;
+			await writeDatasetFiles(prepared, path.join(side, prepared.dataset.mount));
 		}
 		return {
 			artifacts,
@@ -126,21 +283,50 @@ async function createTrialWorkspace(context, fixtures = [], persistentRoot) {
 		throw error;
 	}
 }
+async function writeDatasetFiles(prepared, target) {
+	const { files, dataset } = prepared;
+	if (files.kind === "none") return;
+	await mkdir(target, { recursive: true });
+	const entries = files.kind === "directory" ? (await listFiles(files.source)).map((file) => ({
+		path: file,
+		kind: "file",
+		source: path.join(files.source, file)
+	})) : files.entries;
+	for (const entry of entries) await writeEntry(dataset.id, target, entry);
+}
+async function writeEntry(datasetId, target, entry) {
+	const destination = path.join(target, ...entry.path.split("/"));
+	await mkdir(path.dirname(destination), { recursive: true });
+	await (entry.kind === "bytes" ? writeFile(destination, entry.data, { flag: "wx" }) : copyFile(entry.source, destination, 1)).catch((cause) => {
+		throw collision(datasetId, entry.path, cause);
+	});
+}
+function collision(datasetId, file, cause) {
+	return cause.code === "EEXIST" ? /* @__PURE__ */ new Error(`Dataset ${datasetId} would overwrite ${file}, which another dataset already wrote`) : cause instanceof Error ? cause : new Error(String(cause));
+}
 //#endregion
 //#region ../runner/src/score.ts
-/** Shared normalization for inline and final predicate and judge rules. */
-function normalizeScore(value) {
+/**
+* Shared normalization for inline and final predicate and judge rules.
+* `passAt` turns a numeric value into a verdict; an explicit `passed` wins.
+*/
+function normalizeScore(value, passAt) {
+	if (typeof value === "object" && value !== null && "skipped" in value && value.skipped === true) return {
+		skipped: true,
+		explanation: value.reason
+	};
 	const normalized = typeof value === "boolean" ? { value: Number(value) } : typeof value === "number" ? { value } : value;
 	if (!normalized || !Number.isFinite(normalized.value) || normalized.value < 0 || normalized.value > 1) throw new Error(`Score value must be a finite number from 0 to 1; received ${normalized?.value}`);
 	if ("judge" in normalized && normalized.judge) {
 		const { usage } = normalized.judge;
 		if (usage && Object.values(usage).some((tokens) => !Number.isSafeInteger(tokens) || tokens < 0)) throw new Error("Judge token usage must contain nonnegative integers");
 	}
+	const passed = "passed" in normalized && normalized.passed !== void 0 ? normalized.passed : normalized.value >= (passAt ?? 1);
 	return {
 		value: normalized.value,
-		passed: normalized.passed ?? normalized.value === 1,
-		...normalized.explanation ? { explanation: normalized.explanation } : {},
-		...normalized.evidence === void 0 ? {} : { evidence: normalized.evidence },
+		passed,
+		..."explanation" in normalized && normalized.explanation ? { explanation: normalized.explanation } : {},
+		..."evidence" in normalized && normalized.evidence !== void 0 ? { evidence: toJson(normalized.evidence, "evidence") } : {},
 		..."judge" in normalized && normalized.judge !== void 0 ? { judge: normalized.judge } : {}
 	};
 }
@@ -156,6 +342,7 @@ function runJudgeAgent(rule, agent, input, placement) {
 	const events = [];
 	const context = {
 		...input.context,
+		data: input.data,
 		workspace: input.artifacts.evaluator,
 		evaluatorWorkspace: input.artifacts.evaluator
 	};
@@ -173,7 +360,7 @@ function runJudgeAgent(rule, agent, input, placement) {
 				context,
 				onEvent: (event) => {
 					const observed = Schema.decodeUnknownSync(TrajectoryEventSchema)({
-						...event,
+						...normalizeAutEvent(event, () => /* @__PURE__ */ new Date()),
 						source: "aut"
 					});
 					if (observed.source !== "aut") throw new Error("Invalid judge event source");
@@ -183,10 +370,13 @@ function runJudgeAgent(rule, agent, input, placement) {
 			}),
 			catch: (cause) => failure$2(rule.name, cause)
 		}), (session) => Effect.tryPromise({
-			try: () => session.send(prompt),
+			try: async () => {
+				const result = await session.send(prompt);
+				if (result !== void 0) events.push(...turnResultEvents(result, 1, () => /* @__PURE__ */ new Date()));
+			},
 			catch: (cause) => failure$2(rule.name, cause)
 		}), (session) => Effect.tryPromise({
-			try: () => session.close(),
+			try: async () => session.close?.(),
 			catch: (cause) => failure$2(rule.name, cause)
 		}).pipe(Effect.catchAll((error) => Effect.sync(() => {
 			closeError = error;
@@ -229,7 +419,7 @@ function evaluateRule(rule, input, judgeAgent, placement) {
 			catch: (cause) => failure$1(rule.name, cause)
 		}) } : yield* runJudgeAgent(rule, judgeAgent, input, placement);
 		const score = yield* Effect.try({
-			try: () => normalizeScore(evaluated.value),
+			try: () => normalizeScore(evaluated.value, rule.kind === "predicate" ? rule.passAt : void 0),
 			catch: (cause) => failure$1(rule.name, cause)
 		});
 		const withJudge = rule.kind === "judge" && "judge" in evaluated ? {
@@ -242,6 +432,10 @@ function evaluateRule(rule, input, judgeAgent, placement) {
 			durationMs: 0,
 			...withJudge
 		}).pipe(Effect.mapError((cause) => failure$1(rule.name, cause)));
+		if (validated.skipped) return {
+			skipped: true,
+			...validated.explanation === void 0 ? {} : { explanation: validated.explanation }
+		};
 		return {
 			value: validated.value,
 			passed: validated.passed,
@@ -315,24 +509,26 @@ function evaluateCheckpoint(step, index, input, now, judgeAgent) {
 			try: () => normalizeScore(Boolean(match)),
 			catch: (cause) => failure(index, step.name, cause)
 		}) : yield* evaluateRule(step, input, judgeAgent, "transcript").pipe(Effect.mapError((cause) => failure(index, step.name, cause)));
+		const status = score.skipped ? "skipped" : score.passed ? "passed" : "failed";
+		const { skipped: _, ...recorded } = score;
 		const result = yield* Schema.decodeUnknown(CheckpointResultSchema)({
 			step: index,
 			kind: step.kind,
 			name: step.name,
-			status: score.passed ? "passed" : "failed",
+			status,
 			durationMs: now().getTime() - started.getTime(),
-			...score,
+			...recorded,
 			...match ? { matchedToolCall: {
 				eventIndex: match.eventIndex,
 				id: match.id
 			} } : {}
 		}).pipe(Effect.mapError((cause) => failure(index, step.name, cause)));
-		yield* Effect.logInfo("checkpoint completed").pipe(Effect.annotateLogs({ status: result.status }));
+		yield* Effect.logDebug("checkpoint completed").pipe(Effect.annotateLogs({ status: result.status }));
 		return {
 			...result,
-			status: score.passed ? "passed" : "failed"
+			status
 		};
-	}).pipe(Effect.tapError((error) => Effect.logError("checkpoint execution error").pipe(Effect.annotateLogs({ errorName: error._tag }))), Effect.annotateLogs(annotations));
+	}).pipe(Effect.tapError((error) => Effect.logDebug("checkpoint execution error").pipe(Effect.annotateLogs({ errorName: error._tag }))), Effect.annotateLogs(annotations));
 }
 //#endregion
 //#region ../runner/src/effect-boundary.ts
@@ -346,7 +542,9 @@ async function runBoundary(effect) {
 //#region ../runner/src/transcript.ts
 async function executeTranscript(definition, session, input) {
 	const { now, emitRunner, progress } = input;
+	let turn = 0;
 	for (const [index, step] of definition.transcript.entries()) {
+		if (input.signal.aborted) throw cancelled();
 		await emitRunner({
 			kind: "transcript-step-started",
 			step: index,
@@ -354,7 +552,9 @@ async function executeTranscript(definition, session, input) {
 		});
 		if (step.kind === "user") {
 			const cursor = input.events.length;
-			await sendUserTurn(session, step.message.replaceAll("{{randomSeed}}", String(input.randomSeed)));
+			turn++;
+			const result = await sendUserTurn(session, step.message.replaceAll("{{randomSeed}}", String(input.randomSeed)), input.signal);
+			if (result !== void 0) for (const event of turnResultEvents(result, turn, now)) await input.emitAut(event);
 			progress.lastTurn = turnView(input.events, cursor, input.events.length, index);
 		} else if (step.kind === "predicate" || step.kind === "judge" || step.kind === "expect-tool-call") {
 			const result = await runCheckpoint(definition, step, index, input);
@@ -370,7 +570,7 @@ async function executeTranscript(definition, session, input) {
 				step: index,
 				timestamp: now().toISOString()
 			});
-			if (!result.passed && definition.policy?.failfast) {
+			if (result.status === "failed" && definition.policy?.failfast) {
 				progress.stoppedEarly = true;
 				await skipRemainingSteps(definition, index, input);
 				break;
@@ -384,10 +584,10 @@ async function executeTranscript(definition, session, input) {
 		});
 	}
 }
-async function sendUserTurn(session, message) {
-	await runBoundary(Effect.tryPromise({
-		try: () => session.send(message),
-		catch: (cause) => cause instanceof ReportError ? cause : new AutExecutionError({
+async function sendUserTurn(session, message, signal) {
+	return runBoundary(Effect.tryPromise({
+		try: () => unlessAborted(() => session.send(message), signal),
+		catch: (cause) => cause instanceof ReportError || cause instanceof CancelledError ? cause : new AutExecutionError({
 			cause,
 			message: cause instanceof Error ? cause.message : String(cause)
 		})
@@ -407,7 +607,9 @@ async function runCheckpoint(definition, step, index, input) {
 		context: input.context,
 		artifacts: input.artifacts,
 		trajectory: { events: input.events },
-		turn: progress.lastTurn
+		turn: progress.lastTurn,
+		...turnOutput(progress.lastTurn),
+		data: input.data
 	}, now, definition.judge)).catch(async (error) => {
 		const recorded = recordError(error);
 		progress.checkpoints.push({
@@ -445,24 +647,29 @@ async function skipRemainingSteps(definition, after, { now, emitRunner, progress
 		});
 	}
 }
+/** The scorer-facing `output`: the content of the turn's last assistant message. */
+function turnOutput(turn) {
+	const last = turn?.assistantMessages.at(-1);
+	return last === void 0 ? {} : { output: last.content };
+}
 //#endregion
 //#region ../runner/src/final-scoring.ts
 function scoreSummary(results, checkpoints, skippedScorers) {
-	const valid = results.filter((result) => result.value !== void 0);
+	const valid = results.filter((result) => result.value !== void 0 && !result.skipped);
 	const hasError = results.some((result) => result.error !== void 0);
 	return {
 		results,
 		...checkpoints.length ? { checkpoints } : {},
 		...skippedScorers.length ? { skippedScorers } : {},
 		...valid.length === 0 ? {} : { overall: valid.reduce((sum, result) => sum + (result.value ?? 0), 0) / valid.length },
-		passed: !hasError && results.every((result) => result.passed === true) && checkpoints.every((result) => result.status === "passed") && !skippedScorers.length
+		passed: !hasError && results.every((result) => result.skipped || result.passed === true) && checkpoints.every((result) => result.status === "passed" || result.status === "skipped") && !skippedScorers.length
 	};
 }
 /** Append completed scores to the supplied arrays, even when a later scorer or report write fails. */
 async function evaluateFinalScorers(definition, input) {
 	const { context, workspace, events, lastTurn, results, skippedScorers, now, emitRunner } = input;
 	for (const scorer of definition.scoring) {
-		if (input.reportingFailed || !context || !workspace || (input.primaryError || input.stoppedEarly) && (!("supportsPartial" in scorer) || !scorer.supportsPartial)) {
+		if (input.cancelled || input.reportingFailed || !context || !workspace || (input.primaryError || input.stoppedEarly) && (!("supportsPartial" in scorer) || !scorer.supportsPartial)) {
 			skippedScorers.push(scorer.name);
 			continue;
 		}
@@ -478,7 +685,9 @@ async function evaluateFinalScorers(definition, input) {
 				context,
 				artifacts: workspace.artifacts,
 				trajectory: { events },
-				...lastTurn ? { turn: lastTurn } : {}
+				...lastTurn ? { turn: lastTurn } : {},
+				...turnOutput(lastTurn),
+				data: input.data
 			}, definition.judge, "scoring").pipe(Effect.mapError((cause) => new ScoringError({
 				cause,
 				message: cause.message
@@ -510,7 +719,7 @@ async function evaluateFinalScorers(definition, input) {
 		await emitRunner({
 			kind: "scorer-completed",
 			scorer: scorer.name,
-			value: result.value ?? 0,
+			...result.skipped ? { skipped: true } : { value: result.value ?? 0 },
 			timestamp: now().toISOString()
 		});
 	}
@@ -618,6 +827,20 @@ const readTrialScoring = (root, runId, trialId) => decode(join(root, runId, "tri
 async function readTrialEvents(root, runId, trialId) {
 	return parseTrajectoryJsonl(await readFile(join(root, runId, "trials", trialId, "trajectory.jsonl"), "utf8"));
 }
+/**
+* True when a run's owning process is known to be gone: it ran on this host
+* and its PID no longer exists. Runs from other hosts are never judged.
+*/
+function ownerIsGone(owner) {
+	if (!owner || owner.hostname !== os.hostname()) return false;
+	if (owner.pid === process.pid) return false;
+	try {
+		process.kill(owner.pid, 0);
+		return false;
+	} catch (error) {
+		return error.code === "ESRCH";
+	}
+}
 //#endregion
 //#region ../runner/src/matrix.ts
 /** Bounded workers pull cells from the iterator; no Cartesian job array is allocated. */
@@ -630,10 +853,11 @@ function runMatrix(matrix, options) {
 		const summary = {
 			cells: 0,
 			passed: 0,
-			failed: 0
+			failed: 0,
+			cancelled: 0
 		};
 		yield* Effect.all(Array.from({ length: Math.min(count, concurrency) }, () => Effect.gen(function* () {
-			while (true) {
+			while (!options.signal?.aborted) {
 				const next = iterator.next();
 				if (next.done) return;
 				const cell = next.value;
@@ -641,9 +865,11 @@ function runMatrix(matrix, options) {
 					"local",
 					"remote",
 					"sandbox"
-				].find((name) => cell.eval.agent.runtimes?.[name]);
+				].find((name) => agentRuntimes(cell.eval.agent)[name]);
+				const suiteId = options.suiteId ?? options.suiteFor?.(cell);
 				const result = yield* runEval(cell.eval, {
 					...options,
+					...suiteId ? { suiteId } : {},
 					concurrency: 1,
 					runtime,
 					parameters: cell.parameters,
@@ -653,7 +879,8 @@ function runMatrix(matrix, options) {
 					}
 				});
 				summary.cells++;
-				if (result.status === "completed" && (result.aggregateScoring?.passRate === 1 || result.scoring?.passed)) summary.passed++;
+				if (result.status === "cancelled") summary.cancelled++;
+				else if (result.status === "completed" && (result.aggregateScoring?.passRate === 1 || result.scoring?.passed)) summary.passed++;
 				else summary.failed++;
 				if (options.onResult) yield* Effect.tryPromise(async () => options.onResult(cell, result));
 			}
@@ -694,12 +921,16 @@ function assertUuid(value, label) {
 }
 function runMetadata(definition, options, runId, startedAt) {
 	return {
-		schemaVersion: 4,
+		schemaVersion: 5,
 		runId,
 		runUri: canonicalId("run", runId),
 		evalId: definition.id,
 		...options.suiteId ? { suiteId: options.suiteId } : {},
 		...options.source ? { source: options.source } : {},
+		process: {
+			pid: process.pid,
+			hostname: os.hostname()
+		},
 		...definition.agent.identity ? { aut: definition.agent.identity } : {},
 		parameters: options.parameters,
 		matrix: options.matrix,
@@ -731,9 +962,10 @@ async function executeRun(definition, options) {
 	const aggregateStartedAt = now();
 	const runId = options.runId ?? createId("run");
 	const runWriter = await options.report.startRun(runMetadata(definition, options, runId, aggregateStartedAt.toISOString()), { catalog: options.catalog });
+	const signal = options.signal;
 	const trialEffects = Array.from({ length: requestedTrials }, (_, trialIndex) => {
 		const trial = Effect.tryPromise({
-			try: () => runTrial(definition, {
+			try: async () => signal?.aborted ? void 0 : runTrial(definition, {
 				...options,
 				runId,
 				trialId: createId("trial"),
@@ -745,18 +977,38 @@ async function executeRun(definition, options) {
 		});
 		return options.semaphore ? options.semaphore.withPermits(1)(trial) : trial;
 	});
-	const trials = await Effect.runPromise(Effect.all(trialEffects, { concurrency: options.concurrency ?? 32 }));
-	const failed = trials.filter((trial) => trial.status !== "completed" || !trial.scoring?.passed).length;
-	const status = trials.some((trial) => trial.status === "failed") ? "failed" : "completed";
+	const trials = (await Effect.runPromise(Effect.all(trialEffects, { concurrency: options.concurrency ?? 32 }))).filter((trial) => trial !== void 0);
+	const counts = trialCounts(trials);
+	const status = counts.cancelled || signal?.aborted ? "cancelled" : trials.some((trial) => trial.status === "failed") ? "failed" : "completed";
 	const endedAt = now();
 	await runWriter.finalize({
 		status,
 		endedAt: endedAt.toISOString(),
 		durationMs: endedAt.getTime() - aggregateStartedAt.getTime(),
 		trialCount: trials.length,
-		passed: trials.length - failed,
-		failed
+		passed: counts.passed,
+		failed: counts.failed,
+		...counts.cancelled ? { cancelled: counts.cancelled } : {}
 	});
+	if (!trials.length) return {
+		runId,
+		runUri: canonicalId("run", runId),
+		trialId: "",
+		trialUri: canonicalId("trial", runId),
+		trialIndex: 0,
+		evalId: authoringId(definition),
+		status,
+		reportLocation: runWriter.location,
+		trialCount: 0,
+		passed: 0,
+		failed: 0,
+		aggregateScoring: {
+			passed: 0,
+			failed: 0,
+			passRate: 0
+		},
+		trials: []
+	};
 	const overallScores = trials.map((trial) => trial.scoring?.overall).filter((score) => score !== void 0);
 	return {
 		...trials[0],
@@ -764,12 +1016,13 @@ async function executeRun(definition, options) {
 		evalId: authoringId(definition),
 		status,
 		trialCount: trials.length,
-		passed: trials.length - failed,
-		failed,
+		passed: counts.passed,
+		failed: counts.failed,
 		aggregateScoring: {
-			passed: trials.length - failed,
-			failed,
-			passRate: (trials.length - failed) / trials.length,
+			passed: counts.passed,
+			failed: counts.failed,
+			...counts.cancelled ? { cancelled: counts.cancelled } : {},
+			passRate: counts.passed / trials.length,
 			...overallScores.length ? { overall: overallScores.reduce((sum, score) => sum + score, 0) / overallScores.length } : {}
 		},
 		trials
@@ -781,25 +1034,45 @@ async function runTrial(definition, options) {
 	const trialId = options.trialId ?? createId("trial");
 	const startedAt = now().toISOString();
 	const selectedRuntime = options.runtime;
-	const runtime = selectedRuntime ? definition.agent.runtimes?.[selectedRuntime] : void 0;
+	const runtime = selectedRuntime ? agentRuntimes(definition.agent)[selectedRuntime] : void 0;
 	if (selectedRuntime && !runtime) throw new Error(`AUT does not declare the requested ${selectedRuntime} runtime`);
 	const randomSeed = crypto.getRandomValues(/* @__PURE__ */ new Uint32Array(1))[0];
-	const fixtureContext = {
+	const trialIndex = options.trialIndex ?? 0;
+	const signal = options.signal ?? new AbortController().signal;
+	const root = options.root ?? process.cwd();
+	const baseContext = {
 		runId,
 		evalId: authoringId(definition),
 		trialId,
-		trialIndex: options.trialIndex ?? 0,
+		trialIndex,
 		metadata: {
-			...definition.metadata,
+			...toJsonObject(definition.metadata ?? {}, `${definition.id}.metadata`),
 			randomSeed
 		},
 		...options.parameters ? { parameters: options.parameters } : {},
 		...selectedRuntime ? { runtime: selectedRuntime } : {}
 	};
+	let datasetsError;
+	let prepared = [];
+	try {
+		prepared = await prepareDatasets(definition.datasets ?? [], root, {
+			evalId: baseContext.evalId,
+			runId,
+			trialId,
+			trialIndex,
+			seed: randomSeed,
+			parameters: options.parameters ?? {},
+			signal
+		});
+	} catch (error) {
+		datasetsError = error;
+	}
+	const candidateData = datasetValues(prepared, "candidate");
+	const scoringData = datasetValues(prepared, "all");
 	const events = [];
 	const runWriter = options.runWriter ?? await options.report.startRun(runMetadata(definition, options, runId, startedAt), { catalog: options.catalog });
 	const trialWriter = await runWriter.startTrial({
-		schemaVersion: 4,
+		schemaVersion: 5,
 		runId,
 		runUri: canonicalId("run", runId),
 		trialId,
@@ -809,15 +1082,30 @@ async function runTrial(definition, options) {
 		...definition.agent.identity ? { aut: definition.agent.identity } : {},
 		parameters: options.parameters,
 		matrix: options.matrix,
-		startedAt
+		startedAt,
+		...prepared.length ? { datasets: trialDatasets(prepared) } : {}
 	});
 	let writeChain = Promise.resolve();
 	let reportingFailed = false;
+	const trialRef = {
+		runId,
+		trialId,
+		trialIndex,
+		evalId: definition.id,
+		...options.suiteId ? { suiteId: options.suiteId } : {},
+		...options.parameters ? { parameters: options.parameters } : {}
+	};
+	const observe = (event) => {
+		try {
+			options.onEvent?.(trialRef, event);
+		} catch {}
+	};
 	const append = async (event) => {
 		events.push(event);
 		writeChain = writeChain.then(() => trialWriter.appendEvent(event));
 		try {
 			await writeChain;
+			observe(event);
 		} catch (cause) {
 			reportingFailed = true;
 			throw new ReportError({
@@ -830,10 +1118,11 @@ async function runTrial(definition, options) {
 		...event,
 		source: "runner"
 	});
-	const onAutEvent = (event) => append({
+	const emitAut = (event) => append({
 		...event,
 		source: "aut"
 	});
+	const onAutEvent = (event) => emitAut(normalizeAutEvent(event, now));
 	let workspace;
 	let context;
 	let primaryError;
@@ -852,15 +1141,20 @@ async function runTrial(definition, options) {
 			kind: "trial-started",
 			timestamp: now().toISOString()
 		});
+		if (signal.aborted) throw cancelled();
+		if (datasetsError) throw new DatasetError({
+			cause: datasetsError,
+			message: datasetsError instanceof Error ? datasetsError.message : String(datasetsError)
+		});
 		workspace = await runBoundary(Effect.acquireRelease(Effect.tryPromise({
-			try: () => createTrialWorkspace(fixtureContext, definition.fixtures, options.workspaceRoot),
-			catch: (cause) => new FixtureError({
+			try: () => createTrialWorkspace(trialId, prepared, options.workspaceRoot),
+			catch: (cause) => new DatasetError({
 				cause,
 				message: cause instanceof Error ? cause.message : String(cause)
 			})
 		}), (created) => Effect.tryPromise({
 			try: () => created.cleanup(),
-			catch: (cause) => new FixtureError({
+			catch: (cause) => new DatasetError({
 				cause,
 				message: cause instanceof Error ? cause.message : String(cause)
 			})
@@ -869,7 +1163,8 @@ async function runTrial(definition, options) {
 		}).pipe(Effect.tap(() => Effect.logError("trial workspace cleanup failed")))))).pipe(Effect.provideService(Scope.Scope, workspaceScope)));
 		const trialWorkspace = workspace;
 		const trialContext = {
-			...fixtureContext,
+			...baseContext,
+			data: candidateData,
 			workspace: trialWorkspace.artifacts.candidate,
 			evaluatorWorkspace: trialWorkspace.artifacts.evaluator
 		};
@@ -892,11 +1187,14 @@ async function runTrial(definition, options) {
 				now,
 				randomSeed,
 				emitRunner,
+				emitAut,
+				data: scoringData,
+				signal,
 				progress
 			}),
 			catch: (cause) => cause
 		}), (opened) => Effect.tryPromise({
-			try: () => opened.close(),
+			try: async () => opened.close?.(),
 			catch: (cause) => new AutExecutionError({
 				cause,
 				message: cause instanceof Error ? cause.message : String(cause)
@@ -916,6 +1214,7 @@ async function runTrial(definition, options) {
 		if (!primaryError) primaryError = closeError;
 		await tryEmitError(closeError, emitRunner, now);
 	}
+	const wasCancelled = primaryError instanceof CancelledError;
 	const { checkpoints, stoppedEarly, lastTurn } = progress;
 	const scoreResults = [];
 	const skippedScorers = [];
@@ -927,9 +1226,11 @@ async function runTrial(definition, options) {
 			lastTurn,
 			primaryError,
 			stoppedEarly,
+			cancelled: wasCancelled,
 			reportingFailed,
 			results: scoreResults,
 			skippedScorers,
+			data: scoringData,
 			now,
 			emitRunner
 		});
@@ -960,7 +1261,7 @@ async function runTrial(definition, options) {
 	} catch (error) {
 		primaryError = error;
 	}
-	status = primaryError ? "failed" : "completed";
+	status = wasCancelled ? "cancelled" : primaryError ? "failed" : "completed";
 	const error = primaryError ? recordError(primaryError) : void 0;
 	const endedAt = now().toISOString();
 	await trialWriter.finalize({
@@ -971,14 +1272,22 @@ async function runTrial(definition, options) {
 		...artifacts ? { artifacts } : {},
 		...error ? { error } : {}
 	});
-	if (!options.runWriter) await runWriter.finalize({
-		status,
-		endedAt,
-		trialCount: 1,
-		passed: status === "completed" && scoring.passed ? 1 : 0,
-		failed: status === "failed" || !scoring.passed ? 1 : 0,
-		...error ? { error } : {}
-	});
+	if (!options.runWriter) {
+		const counts = trialCounts([{
+			status,
+			scoring
+		}]);
+		await runWriter.finalize({
+			status,
+			endedAt,
+			durationMs: new Date(endedAt).getTime() - new Date(startedAt).getTime(),
+			trialCount: 1,
+			passed: counts.passed,
+			failed: counts.failed,
+			...counts.cancelled ? { cancelled: counts.cancelled } : {},
+			...error ? { error } : {}
+		});
+	}
 	return {
 		runId,
 		runUri: canonicalId("run", runId),
@@ -993,7 +1302,28 @@ async function runTrial(definition, options) {
 		...error ? { error } : {}
 	};
 }
+function datasetValues(prepared, audience) {
+	const values = {};
+	for (const { dataset, value } of prepared) {
+		if (value === void 0) continue;
+		if (audience === "candidate" && dataset.visibility !== "candidate") continue;
+		values[dataset.id] = value;
+	}
+	return values;
+}
+/** A cancelled trial is neither passed nor failed. */
+function trialCounts(trials) {
+	let passed = 0;
+	let failed = 0;
+	let cancelledTrials = 0;
+	for (const trial of trials) if (trial.status === "cancelled") cancelledTrials++;
+	else if (trial.status === "completed" && trial.scoring?.passed) passed++;
+	else failed++;
+	return {
+		passed,
+		failed,
+		cancelled: cancelledTrials
+	};
+}
 //#endregion
-export { readTrialEvents as a, readTrialSummary as c, CheckpointExecutionError as d, FixtureError as f, readRunSummary as i, localReportStore as l, ScoringError as m, runMatrix as n, readTrialManifest as o, ReportError as p, readRunManifest as r, readTrialScoring as s, runEval as t, AutExecutionError as u };
-
-//# sourceMappingURL=src-DoPs9Yug.mjs.map
+export { ReportError as _, readRunSummary as a, readTrialScoring as c, staticDatasetFile as d, staticDatasetFiles as f, DatasetError as g, CheckpointExecutionError as h, readRunManifest as i, readTrialSummary as l, CancelledError as m, runMatrix as n, readTrialEvents as o, AutExecutionError as p, ownerIsGone as r, readTrialManifest as s, runEval as t, localReportStore as u, ScoringError as v };