@leostera/evalkit 0.5.0 → 0.6.1

package/dist/{src-ButXPY8P.mjs → src-CZPufWN_.mjs}

@@ -31,6 +31,291 @@ function resourceUuid(value, expectedKind) {
 	return parseResourceUri(value, expectedKind).uuid;
 }
 //#endregion
+//#region ../core/src/definition-site.ts
+const SITE = Symbol.for("evalkit.definitionSite");
+const internalFiles = /* @__PURE__ */ new Set();
+/** Call from each module that defines helpers, so its frames are skipped. */
+function internalModule(url) {
+	if (url) internalFiles.add(filePath(url));
+}
+internalModule(import.meta.url);
+function markDefinitionSite(value) {
+	if (Object.hasOwn(value, SITE)) return value;
+	const site = callerSite();
+	if (site) Object.defineProperty(value, SITE, {
+		value: site,
+		enumerable: false
+	});
+	return value;
+}
+function definitionSite(value) {
+	return typeof value === "object" && value !== null ? value[SITE] ?? void 0 : void 0;
+}
+/** `file:line` relative to `root`, or undefined outside it (never an absolute path). */
+function relativeSite(site, root) {
+	if (!site || !root) return void 0;
+	const base = root.replace(/[\\/]+$/, "") + "/";
+	const file = site.file.replace(/\\/g, "/");
+	const prefix = base.replace(/\\/g, "/");
+	if (!file.startsWith(prefix)) return void 0;
+	return `${file.slice(prefix.length)}:${site.line}`;
+}
+const FRAME = /\(?((?:file:\/\/)?(?:\/|[A-Za-z]:[\\/])[^():]*):(\d+):\d+\)?\s*$/;
+function callerSite() {
+	const stack = (/* @__PURE__ */ new Error()).stack?.split("\n").slice(1) ?? [];
+	for (const frame of stack) {
+		const match = FRAME.exec(frame);
+		if (!match) continue;
+		const file = filePath(match[1]);
+		if (internalFiles.has(file) || file.includes("/node_modules/")) continue;
+		return {
+			file,
+			line: Number(match[2])
+		};
+	}
+}
+function filePath(location) {
+	if (!location.startsWith("file://")) return location;
+	try {
+		return decodeURIComponent(new URL(location).pathname);
+	} catch {
+		return location.slice(7);
+	}
+}
+//#endregion
+//#region ../core/src/dataset.ts
+internalModule(import.meta.url);
+/** Normalizes a relative POSIX path and rejects absolute or escaping ones. */
+function relativeDatasetPath(value, label) {
+	const parts = [];
+	if (!value || value.startsWith("/") || /^[a-zA-Z]:[\\/]/.test(value)) throw new Error(`${label} must be a non-empty relative path: ${value}`);
+	for (const part of value.replace(/\\/g, "/").split("/")) {
+		if (part === "" || part === ".") continue;
+		if (part === "..") {
+			if (!parts.length) throw new Error(`${label} escapes its workspace: ${value}`);
+			parts.pop();
+			continue;
+		}
+		parts.push(part);
+	}
+	return parts.length ? parts.join("/") : ".";
+}
+/**
+* Declares named data that scenarios use: files placed in each trial's
+* workspace, a value for the agent and scorers, or both. Static datasets give
+* `files`, `value` or `valueFile`; computed datasets give `onLoad` (cached per
+* process) and optionally `onRun` (per trial).
+*/
+function defineDataset(definition) {
+	return markDefinitionSite(checkedDataset(definition));
+}
+function checkedDataset(definition) {
+	if ("uri" in definition || "uuid" in definition || "slug" in definition) throw new Error("Authored resources use id, not uri, uuid, or slug");
+	const id = authorId(definition.id);
+	const visibility = definition.visibility ?? "candidate";
+	if (visibility !== "candidate" && visibility !== "evaluator") throw new Error(`Dataset ${id} visibility must be candidate or evaluator; received ${String(visibility)}`);
+	const mount = relativeDatasetPath(definition.mount ?? id, `Dataset ${id} mount`);
+	const common = {
+		kind: "dataset",
+		id,
+		...definition.name ? { name: definition.name } : {},
+		visibility,
+		mount
+	};
+	if (definition.onLoad !== void 0) {
+		if (typeof definition.onLoad !== "function") throw new Error(`Dataset ${id} onLoad must be a function`);
+		if (definition.files !== void 0 || definition.value !== void 0 || definition.valueFile !== void 0) throw new Error(`Dataset ${id} is computed (onLoad); move files and values into what onLoad returns`);
+		if (definition.onRun !== void 0 && typeof definition.onRun !== "function") throw new Error(`Dataset ${id} onRun must be a function`);
+		return {
+			...common,
+			source: {
+				kind: "computed",
+				onLoad: definition.onLoad,
+				...definition.onRun ? { onRun: definition.onRun } : {}
+			}
+		};
+	}
+	if (definition.onRun !== void 0) throw new Error(`Dataset ${id} has onRun without onLoad`);
+	if (definition.value !== void 0 && definition.valueFile !== void 0) throw new Error(`Dataset ${id} has both value and valueFile; give one`);
+	if (definition.files === void 0 && definition.value === void 0 && definition.valueFile === void 0) throw new Error(`Dataset ${id} is empty; give files, value, valueFile or onLoad`);
+	return {
+		...common,
+		source: {
+			kind: "static",
+			...definition.files !== void 0 ? { files: definition.files } : {},
+			...definition.value !== void 0 ? { value: definition.value } : {},
+			...definition.valueFile !== void 0 ? { valueFile: definition.valueFile } : {}
+		}
+	};
+}
+function isDataset(value) {
+	return typeof value === "object" && value !== null && value.kind === "dataset" && typeof value.source === "object";
+}
+//#endregion
+//#region ../core/src/json.ts
+/** A value that cannot be stored in a report, with the path of the offending part. */
+var NotJsonError = class extends Error {
+	path;
+	reason;
+	name = "NotJsonError";
+	constructor(path, reason) {
+		super(`${path}: ${reason}`);
+		this.path = path;
+		this.reason = reason;
+	}
+};
+/**
+* Converts an author-supplied value to JSON once, where it enters EvalKit.
+* Honours `toJSON()` (so `Date` becomes an ISO string) and drops `undefined`
+* object fields. Rejects functions, symbols, bigints, non-finite numbers and
+* cycles, naming the path of the bad value.
+*/
+function toJson(value, path = "value") {
+	return convert(value, path, /* @__PURE__ */ new Set());
+}
+function convert(value, path, seen) {
+	if (value === null || typeof value === "string" || typeof value === "boolean") return value;
+	if (typeof value === "number") {
+		if (!Number.isFinite(value)) throw new NotJsonError(path, `${value} is not JSON`);
+		return value;
+	}
+	if (value === void 0) throw new NotJsonError(path, "undefined is not JSON");
+	if (typeof value !== "object") throw new NotJsonError(path, `${typeof value} is not JSON`);
+	if (seen.has(value)) throw new NotJsonError(path, "cycle is not JSON");
+	const withToJson = value;
+	if (typeof withToJson.toJSON === "function") return convert(withToJson.toJSON(), path, seen);
+	seen.add(value);
+	try {
+		if (Array.isArray(value)) return value.map((item, index) => convert(item === void 0 ? null : item, `${path}[${index}]`, seen));
+		if (value instanceof Map || value instanceof Set) throw new NotJsonError(path, `${value.constructor.name} is not JSON`);
+		const result = {};
+		for (const [key, field] of Object.entries(value)) {
+			if (field === void 0) continue;
+			result[key] = convert(field, `${path}.${key}`, seen);
+		}
+		return result;
+	} finally {
+		seen.delete(value);
+	}
+}
+/** `toJson` for values that must be objects, such as metadata. */
+function toJsonObject(value, path) {
+	const converted = toJson(value, path);
+	if (converted === null || typeof converted !== "object" || Array.isArray(converted)) throw new NotJsonError(path, "must be a JSON object");
+	return converted;
+}
+//#endregion
+//#region ../core/src/catalog.ts
+function catalogEntry(evaluation, suite, options = {}) {
+	const at = (value) => relativeSite(definitionSite(value), options.root);
+	const definedAt = at(evaluation);
+	const judgeAgent = evaluation.judge ? catalogAgent(evaluation.judge, at) : void 0;
+	const policy = evaluation.policy ?? {};
+	return {
+		id: evaluation.id,
+		path: suite ? `${suite.id}#${evaluation.id}` : evaluation.id,
+		...evaluation.name ? { name: evaluation.name } : {},
+		...suite ? { suiteId: authoringId(suite) } : {},
+		agent: catalogAgent(evaluation.agent, at),
+		...judgeAgent ? { judgeAgent } : {},
+		datasets: (evaluation.datasets ?? []).map((dataset) => catalogDataset(dataset, at)),
+		trialCount: policy.trials ?? 1,
+		policy: {
+			trials: policy.trials ?? 1,
+			...policy.timeoutMs !== void 0 ? { timeoutMs: policy.timeoutMs } : {},
+			...policy.failfast !== void 0 ? { failfast: policy.failfast } : {}
+		},
+		transcript: evaluation.transcript.map((step) => catalogStep(step, at)),
+		scorers: evaluation.scoring.map((rule) => catalogScorer(rule, at)),
+		...evaluation.metadata !== void 0 ? { metadata: toJson(evaluation.metadata, `${evaluation.id}.metadata`) } : {},
+		...definedAt ? { definedAt } : {}
+	};
+}
+function catalogAgent(agent, at) {
+	const identity = agent.identity;
+	const definedAt = at(agent);
+	return {
+		...identity?.name ? { name: identity.name } : {},
+		kind: identity?.kind ?? "adapter",
+		...identity?.id ? { id: identity.id } : {},
+		...identity?.version ? { version: identity.version } : {},
+		runtimes: Object.entries(agentRuntimes(agent)).map(([name, runtime]) => ({
+			name,
+			kind: runtime.kind
+		})),
+		...definedAt ? { definedAt } : {}
+	};
+}
+function catalogDataset(dataset, at) {
+	const definedAt = at(dataset);
+	const source = dataset.source;
+	const base = {
+		id: dataset.id,
+		...dataset.name ? { name: dataset.name } : {},
+		visibility: dataset.visibility,
+		source: source.kind,
+		mount: dataset.mount,
+		...definedAt ? { definedAt } : {}
+	};
+	if (source.kind === "computed") return {
+		...base,
+		hooks: {
+			onLoad: true,
+			onRun: !!source.onRun
+		}
+	};
+	return {
+		...base,
+		...source.files === void 0 ? {} : typeof source.files === "string" ? { files: { directory: source.files } } : { files: { paths: Object.keys(source.files).sort() } },
+		...source.valueFile !== void 0 ? { value: { file: source.valueFile } } : source.value !== void 0 ? { value: { shape: shapeOf(source.value) } } : {}
+	};
+}
+function catalogScorer(rule, at) {
+	const definedAt = at(rule);
+	return {
+		name: rule.name,
+		kind: rule.kind,
+		...rule.kind === "predicate" && rule.passAt !== void 0 ? { passAt: rule.passAt } : {},
+		...rule.supportsPartial !== void 0 ? { supportsPartial: rule.supportsPartial } : {},
+		...rule.kind === "judge" ? { rubric: rule.rubric } : {},
+		...definedAt ? { definedAt } : {}
+	};
+}
+function catalogStep(step, at) {
+	switch (step.kind) {
+		case "user": return {
+			kind: "user",
+			message: step.message
+		};
+		case "agent": return {
+			kind: "agent",
+			expectation: step.expectation
+		};
+		case "expect-tool-call": return {
+			kind: "expect-tool-call",
+			name: step.expected.name,
+			...step.expected.arguments !== void 0 ? { arguments: step.expected.arguments } : {}
+		};
+		default: return {
+			kind: "checkpoint",
+			scorer: catalogScorer(step, at)
+		};
+	}
+}
+function shapeOf(value) {
+	const json = toJson(value, "dataset value");
+	if (Array.isArray(json)) return {
+		type: "array",
+		items: json.length
+	};
+	if (json === null) return { type: "null" };
+	if (typeof json === "object") return {
+		type: "object",
+		keys: Object.keys(json).sort()
+	};
+	return { type: typeof json };
+}
+//#endregion
 //#region ../core/src/canonical-json.ts
 /** Canonical JSON: keys sorted recursively, array order preserved. Rejects non-JSON inputs. */
 function canonicalParameters(value) {
@@ -176,6 +461,83 @@ function defineEvalMatrix(definition) {
 	};
 }
 //#endregion
+//#region ../core/src/aut-events.ts
+/** Converts an adapter event into a recorded `AutEvent`. */
+function normalizeAutEvent(input, now) {
+	if (typeof input !== "object" || input === null) throw new Error("AUT events must be objects");
+	const timestamp = input.timestamp ?? now().toISOString();
+	switch (input.kind) {
+		case "message": return {
+			kind: "message",
+			role: input.role,
+			content: toJson(input.content, "event.content"),
+			timestamp
+		};
+		case "tool-call": return {
+			kind: "tool-call",
+			id: input.id,
+			name: input.name,
+			arguments: input.arguments === void 0 ? {} : toJson(input.arguments, "event.arguments"),
+			timestamp
+		};
+		case "tool-result": return {
+			kind: "tool-result",
+			id: input.id,
+			...input.name === void 0 ? {} : { name: input.name },
+			result: input.result === void 0 ? null : toJson(input.result, "event.result"),
+			timestamp
+		};
+		case "completed": return {
+			kind: "completed",
+			...input.output === void 0 ? {} : { output: toJson(input.output, "event.output") },
+			timestamp
+		};
+		case "started":
+		case "turn-started":
+		case "turn-completed":
+		case "error": return {
+			...input,
+			timestamp
+		};
+		default: throw new Error(`Unknown AUT event kind: ${String(input.kind)}`);
+	}
+}
+/**
+* The events a returned turn result stands for: each tool call (and its
+* result), then the reply, then the usage as a `turn-completed` event.
+*/
+function turnResultEvents(result, turn, now) {
+	if (typeof result === "string") result = { reply: result };
+	if (typeof result !== "object" || result === null) throw new Error("send() must return nothing, a string, or a turn result");
+	const events = [];
+	for (const [index, call] of (result.toolCalls ?? []).entries()) {
+		const id = call.id ?? `turn-${turn}-call-${index + 1}`;
+		events.push(normalizeAutEvent({
+			kind: "tool-call",
+			id,
+			name: call.name,
+			arguments: call.arguments
+		}, now));
+		if (call.result !== void 0) events.push(normalizeAutEvent({
+			kind: "tool-result",
+			id,
+			name: call.name,
+			result: call.result
+		}, now));
+	}
+	if (result.reply !== void 0) events.push(normalizeAutEvent({
+		kind: "message",
+		role: "assistant",
+		content: result.reply
+	}, now));
+	if (result.usage) events.push(normalizeAutEvent({
+		kind: "turn-completed",
+		turn,
+		usage: result.usage
+	}, now));
+	return events;
+}
+//#endregion
 //#region ../core/src/schema.ts
 /** Runtime schemas for serialized EvalKit report and API boundaries. */
 const RunStatusSchema = Schema.Literal("running", "completed", "failed", "cancelled");
@@ -217,7 +579,7 @@ const RunSourceSchema = Schema.Struct({
 	}))
 });
 const RunMetadataSchema = Schema.Struct({
-	schemaVersion: Schema.Literal(2, 3, 4),
+	schemaVersion: Schema.Literal(2, 3, 4, 5),
 	runUri: resourceUriSchema("run"),
 	evalId: Schema.String,
 	suiteId: Schema.optional(Schema.String),
@@ -225,10 +587,14 @@ const RunMetadataSchema = Schema.Struct({
 	matrix: Schema.optional(MatrixCellSchema),
 	aut: Schema.optional(AutIdentitySchema),
 	startedAt: Schema.String,
-	source: Schema.optional(RunSourceSchema)
+	source: Schema.optional(RunSourceSchema),
+	process: Schema.optional(Schema.Struct({
+		pid: Schema.Number,
+		hostname: Schema.String
+	}))
 });
 const TrialMetadataSchema = Schema.Struct({
-	schemaVersion: Schema.Literal(2, 3, 4),
+	schemaVersion: Schema.Literal(2, 3, 4, 5),
 	runUri: resourceUriSchema("run"),
 	trialUri: resourceUriSchema("trial"),
 	trialIndex: Schema.Number,
@@ -236,7 +602,13 @@ const TrialMetadataSchema = Schema.Struct({
 	parameters: Schema.optional(ParametersSchema),
 	matrix: Schema.optional(MatrixCellSchema),
 	aut: Schema.optional(AutIdentitySchema),
-	startedAt: Schema.String
+	startedAt: Schema.String,
+	datasets: Schema.optional(Schema.mutable(Schema.Array(Schema.Struct({
+		id: Schema.String,
+		visibility: Schema.Literal("candidate", "evaluator"),
+		source: Schema.Literal("static", "computed"),
+		digest: Schema.String
+	}))))
 });
 /** On-disk manifests include a lifecycle status; older finalized manifests may still say running. */
 const RunManifestSchema = Schema.Struct({
@@ -271,7 +643,8 @@ const ScoreResultSchema = Schema.Struct({
 	evidence: Schema.optional(JsonValueSchema),
 	judge: Schema.optional(JudgeRunInfoSchema),
 	durationMs: Schema.Number,
-	error: Schema.optional(RecordedErrorSchema)
+	error: Schema.optional(RecordedErrorSchema),
+	skipped: Schema.optional(Schema.Literal(true))
 });
 const CheckpointResultSchema = Schema.Struct({
 	step: Schema.Number,
@@ -317,6 +690,7 @@ const RunSummarySchema = Schema.Struct({
 	trialCount: Schema.Number,
 	passed: Schema.Number,
 	failed: Schema.Number,
+	cancelled: Schema.optional(Schema.Number),
 	error: Schema.optional(RecordedErrorSchema)
 });
 const autEvent = {
@@ -385,6 +759,7 @@ const runnerEvent = {
 		source: Schema.Literal("runner"),
 		kind: Schema.Literal("transcript-step-started"),
 		step: Schema.Number,
+		message: Schema.optional(Schema.String),
 		timestamp: Schema.String
 	}),
 	stepCompleted: Schema.Struct({
@@ -410,7 +785,7 @@ const runnerEvent = {
 		source: Schema.Literal("runner"),
 		kind: Schema.Literal("checkpoint-completed"),
 		step: Schema.Number,
-		status: Schema.Literal("passed", "failed"),
+		status: Schema.Literal("passed", "failed", "skipped"),
 		timestamp: Schema.String
 	}),
 	checkpointError: Schema.Struct({
@@ -430,7 +805,8 @@ const runnerEvent = {
 		source: Schema.Literal("runner"),
 		kind: Schema.Literal("scorer-completed"),
 		scorer: Schema.String,
-		value: Schema.Number,
+		value: Schema.optional(Schema.Number),
+		skipped: Schema.optional(Schema.Literal(true)),
 		timestamp: Schema.String
 	}),
 	scorerFailed: Schema.Struct({
@@ -474,52 +850,20 @@ function parseTrajectoryJsonl(contents) {
 }
 //#endregion
 //#region ../core/src/index.ts
+internalModule(import.meta.url);
+/** Used when an agent declares no runtimes. */
+const DEFAULT_AGENT_RUNTIMES = { local: { kind: "in-process" } };
+/** The runtimes an agent can run in, applying the default. */
+function agentRuntimes(agent) {
+	return agent.runtimes && Object.keys(agent.runtimes).length ? agent.runtimes : DEFAULT_AGENT_RUNTIMES;
+}
 /**
 * Defines an Agent Under Test without coupling it to a runtime or transport.
 * It preserves the adapter's concrete type for application-specific helpers.
 */
 function defineAgent(agent) {
 	if (agent.identity) authoringId(agent.identity);
-	return agent;
-}
-function defaultFixtureDestination(src) {
-	const normalized = src.replace(/\\/g, "/").replace(/\/+$/, "");
-	const destination = normalized.slice(normalized.lastIndexOf("/") + 1);
-	if (!destination || destination === "." || destination === "..") throw new Error(`Fixture source needs a basename: ${src}`);
-	return destination;
-}
-/**
-* Declares a directory relative to the invoking project's working directory.
-* The short form is candidate-visible and copies beneath the source basename.
-*/
-function directory(src, options = {}) {
-	return {
-		kind: "directory",
-		src,
-		dst: options.dst ?? defaultFixtureDestination(src),
-		visibility: options.visibility ?? "candidate"
-	};
-}
-function file(src, options) {
-	return {
-		kind: "file",
-		src,
-		...options
-	};
-}
-function inlineFile(file, data, visibility) {
-	return {
-		kind: "inline",
-		file,
-		data,
-		visibility
-	};
-}
-function dynamic(create) {
-	return {
-		kind: "dynamic",
-		create
-	};
+	return markDefinitionSite(agent);
 }
 /** Observes a tool call in the preceding turn; never invokes a tool. */
 function expectToolCall(expected) {
@@ -542,30 +886,51 @@ function agent(expectation) {
 		expectation
 	};
 }
-function predicate(name, run, options = {}) {
+/**
+* Marks a scorer as not applicable to this trial. A skipped result neither
+* passes nor fails the trial and is left out of score averages.
+*/
+function skip(reason) {
+	if (!reason.trim()) throw new Error("skip() needs a reason");
 	return {
+		skipped: true,
+		reason
+	};
+}
+function predicate(name, run, options = {}) {
+	if (!name.trim()) throw new Error("Predicate name must not be empty");
+	if (options.passAt !== void 0 && !(Number.isFinite(options.passAt) && options.passAt >= 0 && options.passAt <= 1)) throw new Error(`Predicate ${name} passAt must be a number from 0 to 1; received ${options.passAt}`);
+	return markDefinitionSite({
 		kind: "predicate",
 		name,
 		run,
 		...options
-	};
+	});
 }
 function judge(name, options) {
 	if (!name.trim() || !options.rubric.trim()) throw new Error("Judge name and rubric must not be empty");
-	return {
+	return markDefinitionSite({
 		kind: "judge",
 		name,
 		...options
-	};
+	});
 }
 function defineEval(definition) {
 	authoringId(definition);
+	if ("fixtures" in definition) throw new Error(`Eval ${definition.id} uses fixtures, which were replaced by datasets: [defineDataset(...)]`);
+	if (definition.metadata !== void 0) toJsonObject(definition.metadata, `${definition.id}.metadata`);
+	const datasetIds = /* @__PURE__ */ new Set();
+	for (const [index, dataset] of (definition.datasets ?? []).entries()) {
+		if (!isDataset(dataset)) throw new Error(`Eval ${definition.id} datasets[${index}] is not a dataset; create it with defineDataset(...)`);
+		if (datasetIds.has(dataset.id)) throw new Error(`Eval ${definition.id} uses dataset ${dataset.id} more than once`);
+		datasetIds.add(dataset.id);
+	}
 	let hasUser = false;
 	for (const step of definition.transcript) if (step.kind === "user") hasUser = true;
 	else if (step.kind !== "agent" && !hasUser) throw new Error("Checkpoint requires a preceding user step");
 	if ((definition.transcript.some((step) => step.kind === "judge") || definition.scoring.some((rule) => rule.kind === "judge")) && !definition.judge) throw new Error("Judge rules require an eval judge agent");
 	if (definition.judge && definition.judge === definition.agent) throw new Error("Judge agent must be distinct from the agent under test");
-	return definition;
+	return markDefinitionSite(definition);
 }
 function defineSuite(suite) {
 	authoringId(suite);
@@ -633,45 +998,7 @@ function registerEvals(registrations) {
 			evalIds: suite.evals.map(authoringId),
 			...suite.name ? { name: suite.name } : {}
 		})),
-		catalog: () => evals.map((evaluation) => {
-			const suite = suiteIdByEvalId.get(authoringId(evaluation));
-			const identity = evaluation.agent.identity;
-			return {
-				id: evaluation.id,
-				path: suite ? `${suite.id}#${evaluation.id}` : evaluation.id,
-				...evaluation.name ? { name: evaluation.name } : {},
-				...suite ? { suiteId: suite.id } : {},
-				agent: {
-					...identity?.name ? { name: identity.name } : {},
-					kind: identity?.kind ?? "adapter",
-					...identity?.id ? { id: identity.id } : {},
-					...identity?.version ? { version: identity.version } : {},
-					runtimes: Object.entries(evaluation.agent.runtimes ?? {}).map(([name, runtime]) => ({
-						name,
-						kind: runtime.kind
-					}))
-				},
-				fixtures: (evaluation.fixtures ?? []).map((fixture) => fixture.kind === "dynamic" ? {
-					kind: "dynamic",
-					source: "dynamic"
-				} : fixture.kind === "inline" ? {
-					kind: "inline",
-					source: fixture.file,
-					destination: fixture.file,
-					visibility: fixture.visibility
-				} : {
-					kind: fixture.kind,
-					source: fixture.src,
-					destination: fixture.dst,
-					visibility: fixture.visibility
-				}),
-				trialCount: evaluation.policy?.trials ?? 1,
-				scorers: evaluation.scoring.map(({ name, kind }) => ({
-					name,
-					kind
-				}))
-			};
-		})
+		catalog: (options) => evals.map((evaluation) => catalogEntry(evaluation, suiteIdByEvalId.get(authoringId(evaluation)), options))
 	};
 }
 function recordError(error) {
@@ -686,6 +1013,4 @@ function recordError(error) {
 	};
 }
 //#endregion
-export { TrajectoryEventSchema as A, UuidSchema as B, RecordedErrorSchema as C, RunStatusSchema as D, RunSourceSchema as E, UsageSchema as F, resourceUuid as G, parseResourceUri as H, defineEvalMatrix as I, canonicalParameters as L, TrialMetadataSchema as M, TrialScoringSchema as N, RunSummarySchema as O, TrialSummarySchema as P, ResourceKindSchema as R, JudgeRunInfoSchema as S, RunMetadataSchema as T, resourceUri as U, authorId as V, resourceUriSchema as W, ApiErrorSchema as _, defineSuite as a, CheckpointResultSchema as b, expectToolCall as c, judge as d, predicate as f, parseTrajectoryJsonl as g, user as h, defineEval as i, TrialManifestSchema as j, ScoreResultSchema as k, file as l, registerEvals as m, authoringId as n, directory as o, recordError as p, defineAgent as r, dynamic as s, agent as t, inlineFile as u, ArtifactEntrySchema as v, RunManifestSchema as w, JsonValueSchema as x, AutIdentitySchema as y, ResourceUriSchema as z };
-
-//# sourceMappingURL=src-ButXPY8P.mjs.map
+export { resourceUriSchema as $, TrialManifestSchema as A, NotJsonError as B, RunManifestSchema as C, RunSummarySchema as D, RunStatusSchema as E, normalizeAutEvent as F, relativeDatasetPath as G, toJsonObject as H, turnResultEvents as I, ResourceUriSchema as J, definitionSite as K, defineEvalMatrix as L, TrialScoringSchema as M, TrialSummarySchema as N, ScoreResultSchema as O, UsageSchema as P, resourceUri as Q, canonicalParameters as R, RecordedErrorSchema as S, RunSourceSchema as T, defineDataset as U, toJson as V, isDataset as W, authorId as X, UuidSchema as Y, parseResourceUri as Z, ArtifactEntrySchema as _, defineAgent as a, JsonValueSchema as b, expectToolCall as c, recordError as d, resourceUuid as et, registerEvals as f, ApiErrorSchema as g, parseTrajectoryJsonl as h, authoringId as i, TrialMetadataSchema as j, TrajectoryEventSchema as k, judge as l, user as m, agent as n, defineEval as o, skip as p, ResourceKindSchema as q, agentRuntimes as r, defineSuite as s, DEFAULT_AGENT_RUNTIMES as t, predicate as u, AutIdentitySchema as v, RunMetadataSchema as w, JudgeRunInfoSchema as x, CheckpointResultSchema as y, catalogEntry as z };