@leostera/evalkit 0.8.0 → 0.10.0

package/dist/{src-LeGSyDD2.mjs → src-B14fXAGp.mjs}

@@ -1,6 +1,7 @@
+import { n as assertNoRemovedOptions } from "./upgrade-G-uFPJKg.mjs";
 import * as Schema from "effect/Schema";
 //#region ../core/src/identity.ts
-const ResourceKindSchema = Schema.Literal("run", "trial", "artifact");
+const ResourceKindSchema = Schema.Literal("run", "artifact");
 /** Human-authored project-local identifier; never a generated resource URI. */
 function authorId(value) {
 	if (!/^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(value)) throw new Error(`ID must be lowercase kebab-case: ${value}`);
@@ -8,7 +9,7 @@ function authorId(value) {
 }
 /** RFC 4122 textual UUID. EvalKit accepts v4/v7 and other valid UUID versions. */
 const UuidSchema = Schema.String.pipe(Schema.pattern(/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i));
-const ResourceUriSchema = Schema.String.pipe(Schema.pattern(/^evalkit:(run|trial|artifact):[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i));
+const ResourceUriSchema = Schema.String.pipe(Schema.pattern(/^evalkit:(run|artifact):[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i));
 function resourceUriSchema(kind) {
 	return Schema.String.pipe(Schema.pattern(new RegExp(`^evalkit:${kind}:[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$`, "i")));
 }
@@ -16,7 +17,7 @@ function resourceUri(kind, uuid) {
 	return `evalkit:${kind}:${uuid}`;
 }
 function parseResourceUri(value, expectedKind) {
-	const match = /^evalkit:(run|trial|artifact):(.+)$/.exec(value);
+	const match = /^evalkit:(run|artifact):(.+)$/.exec(value);
 	if (!match) throw new Error(`Invalid EvalKit resource URI: ${value}`);
 	const kind = match[1];
 	if (expectedKind && kind !== expectedKind) throw new Error(`Expected an EvalKit ${expectedKind} URI; received ${value}`);
@@ -101,10 +102,10 @@ function relativeDatasetPath(value, label) {
 	return parts.length ? parts.join("/") : ".";
 }
 /**
-* Declares named data that scenarios use: files placed in each trial's
+* Declares named data that scenarios use: files placed in each run's
 * workspace, a value for the agent and scorers, or both. Static datasets give
 * `files`, `value` or `valueFile`; computed datasets give `onLoad` (cached per
-* process) and optionally `onRun` (per trial).
+* process) and optionally `onRun` (per run).
 */
 function defineDataset(definition) {
 	return markDefinitionSite(checkedDataset(definition));
@@ -205,11 +206,194 @@ function toJsonObject(value, path) {
 	return converted;
 }
 //#endregion
+//#region ../core/src/judge.ts
+const KEY = /^[A-Za-z0-9][A-Za-z0-9_-]{0,31}$/;
+function unitInterval(value) {
+	return typeof value === "number" && Number.isFinite(value) && value >= 0 && value <= 1;
+}
+/**
+* A judge that answers with one of these choices, in this order:
+* `choices({ A: ['Accurate and helpful', 1], B: ['Partly right', 0.5], C: ['Wrong', 0] })`.
+*/
+function choices(options) {
+	const entries = Object.entries(options);
+	if (entries.length < 2) throw new Error("choices(...) needs at least two choices");
+	const seen = /* @__PURE__ */ new Set();
+	return {
+		kind: "choices",
+		choices: entries.map(([key, [label, value]]) => {
+			if (!KEY.test(key)) throw new Error(`Choice ${JSON.stringify(key)} must be a short word of letters, digits, "-" or "_"`);
+			if (seen.has(key.toUpperCase())) throw new Error(`Choice ${key} is listed twice`);
+			seen.add(key.toUpperCase());
+			if (!label.trim()) throw new Error(`Choice ${key} needs a label`);
+			if (!unitInterval(value)) throw new Error(`Choice ${key} must be worth a number from 0 to 1; received ${String(value)}`);
+			return {
+				key,
+				label,
+				value
+			};
+		})
+	};
+}
+/**
+* A judge that scores each criterion from 0 to 1; the score is the weighted
+* mean. A number is the weight; `{ weight, description }` also says what the
+* criterion means. Weights default to 1.
+*/
+function criteria(options) {
+	return {
+		kind: "criteria",
+		criteria: declaredCriteria(options)
+	};
+}
+/** Checked criteria declarations, for `criteria()` and `predicate(..., { criteria })`. */
+function declaredCriteria(options) {
+	const entries = Object.entries(options);
+	if (!entries.length) throw new Error("criteria(...) needs a criterion");
+	return entries.map(([name, option]) => {
+		if (!name.trim()) throw new Error("A criterion needs a name");
+		const weight = typeof option === "number" ? option : option.weight ?? 1;
+		if (!Number.isFinite(weight) || weight <= 0) throw new Error(`Criterion ${name} must have a positive weight; received ${String(weight)}`);
+		const description = typeof option === "number" ? void 0 : option.description;
+		return {
+			name,
+			weight,
+			...description ? { description } : {}
+		};
+	});
+}
+/** What a criterion counts for in the weighted mean. */
+const criterionValue = (result) => "value" in result ? result.value : 0;
+/** The weighted mean of criteria; weights default to 1, a criterion not reached counts 0. */
+function weightedMean(results) {
+	let total = 0;
+	let weights = 0;
+	for (const result of results) {
+		const weight = result.weight ?? 1;
+		total += weight * criterionValue(result);
+		weights += weight;
+	}
+	if (!weights) throw new Error("Criteria need a positive total weight");
+	return total / weights;
+}
+/** The inputs a judge sees when the rule names none. */
+const DEFAULT_JUDGE_INPUTS = ["conversation", "reply"];
+/** The names of the inputs a judge sees, in prompt order. */
+function judgeInputNames(rule) {
+	return rule.inputs ? Object.keys(rule.inputs) : [...DEFAULT_JUDGE_INPUTS];
+}
+/**
+* The prompt a judge agent receives. `values` holds each input's text;
+* an input without one shows as a `{placeholder}`, which is how the
+* dashboard shows the prompt before any run.
+*/
+function judgePrompt(options) {
+	const inputs = options.inputs.map((name) => `${name}:\n${options.values?.[name] ?? `{${name}}`}`).join("\n\n");
+	return [
+		"You are judging one run of an AI agent. Judge only by the rubric, using only the inputs below.",
+		`Rubric:\n${options.rubric.trim()}`,
+		inputs,
+		answerInstruction(options.answer)
+	].join("\n\n");
+}
+function answerInstruction(answer) {
+	switch (answer.kind) {
+		case "verdict": return "Answer with only a JSON object: {\"value\": <a number from 0 to 1>, \"explanation\": \"<one or two sentences why>\"}";
+		case "choices": return [`Answer with one of ${answer.choices.map((choice) => choice.key).join(", ")} on the first line, then one sentence why on the next.`, ...answer.choices.map((choice) => `${choice.key}) ${choice.label}`)].join("\n");
+		case "criteria": return [
+			"Score each criterion from 0 to 1:",
+			...answer.criteria.map((criterion) => `- ${criterion.name}${criterion.description ? `: ${criterion.description}` : ""}`),
+			`Answer with only a JSON object: {"criteria": {${answer.criteria.map((criterion) => `${JSON.stringify(criterion.name)}: {"value": <0 to 1>, "explanation": "<why>"}`).join(", ")}}, "explanation": "<one sentence overall>"}`
+		].join("\n");
+	}
+}
+/** Thrown when a judge's reply is not in the format asked for; the message says why. */
+var UnreadableAnswer = class extends Error {};
+/**
+* Reads a judge's reply in the format its rule asked for. Throws
+* `UnreadableAnswer` with a reason the judge can be told, so it can answer again.
+*/
+function readJudgeAnswer(answer, reply) {
+	if (answer.kind === "choices") return readChoice(answer.choices, reply);
+	const object = jsonObject(reply);
+	if (answer.kind === "verdict") {
+		if (!unitInterval(object.value)) throw new UnreadableAnswer("\"value\" must be a number from 0 to 1");
+		return {
+			value: object.value,
+			...typeof object.explanation === "string" && object.explanation.trim() ? { explanation: object.explanation.trim() } : {},
+			...object.evidence !== void 0 ? { evidence: object.evidence } : {}
+		};
+	}
+	const scored = object.criteria;
+	if (typeof scored !== "object" || scored === null || Array.isArray(scored)) throw new UnreadableAnswer("\"criteria\" must be an object with one entry per criterion");
+	const results = answer.criteria.map((criterion) => {
+		const entry = scored[criterion.name];
+		const value = typeof entry === "number" ? entry : typeof entry === "object" && entry !== null && !Array.isArray(entry) ? entry.value : void 0;
+		if (!unitInterval(value)) throw new UnreadableAnswer(`criterion ${JSON.stringify(criterion.name)} needs a "value" from 0 to 1`);
+		const explanation = typeof entry === "object" && entry !== null && !Array.isArray(entry) && typeof entry.explanation === "string" && entry.explanation.trim() ? entry.explanation.trim() : void 0;
+		return {
+			name: criterion.name,
+			value,
+			weight: criterion.weight,
+			...explanation ? { explanation } : {}
+		};
+	});
+	return {
+		value: weightedMean(results),
+		criteria: results,
+		...typeof object.explanation === "string" && object.explanation.trim() ? { explanation: object.explanation.trim() } : {}
+	};
+}
+function readChoice(options, reply) {
+	if (typeof reply !== "string") throw new UnreadableAnswer("answer with plain text, not JSON");
+	const lines = reply.split("\n").map((line) => line.trim()).filter(Boolean);
+	const first = lines[0] ?? "";
+	const match = /^(?:answer\s*[:-]?\s*)?[*_("'[\s]*([A-Za-z0-9][A-Za-z0-9_-]*)/i.exec(first);
+	const said = match?.[1]?.toUpperCase();
+	const choice = options.find((option) => option.key.toUpperCase() === said);
+	if (!choice) throw new UnreadableAnswer(`the first line must be one of ${options.map((option) => option.key).join(", ")}`);
+	const explanation = [first.slice(match[0].length).replace(/^[*_)"'\].:\s-]+/, "").trim(), ...lines.slice(1)].filter(Boolean).join(" ");
+	return {
+		value: choice.value,
+		choice: choice.key,
+		...explanation ? { explanation } : {}
+	};
+}
+/** The JSON object in a reply: the reply itself, or the first `{…}` in its text (code fences allowed). */
+function jsonObject(reply) {
+	if (typeof reply === "object" && reply !== null && !Array.isArray(reply)) return reply;
+	if (typeof reply !== "string") throw new UnreadableAnswer("answer with a JSON object");
+	const start = reply.indexOf("{");
+	const end = reply.lastIndexOf("}");
+	if (start === -1 || end < start) throw new UnreadableAnswer("answer with only a JSON object");
+	let parsed;
+	try {
+		parsed = JSON.parse(reply.slice(start, end + 1));
+	} catch {
+		throw new UnreadableAnswer("the JSON object could not be parsed");
+	}
+	if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) throw new UnreadableAnswer("answer with a JSON object");
+	return parsed;
+}
+/**
+* The pass mark in words: `only 1`, `≥ 0.5`, or for choices the ones that
+* pass, such as `A or B`.
+*/
+function passMark(passAt, answer) {
+	const at = passAt ?? 1;
+	if (answer?.kind === "choices") {
+		const passing = answer.choices.filter((choice) => choice.value >= at).map((choice) => choice.key);
+		if (!passing.length) return "no choice passes";
+		if (passing.length === answer.choices.length) return "any choice";
+		return passing.length === 1 ? `only ${passing[0]}` : `${passing.slice(0, -1).join(", ")} or ${passing.at(-1)}`;
+	}
+	return at === 1 ? "only 1" : `≥ ${at}`;
+}
+//#endregion
 //#region ../core/src/catalog.ts
 function catalogEntry(evaluation, suite, options = {}) {
 	const at = (value) => relativeSite(definitionSite(value), options.root);
 	const definedAt = at(evaluation);
-	const judgeAgent = evaluation.judge ? catalogAgent(evaluation.judge, at) : void 0;
 	const policy = evaluation.policy ?? {};
 	return {
 		id: evaluation.id,
@@ -217,16 +401,14 @@ function catalogEntry(evaluation, suite, options = {}) {
 		...evaluation.name ? { name: evaluation.name } : {},
 		...suite ? { suiteId: authoringId(suite) } : {},
 		agent: catalogAgent(evaluation.agent, at),
-		...judgeAgent ? { judgeAgent } : {},
 		datasets: (evaluation.datasets ?? []).map((dataset) => catalogDataset(dataset, at)),
-		trialCount: policy.trials ?? 1,
+		repeat: evaluation.repeat ?? 1,
 		policy: {
-			trials: policy.trials ?? 1,
 			...policy.timeoutMs !== void 0 ? { timeoutMs: policy.timeoutMs } : {},
 			...policy.failfast !== void 0 ? { failfast: policy.failfast } : {}
 		},
-		transcript: evaluation.transcript.map((step) => catalogStep(step, at)),
-		scorers: evaluation.scoring.map((rule) => catalogScorer(rule, at)),
+		transcript: evaluation.transcript.map((step) => catalogStep(step, at, options.judge)),
+		scorers: evaluation.scoring.map((rule) => catalogScorer(rule, at, options.judge)),
 		...evaluation.metadata !== void 0 ? { metadata: toJson(evaluation.metadata, `${evaluation.id}.metadata`) } : {},
 		...definedAt ? { definedAt } : {}
 	};
@@ -270,18 +452,37 @@ function catalogDataset(dataset, at) {
 		...source.valueFile !== void 0 ? { value: { file: source.valueFile } } : source.value !== void 0 ? { value: { shape: shapeOf(source.value) } } : {}
 	};
 }
-function catalogScorer(rule, at) {
+function catalogScorer(rule, at, defaultJudge) {
 	const definedAt = at(rule);
 	return {
 		name: rule.name,
 		kind: rule.kind,
-		...rule.kind === "predicate" && rule.passAt !== void 0 ? { passAt: rule.passAt } : {},
+		...rule.description ? { description: rule.description } : {},
+		...rule.passAt !== void 0 ? { passAt: rule.passAt } : {},
 		...rule.supportsPartial !== void 0 ? { supportsPartial: rule.supportsPartial } : {},
-		...rule.kind === "judge" ? { rubric: rule.rubric } : {},
+		...rule.kind === "judge" ? catalogJudge(rule, at, defaultJudge) : rule.criteria ? { answer: {
+			kind: "criteria",
+			criteria: rule.criteria
+		} } : {},
 		...definedAt ? { definedAt } : {}
 	};
 }
-function catalogStep(step, at) {
+function catalogJudge(rule, at, defaultJudge) {
+	const agent = rule.agent ?? defaultJudge;
+	const inputs = judgeInputNames(rule);
+	return {
+		rubric: rule.rubric,
+		...agent ? { agent: catalogAgent(agent, at) } : {},
+		inputs,
+		answer: rule.answer,
+		prompt: judgePrompt({
+			rubric: rule.rubric,
+			answer: rule.answer,
+			inputs
+		})
+	};
+}
+function catalogStep(step, at, defaultJudge) {
 	switch (step.kind) {
 		case "user": return typeof step.message === "string" ? {
 			kind: "user",
@@ -298,7 +499,7 @@ function catalogStep(step, at) {
 		};
 		default: return {
 			kind: "checkpoint",
-			scorer: catalogScorer(step, at)
+			scorer: catalogScorer(step, at, defaultJudge)
 		};
 	}
 }
@@ -317,6 +518,10 @@ function shapeOf(value) {
 }
 //#endregion
 //#region ../core/src/config.ts
+const DEFAULT_ATTACHMENT_LIMITS = {
+	maxFileBytes: 5e6,
+	maxRunBytes: 2e7
+};
 /** Left out of candidate snapshots unless `artifacts.exclude` says otherwise. */
 const DEFAULT_ARTIFACT_EXCLUDE = [
 	".git",
@@ -332,6 +537,18 @@ function defineConfig(configuration) {
 	return configuration;
 }
 //#endregion
+//#region ../core/src/usage.ts
+/**
+* The tokens a turn counts toward the `tokens` metric: `inputTokens +
+* outputTokens` (cached input included), or `totalTokens` when neither is
+* reported. Undefined when the turn reported no usage.
+*/
+function turnTokens(usage) {
+	if (!usage) return void 0;
+	if (usage.inputTokens !== void 0 || usage.outputTokens !== void 0) return (usage.inputTokens ?? 0) + (usage.outputTokens ?? 0);
+	return usage.totalTokens;
+}
+//#endregion
 //#region ../core/src/canonical-json.ts
 /** Canonical JSON: keys sorted recursively, array order preserved. Rejects non-JSON inputs. */
 function canonicalParameters(value) {
@@ -546,17 +763,20 @@ function turnResultEvents(result, turn, now) {
 		role: "assistant",
 		content: result.reply
 	}, now));
-	if (result.usage) events.push(normalizeAutEvent({
+	if (result.usage || result.model) events.push(normalizeAutEvent({
 		kind: "turn-completed",
 		turn,
-		usage: result.usage
+		...result.usage ? { usage: result.usage } : {},
+		...result.model ? { model: result.model } : {}
 	}, now));
 	return events;
 }
 //#endregion
 //#region ../core/src/schema.ts
 /** Runtime schemas for serialized EvalKit report and API boundaries. */
-const RunStatusSchema = Schema.Literal("running", "completed", "failed", "cancelled");
+const RunStatusSchema = Schema.Literal("running", "passed", "failed", "errored", "cancelled");
+/** The report schema version this EvalKit writes and reads. */
+const REPORT_SCHEMA_VERSION = 6;
 const RecordedErrorSchema = Schema.Struct({
 	name: Schema.String,
 	message: Schema.String,
@@ -594,8 +814,19 @@ const RunSourceSchema = Schema.Struct({
 		url: Schema.optional(Schema.String)
 	}))
 });
+const BatchSchema = Schema.Struct({
+	id: Schema.String,
+	index: Schema.Number,
+	size: Schema.Number
+});
+const RunDatasetSchema = Schema.Struct({
+	id: Schema.String,
+	visibility: Schema.Literal("candidate", "evaluator"),
+	source: Schema.Literal("static", "computed"),
+	digest: Schema.String
+});
 const RunMetadataSchema = Schema.Struct({
-	schemaVersion: Schema.Literal(2, 3, 4, 5),
+	schemaVersion: Schema.Literal(6),
 	runUri: resourceUriSchema("run"),
 	evalId: Schema.String,
 	suiteId: Schema.optional(Schema.String),
@@ -607,39 +838,21 @@ const RunMetadataSchema = Schema.Struct({
 	process: Schema.optional(Schema.Struct({
 		pid: Schema.Number,
 		hostname: Schema.String
-	}))
-});
-const TrialMetadataSchema = Schema.Struct({
-	schemaVersion: Schema.Literal(2, 3, 4, 5),
-	runUri: resourceUriSchema("run"),
-	trialUri: resourceUriSchema("trial"),
-	trialIndex: Schema.Number,
-	evalId: Schema.String,
-	parameters: Schema.optional(ParametersSchema),
-	matrix: Schema.optional(MatrixCellSchema),
-	aut: Schema.optional(AutIdentitySchema),
-	startedAt: Schema.String,
+	})),
+	batch: Schema.optional(BatchSchema),
 	/** The effective snapshot ignore patterns: the project's, then the scenario's. */
 	workspaceIgnore: Schema.optional(Schema.mutable(Schema.Array(Schema.String))),
-	datasets: Schema.optional(Schema.mutable(Schema.Array(Schema.Struct({
-		id: Schema.String,
-		visibility: Schema.Literal("candidate", "evaluator"),
-		source: Schema.Literal("static", "computed"),
-		digest: Schema.String
-	}))))
+	datasets: Schema.optional(Schema.mutable(Schema.Array(RunDatasetSchema)))
 });
-/** On-disk manifests include a lifecycle status; older finalized manifests may still say running. */
+/** On-disk manifests include a lifecycle status. */
 const RunManifestSchema = Schema.Struct({
 	...RunMetadataSchema.fields,
 	status: Schema.optional(RunStatusSchema)
 });
-const TrialManifestSchema = Schema.Struct({
-	...TrialMetadataSchema.fields,
-	status: Schema.optional(RunStatusSchema)
-});
 const UsageSchema = Schema.Struct({
 	inputTokens: Schema.optional(Schema.Number),
 	outputTokens: Schema.optional(Schema.Number),
+	cachedInputTokens: Schema.optional(Schema.Number),
 	totalTokens: Schema.optional(Schema.Number)
 });
 const JudgeRunInfoSchema = Schema.Struct({
@@ -649,9 +862,32 @@ const JudgeRunInfoSchema = Schema.Struct({
 		name: Schema.optional(Schema.String),
 		version: Schema.optional(Schema.String)
 	})),
+	model: Schema.optional(Schema.String),
 	usage: Schema.optional(UsageSchema),
+	prompt: Schema.optional(Schema.String),
+	reply: Schema.optional(JsonValueSchema),
+	attempts: Schema.optional(Schema.Number),
 	events: Schema.optional(Schema.mutable(Schema.Array(JsonValueSchema)))
 });
+const CriterionResultSchema = Schema.Union(Schema.Struct({
+	name: Schema.String,
+	value: Schema.Number,
+	weight: Schema.optional(Schema.Number),
+	explanation: Schema.optional(Schema.String),
+	evidence: Schema.optional(JsonValueSchema)
+}), Schema.Struct({
+	name: Schema.String,
+	reached: Schema.Literal(false),
+	weight: Schema.optional(Schema.Number),
+	explanation: Schema.optional(Schema.String)
+}));
+/** What every score and checkpoint result can carry about its verdict. */
+const verdictFields = {
+	passAt: Schema.optional(Schema.Number),
+	choice: Schema.optional(Schema.String),
+	criteria: Schema.optional(Schema.mutable(Schema.Array(CriterionResultSchema))),
+	attachments: Schema.optional(Schema.mutable(Schema.Array(Schema.String)))
+};
 const ScoreResultSchema = Schema.Struct({
 	name: Schema.String,
 	kind: Schema.Literal("predicate", "judge"),
@@ -659,6 +895,7 @@ const ScoreResultSchema = Schema.Struct({
 	passed: Schema.optional(Schema.Boolean),
 	explanation: Schema.optional(Schema.String),
 	evidence: Schema.optional(JsonValueSchema),
+	...verdictFields,
 	judge: Schema.optional(JudgeRunInfoSchema),
 	durationMs: Schema.Number,
 	error: Schema.optional(RecordedErrorSchema),
@@ -673,6 +910,7 @@ const CheckpointResultSchema = Schema.Struct({
 	passed: Schema.optional(Schema.Boolean),
 	explanation: Schema.optional(Schema.String),
 	evidence: Schema.optional(JsonValueSchema),
+	...verdictFields,
 	judge: Schema.optional(JudgeRunInfoSchema),
 	durationMs: Schema.optional(Schema.Number),
 	error: Schema.optional(RecordedErrorSchema),
@@ -689,7 +927,7 @@ const MetricResultSchema = Schema.Struct({
 	builtIn: Schema.optional(Schema.Literal(true)),
 	error: Schema.optional(RecordedErrorSchema)
 });
-const TrialScoringSchema = Schema.Struct({
+const RunScoringSchema = Schema.Struct({
 	results: Schema.mutable(Schema.Array(ScoreResultSchema)),
 	metrics: Schema.optional(Schema.mutable(Schema.Array(MetricResultSchema))),
 	checkpoints: Schema.optional(Schema.mutable(Schema.Array(CheckpointResultSchema))),
@@ -702,22 +940,18 @@ const ArtifactEntrySchema = Schema.Struct({
 	kind: Schema.Literal("file", "directory"),
 	size: Schema.optional(Schema.Number)
 });
-const TrialSummarySchema = Schema.Struct({
-	status: RunStatusSchema,
-	endedAt: Schema.String,
-	durationMs: Schema.optional(Schema.Number),
-	scoring: Schema.optional(TrialScoringSchema),
-	artifacts: Schema.optional(Schema.mutable(Schema.Array(ArtifactEntrySchema))),
-	error: Schema.optional(RecordedErrorSchema)
+const AttachmentEntrySchema = Schema.Struct({
+	name: Schema.String,
+	size: Schema.Number,
+	truncatedFrom: Schema.optional(Schema.Number)
 });
 const RunSummarySchema = Schema.Struct({
 	status: RunStatusSchema,
 	endedAt: Schema.String,
 	durationMs: Schema.optional(Schema.Number),
-	trialCount: Schema.Number,
-	passed: Schema.Number,
-	failed: Schema.Number,
-	cancelled: Schema.optional(Schema.Number),
+	scoring: Schema.optional(RunScoringSchema),
+	artifacts: Schema.optional(Schema.mutable(Schema.Array(ArtifactEntrySchema))),
+	attachments: Schema.optional(Schema.mutable(Schema.Array(AttachmentEntrySchema))),
 	error: Schema.optional(RecordedErrorSchema)
 });
 const autEvent = {
@@ -761,7 +995,8 @@ const autEvent = {
 		turn: Schema.Number,
 		timestamp: Schema.String,
 		latencyMs: Schema.optional(Schema.Number),
-		usage: Schema.optional(UsageSchema)
+		usage: Schema.optional(UsageSchema),
+		model: Schema.optional(Schema.String)
 	}),
 	completed: Schema.Struct({
 		source: Schema.Literal("aut"),
@@ -777,9 +1012,9 @@ const autEvent = {
 	})
 };
 const runnerEvent = {
-	trialStarted: Schema.Struct({
+	runStarted: Schema.Struct({
 		source: Schema.Literal("runner"),
-		kind: Schema.Literal("trial-started"),
+		kind: Schema.Literal("run-started"),
 		timestamp: Schema.String
 	}),
 	stepStarted: Schema.Struct({
@@ -843,9 +1078,9 @@ const runnerEvent = {
 		error: RecordedErrorSchema,
 		timestamp: Schema.String
 	}),
-	trialCompleted: Schema.Struct({
+	runCompleted: Schema.Struct({
 		source: Schema.Literal("runner"),
-		kind: Schema.Literal("trial-completed"),
+		kind: Schema.Literal("run-completed"),
 		timestamp: Schema.String
 	}),
 	error: Schema.Struct({
@@ -902,7 +1137,7 @@ function expectToolCall(expected) {
 	};
 }
 /**
-* A user message. Pass a function to compute it for each trial, e.g. from a
+* A user message. Pass a function to compute it for each run, e.g. from a
 * matrix axis: `user(({ parameters }) => `Set up OCaml with ${parameters.toolchain}.`)`.
 */
 function user(message) {
@@ -918,8 +1153,8 @@ function agent(expectation) {
 	};
 }
 /**
-* Marks a scorer as not applicable to this trial. A skipped result neither
-* passes nor fails the trial and is left out of score averages.
+* Marks a scorer as not applicable to this run. A skipped result neither
+* passes nor fails the run and is left out of score averages.
 */
 function skip(reason) {
 	if (!reason.trim()) throw new Error("skip() needs a reason");
@@ -928,16 +1163,20 @@ function skip(reason) {
 		reason
 	};
 }
+/** A scorer in code. Its `evidence` type carries through to `checkpoint(rule)`. */
 function predicate(name, run, options = {}) {
 	if (!name.trim()) throw new Error("Predicate name must not be empty");
-	if (options.passAt !== void 0 && !(Number.isFinite(options.passAt) && options.passAt >= 0 && options.passAt <= 1)) throw new Error(`Predicate ${name} passAt must be a number from 0 to 1; received ${options.passAt}`);
+	assertPassAt(`Predicate ${name}`, options.passAt);
+	const { criteria: declared, ...rest } = options;
 	return markDefinitionSite({
 		kind: "predicate",
 		name,
 		run,
-		...options
+		...rest,
+		...declared ? { criteria: declaredCriteria(Object.fromEntries(Object.entries(declared).map(([key, option]) => [key, typeof option === "string" ? { description: option } : option]))) } : {}
 	});
 }
+/** A built-in metric's name is a type error here, as well as at definition time. */
 function metric(name, run, options = {}) {
 	if (!name.trim()) throw new Error("Metric name must not be empty");
 	return markDefinitionSite({
@@ -947,17 +1186,26 @@ function metric(name, run, options = {}) {
 		...options
 	});
 }
+const INPUT_NAME = /^[a-z][a-zA-Z0-9_-]{0,31}$/;
 function judge(name, options) {
-	if (!name.trim() || !options.rubric.trim()) throw new Error("Judge name and rubric must not be empty");
+	if (!name.trim() || !options.rubric?.trim()) throw new Error("Judge name and rubric must not be empty");
+	assertPassAt(`Judge ${name}`, options.passAt);
+	for (const input of Object.keys(options.inputs ?? {})) if (!INPUT_NAME.test(input)) throw new Error(`Judge ${name} input ${JSON.stringify(input)} must start with a lowercase letter and use letters, digits, "-" or "_"`);
+	if (options.inputs && !Object.keys(options.inputs).length) throw new Error(`Judge ${name} inputs must name at least one input`);
+	if (options.agent && typeof options.agent.start !== "function") throw new Error(`Judge ${name} agent must be made with defineAgent()`);
 	return markDefinitionSite({
 		kind: "judge",
 		name,
-		...options
+		...options,
+		answer: options.answer ?? { kind: "verdict" }
 	});
 }
+function assertPassAt(owner, passAt) {
+	if (passAt !== void 0 && !(Number.isFinite(passAt) && passAt >= 0 && passAt <= 1)) throw new Error(`${owner} passAt must be a number from 0 to 1; received ${passAt}`);
+}
 function defineEval(definition) {
 	authoringId(definition);
-	if ("fixtures" in definition) throw new Error(`Eval ${definition.id} uses fixtures, which were replaced by datasets: [defineDataset(...)]`);
+	assertNoRemovedOptions(definition);
 	if (definition.metadata !== void 0) toJsonObject(definition.metadata, `${definition.id}.metadata`);
 	for (const [index, pattern] of (definition.workspace?.ignore ?? []).entries()) if (typeof pattern !== "string" || pattern.replace(/^!/, "").trim() === "") throw new Error(`Eval ${definition.id} workspace.ignore[${index}] must be a non-empty glob, got ${JSON.stringify(pattern)}`);
 	const datasetIds = /* @__PURE__ */ new Set();
@@ -975,10 +1223,13 @@ function defineEval(definition) {
 	let hasUser = false;
 	for (const step of definition.transcript) if (step.kind === "user") hasUser = true;
 	else if (step.kind !== "agent" && !hasUser) throw new Error("Checkpoint requires a preceding user step");
-	if ((definition.transcript.some((step) => step.kind === "judge") || definition.scoring.some((rule) => rule.kind === "judge")) && !definition.judge) throw new Error("Judge rules require an eval judge agent");
-	if (definition.judge && definition.judge === definition.agent) throw new Error("Judge agent must be distinct from the agent under test");
+	assertJudgesIndependent(definition);
 	return markDefinitionSite(definition);
 }
+/** A judge never grades with the agent it is judging. */
+function assertJudgesIndependent(definition, defaultJudge) {
+	for (const rule of [...definition.transcript, ...definition.scoring]) if (rule.kind === "judge" && (rule.agent ?? defaultJudge) === definition.agent) throw new Error(`Judge ${rule.name} in ${definition.id} uses the agent under test; a judge needs its own agent`);
+}
 function defineSuite(suite) {
 	authoringId(suite);
 	return suite;
@@ -1048,11 +1299,21 @@ function registerEvals(registrations) {
 		catalog: (options) => evals.map((evaluation) => catalogEntry(evaluation, suiteIdByEvalId.get(authoringId(evaluation)), options))
 	};
 }
-/** Metrics EvalKit records for every trial; scenario metrics may not reuse these names. */
+/**
+* Metrics EvalKit records for every run; scenario metrics may not reuse
+* these names.
+*
+* - `agent time` (ms): from the run's first event to its last one before
+*   final scoring.
+* - `tokens`: the sum of `turnTokens(usage)` over the turns that reported
+*   usage, cached input included. Absent when no turn reported any.
+* - `tool calls`: the agent's `tool-call` events.
+*/
 const BUILT_IN_METRICS = [
 	"agent time",
 	"tokens",
-	"tool calls"
+	"tool calls",
+	"judge tokens"
 ];
 function recordError(error) {
 	if (error instanceof Error) return {
@@ -1066,4 +1327,4 @@ function recordError(error) {
 	};
 }
 //#endregion
-export { ResourceUriSchema as $, RunSummarySchema as A, defineEvalMatrix as B, JudgeRunInfoSchema as C, RunMetadataSchema as D, RunManifestSchema as E, TrialScoringSchema as F, NotJsonError as G, DEFAULT_ARTIFACT_EXCLUDE as H, TrialSummarySchema as I, defineDataset as J, toJson as K, UsageSchema as L, TrajectoryEventSchema as M, TrialManifestSchema as N, RunSourceSchema as O, TrialMetadataSchema as P, ResourceKindSchema as Q, normalizeAutEvent as R, JsonValueSchema as S, RecordedErrorSchema as T, defineConfig as U, canonicalParameters as V, catalogEntry as W, relativeDatasetPath as X, isDataset as Y, definitionSite as Z, parseTrajectoryJsonl as _, authoringId as a, resourceUuid as at, AutIdentitySchema as b, defineSuite as c, metric as d, UuidSchema as et, predicate as f, user as g, skip as h, agentRuntimes as i, resourceUriSchema as it, ScoreResultSchema as j, RunStatusSchema as k, expectToolCall as l, registerEvals as m, DEFAULT_AGENT_RUNTIMES as n, parseResourceUri as nt, defineAgent as o, recordError as p, toJsonObject as q, agent as r, resourceUri as rt, defineEval as s, BUILT_IN_METRICS as t, authorId as tt, judge as u, ApiErrorSchema as v, MetricResultSchema as w, CheckpointResultSchema as x, ArtifactEntrySchema as y, turnResultEvents as z };
+export { judgeInputNames as $, RunDatasetSchema as A, normalizeAutEvent as B, BatchSchema as C, MetricResultSchema as D, JudgeRunInfoSchema as E, RunStatusSchema as F, DEFAULT_ARTIFACT_EXCLUDE as G, defineEvalMatrix as H, RunSummarySchema as I, catalogEntry as J, DEFAULT_ATTACHMENT_LIMITS as K, ScoreResultSchema as L, RunMetadataSchema as M, RunScoringSchema as N, REPORT_SCHEMA_VERSION as O, RunSourceSchema as P, criteria as Q, TrajectoryEventSchema as R, AutIdentitySchema as S, JsonValueSchema as T, canonicalParameters as U, turnResultEvents as V, turnTokens as W, answerInstruction as X, UnreadableAnswer as Y, choices as Z, user as _, resourceUriSchema as _t, assertJudgesIndependent as a, toJson as at, ArtifactEntrySchema as b, defineEval as c, isDataset as ct, judge as d, ResourceKindSchema as dt, judgePrompt as et, metric as f, ResourceUriSchema as ft, skip as g, resourceUri as gt, registerEvals as h, parseResourceUri as ht, agentRuntimes as i, NotJsonError as it, RunManifestSchema as j, RecordedErrorSchema as k, defineSuite as l, relativeDatasetPath as lt, recordError as m, authorId as mt, DEFAULT_AGENT_RUNTIMES as n, readJudgeAnswer as nt, authoringId as o, toJsonObject as ot, predicate as p, UuidSchema as pt, defineConfig as q, agent as r, weightedMean as rt, defineAgent as s, defineDataset as st, BUILT_IN_METRICS as t, passMark as tt, expectToolCall as u, definitionSite as ut, parseTrajectoryJsonl as v, resourceUuid as vt, CheckpointResultSchema as w, AttachmentEntrySchema as x, ApiErrorSchema as y, UsageSchema as z };