@leostera/evalkit 0.10.0 → 0.11.0

package/dist/{src-B14fXAGp.mjs → src-D6Ax2t9b.mjs}

@@ -1,5 +1,6 @@
-import { n as assertNoRemovedOptions } from "./upgrade-G-uFPJKg.mjs";
+import { n as assertNoRemovedOptions, u as transcriptUsage } from "./upgrade-rMsGzUEJ.mjs";
 import * as Schema from "effect/Schema";
+import { buildContext } from "@leostera/atxf";
 //#region ../core/src/identity.ts
 const ResourceKindSchema = Schema.Literal("run", "artifact");
 /** Human-authored project-local identifier; never a generated resource URI. */
@@ -153,6 +154,188 @@ function isDataset(value) {
 	return typeof value === "object" && value !== null && value.kind === "dataset" && typeof value.source === "object";
 }
 //#endregion
+//#region ../core/src/limits.ts
+const UNITS = {
+	ms: 1,
+	s: 1e3,
+	m: 6e4,
+	h: 36e5
+};
+/** `'20m'` → 1_200_000. A number is milliseconds. */
+function parseDuration(value, owner) {
+	if (typeof value === "number") return positive(value, owner);
+	const match = /^\s*(\d+(?:\.\d+)?)\s*(ms|s|m|h)\s*$/.exec(value);
+	if (!match) throw new Error(`${owner} must be milliseconds or a duration such as 90s, 20m or 1h; received ${JSON.stringify(value)}`);
+	return positive(Number(match[1]) * UNITS[match[2]], owner);
+}
+function positive(value, owner) {
+	if (!Number.isFinite(value) || value <= 0) throw new Error(`${owner} must be a positive number; received ${value}`);
+	return value;
+}
+/** Checks and normalizes limits; `owner` names where they were written. */
+function parseLimits(input, owner) {
+	if (input === void 0) return {};
+	if (typeof input !== "object" || input === null || Array.isArray(input)) throw new Error(`${owner} limits must be an object`);
+	const limits = {};
+	for (const [key, value] of Object.entries(input)) {
+		if (value === void 0) continue;
+		if (key === "time") limits.time = parseDuration(value, `${owner} limits.time`);
+		else if (key === "steps" || key === "tokens") {
+			if (!Number.isInteger(value) || value <= 0) throw new Error(`${owner} limits.${key} must be a positive whole number; received ${String(value)}`);
+			limits[key] = value;
+		} else throw new Error(`${owner} has an unknown limit ${JSON.stringify(key)}; limits are steps, tokens and time`);
+	}
+	return limits;
+}
+/** Later layers win, kind by kind: config, then scenario, then matrix cell, then the command line. */
+function resolveLimits(...layers) {
+	return Object.assign({}, ...layers);
+}
+/** A matrix's `limits` axes are parameters named `limits.<kind>`. */
+const LIMIT_PARAMETER_PREFIX = "limits.";
+/** The limits a matrix cell's `limits.<kind>` parameters set. */
+function limitsFromParameters(parameters) {
+	const input = {};
+	for (const [key, value] of Object.entries(parameters ?? {})) if (key.startsWith("limits.")) input[key.slice(7)] = value;
+	return parseLimits(input, "The matrix cell");
+}
+/** `steps=80` or `time=1h`, from `--limit`. */
+function parseLimitFlag(flag) {
+	const index = flag.indexOf("=");
+	if (index < 1) throw new Error(`--limit takes kind=value, such as steps=80 or time=20m; received ${JSON.stringify(flag)}`);
+	const kind = flag.slice(0, index).trim();
+	const raw = flag.slice(index + 1).trim();
+	return parseLimits({ [kind]: kind === "time" && !/^\d+$/.test(raw) ? raw : Number(raw) }, "--limit");
+}
+/** `40 steps`, `2,000,000 tokens`, `20m`. */
+function formatLimit(kind, value) {
+	if (kind === "time") {
+		if (value % 36e5 === 0) return `${value / 36e5}h`;
+		if (value % 6e4 === 0) return `${value / 6e4}m`;
+		if (value % 1e3 === 0) return `${value / 1e3}s`;
+		return `${value}ms`;
+	}
+	return `${value.toLocaleString("en-US")} ${kind}`;
+}
+//#endregion
+//#region ../core/src/transcript.ts
+const TREE = /* @__PURE__ */ new Set([
+	"message",
+	"context",
+	"settings",
+	"compaction",
+	"context_edit",
+	"branch_summary",
+	"turn",
+	"tool_status",
+	"usage",
+	"checkpoint",
+	"agent_link",
+	"custom"
+]);
+function isTreeRecord(record) {
+	return TREE.has(record.type);
+}
+/**
+* The records from the transcript's root to its active leaf, in order: what
+* the session is now, without abandoned branches (ATXF §4).
+*/
+function activePath(transcript) {
+	const byId = /* @__PURE__ */ new Map();
+	let leaf = null;
+	for (const record of transcript.records) if (isTreeRecord(record)) {
+		byId.set(record.id, record);
+		leaf = record.id;
+	} else if (record.type === "leaf") leaf = record.targetId;
+	const path = [];
+	const seen = /* @__PURE__ */ new Set();
+	for (let id = leaf; id !== null && !seen.has(id);) {
+		seen.add(id);
+		const record = byId.get(id);
+		if (!record) break;
+		path.push(record);
+		id = record.parentId;
+	}
+	return path.reverse();
+}
+/** Messages on the active path, optionally of one role. */
+function transcriptMessages(transcript, role) {
+	return activePath(transcript).filter((record) => record.type === "message" && (role === void 0 || record.role === role));
+}
+/** Every tool call on the active path, in order, with its result. */
+function transcriptToolCalls(transcript, name) {
+	return callsIn(activePath(transcript)).filter((entry) => name === void 0 || entry.call.name === name).map(({ message, call, result }) => ({
+		id: call.id,
+		name: call.name,
+		arguments: call.arguments ?? {},
+		...call.input !== void 0 ? { input: call.input } : {},
+		...result ? {
+			result: toolResultValue(result),
+			resultId: result.id
+		} : {},
+		isError: result?.isError === true,
+		messageId: message.id
+	}));
+}
+/** Tool calls among `records`, each paired with the first result for it. */
+function callsIn(records) {
+	const calls = [];
+	const open = /* @__PURE__ */ new Map();
+	for (const record of records) {
+		if (record.type !== "message") continue;
+		if (record.role === "assistant") {
+			for (const block of record.content) if (block.type === "tool_call") {
+				const entry = {
+					message: record,
+					call: block
+				};
+				calls.push(entry);
+				open.set(block.id, entry);
+			}
+		}
+		if (record.role === "tool") {
+			const entry = open.get(record.toolCallId);
+			if (entry && !entry.result) entry.result = record;
+		}
+	}
+	return calls;
+}
+/** The text of a message's content: text blocks joined, other blocks left out. */
+function contentText(content) {
+	if (typeof content === "string") return content;
+	return content.flatMap((block) => block.type === "text" ? [block.text] : []).join("\n");
+}
+/** The text of the last assistant message on the active path, or `''`. */
+function transcriptReply(transcript) {
+	const last = transcriptMessages(transcript, "assistant").at(-1);
+	return last ? contentText(last.content) : "";
+}
+/**
+* A tool result's value: the JSON an adapter returned (kept in
+* `details.result`), else the result's text.
+*/
+function toolResultValue(result) {
+	if (result.details && "result" in result.details) return result.details.result;
+	return contentText(result.content);
+}
+/** `provider/id`, or `id` when the provider is unknown. */
+function modelName(model) {
+	if (!model) return void 0;
+	return model.provider === "unknown" ? model.id : `${model.provider}/${model.id}`;
+}
+/** `openai/gpt-5` → `{ provider: 'openai', id: 'gpt-5' }`; without a provider, `unknown`. */
+function parseModel(model) {
+	if (typeof model !== "string") return model;
+	const slash = model.indexOf("/");
+	return slash > 0 && slash < model.length - 1 ? {
+		provider: model.slice(0, slash),
+		id: model.slice(slash + 1)
+	} : {
+		provider: "unknown",
+		id: model
+	};
+}
+//#endregion
 //#region ../core/src/json.ts
 /** A value that cannot be stored in a report, with the path of the offending part. */
 var NotJsonError = class extends Error {
@@ -403,10 +586,8 @@ function catalogEntry(evaluation, suite, options = {}) {
 		agent: catalogAgent(evaluation.agent, at),
 		datasets: (evaluation.datasets ?? []).map((dataset) => catalogDataset(dataset, at)),
 		repeat: evaluation.repeat ?? 1,
-		policy: {
-			...policy.timeoutMs !== void 0 ? { timeoutMs: policy.timeoutMs } : {},
-			...policy.failfast !== void 0 ? { failfast: policy.failfast } : {}
-		},
+		policy: policy.failfast !== void 0 ? { failfast: policy.failfast } : {},
+		...evaluation.limits ? { limits: parseLimits(evaluation.limits, `Eval ${evaluation.id}`) } : {},
 		transcript: evaluation.transcript.map((step) => catalogStep(step, at, options.judge)),
 		scorers: evaluation.scoring.map((rule) => catalogScorer(rule, at, options.judge)),
 		...evaluation.metadata !== void 0 ? { metadata: toJson(evaluation.metadata, `${evaluation.id}.metadata`) } : {},
@@ -537,18 +718,6 @@ function defineConfig(configuration) {
 	return configuration;
 }
 //#endregion
-//#region ../core/src/usage.ts
-/**
-* The tokens a turn counts toward the `tokens` metric: `inputTokens +
-* outputTokens` (cached input included), or `totalTokens` when neither is
-* reported. Undefined when the turn reported no usage.
-*/
-function turnTokens(usage) {
-	if (!usage) return void 0;
-	if (usage.inputTokens !== void 0 || usage.outputTokens !== void 0) return (usage.inputTokens ?? 0) + (usage.outputTokens ?? 0);
-	return usage.totalTokens;
-}
-//#endregion
 //#region ../core/src/canonical-json.ts
 /** Canonical JSON: keys sorted recursively, array order preserved. Rejects non-JSON inputs. */
 function canonicalParameters(value) {
@@ -561,10 +730,27 @@ function canonicalParameters(value) {
 //#endregion
 //#region ../core/src/matrix.ts
 const COUNT_BUDGET = 1e5;
+/** The matrix's parameters with its `limits` as `limits.<kind>` axes. */
+function withLimitAxes(definition) {
+	if (definition.limits === void 0) return definition.parameters;
+	const parameters = { ...definition.parameters };
+	for (const [kind, values] of Object.entries(definition.limits)) {
+		if (values === void 0) continue;
+		const axis = `${LIMIT_PARAMETER_PREFIX}${kind}`;
+		if (Object.hasOwn(parameters, axis)) throw new Error(`Matrix declares ${axis} as both a parameter and a limit`);
+		for (const value of values) parseLimits({ [kind]: value }, `Matrix ${definition.id}`);
+		parameters[axis] = values;
+	}
+	return parameters;
+}
 const equal = (a, b) => canonicalParameters(a) === canonicalParameters(b);
 const matches = (pattern, values) => Object.entries(pattern).every(([key, value]) => Object.hasOwn(values, key) && equal(values[key], value));
-function defineEvalMatrix(definition) {
-	authoringId(definition);
+function defineEvalMatrix(input) {
+	authoringId(input);
+	const definition = {
+		...input,
+		parameters: withLimitAxes(input)
+	};
 	const identities = /* @__PURE__ */ new Set();
 	for (const evaluation of definition.evals) {
 		const id = authoringId(evaluation);
@@ -601,7 +787,7 @@ function defineEvalMatrix(definition) {
 	const caseChoices = new Map(caseKeys.map((key) => [key, cases.map((assignment) => assignment[key])]));
 	function select(selection = {}) {
 		for (const key of Object.keys(selection.parameters ?? {})) if (!declared.has(key)) throw new Error(`Unknown matrix axis: ${key}`);
-		for (const key of Object.keys(selection.overrides ?? {})) if (declared.has(key)) throw new Error(`Select axis ${key}; do not override it`);
+		for (const key of Object.keys(selection.overrides ?? {})) if (declared.has(key)) throw new Error(`${key} is a matrix axis: pick its value with -a ${key}=<value> (--select), not --param`);
 		canonicalParameters(selection.overrides ?? {});
 		for (const name of selection.evals ?? []) if (!definition.evals.some((e) => e.id === name)) throw new Error(`Unknown matrix eval: ${name}`);
 		const evals = definition.evals.filter((e) => !selection.evals || selection.evals.includes(e.id));
@@ -694,89 +880,76 @@ function defineEvalMatrix(definition) {
 	};
 }
 //#endregion
-//#region ../core/src/aut-events.ts
-/** Converts an adapter event into a recorded `AutEvent`. */
-function normalizeAutEvent(input, now) {
-	if (typeof input !== "object" || input === null) throw new Error("AUT events must be objects");
-	const timestamp = input.timestamp ?? now().toISOString();
-	switch (input.kind) {
-		case "message": return {
-			kind: "message",
-			role: input.role,
-			content: toJson(input.content, "event.content"),
-			timestamp
-		};
-		case "tool-call": return {
-			kind: "tool-call",
-			id: input.id,
-			name: input.name,
-			arguments: input.arguments === void 0 ? {} : toJson(input.arguments, "event.arguments"),
-			timestamp
-		};
-		case "tool-result": return {
-			kind: "tool-result",
-			id: input.id,
-			...input.name === void 0 ? {} : { name: input.name },
-			result: input.result === void 0 ? null : toJson(input.result, "event.result"),
-			timestamp
-		};
-		case "completed": return {
-			kind: "completed",
-			...input.output === void 0 ? {} : { output: toJson(input.output, "event.output") },
-			timestamp
-		};
-		case "started":
-		case "turn-started":
-		case "turn-completed":
-		case "error": return {
-			...input,
-			timestamp
-		};
-		default: throw new Error(`Unknown AUT event kind: ${String(input.kind)}`);
-	}
+//#region ../core/src/isolation.ts
+const NETWORKS = [
+	"none",
+	"fixtures",
+	"full"
+];
+const ENV_NAME = /^[A-Za-z_][A-Za-z0-9_]*$/;
+/** No isolation: commands run on the host in the run's workspace. */
+function none() {
+	return { kind: "none" };
 }
 /**
-* The events a returned turn result stands for: each tool call (and its
-* result), then the reply, then the usage as a `turn-completed` event.
+* Container isolation through the `docker` CLI, or a compatible one such as
+* podman's: a container per run, the candidate workspace mounted at
+* `/workspace`, the evaluator workspace never mounted, and its own process
+* table, so a `pkill` reaches only that run.
 */
-function turnResultEvents(result, turn, now) {
-	if (typeof result === "string") result = { reply: result };
-	if (typeof result !== "object" || result === null) throw new Error("send() must return nothing, a string, or a turn result");
-	const events = [];
-	for (const [index, call] of (result.toolCalls ?? []).entries()) {
-		const id = call.id ?? `turn-${turn}-call-${index + 1}`;
-		events.push(normalizeAutEvent({
-			kind: "tool-call",
-			id,
-			name: call.name,
-			arguments: call.arguments
-		}, now));
-		if (call.result !== void 0) events.push(normalizeAutEvent({
-			kind: "tool-result",
-			id,
-			name: call.name,
-			result: call.result
-		}, now));
-	}
-	if (result.reply !== void 0) events.push(normalizeAutEvent({
-		kind: "message",
-		role: "assistant",
-		content: result.reply
-	}, now));
-	if (result.usage || result.model) events.push(normalizeAutEvent({
-		kind: "turn-completed",
-		turn,
-		...result.usage ? { usage: result.usage } : {},
-		...result.model ? { model: result.model } : {}
-	}, now));
-	return events;
+function container(options) {
+	if (typeof options?.image !== "string" || !options.image.trim()) throw new Error("container() needs an image: a name such as node:22, or a Dockerfile path");
+	const network = options.network ?? "none";
+	if (!NETWORKS.includes(network)) throw new Error(`container() network must be ${NETWORKS.join(", ")}; received ${String(network)}`);
+	const env = [...options.env ?? []];
+	for (const name of env) if (!ENV_NAME.test(name)) throw new Error(`container() env takes variable names, such as API_KEY; received ${JSON.stringify(name)}`);
+	const mounts = (options.mounts ?? []).map(parseMount);
+	return {
+		kind: "container",
+		image: options.image.trim(),
+		network,
+		runtime: options.runtime ?? "docker",
+		env,
+		mounts
+	};
+}
+function parseMount(mount) {
+	const parsed = typeof mount === "string" ? (() => {
+		const at = mount.lastIndexOf(":");
+		return at > 0 ? {
+			host: mount.slice(0, at),
+			path: mount.slice(at + 1)
+		} : void 0;
+	})() : mount;
+	if (!parsed?.host || !parsed.path?.startsWith("/")) throw new Error(`container() mounts take { host, path } with an absolute container path, or 'host:/path'; received ${JSON.stringify(mount)}`);
+	if (parsed.path === "/workspace" || parsed.path.startsWith("/workspace/")) throw new Error(`container() cannot mount over /workspace, the run's workspace: ${parsed.path}`);
+	return {
+		host: parsed.host,
+		path: parsed.path
+	};
+}
+/**
+* macOS `sandbox-exec` isolation, for projects that cannot use containers:
+* the run cannot read or write the project, the home directory or other
+* runs' workspaces, except its own workspace and `allow`, and can signal
+* only processes it started.
+*/
+function sandboxExec(options = {}) {
+	for (const name of options.env ?? []) if (!ENV_NAME.test(name)) throw new Error(`sandboxExec() env takes variable names, such as API_KEY; received ${JSON.stringify(name)}`);
+	return {
+		kind: "sandbox-exec",
+		allow: [...options.allow ?? []],
+		read: [...options.read ?? []],
+		deny: [...options.deny ?? []],
+		...options.env ? { env: [...options.env] } : {}
+	};
 }
 //#endregion
 //#region ../core/src/schema.ts
 /** Runtime schemas for serialized EvalKit report and API boundaries. */
-const RunStatusSchema = Schema.Literal("running", "passed", "failed", "errored", "cancelled");
+const RunStatusSchema = Schema.Literal("running", "passed", "failed", "errored", "cancelled", "limited");
 /** The report schema version this EvalKit writes and reads. */
-const REPORT_SCHEMA_VERSION = 6;
+const REPORT_SCHEMA_VERSION = 7;
 const RecordedErrorSchema = Schema.Struct({
 	name: Schema.String,
 	message: Schema.String,
@@ -825,8 +998,23 @@ const RunDatasetSchema = Schema.Struct({
 	source: Schema.Literal("static", "computed"),
 	digest: Schema.String
 });
+const LimitsSchema = Schema.Struct({
+	steps: Schema.optional(Schema.Number),
+	tokens: Schema.optional(Schema.Number),
+	time: Schema.optional(Schema.Number)
+});
+const LimitReachedSchema = Schema.Struct({
+	limit: Schema.Literal("steps", "tokens", "time"),
+	value: Schema.Number,
+	used: Schema.Number
+});
+const RunIsolationSchema = Schema.Union(Schema.Struct({
+	kind: Schema.Literal("container"),
+	image: Schema.String,
+	network: Schema.Literal("none", "fixtures", "full")
+}), Schema.Struct({ kind: Schema.Literal("sandbox-exec") }));
 const RunMetadataSchema = Schema.Struct({
-	schemaVersion: Schema.Literal(6),
+	schemaVersion: Schema.Literal(7),
 	runUri: resourceUriSchema("run"),
 	evalId: Schema.String,
 	suiteId: Schema.optional(Schema.String),
@@ -842,19 +1030,37 @@ const RunMetadataSchema = Schema.Struct({
 	batch: Schema.optional(BatchSchema),
 	/** The effective snapshot ignore patterns: the project's, then the scenario's. */
 	workspaceIgnore: Schema.optional(Schema.mutable(Schema.Array(Schema.String))),
-	datasets: Schema.optional(Schema.mutable(Schema.Array(RunDatasetSchema)))
+	datasets: Schema.optional(Schema.mutable(Schema.Array(RunDatasetSchema))),
+	limits: Schema.optional(LimitsSchema),
+	isolation: Schema.optional(RunIsolationSchema)
 });
 /** On-disk manifests include a lifecycle status. */
 const RunManifestSchema = Schema.Struct({
 	...RunMetadataSchema.fields,
 	status: Schema.optional(RunStatusSchema)
 });
+/** ATXF usage: disjoint buckets, `input` without cached tokens. */
 const UsageSchema = Schema.Struct({
-	inputTokens: Schema.optional(Schema.Number),
-	outputTokens: Schema.optional(Schema.Number),
-	cachedInputTokens: Schema.optional(Schema.Number),
-	totalTokens: Schema.optional(Schema.Number)
+	input: Schema.Number,
+	output: Schema.Number,
+	cacheRead: Schema.optional(Schema.Number),
+	cacheWrite: Schema.optional(Schema.Number),
+	reasoning: Schema.optional(Schema.Number),
+	total: Schema.optional(Schema.Number),
+	cost: Schema.optional(Schema.Struct({
+		currency: Schema.optional(Schema.String),
+		input: Schema.optional(Schema.Number),
+		output: Schema.optional(Schema.Number),
+		cacheRead: Schema.optional(Schema.Number),
+		cacheWrite: Schema.optional(Schema.Number),
+		total: Schema.Number
+	}))
 });
+/**
+* An ATXF transcript in JSON (`{ header, records }`). Its records are checked
+* by the ATXF validator where they are written, not here.
+*/
+const TranscriptSchema = Schema.declare((value) => typeof value === "object" && value !== null && typeof value.header === "object" && value.header?.type === "transcript" && Array.isArray(value.records), { identifier: "Transcript" });
 const JudgeRunInfoSchema = Schema.Struct({
 	agent: Schema.optional(Schema.Struct({
 		id: Schema.String,
@@ -867,7 +1073,8 @@ const JudgeRunInfoSchema = Schema.Struct({
 	prompt: Schema.optional(Schema.String),
 	reply: Schema.optional(JsonValueSchema),
 	attempts: Schema.optional(Schema.Number),
-	events: Schema.optional(Schema.mutable(Schema.Array(JsonValueSchema)))
+	/** The judge's own session, an ATXF transcript (checked when written). */
+	transcript: Schema.optional(TranscriptSchema)
 });
 const CriterionResultSchema = Schema.Union(Schema.Struct({
 	name: Schema.String,
@@ -915,7 +1122,7 @@ const CheckpointResultSchema = Schema.Struct({
 	durationMs: Schema.optional(Schema.Number),
 	error: Schema.optional(RecordedErrorSchema),
 	matchedToolCall: Schema.optional(Schema.Struct({
-		eventIndex: Schema.Number,
+		messageId: Schema.String,
 		id: Schema.String
 	}))
 });
@@ -952,165 +1159,92 @@ const RunSummarySchema = Schema.Struct({
 	scoring: Schema.optional(RunScoringSchema),
 	artifacts: Schema.optional(Schema.mutable(Schema.Array(ArtifactEntrySchema))),
 	attachments: Schema.optional(Schema.mutable(Schema.Array(AttachmentEntrySchema))),
-	error: Schema.optional(RecordedErrorSchema)
+	error: Schema.optional(RecordedErrorSchema),
+	limit: Schema.optional(LimitReachedSchema)
 });
-const autEvent = {
-	started: Schema.Struct({
-		source: Schema.Literal("aut"),
-		kind: Schema.Literal("started"),
-		timestamp: Schema.String
-	}),
-	message: Schema.Struct({
-		source: Schema.Literal("aut"),
-		kind: Schema.Literal("message"),
-		role: Schema.Literal("system", "user", "assistant", "tool"),
-		content: JsonValueSchema,
-		timestamp: Schema.String
-	}),
-	toolCall: Schema.Struct({
-		source: Schema.Literal("aut"),
-		kind: Schema.Literal("tool-call"),
-		id: Schema.String,
-		name: Schema.String,
-		arguments: JsonValueSchema,
-		timestamp: Schema.String
-	}),
-	toolResult: Schema.Struct({
-		source: Schema.Literal("aut"),
-		kind: Schema.Literal("tool-result"),
-		id: Schema.String,
-		name: Schema.optional(Schema.String),
-		result: JsonValueSchema,
-		timestamp: Schema.String
-	}),
-	turnStarted: Schema.Struct({
-		source: Schema.Literal("aut"),
-		kind: Schema.Literal("turn-started"),
-		turn: Schema.Number,
-		timestamp: Schema.String
-	}),
-	turnCompleted: Schema.Struct({
-		source: Schema.Literal("aut"),
-		kind: Schema.Literal("turn-completed"),
-		turn: Schema.Number,
-		timestamp: Schema.String,
-		latencyMs: Schema.optional(Schema.Number),
-		usage: Schema.optional(UsageSchema),
-		model: Schema.optional(Schema.String)
-	}),
-	completed: Schema.Struct({
-		source: Schema.Literal("aut"),
-		kind: Schema.Literal("completed"),
-		output: Schema.optional(JsonValueSchema),
-		timestamp: Schema.String
-	}),
-	error: Schema.Struct({
-		source: Schema.Literal("aut"),
-		kind: Schema.Literal("error"),
-		error: RecordedErrorSchema,
-		timestamp: Schema.String
-	})
+const runEventFields = {
+	/** The transcript record it happened after; null before the first. */
+	at: Schema.NullOr(Schema.String),
+	timestamp: Schema.String
 };
 const runnerEvent = {
 	runStarted: Schema.Struct({
-		source: Schema.Literal("runner"),
 		kind: Schema.Literal("run-started"),
-		timestamp: Schema.String
+		...runEventFields
 	}),
 	stepStarted: Schema.Struct({
-		source: Schema.Literal("runner"),
 		kind: Schema.Literal("transcript-step-started"),
 		step: Schema.Number,
 		message: Schema.optional(Schema.String),
-		timestamp: Schema.String
+		...runEventFields
 	}),
 	stepCompleted: Schema.Struct({
-		source: Schema.Literal("runner"),
 		kind: Schema.Literal("transcript-step-completed"),
 		step: Schema.Number,
-		timestamp: Schema.String
+		...runEventFields
 	}),
 	stepSkipped: Schema.Struct({
-		source: Schema.Literal("runner"),
 		kind: Schema.Literal("transcript-step-skipped"),
 		step: Schema.Number,
-		timestamp: Schema.String
+		...runEventFields
 	}),
 	checkpointStarted: Schema.Struct({
-		source: Schema.Literal("runner"),
 		kind: Schema.Literal("checkpoint-started"),
 		step: Schema.Number,
 		name: Schema.String,
-		timestamp: Schema.String
+		...runEventFields
 	}),
 	checkpointCompleted: Schema.Struct({
-		source: Schema.Literal("runner"),
 		kind: Schema.Literal("checkpoint-completed"),
 		step: Schema.Number,
 		status: Schema.Literal("passed", "failed", "skipped"),
-		timestamp: Schema.String
+		...runEventFields
 	}),
 	checkpointError: Schema.Struct({
-		source: Schema.Literal("runner"),
 		kind: Schema.Literal("checkpoint-error"),
 		step: Schema.Number,
 		error: RecordedErrorSchema,
-		timestamp: Schema.String
+		...runEventFields
 	}),
 	scorerStarted: Schema.Struct({
-		source: Schema.Literal("runner"),
 		kind: Schema.Literal("scorer-started"),
 		scorer: Schema.String,
-		timestamp: Schema.String
+		...runEventFields
 	}),
 	scorerCompleted: Schema.Struct({
-		source: Schema.Literal("runner"),
 		kind: Schema.Literal("scorer-completed"),
 		scorer: Schema.String,
 		value: Schema.optional(Schema.Number),
 		skipped: Schema.optional(Schema.Literal(true)),
-		timestamp: Schema.String
+		...runEventFields
 	}),
 	scorerFailed: Schema.Struct({
-		source: Schema.Literal("runner"),
 		kind: Schema.Literal("scorer-failed"),
 		scorer: Schema.String,
 		error: RecordedErrorSchema,
-		timestamp: Schema.String
+		...runEventFields
+	}),
+	limitReached: Schema.Struct({
+		kind: Schema.Literal("limit-reached"),
+		...LimitReachedSchema.fields,
+		...runEventFields
 	}),
 	runCompleted: Schema.Struct({
-		source: Schema.Literal("runner"),
 		kind: Schema.Literal("run-completed"),
-		timestamp: Schema.String
+		...runEventFields
 	}),
 	error: Schema.Struct({
-		source: Schema.Literal("runner"),
 		kind: Schema.Literal("error"),
 		error: RecordedErrorSchema,
-		timestamp: Schema.String
+		...runEventFields
 	})
 };
-const TrajectoryEventSchema = Schema.Union(...Object.values(autEvent), ...Object.values(runnerEvent));
+const RunEventSchema = Schema.Union(...Object.values(runnerEvent));
 const ApiErrorSchema = Schema.Struct({
 	error: Schema.String,
 	message: Schema.String
 });
 //#endregion
-//#region ../core/src/trajectory.ts
-/** Decode one private report's JSONL trajectory; reject malformed/non-event lines with their line number. */
-function parseTrajectoryJsonl(contents) {
-	const events = [];
-	for (const [index, line] of contents.split(/\r?\n/).entries()) {
-		if (!line.trim()) continue;
-		try {
-			events.push(Schema.decodeUnknownSync(TrajectoryEventSchema)(JSON.parse(line)));
-		} catch (cause) {
-			throw new Error(`Invalid trajectory JSONL at line ${index + 1}`, { cause });
-		}
-	}
-	return events;
-}
-//#endregion
 //#region ../core/src/index.ts
 internalModule(import.meta.url);
 /** Used when an agent declares no runtimes. */
@@ -1207,6 +1341,7 @@ function defineEval(definition) {
 	authoringId(definition);
 	assertNoRemovedOptions(definition);
 	if (definition.metadata !== void 0) toJsonObject(definition.metadata, `${definition.id}.metadata`);
+	parseLimits(definition.limits, `Eval ${definition.id}`);
 	for (const [index, pattern] of (definition.workspace?.ignore ?? []).entries()) if (typeof pattern !== "string" || pattern.replace(/^!/, "").trim() === "") throw new Error(`Eval ${definition.id} workspace.ignore[${index}] must be a non-empty glob, got ${JSON.stringify(pattern)}`);
 	const datasetIds = /* @__PURE__ */ new Set();
 	for (const [index, dataset] of (definition.datasets ?? []).entries()) {
@@ -1303,15 +1438,23 @@ function registerEvals(registrations) {
 * Metrics EvalKit records for every run; scenario metrics may not reuse
 * these names.
 *
-* - `agent time` (ms): from the run's first event to its last one before
-*   final scoring.
-* - `tokens`: the sum of `turnTokens(usage)` over the turns that reported
-*   usage, cached input included. Absent when no turn reported any.
-* - `tool calls`: the agent's `tool-call` events.
+* - `agent time` (ms): from the session's start to the end of its last turn.
+* - `tokens`: uncached input, cache writes and output, summed over the
+*   transcript (`uncachedTokens`). Absent when nothing reported usage.
+* - `cached tokens`: input tokens read from a prompt cache.
+* - `output tokens`: output, reasoning included.
+* - `cache hit rate`: the share of input tokens read from a cache, 0 to 1.
+* - `steps`: assistant messages, one per model call.
+* - `tool calls`: tool calls in the transcript.
+* - `judge tokens`: `tokens` over the judges' transcripts, never the agent's.
 */
 const BUILT_IN_METRICS = [
 	"agent time",
 	"tokens",
+	"cached tokens",
+	"output tokens",
+	"cache hit rate",
+	"steps",
 	"tool calls",
 	"judge tokens"
 ];
@@ -1326,5 +1469,35 @@ function recordError(error) {
 		message: String(error)
 	};
 }
+/** Messages on the active path of what the agent did, optionally of one role. */
+function messages(scoring, role) {
+	return transcriptMessages(scoring.transcript, role).map((record) => ({
+		id: record.id,
+		role: record.role,
+		text: contentText(record.content),
+		time: record.time,
+		record
+	}));
+}
+/** The agent's tool calls, in order, each with its result when recorded; optionally of one tool. */
+function toolCalls(scoring, name) {
+	return transcriptToolCalls(scoring.transcript, name);
+}
+/** The text of the agent's last reply, or `''`. */
+function lastReply(scoring) {
+	return transcriptReply(scoring.transcript);
+}
+/** Everything the agent's transcript reported using; undefined when nothing reported usage. */
+function usage(scoring) {
+	return transcriptUsage(scoring.transcript);
+}
+/**
+* What the model saw at the end of the agent's transcript, rebuilt by ATXF
+* (compactions and context edits applied): its instructions, tools and
+* messages.
+*/
+function modelContext(scoring) {
+	return buildContext(scoring.transcript);
+}
 //#endregion
-export { judgeInputNames as $, RunDatasetSchema as A, normalizeAutEvent as B, BatchSchema as C, MetricResultSchema as D, JudgeRunInfoSchema as E, RunStatusSchema as F, DEFAULT_ARTIFACT_EXCLUDE as G, defineEvalMatrix as H, RunSummarySchema as I, catalogEntry as J, DEFAULT_ATTACHMENT_LIMITS as K, ScoreResultSchema as L, RunMetadataSchema as M, RunScoringSchema as N, REPORT_SCHEMA_VERSION as O, RunSourceSchema as P, criteria as Q, TrajectoryEventSchema as R, AutIdentitySchema as S, JsonValueSchema as T, canonicalParameters as U, turnResultEvents as V, turnTokens as W, answerInstruction as X, UnreadableAnswer as Y, choices as Z, user as _, resourceUriSchema as _t, assertJudgesIndependent as a, toJson as at, ArtifactEntrySchema as b, defineEval as c, isDataset as ct, judge as d, ResourceKindSchema as dt, judgePrompt as et, metric as f, ResourceUriSchema as ft, skip as g, resourceUri as gt, registerEvals as h, parseResourceUri as ht, agentRuntimes as i, NotJsonError as it, RunManifestSchema as j, RecordedErrorSchema as k, defineSuite as l, relativeDatasetPath as lt, recordError as m, authorId as mt, DEFAULT_AGENT_RUNTIMES as n, readJudgeAnswer as nt, authoringId as o, toJsonObject as ot, predicate as p, UuidSchema as pt, defineConfig as q, agent as r, weightedMean as rt, defineAgent as s, defineDataset as st, BUILT_IN_METRICS as t, passMark as tt, expectToolCall as u, definitionSite as ut, parseTrajectoryJsonl as v, resourceUuid as vt, CheckpointResultSchema as w, AttachmentEntrySchema as x, ApiErrorSchema as y, UsageSchema as z };
+export { DEFAULT_ARTIFACT_EXCLUDE as $, JudgeRunInfoSchema as A, definitionSite as At, RunMetadataSchema as B, ApiErrorSchema as C, limitsFromParameters as Ct, BatchSchema as D, defineDataset as Dt, AutIdentitySchema as E, resolveLimits as Et, RecordedErrorSchema as F, parseResourceUri as Ft, ScoreResultSchema as G, RunSourceSchema as H, RunDatasetSchema as I, resourceUri as It, container as J, TranscriptSchema as K, RunEventSchema as L, resourceUriSchema as Lt, LimitsSchema as M, ResourceUriSchema as Mt, MetricResultSchema as N, UuidSchema as Nt, CheckpointResultSchema as O, isDataset as Ot, REPORT_SCHEMA_VERSION as P, authorId as Pt, canonicalParameters as Q, RunIsolationSchema as R, resourceUuid as Rt, user as S, formatLimit as St, AttachmentEntrySchema as T, parseLimits as Tt, RunStatusSchema as U, RunScoringSchema as V, RunSummarySchema as W, sandboxExec as X, none as Y, defineEvalMatrix as Z, recordError as _, contentText as _t, assertJudgesIndependent as a, choices as at, toolCalls as b, parseModel as bt, defineEval as c, judgePrompt as ct, judge as d, weightedMean as dt, DEFAULT_ATTACHMENT_LIMITS as et, lastReply as f, NotJsonError as ft, predicate as g, callsIn as gt, modelContext as h, activePath as ht, agentRuntimes as i, answerInstruction as it, LimitReachedSchema as j, ResourceKindSchema as jt, JsonValueSchema as k, relativeDatasetPath as kt, defineSuite as l, passMark as lt, metric as m, toJsonObject as mt, DEFAULT_AGENT_RUNTIMES as n, catalogEntry as nt, authoringId as o, criteria as ot, messages as p, toJson as pt, UsageSchema as q, agent as r, UnreadableAnswer as rt, defineAgent as s, judgeInputNames as st, BUILT_IN_METRICS as t, defineConfig as tt, expectToolCall as u, readJudgeAnswer as ut, registerEvals as v, isTreeRecord as vt, ArtifactEntrySchema as w, parseLimitFlag as wt, usage as x, toolResultValue as xt, skip as y, modelName as yt, RunManifestSchema as z };