@leostera/evalkit 0.9.0 → 0.11.0

package/dist/upgrade-rMsGzUEJ.mjs

@@ -0,0 +1,230 @@
+//#region ../core/src/usage.ts
+const BUCKETS = [
+	"input",
+	"output",
+	"cacheRead",
+	"cacheWrite",
+	"reasoning",
+	"total"
+];
+/** ATXF usage from what an adapter reported; undefined when it reported nothing. */
+function normalizeUsage(input) {
+	if (!input) return void 0;
+	for (const key of BUCKETS) {
+		const value = input[key];
+		if (value !== void 0 && (!Number.isFinite(value) || value < 0)) throw new Error(`Usage ${key} must be a non-negative number; received ${String(value)}`);
+	}
+	if (BUCKETS.every((key) => input[key] === void 0)) return void 0;
+	return {
+		input: input.input ?? 0,
+		output: input.output ?? 0,
+		...input.cacheRead === void 0 ? {} : { cacheRead: input.cacheRead },
+		...input.cacheWrite === void 0 ? {} : { cacheWrite: input.cacheWrite },
+		...input.reasoning === void 0 ? {} : { reasoning: input.reasoning },
+		...input.total === void 0 ? {} : { total: input.total }
+	};
+}
+/**
+* Tokens the model had to process: uncached input, cache writes and output.
+* Cache reads are left out; see `cachedTokens`. Usage that has only a total
+* (from a provider, or a run saved before 0.11) counts its total less any
+* cache reads.
+*/
+function uncachedTokens(usage) {
+	const counted = usage.input + (usage.cacheWrite ?? 0) + usage.output;
+	if (counted === 0 && usage.total !== void 0) return Math.max(0, usage.total - (usage.cacheRead ?? 0));
+	return counted;
+}
+/** Bucket by bucket; a bucket absent from both stays absent. */
+function addUsage(a, b) {
+	if (!a) return b;
+	if (!b) return a;
+	const sum = {
+		input: a.input + b.input,
+		output: a.output + b.output
+	};
+	for (const key of [
+		"cacheRead",
+		"cacheWrite",
+		"reasoning",
+		"total"
+	]) if (a[key] !== void 0 || b[key] !== void 0) sum[key] = (a[key] ?? 0) + (b[key] ?? 0);
+	return sum;
+}
+/**
+* A transcript's usage as ATXF totals it: usage on assistant messages,
+* compactions and branch summaries, and usage records that are not running
+* totals. Every record counts, abandoned branches too, since they were paid
+* for. Undefined when nothing reported usage.
+*/
+function transcriptUsage(transcript) {
+	let total;
+	for (const record of transcript.records) if (record.type === "message" && record.role === "assistant") total = addUsage(total, record.usage);
+	else if (record.type === "compaction" || record.type === "branch_summary") total = addUsage(total, record.usage);
+	else if (record.type === "usage" && !record.cumulative) total = addUsage(total, record.usage);
+	return total;
+}
+/**
+* The built-in token metrics for the usage a transcript reported, each usage
+* counted on its own and then summed; none when nothing reported usage.
+*/
+function usageMetrics(usages) {
+	if (!usages.length) return [];
+	const sum = (read) => usages.reduce((total, usage) => total + read(usage), 0);
+	const cacheRead = sum((usage) => usage.cacheRead ?? 0);
+	const input = sum((usage) => usage.input + (usage.cacheRead ?? 0) + (usage.cacheWrite ?? 0));
+	return [
+		{
+			name: "tokens",
+			value: sum(uncachedTokens),
+			unit: "tokens",
+			better: "lower",
+			builtIn: true
+		},
+		{
+			name: "cached tokens",
+			value: cacheRead,
+			unit: "tokens",
+			builtIn: true
+		},
+		{
+			name: "output tokens",
+			value: sum((usage) => usage.output),
+			unit: "tokens",
+			better: "lower",
+			builtIn: true
+		},
+		...input > 0 ? [{
+			name: "cache hit rate",
+			value: cacheRead / input,
+			better: "higher",
+			builtIn: true
+		}] : []
+	];
+}
+/**
+* Every usage a transcript reports that ATXF counts toward its total: on
+* assistant messages, compactions and branch summaries, and usage records
+* that are not running totals.
+*/
+function reportedUsages(transcript) {
+	const usages = [];
+	for (const record of transcript.records) {
+		const usage = record.type === "message" && record.role === "assistant" ? record.usage : record.type === "compaction" || record.type === "branch_summary" ? record.usage : record.type === "usage" && !record.cumulative ? record.usage : void 0;
+		if (usage) usages.push(usage);
+	}
+	return usages;
+}
+//#endregion
+//#region ../core/src/upgrade.ts
+/** Report schema versions `evalkit upgrade` converts to version 6 first. */
+const UPGRADABLE_SCHEMA_VERSIONS = [
+	2,
+	3,
+	4,
+	5
+];
+const isRecord = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
+/**
+* A version 6 run status from an old trial's status and whether it passed.
+* Old `completed` meant "finished", passed or not; old `failed` meant it
+* could not finish, which version 6 calls `errored`.
+*/
+function upgradeStatus(status, passed) {
+	switch (status) {
+		case "completed": return passed ? "passed" : "failed";
+		case "failed": return "errored";
+		case "cancelled": return "cancelled";
+		default: return "running";
+	}
+}
+/** A trajectory event with the old trial lifecycle events renamed. */
+function upgradeEvent(event) {
+	if (!isRecord(event)) return event;
+	if (event.kind === "trial-started") return {
+		...event,
+		kind: "run-started"
+	};
+	if (event.kind === "trial-completed") return {
+		...event,
+		kind: "run-completed"
+	};
+	return event;
+}
+/** A whole `trajectory.jsonl`, line by line; lines that are not JSON stay as they are. */
+function upgradeTrajectory(jsonl) {
+	return jsonl.split("\n").map((line) => {
+		if (!line.trim()) return line;
+		try {
+			return JSON.stringify(upgradeEvent(JSON.parse(line)));
+		} catch {
+			return line;
+		}
+	}).join("\n");
+}
+/**
+* A catalog (`catalog.json`, or its `catalog` list on its own) with each
+* scenario's `trialCount` and `policy.trials` replaced by `repeat`.
+*/
+function upgradeCatalog(catalog) {
+	const entry = (value) => {
+		if (!isRecord(value)) return value;
+		const { trialCount, policy, ...rest } = value;
+		const { trials, ...keptPolicy } = isRecord(policy) ? policy : {};
+		const repeat = typeof trialCount === "number" ? trialCount : typeof trials === "number" ? trials : value.repeat;
+		return {
+			...rest,
+			...repeat === void 0 ? {} : { repeat },
+			...isRecord(policy) ? { policy: keptPolicy } : {}
+		};
+	};
+	if (Array.isArray(catalog)) return catalog.map(entry);
+	if (isRecord(catalog) && Array.isArray(catalog.catalog)) return {
+		...catalog,
+		catalog: catalog.catalog.map(entry)
+	};
+	return catalog;
+}
+/** The fields of an old run manifest and trial manifest that a version 6 manifest keeps. */
+function upgradeManifest(input) {
+	const { run, trial = {}, runId, batch, status } = input;
+	const pick = (source, keys) => Object.fromEntries(keys.flatMap((key) => source[key] === void 0 ? [] : [[key, source[key]]]));
+	return {
+		schemaVersion: 6,
+		runUri: `evalkit:run:${runId}`,
+		...pick(run, [
+			"evalId",
+			"suiteId",
+			"parameters",
+			"matrix",
+			"aut",
+			"startedAt",
+			"source",
+			"process"
+		]),
+		...pick(trial, [
+			"evalId",
+			"parameters",
+			"matrix",
+			"aut",
+			"startedAt",
+			"workspaceIgnore",
+			"datasets"
+		]),
+		...batch ? { batch } : {},
+		status
+	};
+}
+/**
+* Scenario options from before 0.9 that plain JavaScript scenarios can still
+* carry. Ignoring them would silently change what runs, so both
+* `defineEval` and the runner reject them with the replacement.
+*/
+function assertNoRemovedOptions(definition) {
+	if ("fixtures" in definition) throw new Error(`Eval ${String(definition.id)} uses fixtures, which were replaced by datasets: [defineDataset(...)]`);
+	if ("judge" in definition) throw new Error(`Eval ${String(definition.id)} sets judge, which moved to each rule: judge(name, { agent, rubric }), or judge in evalkit.config.ts for every judge (EvalKit 0.10)`);
+	if (isRecord(definition.policy) && "timeoutMs" in definition.policy) throw new Error(`Eval ${String(definition.id)} sets policy.timeoutMs, which was replaced by limits: { time: ${JSON.stringify(definition.policy.timeoutMs)} }, enforced since EvalKit 0.11`);
+	if (isRecord(definition.policy) && "trials" in definition.policy) throw new Error(`Eval ${String(definition.id)} sets policy.trials, which was replaced by repeat: ${JSON.stringify(definition.policy.trials)} (EvalKit 0.9)`);
+}
+//#endregion
+export { upgradeManifest as a, normalizeUsage as c, uncachedTokens as d, usageMetrics as f, upgradeEvent as i, reportedUsages as l, assertNoRemovedOptions as n, upgradeStatus as o, upgradeCatalog as r, upgradeTrajectory as s, UPGRADABLE_SCHEMA_VERSIONS as t, transcriptUsage as u };

package/dist/upgrade-transcript-C8DUykLd.mjs

@@ -0,0 +1,402 @@
+import { d as uncachedTokens, f as usageMetrics, l as reportedUsages } from "./upgrade-rMsGzUEJ.mjs";
+//#region ../core/src/upgrade-transcript.ts
+const isRecord = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
+/** The `source.format` of a transcript converted from a version 6 trajectory. */
+const V6_TRAJECTORY_FORMAT = "evalkit-trajectory-v6";
+/** Old usage: input counted cached tokens, which were reported as a subset. */
+function upgradeUsage(value) {
+	if (!isRecord(value)) return void 0;
+	const num = (key) => typeof value[key] === "number" ? value[key] : void 0;
+	const input = num("inputTokens");
+	const output = num("outputTokens");
+	const cached = num("cachedInputTokens");
+	const total = num("totalTokens");
+	if (input === void 0 && output === void 0) {
+		if (total === void 0) return void 0;
+		return {
+			input: 0,
+			output: 0,
+			total
+		};
+	}
+	const cacheRead = Math.min(cached ?? 0, input ?? 0);
+	return {
+		input: (input ?? 0) - cacheRead,
+		output: output ?? 0,
+		...cacheRead ? { cacheRead } : {},
+		...total === void 0 ? {} : { total }
+	};
+}
+function parseModel(model) {
+	if (typeof model !== "string" || !model) return void 0;
+	const slash = model.indexOf("/");
+	return slash > 0 && slash < model.length - 1 ? {
+		provider: model.slice(0, slash),
+		id: model.slice(slash + 1)
+	} : {
+		provider: "unknown",
+		id: model
+	};
+}
+const text = (value) => typeof value === "string" ? value : JSON.stringify(value ?? null);
+/**
+* Splits a version 6 trajectory (its events, parsed) into an ATXF
+* transcript of the agent's events and EvalKit's events placed in it.
+* Every record carries the old events it came from in `native`.
+*/
+function convertTrajectory(events, run) {
+	const records = [];
+	const runEvents = [];
+	const recordOfEvent = /* @__PURE__ */ new Map();
+	let lastId = null;
+	let next = 0;
+	let sawUsage = false;
+	let sawTurns = false;
+	/** The assistant message the next tool call joins, while nothing came after it. */
+	let joinable;
+	let openTurn;
+	/**
+	* 0.10 recorded EvalKit's prompt only on its step event; it becomes the
+	* user message, unless the adapter recorded the same message itself.
+	*/
+	let prompt;
+	const native = (...old) => ({
+		source: "evalkit",
+		format: V6_TRAJECTORY_FORMAT,
+		records: old
+	});
+	const push = (record, index) => {
+		const full = {
+			...record,
+			id: `r${++next}`,
+			parentId: lastId
+		};
+		records.push(full);
+		lastId = full.id;
+		recordOfEvent.set(index, full.id);
+		joinable = void 0;
+		return full;
+	};
+	const flushPrompt = () => {
+		if (!prompt) return;
+		const { content, time, index, event } = prompt;
+		prompt = void 0;
+		push({
+			type: "message",
+			time,
+			role: "user",
+			content,
+			native: native(event)
+		}, index);
+	};
+	events.forEach((event, index) => {
+		if (!isRecord(event)) return;
+		const time = typeof event.timestamp === "string" ? event.timestamp : run.startedAt;
+		if (event.source !== "aut") {
+			const { source: _, ...rest } = event;
+			if (event.kind !== "transcript-step-started") flushPrompt();
+			runEvents.push({
+				...rest,
+				at: lastId
+			});
+			if (event.kind === "transcript-step-started" && typeof event.message === "string") prompt = {
+				content: event.message,
+				time,
+				index,
+				event
+			};
+			return;
+		}
+		if (prompt && event.kind === "message" && event.role === "user" && event.content === prompt.content) prompt = void 0;
+		else flushPrompt();
+		switch (event.kind) {
+			case "message": {
+				const role = event.role;
+				const content = event.content;
+				const extra = typeof content === "string" ? {} : { content };
+				const ext = Object.keys(extra).length ? { ext: { evalkit: extra } } : {};
+				if (role === "assistant") joinable = push({
+					type: "message",
+					time,
+					role: "assistant",
+					content: [{
+						type: "text",
+						text: text(content)
+					}],
+					native: native(event),
+					...ext
+				}, index);
+				else if (role === "tool") push({
+					type: "message",
+					time,
+					role: "tool",
+					toolCallId: `unknown-${index}`,
+					content: text(content),
+					native: native(event),
+					...ext
+				}, index);
+				else push({
+					type: "message",
+					time,
+					role: role === "system" ? "system" : "user",
+					content: text(content),
+					native: native(event),
+					...ext
+				}, index);
+				return;
+			}
+			case "tool-call": {
+				const block = {
+					type: "tool_call",
+					id: String(event.id ?? `call-${index}`),
+					name: String(event.name ?? "tool"),
+					kind: "function",
+					arguments: isRecord(event.arguments) ? event.arguments : { value: event.arguments ?? null }
+				};
+				if (joinable) {
+					joinable.content.push(block);
+					joinable.native.records.push(event);
+					recordOfEvent.set(index, joinable.id);
+					return;
+				}
+				joinable = push({
+					type: "message",
+					time,
+					role: "assistant",
+					content: [block],
+					native: native(event)
+				}, index);
+				return;
+			}
+			case "tool-result": {
+				const result = event.result ?? null;
+				push({
+					type: "message",
+					time,
+					role: "tool",
+					toolCallId: String(event.id ?? `call-${index}`),
+					...typeof event.name === "string" ? { toolName: event.name } : {},
+					content: text(result),
+					...typeof result === "string" ? {} : { details: { result } },
+					native: native(event)
+				}, index);
+				return;
+			}
+			case "turn-started": {
+				sawTurns = true;
+				const turnId = `turn-${String(event.turn ?? index)}`;
+				push({
+					type: "turn",
+					time,
+					phase: "start",
+					turnId,
+					native: native(event)
+				}, index);
+				openTurn = {
+					turnId,
+					from: records.length
+				};
+				return;
+			}
+			case "turn-completed": {
+				const usage = upgradeUsage(event.usage);
+				const model = parseModel(event.model);
+				const from = openTurn?.from ?? 0;
+				let holder;
+				for (let at = records.length - 1; at >= from; at -= 1) {
+					const record = records[at];
+					if (record.type === "message" && record.role === "assistant") {
+						holder = record;
+						break;
+					}
+				}
+				if (usage) sawUsage = true;
+				const totalOnly = usage && usage.input === 0 && usage.output === 0 && !usage.cacheRead;
+				if (holder && !(usage && holder.usage)) {
+					if (usage) holder.usage = usage;
+					if (model) holder.model = model;
+					if (totalOnly) holder.ext = {
+						...holder.ext,
+						evalkit: {
+							...holder.ext?.evalkit,
+							totalOnly: true
+						}
+					};
+				} else if (usage) push({
+					type: "usage",
+					time,
+					kind: "turn",
+					usage,
+					...model ? { model } : {},
+					...totalOnly ? { ext: { evalkit: { totalOnly: true } } } : {}
+				}, index);
+				const latency = typeof event.latencyMs === "number" ? { ext: { evalkit: { latencyMs: event.latencyMs } } } : {};
+				if (openTurn) {
+					push({
+						type: "turn",
+						time,
+						phase: "end",
+						turnId: openTurn.turnId,
+						outcome: "succeeded",
+						native: native(event),
+						...latency
+					}, index);
+					openTurn = void 0;
+				} else if (holder) {
+					holder.native.records.push(event);
+					recordOfEvent.set(index, holder.id);
+				} else push({
+					type: "custom",
+					time,
+					namespace: "evalkit",
+					name: "turn-completed",
+					data: event,
+					native: native(event)
+				}, index);
+				return;
+			}
+			default: {
+				const { source: _, kind, timestamp: __, ...data } = event;
+				push({
+					type: "custom",
+					time,
+					namespace: "evalkit",
+					name: String(kind),
+					...Object.keys(data).length ? { data } : {},
+					native: native(event)
+				}, index);
+			}
+		}
+	});
+	flushPrompt();
+	const firstTime = records[0]?.time ?? run.startedAt;
+	return {
+		transcript: {
+			header: {
+				type: "transcript",
+				atxf: "0.1",
+				id: run.runId,
+				time: firstTime,
+				source: {
+					agent: run.agent?.id ?? "unknown",
+					...run.agent?.version ? { version: run.agent.version } : {},
+					format: V6_TRAJECTORY_FORMAT
+				},
+				fidelity: {
+					topology: "linear",
+					instructions: "none",
+					tools: "none",
+					reasoning: "none",
+					toolLifecycle: false,
+					turns: sawTurns,
+					usage: sawUsage ? "per-response" : "none",
+					native: "all",
+					ids: "synthetic"
+				},
+				ext: { evalkit: {
+					runId: run.runId,
+					...run.evalId ? { scenarioId: run.evalId } : {}
+				} }
+			},
+			records
+		},
+		runEvents,
+		recordOfEvent
+	};
+}
+/**
+* A version 6 `scoring.json` for version 7: each judge's events become its
+* transcript and its usage moves to ATXF's buckets, each checkpoint's
+* matched tool call points at the record holding it, the token metrics
+* are recomputed from the converted transcript, and `steps` is added.
+* Throws when the recomputed tokens (uncached plus cached)
+* do not add up to the saved ones, so a run is never converted wrongly.
+*/
+function upgradeScoring(scoring, run, recordOfEvent, agent) {
+	if (!isRecord(scoring)) throw new Error("scoring.json is not an object");
+	const judgeUsage = [];
+	const verdict = (entry, index) => {
+		if (!isRecord(entry)) return entry;
+		const result = { ...entry };
+		if (isRecord(result.matchedToolCall)) {
+			const { eventIndex, id } = result.matchedToolCall;
+			const messageId = typeof eventIndex === "number" ? recordOfEvent.get(eventIndex) : void 0;
+			if (messageId && typeof id === "string") result.matchedToolCall = {
+				messageId,
+				id
+			};
+			else delete result.matchedToolCall;
+		}
+		if (isRecord(result.judge)) {
+			const { events, usage, ...judge } = result.judge;
+			const converted = Array.isArray(events) ? convertTrajectory(events.map((event) => isRecord(event) ? {
+				...event,
+				source: "aut"
+			} : event), {
+				runId: `${run.runId}-judge-${index}`,
+				startedAt: run.startedAt,
+				...isRecord(judge.agent) && typeof judge.agent.id === "string" ? { agent: { id: judge.agent.id } } : {}
+			}).transcript : void 0;
+			const atxfUsage = upgradeUsage(usage);
+			if (atxfUsage) judgeUsage.push(atxfUsage);
+			result.judge = {
+				...judge,
+				...atxfUsage ? { usage: atxfUsage } : {},
+				...converted ? { transcript: converted } : {}
+			};
+		}
+		return result;
+	};
+	const results = Array.isArray(scoring.results) ? scoring.results.map((entry, index) => verdict(entry, index)) : [];
+	const checkpoints = Array.isArray(scoring.checkpoints) ? scoring.checkpoints.map((entry, index) => verdict(entry, results.length + index)) : void 0;
+	const metrics = Array.isArray(scoring.metrics) ? upgradeMetrics(scoring.metrics, agent, judgeUsage) : void 0;
+	return {
+		...scoring,
+		results,
+		...checkpoints ? { checkpoints } : {},
+		...metrics ? { metrics } : {}
+	};
+}
+/** Built-in metrics recomputed from the converted transcript. */
+const RECOMPUTED_METRICS = /* @__PURE__ */ new Set([
+	"steps",
+	"tokens",
+	"cached tokens",
+	"output tokens",
+	"cache hit rate",
+	"judge tokens"
+]);
+function upgradeMetrics(metrics, transcript, judges) {
+	const agent = reportedUsages(transcript);
+	const steps = transcript.records.filter((record) => record.type === "message" && record.role === "assistant").length;
+	const saved = (name) => {
+		const entry = metrics.find((metric) => isRecord(metric) && metric.name === name);
+		return isRecord(entry) && typeof entry.value === "number" ? entry.value : void 0;
+	};
+	const check = (name, before, usages) => {
+		if (before === void 0) return;
+		const after = usages.reduce((sum, usage) => sum + uncachedTokens(usage) + (usage.cacheRead ?? 0), 0);
+		if (after !== before) throw new Error(`${name} do not add up after converting: ${before} saved, ${after} recomputed`);
+	};
+	check("tokens", saved("tokens"), agent);
+	check("judge tokens", saved("judge tokens"), judges);
+	return [
+		...metrics.filter((metric) => !(isRecord(metric) && RECOMPUTED_METRICS.has(String(metric.name)))),
+		{
+			name: "steps",
+			value: steps,
+			better: "lower",
+			builtIn: true
+		},
+		...usageMetrics(agent).map((metric) => metric),
+		...judges.length ? [{
+			name: "judge tokens",
+			value: judges.reduce((sum, usage) => sum + uncachedTokens(usage), 0),
+			unit: "tokens",
+			better: "lower",
+			builtIn: true
+		}] : []
+	];
+}
+//#endregion
+export { upgradeScoring as n, convertTrajectory as t };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leostera/evalkit",
-  "version": "0.9.0",
+  "version": "0.11.0",
   "description": "Author and run agent evals as code",
   "type": "module",
   "license": "MIT",
@@ -46,6 +46,7 @@
     "evalkit": "dist/cli.mjs"
   },
   "dependencies": {
+    "@leostera/atxf": "0.1.1",
     "effect": "^3.22.2",
     "hono": "^4.11.6",
     "jiti": "^2.7.0"