@leostera/evalkit 0.9.0 → 0.10.0

package/dist/{src-BrfIXfO2.mjs → src-B14fXAGp.mjs}

@@ -1,4 +1,4 @@
-import { n as assertNoRemovedOptions } from "./upgrade-60SIRius.mjs";
+import { n as assertNoRemovedOptions } from "./upgrade-G-uFPJKg.mjs";
 import * as Schema from "effect/Schema";
 //#region ../core/src/identity.ts
 const ResourceKindSchema = Schema.Literal("run", "artifact");
@@ -206,11 +206,194 @@ function toJsonObject(value, path) {
 	return converted;
 }
 //#endregion
+//#region ../core/src/judge.ts
+const KEY = /^[A-Za-z0-9][A-Za-z0-9_-]{0,31}$/;
+function unitInterval(value) {
+	return typeof value === "number" && Number.isFinite(value) && value >= 0 && value <= 1;
+}
+/**
+* A judge that answers with one of these choices, in this order:
+* `choices({ A: ['Accurate and helpful', 1], B: ['Partly right', 0.5], C: ['Wrong', 0] })`.
+*/
+function choices(options) {
+	const entries = Object.entries(options);
+	if (entries.length < 2) throw new Error("choices(...) needs at least two choices");
+	const seen = /* @__PURE__ */ new Set();
+	return {
+		kind: "choices",
+		choices: entries.map(([key, [label, value]]) => {
+			if (!KEY.test(key)) throw new Error(`Choice ${JSON.stringify(key)} must be a short word of letters, digits, "-" or "_"`);
+			if (seen.has(key.toUpperCase())) throw new Error(`Choice ${key} is listed twice`);
+			seen.add(key.toUpperCase());
+			if (!label.trim()) throw new Error(`Choice ${key} needs a label`);
+			if (!unitInterval(value)) throw new Error(`Choice ${key} must be worth a number from 0 to 1; received ${String(value)}`);
+			return {
+				key,
+				label,
+				value
+			};
+		})
+	};
+}
+/**
+* A judge that scores each criterion from 0 to 1; the score is the weighted
+* mean. A number is the weight; `{ weight, description }` also says what the
+* criterion means. Weights default to 1.
+*/
+function criteria(options) {
+	return {
+		kind: "criteria",
+		criteria: declaredCriteria(options)
+	};
+}
+/** Checked criteria declarations, for `criteria()` and `predicate(..., { criteria })`. */
+function declaredCriteria(options) {
+	const entries = Object.entries(options);
+	if (!entries.length) throw new Error("criteria(...) needs a criterion");
+	return entries.map(([name, option]) => {
+		if (!name.trim()) throw new Error("A criterion needs a name");
+		const weight = typeof option === "number" ? option : option.weight ?? 1;
+		if (!Number.isFinite(weight) || weight <= 0) throw new Error(`Criterion ${name} must have a positive weight; received ${String(weight)}`);
+		const description = typeof option === "number" ? void 0 : option.description;
+		return {
+			name,
+			weight,
+			...description ? { description } : {}
+		};
+	});
+}
+/** What a criterion counts for in the weighted mean. */
+const criterionValue = (result) => "value" in result ? result.value : 0;
+/** The weighted mean of criteria; weights default to 1, a criterion not reached counts 0. */
+function weightedMean(results) {
+	let total = 0;
+	let weights = 0;
+	for (const result of results) {
+		const weight = result.weight ?? 1;
+		total += weight * criterionValue(result);
+		weights += weight;
+	}
+	if (!weights) throw new Error("Criteria need a positive total weight");
+	return total / weights;
+}
+/** The inputs a judge sees when the rule names none. */
+const DEFAULT_JUDGE_INPUTS = ["conversation", "reply"];
+/** The names of the inputs a judge sees, in prompt order. */
+function judgeInputNames(rule) {
+	return rule.inputs ? Object.keys(rule.inputs) : [...DEFAULT_JUDGE_INPUTS];
+}
+/**
+* The prompt a judge agent receives. `values` holds each input's text;
+* an input without one shows as a `{placeholder}`, which is how the
+* dashboard shows the prompt before any run.
+*/
+function judgePrompt(options) {
+	const inputs = options.inputs.map((name) => `${name}:\n${options.values?.[name] ?? `{${name}}`}`).join("\n\n");
+	return [
+		"You are judging one run of an AI agent. Judge only by the rubric, using only the inputs below.",
+		`Rubric:\n${options.rubric.trim()}`,
+		inputs,
+		answerInstruction(options.answer)
+	].join("\n\n");
+}
+function answerInstruction(answer) {
+	switch (answer.kind) {
+		case "verdict": return "Answer with only a JSON object: {\"value\": <a number from 0 to 1>, \"explanation\": \"<one or two sentences why>\"}";
+		case "choices": return [`Answer with one of ${answer.choices.map((choice) => choice.key).join(", ")} on the first line, then one sentence why on the next.`, ...answer.choices.map((choice) => `${choice.key}) ${choice.label}`)].join("\n");
+		case "criteria": return [
+			"Score each criterion from 0 to 1:",
+			...answer.criteria.map((criterion) => `- ${criterion.name}${criterion.description ? `: ${criterion.description}` : ""}`),
+			`Answer with only a JSON object: {"criteria": {${answer.criteria.map((criterion) => `${JSON.stringify(criterion.name)}: {"value": <0 to 1>, "explanation": "<why>"}`).join(", ")}}, "explanation": "<one sentence overall>"}`
+		].join("\n");
+	}
+}
+/** Thrown when a judge's reply is not in the format asked for; the message says why. */
+var UnreadableAnswer = class extends Error {};
+/**
+* Reads a judge's reply in the format its rule asked for. Throws
+* `UnreadableAnswer` with a reason the judge can be told, so it can answer again.
+*/
+function readJudgeAnswer(answer, reply) {
+	if (answer.kind === "choices") return readChoice(answer.choices, reply);
+	const object = jsonObject(reply);
+	if (answer.kind === "verdict") {
+		if (!unitInterval(object.value)) throw new UnreadableAnswer("\"value\" must be a number from 0 to 1");
+		return {
+			value: object.value,
+			...typeof object.explanation === "string" && object.explanation.trim() ? { explanation: object.explanation.trim() } : {},
+			...object.evidence !== void 0 ? { evidence: object.evidence } : {}
+		};
+	}
+	const scored = object.criteria;
+	if (typeof scored !== "object" || scored === null || Array.isArray(scored)) throw new UnreadableAnswer("\"criteria\" must be an object with one entry per criterion");
+	const results = answer.criteria.map((criterion) => {
+		const entry = scored[criterion.name];
+		const value = typeof entry === "number" ? entry : typeof entry === "object" && entry !== null && !Array.isArray(entry) ? entry.value : void 0;
+		if (!unitInterval(value)) throw new UnreadableAnswer(`criterion ${JSON.stringify(criterion.name)} needs a "value" from 0 to 1`);
+		const explanation = typeof entry === "object" && entry !== null && !Array.isArray(entry) && typeof entry.explanation === "string" && entry.explanation.trim() ? entry.explanation.trim() : void 0;
+		return {
+			name: criterion.name,
+			value,
+			weight: criterion.weight,
+			...explanation ? { explanation } : {}
+		};
+	});
+	return {
+		value: weightedMean(results),
+		criteria: results,
+		...typeof object.explanation === "string" && object.explanation.trim() ? { explanation: object.explanation.trim() } : {}
+	};
+}
+function readChoice(options, reply) {
+	if (typeof reply !== "string") throw new UnreadableAnswer("answer with plain text, not JSON");
+	const lines = reply.split("\n").map((line) => line.trim()).filter(Boolean);
+	const first = lines[0] ?? "";
+	const match = /^(?:answer\s*[:-]?\s*)?[*_("'[\s]*([A-Za-z0-9][A-Za-z0-9_-]*)/i.exec(first);
+	const said = match?.[1]?.toUpperCase();
+	const choice = options.find((option) => option.key.toUpperCase() === said);
+	if (!choice) throw new UnreadableAnswer(`the first line must be one of ${options.map((option) => option.key).join(", ")}`);
+	const explanation = [first.slice(match[0].length).replace(/^[*_)"'\].:\s-]+/, "").trim(), ...lines.slice(1)].filter(Boolean).join(" ");
+	return {
+		value: choice.value,
+		choice: choice.key,
+		...explanation ? { explanation } : {}
+	};
+}
+/** The JSON object in a reply: the reply itself, or the first `{…}` in its text (code fences allowed). */
+function jsonObject(reply) {
+	if (typeof reply === "object" && reply !== null && !Array.isArray(reply)) return reply;
+	if (typeof reply !== "string") throw new UnreadableAnswer("answer with a JSON object");
+	const start = reply.indexOf("{");
+	const end = reply.lastIndexOf("}");
+	if (start === -1 || end < start) throw new UnreadableAnswer("answer with only a JSON object");
+	let parsed;
+	try {
+		parsed = JSON.parse(reply.slice(start, end + 1));
+	} catch {
+		throw new UnreadableAnswer("the JSON object could not be parsed");
+	}
+	if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) throw new UnreadableAnswer("answer with a JSON object");
+	return parsed;
+}
+/**
+* The pass mark in words: `only 1`, `≥ 0.5`, or for choices the ones that
+* pass, such as `A or B`.
+*/
+function passMark(passAt, answer) {
+	const at = passAt ?? 1;
+	if (answer?.kind === "choices") {
+		const passing = answer.choices.filter((choice) => choice.value >= at).map((choice) => choice.key);
+		if (!passing.length) return "no choice passes";
+		if (passing.length === answer.choices.length) return "any choice";
+		return passing.length === 1 ? `only ${passing[0]}` : `${passing.slice(0, -1).join(", ")} or ${passing.at(-1)}`;
+	}
+	return at === 1 ? "only 1" : `≥ ${at}`;
+}
+//#endregion
 //#region ../core/src/catalog.ts
 function catalogEntry(evaluation, suite, options = {}) {
 	const at = (value) => relativeSite(definitionSite(value), options.root);
 	const definedAt = at(evaluation);
-	const judgeAgent = evaluation.judge ? catalogAgent(evaluation.judge, at) : void 0;
 	const policy = evaluation.policy ?? {};
 	return {
 		id: evaluation.id,
@@ -218,15 +401,14 @@ function catalogEntry(evaluation, suite, options = {}) {
 		...evaluation.name ? { name: evaluation.name } : {},
 		...suite ? { suiteId: authoringId(suite) } : {},
 		agent: catalogAgent(evaluation.agent, at),
-		...judgeAgent ? { judgeAgent } : {},
 		datasets: (evaluation.datasets ?? []).map((dataset) => catalogDataset(dataset, at)),
 		repeat: evaluation.repeat ?? 1,
 		policy: {
 			...policy.timeoutMs !== void 0 ? { timeoutMs: policy.timeoutMs } : {},
 			...policy.failfast !== void 0 ? { failfast: policy.failfast } : {}
 		},
-		transcript: evaluation.transcript.map((step) => catalogStep(step, at)),
-		scorers: evaluation.scoring.map((rule) => catalogScorer(rule, at)),
+		transcript: evaluation.transcript.map((step) => catalogStep(step, at, options.judge)),
+		scorers: evaluation.scoring.map((rule) => catalogScorer(rule, at, options.judge)),
 		...evaluation.metadata !== void 0 ? { metadata: toJson(evaluation.metadata, `${evaluation.id}.metadata`) } : {},
 		...definedAt ? { definedAt } : {}
 	};
@@ -270,18 +452,37 @@ function catalogDataset(dataset, at) {
 		...source.valueFile !== void 0 ? { value: { file: source.valueFile } } : source.value !== void 0 ? { value: { shape: shapeOf(source.value) } } : {}
 	};
 }
-function catalogScorer(rule, at) {
+function catalogScorer(rule, at, defaultJudge) {
 	const definedAt = at(rule);
 	return {
 		name: rule.name,
 		kind: rule.kind,
-		...rule.kind === "predicate" && rule.passAt !== void 0 ? { passAt: rule.passAt } : {},
+		...rule.description ? { description: rule.description } : {},
+		...rule.passAt !== void 0 ? { passAt: rule.passAt } : {},
 		...rule.supportsPartial !== void 0 ? { supportsPartial: rule.supportsPartial } : {},
-		...rule.kind === "judge" ? { rubric: rule.rubric } : {},
+		...rule.kind === "judge" ? catalogJudge(rule, at, defaultJudge) : rule.criteria ? { answer: {
+			kind: "criteria",
+			criteria: rule.criteria
+		} } : {},
 		...definedAt ? { definedAt } : {}
 	};
 }
-function catalogStep(step, at) {
+function catalogJudge(rule, at, defaultJudge) {
+	const agent = rule.agent ?? defaultJudge;
+	const inputs = judgeInputNames(rule);
+	return {
+		rubric: rule.rubric,
+		...agent ? { agent: catalogAgent(agent, at) } : {},
+		inputs,
+		answer: rule.answer,
+		prompt: judgePrompt({
+			rubric: rule.rubric,
+			answer: rule.answer,
+			inputs
+		})
+	};
+}
+function catalogStep(step, at, defaultJudge) {
 	switch (step.kind) {
 		case "user": return typeof step.message === "string" ? {
 			kind: "user",
@@ -298,7 +499,7 @@ function catalogStep(step, at) {
 		};
 		default: return {
 			kind: "checkpoint",
-			scorer: catalogScorer(step, at)
+			scorer: catalogScorer(step, at, defaultJudge)
 		};
 	}
 }
@@ -562,10 +763,11 @@ function turnResultEvents(result, turn, now) {
 		role: "assistant",
 		content: result.reply
 	}, now));
-	if (result.usage) events.push(normalizeAutEvent({
+	if (result.usage || result.model) events.push(normalizeAutEvent({
 		kind: "turn-completed",
 		turn,
-		usage: result.usage
+		...result.usage ? { usage: result.usage } : {},
+		...result.model ? { model: result.model } : {}
 	}, now));
 	return events;
 }
@@ -660,9 +862,32 @@ const JudgeRunInfoSchema = Schema.Struct({
 		name: Schema.optional(Schema.String),
 		version: Schema.optional(Schema.String)
 	})),
+	model: Schema.optional(Schema.String),
 	usage: Schema.optional(UsageSchema),
+	prompt: Schema.optional(Schema.String),
+	reply: Schema.optional(JsonValueSchema),
+	attempts: Schema.optional(Schema.Number),
 	events: Schema.optional(Schema.mutable(Schema.Array(JsonValueSchema)))
 });
+const CriterionResultSchema = Schema.Union(Schema.Struct({
+	name: Schema.String,
+	value: Schema.Number,
+	weight: Schema.optional(Schema.Number),
+	explanation: Schema.optional(Schema.String),
+	evidence: Schema.optional(JsonValueSchema)
+}), Schema.Struct({
+	name: Schema.String,
+	reached: Schema.Literal(false),
+	weight: Schema.optional(Schema.Number),
+	explanation: Schema.optional(Schema.String)
+}));
+/** What every score and checkpoint result can carry about its verdict. */
+const verdictFields = {
+	passAt: Schema.optional(Schema.Number),
+	choice: Schema.optional(Schema.String),
+	criteria: Schema.optional(Schema.mutable(Schema.Array(CriterionResultSchema))),
+	attachments: Schema.optional(Schema.mutable(Schema.Array(Schema.String)))
+};
 const ScoreResultSchema = Schema.Struct({
 	name: Schema.String,
 	kind: Schema.Literal("predicate", "judge"),
@@ -670,6 +895,7 @@ const ScoreResultSchema = Schema.Struct({
 	passed: Schema.optional(Schema.Boolean),
 	explanation: Schema.optional(Schema.String),
 	evidence: Schema.optional(JsonValueSchema),
+	...verdictFields,
 	judge: Schema.optional(JudgeRunInfoSchema),
 	durationMs: Schema.Number,
 	error: Schema.optional(RecordedErrorSchema),
@@ -684,6 +910,7 @@ const CheckpointResultSchema = Schema.Struct({
 	passed: Schema.optional(Schema.Boolean),
 	explanation: Schema.optional(Schema.String),
 	evidence: Schema.optional(JsonValueSchema),
+	...verdictFields,
 	judge: Schema.optional(JudgeRunInfoSchema),
 	durationMs: Schema.optional(Schema.Number),
 	error: Schema.optional(RecordedErrorSchema),
@@ -768,7 +995,8 @@ const autEvent = {
 		turn: Schema.Number,
 		timestamp: Schema.String,
 		latencyMs: Schema.optional(Schema.Number),
-		usage: Schema.optional(UsageSchema)
+		usage: Schema.optional(UsageSchema),
+		model: Schema.optional(Schema.String)
 	}),
 	completed: Schema.Struct({
 		source: Schema.Literal("aut"),
@@ -938,12 +1166,14 @@ function skip(reason) {
 /** A scorer in code. Its `evidence` type carries through to `checkpoint(rule)`. */
 function predicate(name, run, options = {}) {
 	if (!name.trim()) throw new Error("Predicate name must not be empty");
-	if (options.passAt !== void 0 && !(Number.isFinite(options.passAt) && options.passAt >= 0 && options.passAt <= 1)) throw new Error(`Predicate ${name} passAt must be a number from 0 to 1; received ${options.passAt}`);
+	assertPassAt(`Predicate ${name}`, options.passAt);
+	const { criteria: declared, ...rest } = options;
 	return markDefinitionSite({
 		kind: "predicate",
 		name,
 		run,
-		...options
+		...rest,
+		...declared ? { criteria: declaredCriteria(Object.fromEntries(Object.entries(declared).map(([key, option]) => [key, typeof option === "string" ? { description: option } : option]))) } : {}
 	});
 }
 /** A built-in metric's name is a type error here, as well as at definition time. */
@@ -956,14 +1186,23 @@ function metric(name, run, options = {}) {
 		...options
 	});
 }
+const INPUT_NAME = /^[a-z][a-zA-Z0-9_-]{0,31}$/;
 function judge(name, options) {
-	if (!name.trim() || !options.rubric.trim()) throw new Error("Judge name and rubric must not be empty");
+	if (!name.trim() || !options.rubric?.trim()) throw new Error("Judge name and rubric must not be empty");
+	assertPassAt(`Judge ${name}`, options.passAt);
+	for (const input of Object.keys(options.inputs ?? {})) if (!INPUT_NAME.test(input)) throw new Error(`Judge ${name} input ${JSON.stringify(input)} must start with a lowercase letter and use letters, digits, "-" or "_"`);
+	if (options.inputs && !Object.keys(options.inputs).length) throw new Error(`Judge ${name} inputs must name at least one input`);
+	if (options.agent && typeof options.agent.start !== "function") throw new Error(`Judge ${name} agent must be made with defineAgent()`);
 	return markDefinitionSite({
 		kind: "judge",
 		name,
-		...options
+		...options,
+		answer: options.answer ?? { kind: "verdict" }
 	});
 }
+function assertPassAt(owner, passAt) {
+	if (passAt !== void 0 && !(Number.isFinite(passAt) && passAt >= 0 && passAt <= 1)) throw new Error(`${owner} passAt must be a number from 0 to 1; received ${passAt}`);
+}
 function defineEval(definition) {
 	authoringId(definition);
 	assertNoRemovedOptions(definition);
@@ -984,10 +1223,13 @@ function defineEval(definition) {
 	let hasUser = false;
 	for (const step of definition.transcript) if (step.kind === "user") hasUser = true;
 	else if (step.kind !== "agent" && !hasUser) throw new Error("Checkpoint requires a preceding user step");
-	if ((definition.transcript.some((step) => step.kind === "judge") || definition.scoring.some((rule) => rule.kind === "judge")) && !definition.judge) throw new Error("Judge rules require an eval judge agent");
-	if (definition.judge && definition.judge === definition.agent) throw new Error("Judge agent must be distinct from the agent under test");
+	assertJudgesIndependent(definition);
 	return markDefinitionSite(definition);
 }
+/** A judge never grades with the agent it is judging. */
+function assertJudgesIndependent(definition, defaultJudge) {
+	for (const rule of [...definition.transcript, ...definition.scoring]) if (rule.kind === "judge" && (rule.agent ?? defaultJudge) === definition.agent) throw new Error(`Judge ${rule.name} in ${definition.id} uses the agent under test; a judge needs its own agent`);
+}
 function defineSuite(suite) {
 	authoringId(suite);
 	return suite;
@@ -1070,7 +1312,8 @@ function registerEvals(registrations) {
 const BUILT_IN_METRICS = [
 	"agent time",
 	"tokens",
-	"tool calls"
+	"tool calls",
+	"judge tokens"
 ];
 function recordError(error) {
 	if (error instanceof Error) return {
@@ -1084,4 +1327,4 @@ function recordError(error) {
 	};
 }
 //#endregion
-export { relativeDatasetPath as $, RunManifestSchema as A, turnResultEvents as B, CheckpointResultSchema as C, REPORT_SCHEMA_VERSION as D, MetricResultSchema as E, RunSummarySchema as F, DEFAULT_ATTACHMENT_LIMITS as G, canonicalParameters as H, ScoreResultSchema as I, NotJsonError as J, defineConfig as K, TrajectoryEventSchema as L, RunScoringSchema as M, RunSourceSchema as N, RecordedErrorSchema as O, RunStatusSchema as P, isDataset as Q, UsageSchema as R, BatchSchema as S, JudgeRunInfoSchema as T, turnTokens as U, defineEvalMatrix as V, DEFAULT_ARTIFACT_EXCLUDE as W, toJsonObject as X, toJson as Y, defineDataset as Z, parseTrajectoryJsonl as _, authoringId as a, parseResourceUri as at, AttachmentEntrySchema as b, defineSuite as c, resourceUuid as ct, metric as d, definitionSite as et, predicate as f, user as g, skip as h, agentRuntimes as i, authorId as it, RunMetadataSchema as j, RunDatasetSchema as k, expectToolCall as l, registerEvals as m, DEFAULT_AGENT_RUNTIMES as n, ResourceUriSchema as nt, defineAgent as o, resourceUri as ot, recordError as p, catalogEntry as q, agent as r, UuidSchema as rt, defineEval as s, resourceUriSchema as st, BUILT_IN_METRICS as t, ResourceKindSchema as tt, judge as u, ApiErrorSchema as v, JsonValueSchema as w, AutIdentitySchema as x, ArtifactEntrySchema as y, normalizeAutEvent as z };
+export { judgeInputNames as $, RunDatasetSchema as A, normalizeAutEvent as B, BatchSchema as C, MetricResultSchema as D, JudgeRunInfoSchema as E, RunStatusSchema as F, DEFAULT_ARTIFACT_EXCLUDE as G, defineEvalMatrix as H, RunSummarySchema as I, catalogEntry as J, DEFAULT_ATTACHMENT_LIMITS as K, ScoreResultSchema as L, RunMetadataSchema as M, RunScoringSchema as N, REPORT_SCHEMA_VERSION as O, RunSourceSchema as P, criteria as Q, TrajectoryEventSchema as R, AutIdentitySchema as S, JsonValueSchema as T, canonicalParameters as U, turnResultEvents as V, turnTokens as W, answerInstruction as X, UnreadableAnswer as Y, choices as Z, user as _, resourceUriSchema as _t, assertJudgesIndependent as a, toJson as at, ArtifactEntrySchema as b, defineEval as c, isDataset as ct, judge as d, ResourceKindSchema as dt, judgePrompt as et, metric as f, ResourceUriSchema as ft, skip as g, resourceUri as gt, registerEvals as h, parseResourceUri as ht, agentRuntimes as i, NotJsonError as it, RunManifestSchema as j, RecordedErrorSchema as k, defineSuite as l, relativeDatasetPath as lt, recordError as m, authorId as mt, DEFAULT_AGENT_RUNTIMES as n, readJudgeAnswer as nt, authoringId as o, toJsonObject as ot, predicate as p, UuidSchema as pt, defineConfig as q, agent as r, weightedMean as rt, defineAgent as s, defineDataset as st, BUILT_IN_METRICS as t, passMark as tt, expectToolCall as u, definitionSite as ut, parseTrajectoryJsonl as v, resourceUuid as vt, CheckpointResultSchema as w, AttachmentEntrySchema as x, ApiErrorSchema as y, UsageSchema as z };

package/dist/{src-Bn6fnZnF.mjs → src-Bccv9igQ.mjs}

@@ -1,4 +1,4 @@
-import { n as asyncIndex, r as createShareApp, t as ShareIndexDb } from "./index-db-BbVblaHz.mjs";
+import { n as asyncIndex, r as createShareApp, t as ShareIndexDb } from "./index-db-B1FeXzmS.mjs";
 import { mkdir, readFile, readdir, rename, rm, writeFile } from "node:fs/promises";
 import { dirname, join, relative, resolve, sep } from "node:path";
 //#region ../share/src/fs-files.ts