@fevex/core 0.1.0-alpha.10 → 0.1.0-alpha.12

package/dist/evals/index.mjs

@@ -1,3 +1,4 @@
+import { t as FevexRunError } from "../run-error-Cr17wPGH.mjs";
 //#region src/evals/canonical-json.ts
 function canonicalize(value) {
 	if (Array.isArray(value)) return value.map(canonicalize);
@@ -10,9 +11,13 @@ function serializeCanonical(value) {
 }
 //#endregion
 //#region src/evals/evaluation.ts
+const DEFAULT_EVAL_DEADLINE_MS = 10 * 6e4;
 function assertNonEmpty(value, name) {
 	if (typeof value !== "string" || !value.trim()) throw new TypeError(`${name} must be non-empty`);
 }
+function assertDeadline(value, name) {
+	if (value !== void 0 && (!Number.isSafeInteger(value) || value <= 0)) throw new TypeError(`${name} must be a positive safe integer`);
+}
 function errorMessage(error) {
 	return error instanceof Error ? error.message : String(error);
 }
@@ -44,11 +49,22 @@ function validateOptions(options) {
 	assertNonEmpty(options.dataset.name, "dataset.name");
 	assertNonEmpty(options.dataset.version, "dataset.version");
 	if (!Array.isArray(options.dataset.cases) || !options.dataset.cases.length) throw new TypeError("dataset.cases must contain at least one case");
+	assertDeadline(options.deadlineMs, "deadlineMs");
 	const caseIds = /* @__PURE__ */ new Set();
 	for (const testCase of options.dataset.cases) {
 		assertNonEmpty(testCase.id, "case.id");
 		if (caseIds.has(testCase.id)) throw new TypeError(`Eval case "${testCase.id}" is duplicated`);
 		caseIds.add(testCase.id);
+		assertDeadline(testCase.deadlineMs, `case "${testCase.id}" deadlineMs`);
+		for (const approval of testCase.approvals ?? []) {
+			if (approval.decision !== "approve" && approval.decision !== "reject") throw new TypeError(`Eval case "${testCase.id}" has an invalid approval decision`);
+			if (approval.toolName !== void 0 && !approval.toolName.trim()) throw new TypeError(`Eval case "${testCase.id}" approval toolName must be non-empty`);
+			if (approval.actor !== void 0 && !approval.actor.id.trim()) throw new TypeError(`Eval case "${testCase.id}" approval actor must be non-empty`);
+		}
+		for (const elicitation of testCase.elicitations ?? []) {
+			if (elicitation.prompt !== void 0 && !elicitation.prompt.trim()) throw new TypeError(`Eval case "${testCase.id}" elicitation prompt must be non-empty`);
+			if (elicitation.actor !== void 0 && !elicitation.actor.id.trim()) throw new TypeError(`Eval case "${testCase.id}" elicitation actor must be non-empty`);
+		}
 	}
 	if (!Array.isArray(options.scorers) || !options.scorers.length) throw new TypeError("scorers must contain at least one scorer");
 	const scorerIds = /* @__PURE__ */ new Set();
@@ -60,21 +76,187 @@ function validateOptions(options) {
 	}
 	if (options.reporters !== void 0 && (!Array.isArray(options.reporters) || options.reporters.some((reporter) => typeof reporter.report !== "function"))) throw new TypeError("reporters must implement report");
 }
-async function evaluateCase(options, testCase) {
-	options.signal?.throwIfAborted();
-	const startedAt = performance.now();
+function appendEvent(events, seen, event) {
+	if (seen.has(event.id)) return;
+	seen.add(event.id);
+	events.push(event);
+}
+function trajectoryFrom(events) {
+	const trajectory = [];
+	const tools = /* @__PURE__ */ new Map();
+	const toolNames = /* @__PURE__ */ new Map();
+	const elicitationPrompts = /* @__PURE__ */ new Map();
+	for (const event of [...events].sort((left, right) => left.sequence - right.sequence)) switch (event.type) {
+		case "elicitation.requested":
+			elicitationPrompts.set(event.payload.request.id, event.payload.request.prompt);
+			elicitationPrompts.set(event.payload.request.toolCallId, event.payload.request.prompt);
+			break;
+		case "approval.requested":
+			toolNames.set(event.payload.toolCallId, event.payload.toolName);
+			break;
+		case "tool.started": {
+			toolNames.set(event.payload.toolCallId, event.payload.toolName);
+			const startedAttempt = event.payload.attempt ?? 1;
+			for (const previous of tools.values()) if (previous.toolCallId === event.payload.toolCallId && previous.status === "retrying" && previous.attempt < startedAttempt) previous.status = "failed";
+			const entry = {
+				type: "tool",
+				step: event.payload.step,
+				toolCallId: event.payload.toolCallId,
+				name: event.payload.toolName,
+				attempt: event.payload.attempt ?? 1,
+				risk: event.payload.risk ?? "read",
+				...event.payload.declaredRisk === void 0 ? {} : { declaredRisk: event.payload.declaredRisk },
+				status: "started"
+			};
+			tools.set(`${entry.toolCallId}:${entry.attempt}`, entry);
+			trajectory.push(entry);
+			break;
+		}
+		case "tool.completed":
+		case "tool.failed":
+		case "tool.retrying":
+		case "tool.execution_unknown": {
+			const attempt = "attempt" in event.payload ? event.payload.attempt : void 0;
+			const targetAttempt = event.type === "tool.retrying" ? Math.max(1, (attempt ?? 1) - 1) : attempt;
+			const entry = targetAttempt === void 0 ? [...tools.values()].filter(({ toolCallId }) => toolCallId === event.payload.toolCallId).sort((left, right) => right.attempt - left.attempt)[0] : tools.get(`${event.payload.toolCallId}:${targetAttempt}`);
+			if (!entry) break;
+			entry.status = event.type === "tool.completed" ? "completed" : event.type === "tool.retrying" ? "retrying" : event.type === "tool.execution_unknown" ? "unknown" : "failed";
+			break;
+		}
+		case "approval.resolved":
+			trajectory.push({
+				type: "approval",
+				toolCallId: event.payload.toolCallId,
+				toolName: toolNames.get(event.payload.toolCallId) ?? event.payload.toolCallId,
+				decision: event.payload.decision,
+				actorId: event.payload.actorId
+			});
+			break;
+		case "elicitation.resolved": {
+			const prompt = elicitationPrompts.get(event.payload.requestId);
+			trajectory.push({
+				type: "elicitation",
+				requestId: event.payload.requestId,
+				toolCallId: event.payload.toolCallId,
+				...prompt === void 0 ? {} : { prompt },
+				actorId: event.payload.actorId
+			});
+			break;
+		}
+		case "policy.evaluated":
+			if (event.payload.decision !== "allow") trajectory.push({
+				type: "policy",
+				policyName: event.payload.policyName,
+				toolName: event.payload.toolName,
+				...event.payload.toolCallId === void 0 ? {} : { toolCallId: event.payload.toolCallId },
+				action: event.payload.action,
+				decision: event.payload.decision,
+				risk: event.payload.risk,
+				...event.payload.declaredRisk === void 0 ? {} : { declaredRisk: event.payload.declaredRisk }
+			});
+			break;
+	}
+	return trajectory;
+}
+function matchingApproval(approvals, index, toolName) {
+	const approval = approvals[index];
+	if (!approval) return void 0;
+	return approval.toolName === void 0 || approval.toolName === toolName ? approval : void 0;
+}
+function matchingElicitation(elicitations, index, prompt) {
+	const elicitation = elicitations[index];
+	if (!elicitation) return void 0;
+	return elicitation.prompt === void 0 || elicitation.prompt === prompt ? elicitation : void 0;
+}
+async function runCase(app, agentName, testCase, signal, deadlineMs) {
 	const events = [];
+	const seen = /* @__PURE__ */ new Set();
+	const deadlineAt = Date.now() + deadlineMs;
 	let executionError;
+	let timedOut = false;
+	let runId;
 	try {
-		for await (const event of options.app.streamAgent(options.agentName, {
+		for await (const event of app.streamAgent(agentName, {
 			input: testCase.input,
 			context: testCase.context,
-			signal: options.signal
-		})) events.push(event);
+			signal,
+			deadlineMs
+		})) {
+			runId ??= event.runId;
+			appendEvent(events, seen, event);
+		}
 	} catch (error) {
-		if (options.signal?.aborted) throw error;
+		if (signal?.aborted) throw error;
 		executionError = errorMessage(error);
 	}
+	const syncEvents = async () => {
+		if (!runId) return;
+		for (const event of await app.listEvents(runId)) appendEvent(events, seen, event);
+	};
+	await syncEvents();
+	const approvals = testCase.approvals ?? [];
+	const elicitations = testCase.elicitations ?? [];
+	let approvalIndex = 0;
+	let elicitationIndex = 0;
+	while (runId) {
+		signal?.throwIfAborted();
+		const run = await app.getRun(runId);
+		if (run?.status !== "paused") break;
+		let resolution;
+		if (run.pause?.type === "approval") {
+			const pause = run.pause.approval;
+			const approval = matchingApproval(approvals, approvalIndex, pause.toolName);
+			if (!approval) break;
+			approvalIndex += 1;
+			resolution = {
+				type: "approval",
+				approvalId: pause.id,
+				decision: approval.decision,
+				actor: approval.actor ?? { id: "eval" }
+			};
+		} else if (run.pause?.type === "elicitation") {
+			const request = run.pause.request;
+			const elicitation = matchingElicitation(elicitations, elicitationIndex, request.prompt);
+			if (!elicitation) break;
+			elicitationIndex += 1;
+			resolution = {
+				type: "elicitation",
+				requestId: request.id,
+				value: elicitation.value,
+				actor: elicitation.actor ?? { id: "eval" }
+			};
+		} else break;
+		try {
+			await app.resumeRun(runId, resolution);
+			await app.waitForRun(runId, {
+				signal,
+				timeoutMs: Math.max(0, deadlineAt - Date.now())
+			});
+		} catch (error) {
+			if (signal?.aborted) throw error;
+			if (error instanceof FevexRunError && error.code === "RUN_WAIT_TIMEOUT") {
+				timedOut = true;
+				executionError = "timeout";
+				await app.cancelRun(runId).catch(() => {});
+			} else executionError = errorMessage(error);
+			await syncEvents();
+			break;
+		}
+		await syncEvents();
+	}
+	const unused = [...approvals.slice(approvalIndex).map((approval) => approval.toolName === void 0 ? "Unused approval resolution" : `Unused approval resolution for tool "${approval.toolName}"`), ...elicitations.slice(elicitationIndex).map((elicitation) => elicitation.prompt === void 0 ? "Unused elicitation resolution" : `Unused elicitation resolution for prompt "${elicitation.prompt}"`)];
+	return {
+		events,
+		...executionError === void 0 ? {} : { executionError },
+		...timedOut ? { timedOut: true } : {},
+		unused
+	};
+}
+async function evaluateCase(options, testCase) {
+	options.signal?.throwIfAborted();
+	const startedAt = performance.now();
+	const deadlineMs = testCase.deadlineMs ?? options.deadlineMs ?? DEFAULT_EVAL_DEADLINE_MS;
+	const { events, executionError, timedOut, unused } = await runCase(options.app, options.agentName, testCase, options.signal, deadlineMs);
 	const latencyMs = performance.now() - startedAt;
 	const completed = events.find((event) => event.type === "run.completed");
 	const failed = [...events].reverse().find((event) => event.type === "run.failed" || event.type === "run.cancelled");
@@ -87,17 +269,19 @@ async function evaluateCase(options, testCase) {
 		selectedIds.add(event.payload.toolCallId);
 		selectedTools.push(event.payload.toolName);
 	}
+	const trajectory = trajectoryFrom(events);
 	const cost = options.cost && usage ? options.cost.calculate({
 		modelRegistryName: options.cost.modelRegistryName,
 		model: options.cost.model,
 		usage
 	}) : void 0;
 	if (cost && (!Number.isFinite(cost.amount) || cost.amount < 0 || !cost.currency.trim())) throw new TypeError("CostCalculator must return a non-negative amount and currency");
-	const runError = executionError ?? (failed?.type === "run.failed" ? failed.payload.error : failed?.type === "run.cancelled" ? failed.payload.reason : completed ? void 0 : "Run did not complete");
+	const runError = timedOut ? "timeout" : failed?.type === "run.failed" ? failed.payload.error : failed?.type === "run.cancelled" ? failed.payload.reason : executionError ?? (completed ? void 0 : "Run did not complete");
 	const context = {
 		testCase,
 		output,
 		tools: selectedTools,
+		trajectory,
 		events,
 		usage,
 		cost,
@@ -133,10 +317,12 @@ async function evaluateCase(options, testCase) {
 		status: completed ? "completed" : "failed",
 		...output === void 0 ? {} : { output },
 		tools: selectedTools,
+		trajectory,
 		...usage ? { usage } : {},
 		...cost ? { cost } : {},
 		latencyMs,
 		scores,
+		...unused.length ? { warnings: unused } : {},
 		...runError ? { error: runError } : {}
 	};
 }
@@ -159,6 +345,7 @@ function summarize(cases) {
 		failedScoreCount: scores.filter(({ status }) => status === "failed").length,
 		skippedScoreCount: scores.filter(({ status }) => status === "skipped").length,
 		errorScoreCount: scores.filter(({ status }) => status === "error").length,
+		warningCount: cases.reduce((sum, testCase) => sum + (testCase.warnings?.length ?? 0), 0),
 		...numericScores.length ? { averageScore: numericScores.reduce((sum, score) => sum + score, 0) / numericScores.length } : {},
 		...Object.keys(usage).length ? { usage } : {},
 		...totalCost ? { cost: totalCost } : {},
@@ -189,13 +376,37 @@ async function runEvaluation(options) {
 function serializeEvalReport(report) {
 	return `${serializeCanonical(report)}\n`;
 }
+const TOOL_RISKS$1 = /* @__PURE__ */ new Set([
+	"read",
+	"write",
+	"sensitive",
+	"destructive"
+]);
+const TOOL_STATUSES = /* @__PURE__ */ new Set([
+	"started",
+	"completed",
+	"failed",
+	"unknown",
+	"retrying"
+]);
+function isTrajectoryEntry(value) {
+	if (typeof value !== "object" || value === null) return false;
+	const entry = value;
+	if (entry.type === "tool") return typeof entry.step === "number" && Number.isFinite(entry.step) && typeof entry.toolCallId === "string" && typeof entry.name === "string" && typeof entry.attempt === "number" && Number.isSafeInteger(entry.attempt) && TOOL_RISKS$1.has(entry.risk) && (entry.declaredRisk === void 0 || TOOL_RISKS$1.has(entry.declaredRisk)) && TOOL_STATUSES.has(entry.status);
+	if (entry.type === "approval") return typeof entry.toolCallId === "string" && typeof entry.toolName === "string" && (entry.decision === "approve" || entry.decision === "reject") && typeof entry.actorId === "string";
+	if (entry.type === "elicitation") return typeof entry.requestId === "string" && typeof entry.toolCallId === "string" && (entry.prompt === void 0 || typeof entry.prompt === "string") && typeof entry.actorId === "string";
+	if (entry.type === "policy") return typeof entry.policyName === "string" && typeof entry.toolName === "string" && (entry.toolCallId === void 0 || typeof entry.toolCallId === "string") && (entry.action === "tool.execute" || entry.action === "approval.resolve") && (entry.decision === "deny" || entry.decision === "require_approval") && TOOL_RISKS$1.has(entry.risk) && (entry.declaredRisk === void 0 || TOOL_RISKS$1.has(entry.declaredRisk));
+	return false;
+}
 function parseEvalReport(json) {
 	const value = JSON.parse(json);
 	if (typeof value !== "object" || value === null || value.schemaVersion !== "fevex.eval.v1" || typeof value.suiteName !== "string" || typeof value.targetVersion !== "string" || typeof value.agentName !== "string" || typeof value.dataset !== "object" || value.dataset === null || typeof value.dataset.name !== "string" || typeof value.dataset.version !== "string" || !Array.isArray(value.cases) || !Array.isArray(value.scorerIds) || !value.scorerIds.every((scorerId) => typeof scorerId === "string" && Boolean(scorerId)) || typeof value.summary !== "object" || value.summary === null) throw new TypeError("Invalid fevex.eval.v1 report");
+	const warningCount = value.summary.warningCount;
+	if (warningCount !== void 0 && (typeof warningCount !== "number" || !Number.isSafeInteger(warningCount) || warningCount < 0)) throw new TypeError("Invalid fevex.eval.v1 report");
 	const report = value;
 	const caseIds = /* @__PURE__ */ new Set();
 	for (const testCase of report.cases) {
-		if (typeof testCase !== "object" || testCase === null || typeof testCase.caseId !== "string" || !testCase.caseId || caseIds.has(testCase.caseId) || !["completed", "failed"].includes(testCase.status) || !Array.isArray(testCase.tools) || !testCase.tools.every((tool) => typeof tool === "string") || !Number.isFinite(testCase.latencyMs) || testCase.latencyMs < 0 || !Array.isArray(testCase.scores)) throw new TypeError("Invalid fevex.eval.v1 report");
+		if (typeof testCase !== "object" || testCase === null || typeof testCase.caseId !== "string" || !testCase.caseId || caseIds.has(testCase.caseId) || !["completed", "failed"].includes(testCase.status) || !Array.isArray(testCase.tools) || !testCase.tools.every((tool) => typeof tool === "string") || testCase.trajectory !== void 0 && (!Array.isArray(testCase.trajectory) || !testCase.trajectory.every(isTrajectoryEntry)) || testCase.warnings !== void 0 && (!Array.isArray(testCase.warnings) || !testCase.warnings.every((warning) => typeof warning === "string")) || !Number.isFinite(testCase.latencyMs) || testCase.latencyMs < 0 || !Array.isArray(testCase.scores)) throw new TypeError("Invalid fevex.eval.v1 report");
 		caseIds.add(testCase.caseId);
 		for (const score of testCase.scores) if (typeof score !== "object" || score === null || typeof score.scorerId !== "string" || !report.scorerIds.includes(score.scorerId) || ![
 			"passed",
@@ -217,9 +428,10 @@ function textReporter(write) {
 			...report.cases.map((testCase) => {
 				const passed = testCase.scores.filter(({ status }) => status === "passed").length;
 				const failed = testCase.scores.filter(({ status }) => status === "failed" || status === "error").length;
-				return `${testCase.caseId}: ${testCase.status}; ${passed} passed, ${failed} failed`;
+				const warnings = testCase.warnings?.length ? `; ${testCase.warnings.length} warnings` : "";
+				return `${testCase.caseId}: ${testCase.status}; ${passed} passed, ${failed} failed${warnings}`;
 			}),
-			`summary: ${report.summary.completedCount}/${report.summary.caseCount} completed; ${report.summary.failedScoreCount + report.summary.errorScoreCount} failed scores`
+			`summary: ${report.summary.completedCount}/${report.summary.caseCount} completed; ${report.summary.failedScoreCount + report.summary.errorScoreCount} failed scores${report.summary.warningCount ? `; ${report.summary.warningCount} warnings` : ""}`
 		].join("\n")}\n`);
 	} };
 }
@@ -294,6 +506,112 @@ function compareEvaluationReports(baseline, current, options = {}) {
 		regressions
 	};
 }
+function assertRate(value, name) {
+	if (value !== void 0 && (!Number.isFinite(value) || value < 0 || value > 1)) throw new TypeError(`${name} must be between 0 and 1`);
+}
+function assertCount(value, name) {
+	if (value !== void 0 && (!Number.isSafeInteger(value) || value < 0)) throw new TypeError(`${name} must be a non-negative safe integer`);
+}
+function scorecardFrom(report) {
+	const { summary } = report;
+	const scored = summary.passedScoreCount + summary.failedScoreCount + summary.errorScoreCount;
+	return {
+		caseCount: summary.caseCount,
+		completedCount: summary.completedCount,
+		completionRate: summary.caseCount === 0 ? 0 : summary.completedCount / summary.caseCount,
+		passedScoreCount: summary.passedScoreCount,
+		failedScoreCount: summary.failedScoreCount,
+		skippedScoreCount: summary.skippedScoreCount,
+		errorScoreCount: summary.errorScoreCount,
+		warningCount: summary.warningCount ?? 0,
+		passRate: scored === 0 ? 0 : summary.passedScoreCount / scored,
+		...summary.averageScore === void 0 ? {} : { averageScore: summary.averageScore },
+		latencyMs: summary.latencyMs,
+		...summary.cost === void 0 ? {} : { cost: summary.cost }
+	};
+}
+function gateCheck(name, passed, message, expected, actual) {
+	return {
+		name,
+		passed,
+		message,
+		expected,
+		actual
+	};
+}
+/**
+* Evaluates minimum scorecard thresholds and an optional baseline. A trajectory
+* or policy regression appears as a failed score and blocks the gate even when
+* the final output is correct.
+*/
+function evaluateReleaseGate(report, options = {}) {
+	assertRate(options.minCompletionRate, "minCompletionRate");
+	assertRate(options.minPassRate, "minPassRate");
+	assertRate(options.minAverageScore, "minAverageScore");
+	assertCount(options.maxFailedScores, "maxFailedScores");
+	assertCount(options.maxErrorScores, "maxErrorScores");
+	assertCount(options.maxWarnings, "maxWarnings");
+	if (options.maxLatencyMs !== void 0 && (!Number.isFinite(options.maxLatencyMs) || options.maxLatencyMs < 0)) throw new TypeError("maxLatencyMs must be non-negative");
+	if (options.maxCost) {
+		if (!Number.isFinite(options.maxCost.amount) || options.maxCost.amount < 0) throw new TypeError("maxCost.amount must be non-negative");
+		if (!options.maxCost.currency.trim()) throw new TypeError("maxCost.currency must be non-empty");
+	}
+	for (const scorerId of options.requiredScorers ?? []) assertNonEmpty(scorerId, "requiredScorers entry");
+	const scorecard = scorecardFrom(report);
+	const checks = [];
+	const minCompletionRate = options.minCompletionRate ?? 1;
+	checks.push(gateCheck("completion_rate", scorecard.completionRate >= minCompletionRate, `Completion rate ${scorecard.completionRate} (minimum ${minCompletionRate})`, minCompletionRate, scorecard.completionRate));
+	const minPassRate = options.minPassRate ?? 1;
+	checks.push(gateCheck("pass_rate", scorecard.passRate >= minPassRate, `Pass rate ${scorecard.passRate} (minimum ${minPassRate})`, minPassRate, scorecard.passRate));
+	const maxFailedScores = options.maxFailedScores ?? 0;
+	checks.push(gateCheck("failed_scores", scorecard.failedScoreCount <= maxFailedScores, `Failed scores ${scorecard.failedScoreCount} (maximum ${maxFailedScores})`, maxFailedScores, scorecard.failedScoreCount));
+	const maxErrorScores = options.maxErrorScores ?? 0;
+	checks.push(gateCheck("error_scores", scorecard.errorScoreCount <= maxErrorScores, `Scorer errors ${scorecard.errorScoreCount} (maximum ${maxErrorScores})`, maxErrorScores, scorecard.errorScoreCount));
+	const maxWarnings = options.maxWarnings ?? 0;
+	checks.push(gateCheck("warnings", scorecard.warningCount <= maxWarnings, `Warnings ${scorecard.warningCount} (maximum ${maxWarnings})`, maxWarnings, scorecard.warningCount));
+	if (options.minAverageScore !== void 0) checks.push(gateCheck("average_score", scorecard.averageScore !== void 0 && scorecard.averageScore >= options.minAverageScore, `Average score ${scorecard.averageScore ?? "unavailable"} (minimum ${options.minAverageScore})`, options.minAverageScore, scorecard.averageScore ?? null));
+	if (options.requiredScorers?.length) {
+		const present = new Set(report.scorerIds);
+		const passed = new Set(report.cases.flatMap(({ scores }) => scores.filter(({ status }) => status === "passed").map(({ scorerId }) => scorerId)));
+		const missing = options.requiredScorers.filter((scorerId) => !present.has(scorerId));
+		const unverified = options.requiredScorers.filter((scorerId) => present.has(scorerId) && !passed.has(scorerId));
+		const problems = [...missing.length ? [`missing: ${missing.join(", ")}`] : [], ...unverified.length ? [`without a passing result: ${unverified.join(", ")}`] : []];
+		checks.push(gateCheck("required_scorers", problems.length === 0, problems.length ? `Required scorer problems: ${problems.join("; ")}` : "All required scorers passed at least once", [...options.requiredScorers], report.scorerIds));
+	}
+	if (options.maxLatencyMs !== void 0) checks.push(gateCheck("latency", scorecard.latencyMs <= options.maxLatencyMs, `Latency ${scorecard.latencyMs}ms (maximum ${options.maxLatencyMs}ms)`, options.maxLatencyMs, scorecard.latencyMs));
+	if (options.maxCost) {
+		const cost = scorecard.cost;
+		const passed = cost !== void 0 && cost.currency === options.maxCost.currency && cost.amount <= options.maxCost.amount;
+		checks.push(gateCheck("cost", passed, `Cost ${cost === void 0 ? "is unavailable" : `${cost.amount} ${cost.currency}`} (maximum ${options.maxCost.amount} ${options.maxCost.currency})`, options.maxCost.amount, cost?.amount ?? null));
+	}
+	const regressions = options.baseline === void 0 ? [] : compareEvaluationReports(options.baseline, report, options.regression).regressions;
+	for (const regression of regressions) checks.push({
+		name: `regression.${regression.kind}`,
+		passed: false,
+		message: regression.message,
+		...regression.scorerId === void 0 ? {} : { actual: regression.scorerId },
+		expected: null
+	});
+	return {
+		passed: checks.every(({ passed }) => passed),
+		scorecard,
+		checks,
+		regressions
+	};
+}
+var ReleaseGateError = class extends Error {
+	result;
+	name = "ReleaseGateError";
+	constructor(result) {
+		super(`Release gate failed: ${result.checks.filter(({ passed }) => !passed).map(({ message }) => message).join("; ")}`);
+		this.result = result;
+	}
+};
+function assertReleaseGate(report, options = {}) {
+	const result = evaluateReleaseGate(report, options);
+	if (!result.passed) throw new ReleaseGateError(result);
+	return result;
+}
 //#endregion
 //#region src/evals/scorers.ts
 function exactOutputScorer(id = "output.exact") {
@@ -418,5 +736,153 @@ function maxCostScorer(maxAmount, currency, id = "cost.max") {
 		}
 	};
 }
+function matchesExpectation(actual, expected) {
+	if (actual.type !== expected.type) return false;
+	for (const [key, value] of Object.entries(expected)) {
+		if (key === "type" || value === void 0) continue;
+		if (actual[key] !== value) return false;
+	}
+	return true;
+}
+function scoreTrajectory(trajectory, expected, options) {
+	const ordered = options.ordered ?? true;
+	const allowAdditional = options.allowAdditional ?? false;
+	const types = new Set(expected.map(({ type }) => type));
+	const selected = trajectory.filter(({ type }) => types.has(type));
+	let passed;
+	if (ordered && !allowAdditional) passed = expected.every((step, index) => {
+		const actual = selected[index];
+		return actual !== void 0 && matchesExpectation(actual, step);
+	}) && selected.length === expected.length;
+	else if (ordered) {
+		let cursor = 0;
+		passed = expected.every((step) => {
+			while (cursor < selected.length && !matchesExpectation(selected[cursor], step)) cursor += 1;
+			if (cursor >= selected.length) return false;
+			cursor += 1;
+			return true;
+		});
+	} else {
+		const remaining = [...selected];
+		passed = expected.every((step) => {
+			const index = remaining.findIndex((actual) => matchesExpectation(actual, step));
+			if (index < 0) return false;
+			remaining.splice(index, 1);
+			return true;
+		}) && (allowAdditional || remaining.length === 0);
+	}
+	return {
+		passed,
+		score: passed ? 1 : 0,
+		details: {
+			expected,
+			actual: selected
+		}
+	};
+}
+/** Asserts the ordered tool, approval and non-allow policy decisions of a run. */
+function trajectoryScorer(expected, options = {}) {
+	return {
+		id: options.id ?? "trajectory",
+		score: ({ trajectory }) => scoreTrajectory(trajectory, expected, options)
+	};
+}
+/** Reads the expected trajectory from each dataset case. */
+function trajectoryExpectationScorer(options = {}) {
+	return {
+		id: options.id ?? "trajectory",
+		score({ testCase, trajectory }) {
+			const expected = testCase.expected?.trajectory;
+			if (!expected) return {
+				skipped: true,
+				reason: "No expected trajectory"
+			};
+			return scoreTrajectory(trajectory, expected, options);
+		}
+	};
+}
+function approvalScorer(expected, options = {}) {
+	return trajectoryScorer(expected, {
+		...options,
+		id: options.id ?? "approvals"
+	});
+}
+function elicitationScorer(expected, options = {}) {
+	return trajectoryScorer(expected, {
+		...options,
+		id: options.id ?? "elicitations"
+	});
+}
+function policyScorer(expected, options = {}) {
+	return trajectoryScorer(expected, {
+		...options,
+		id: options.id ?? "policies"
+	});
+}
+const TOOL_RISKS = [
+	"read",
+	"write",
+	"sensitive",
+	"destructive"
+];
+/** Asserts that risky tools respected the configured guardrail. */
+function riskGuardrailScorer(options = {}, id = "guardrails.risk") {
+	const deny = new Set(options.deny ?? ["destructive"]);
+	const requireApproval = new Set(options.requireApproval ?? ["sensitive"]);
+	const undeclaredRisk = options.undeclaredRisk ?? "allow";
+	if (![
+		"allow",
+		"require_approval",
+		"deny"
+	].includes(undeclaredRisk)) throw new TypeError("undeclaredRisk must be allow, require_approval or deny");
+	for (const risk of [...deny, ...requireApproval]) if (!TOOL_RISKS.includes(risk)) throw new TypeError(`Invalid tool risk: ${risk}`);
+	return {
+		id,
+		score({ trajectory }) {
+			const violations = [];
+			const approvals = new Map(trajectory.filter((step) => step.type === "approval").map((step) => [step.toolCallId, step]));
+			for (const step of trajectory) {
+				if (step.type !== "tool") continue;
+				const approval = approvals.get(step.toolCallId);
+				if (step.declaredRisk === void 0) {
+					if (undeclaredRisk === "deny") {
+						violations.push({
+							tool: step.name,
+							reason: "undeclared_risk_executed"
+						});
+						continue;
+					}
+					if (undeclaredRisk === "require_approval" && approval?.decision !== "approve") {
+						violations.push({
+							tool: step.name,
+							reason: "undeclared_risk_missing_approval"
+						});
+						continue;
+					}
+				}
+				if (deny.has(step.risk)) {
+					violations.push({
+						tool: step.name,
+						reason: "denied_risk_executed"
+					});
+					continue;
+				}
+				if (requireApproval.has(step.risk) && approval?.decision !== "approve") violations.push({
+					tool: step.name,
+					reason: "missing_approval"
+				});
+				if (approval?.decision === "reject") violations.push({
+					tool: step.name,
+					reason: "rejected_tool_executed"
+				});
+			}
+			return {
+				passed: violations.length === 0,
+				score: violations.length === 0 ? 1 : 0,
+				details: { violations }
+			};
+		}
+	};
+}
 //#endregion
-export { compareEvaluationReports, exactOutputScorer, forbiddenToolsScorer, jsonReporter, maxCostScorer, maxLatencyScorer, maxTokensScorer, parseEvalReport, runEvaluation, serializeCanonical, serializeEvalReport, textReporter, toolSelectionScorer };
+export { ReleaseGateError, approvalScorer, assertReleaseGate, compareEvaluationReports, elicitationScorer, evaluateReleaseGate, exactOutputScorer, forbiddenToolsScorer, jsonReporter, maxCostScorer, maxLatencyScorer, maxTokensScorer, parseEvalReport, policyScorer, riskGuardrailScorer, runEvaluation, serializeCanonical, serializeEvalReport, textReporter, toolSelectionScorer, trajectoryExpectationScorer, trajectoryScorer };

package/dist/{fevex-_KOsOsB8.d.mts → fevex-B4eNkGTa.d.mts}

@@ -1,13 +1,13 @@
-import { S as ConnectionDefinition, j as ToolDefinition, o as ToolEmbeddingCache } from "./index-DHcfIfh-.mjs";
-import { t as ModelGateway } from "./index-D7r0puBi.mjs";
-import { F as ArtifactStore, f as RunId, o as ExecutionContext, t as AgentEvent } from "./types-CckDlhYo.mjs";
-import { r as Sandbox } from "./index-DUhNa7IR.mjs";
-import { B as ResumeRunResolution, I as ListEventsOptions, J as RunRequest, O as CredentialStore, Q as Session, R as PurgeRunsOptions, Y as RunResult, Z as RunStore, a as ToolSearchOptions, d as ContextCompactionStats, dt as WorkflowRun, et as SessionId, q as RunRecord, r as ContextBudget, rt as TeamRun, t as AgentDefinition, u as ContextCompactionOptions, x as AgentRun, z as PurgeRunsResult } from "./index-lCCFokc6.mjs";
-import { c as ContextProvider, f as MemoryStore } from "./index-CIHWc5V1.mjs";
-import { i as ObservabilityStats, r as ObservabilityOptions } from "./index-BEqYWean.mjs";
-import { r as PolicyDefinition } from "./index-CFzrvDQ0.mjs";
-import { n as WorkflowDefinition } from "./index-Btth7rdL.mjs";
-import { t as TeamDefinition } from "./index-26ilbzAs.mjs";
+import { S as ConnectionDefinition, j as ToolDefinition, o as ToolEmbeddingCache } from "./index-AOouohIx.mjs";
+import { t as ModelGateway } from "./index-Cbe1ZiyN.mjs";
+import { F as ArtifactStore, f as RunId, o as ExecutionContext, t as AgentEvent } from "./types-B6hD1o2I.mjs";
+import { r as Sandbox } from "./index-k-gBmfJP.mjs";
+import { B as ResumeRunResolution, I as ListEventsOptions, J as RunRequest, O as CredentialStore, Q as Session, R as PurgeRunsOptions, Y as RunResult, Z as RunStore, a as ToolSearchOptions, d as ContextCompactionStats, dt as WorkflowRun, et as SessionId, q as RunRecord, r as ContextBudget, rt as TeamRun, t as AgentDefinition, u as ContextCompactionOptions, x as AgentRun, z as PurgeRunsResult } from "./index-giFR5t0x.mjs";
+import { c as ContextProvider, f as MemoryStore } from "./index-D1LAhzgC.mjs";
+import { i as ObservabilityStats, r as ObservabilityOptions } from "./index-B7fxPgn_.mjs";
+import { r as PolicyDefinition } from "./index-KKq-_SiK.mjs";
+import { n as WorkflowDefinition } from "./index-CAVMQT9j.mjs";
+import { t as TeamDefinition } from "./index-DMmrhb8l.mjs";
 //#region src/fevex.d.ts
 interface FevexAdmissionRequest {
   kind: 'agent' | 'workflow' | 'team';
@@ -93,6 +93,15 @@ interface Fevex {
   streamTeam<TInput = unknown, TOutput = unknown>(name: string, request: RunRequest<TInput, TOutput>): AsyncIterable<AgentEvent>;
   getTeamRun<TOutput = unknown>(runId: RunId): Promise<TeamRun<TOutput> | undefined>;
   getRun<TOutput = unknown>(runId: RunId): Promise<RunRecord<TOutput> | undefined>;
+  /**
+   * Waits until a run leaves `running` (paused or terminal) and returns its
+   * latest record. Resolves immediately for paused and terminal runs, and with
+   * `undefined` when the run does not exist.
+   */
+  waitForRun<TOutput = unknown>(runId: RunId, options?: {
+    signal?: AbortSignal;
+    timeoutMs?: number;
+  }): Promise<RunRecord<TOutput> | undefined>;
   listEvents(runId: RunId, options?: ListEventsOptions): Promise<AgentEvent[]>;
   cancelRun(runId: RunId): Promise<boolean>;
   resumeRun<TOutput = unknown>(runId: RunId, resolution: ResumeRunResolution): Promise<RunRecord<TOutput>>;

package/dist/http/index.d.mts

@@ -1,7 +1,7 @@
-import { c as ReasoningEffort } from "../index-D7r0puBi.mjs";
-import { D as ArtifactCatalog, F as ArtifactStore, f as RunId, k as ArtifactDescriptor, l as JsonValue, o as ExecutionContext, t as AgentEvent } from "../types-CckDlhYo.mjs";
-import { dt as WorkflowRun, et as SessionId, q as RunRecord, rt as TeamRun, x as AgentRun } from "../index-lCCFokc6.mjs";
-import { t as Fevex } from "../fevex-_KOsOsB8.mjs";
+import { c as ReasoningEffort } from "../index-Cbe1ZiyN.mjs";
+import { D as ArtifactCatalog, F as ArtifactStore, f as RunId, k as ArtifactDescriptor, l as JsonValue, o as ExecutionContext, t as AgentEvent } from "../types-B6hD1o2I.mjs";
+import { dt as WorkflowRun, et as SessionId, q as RunRecord, rt as TeamRun, x as AgentRun } from "../index-giFR5t0x.mjs";
+import { t as Fevex } from "../fevex-B4eNkGTa.mjs";
 //#region src/http/protocol.d.ts
 declare const FEVEX_HTTP_PROTOCOL_VERSION = "3";
 declare const FEVEX_HTTP_PROTOCOL_VERSION_HEADER = "Fevex-Protocol-Version";

package/dist/{index-DHcfIfh-.d.mts → index-AOouohIx.d.mts}

@@ -1,6 +1,6 @@
-import { u as EmbeddingModel } from "./index-D7r0puBi.mjs";
-import { F as ArtifactStore, f as RunId, g as ToolSpec, h as ToolEventSource, l as JsonValue, o as ExecutionContext, s as JsonObject } from "./types-CckDlhYo.mjs";
-import { i as SandboxCapabilities, r as Sandbox } from "./index-DUhNa7IR.mjs";
+import { u as EmbeddingModel } from "./index-Cbe1ZiyN.mjs";
+import { F as ArtifactStore, f as RunId, g as ToolSpec, h as ToolEventSource, l as JsonValue, o as ExecutionContext, s as JsonObject } from "./types-B6hD1o2I.mjs";
+import { i as SandboxCapabilities, r as Sandbox } from "./index-k-gBmfJP.mjs";
 import { StandardSchemaV1 } from "@standard-schema/spec";
 //#region src/internal/exact-definition.d.ts
 type ExactDefinition<TInput, TShape> = TInput & { readonly [TKey in Exclude<keyof TInput, keyof TShape>]: never; };