@kindgi/client 0.1.4-rc.0 → 0.1.4-rc.1

package/dist/index.d.ts

@@ -4238,6 +4238,19 @@ declare namespace Schemas {
 		version: string;
 		conversationId: string;
 	};
+	/**
+	 * Agents and tools a flow runs at other exact versions than the flow version's pins ("this flow, with `acme.scorer` at 0.4.0"), without publishing a new flow version: a comparison's flow candidate (`versions` on the start body and the run's `comparison`), and the flow runs that replay it (a run's `versions`). Each id must be an agent or tool the flow uses.
+	 */
+	export type FlowVersionOverrides = Partial<{
+		/**
+		 * Tool id → exact version.
+		 */
+		tools: Record<string, string>;
+		/**
+		 * Agent id → exact version, for agent nodes with or without a version of their own.
+		 */
+		agents: Record<string, string>;
+	}>;
 	export type Run = {
 		/**
 		 * RunId.
@@ -4277,6 +4290,7 @@ declare namespace Schemas {
 		 * Set on a replay run: the eval run that started it.
 		 */
 		evalRunId?: string;
+		versions?: FlowVersionOverrides;
 		/**
 		 * Only in the response to `POST /v1/runs`, when the deployment issues public run tokens: a read-only token for this run (and its descendants) to hand to a browser, for `GET /v1/runs/{runId}/progress` and its stream.
 		 */
@@ -4394,6 +4408,7 @@ declare namespace Schemas {
 			 * Set on a replay run: the eval run that started it.
 			 */
 			evalRunId?: string;
+			versions?: FlowVersionOverrides;
 			/**
 			 * Only in the response to `POST /v1/runs`, when the deployment issues public run tokens: a read-only token for this run (and its descendants) to hand to a browser, for `GET /v1/runs/{runId}/progress` and its stream.
 			 */
@@ -7115,6 +7130,199 @@ declare namespace Schemas {
 		reads: "recorded" | "live";
 		repetitions: number;
 		k: number;
+		versions?: FlowVersionOverrides;
+	};
+	/**
+	 * One metric, the recorded runs beside the candidate. `null` where a side had no judged evidence; `n` / `weight` are the candidate's evidence (cases with judged items, and the judgment weight behind them), `baselineN` / `baselineWeight` the recorded side's.
+	 */
+	export type ComparisonMetric = {
+		baseline: number | null;
+		candidate: number | null;
+		delta: number | null;
+		n: number;
+		weight: number;
+		baselineN: number;
+		baselineWeight: number;
+		direction: "higher";
+		/**
+		 * `weightedPrecisionAtK`: the ranked items it looked at.
+		 */
+		k?: number;
+		/**
+		 * With more than one repetition: the candidate's max − min across them.
+		 */
+		spread?: number;
+	};
+	/**
+	 * What ran on the cases: an agent version, or a flow version (with any versions it swapped in).
+	 */
+	export type ComparisonCandidate = {
+		kind: "agent";
+		agentId: string;
+		version: string;
+	} | {
+		kind: "flow";
+		flowId: string;
+		version: string;
+		versions?: FlowVersionOverrides;
+	};
+	/**
+	 * What a comparison concluded: the candidate beside the recorded runs, the case counts, and the metrics. What a promotion gate reads.
+	 */
+	export type JudgedComparisonSummary = {
+		evalRunId: string;
+		status: "completed" | "partial" | "failed";
+		completedAt: string;
+		suite: {
+			id: string;
+			version: string;
+		};
+		candidate: ComparisonCandidate;
+		/**
+		 * What the candidate was compared with: `recorded` (the test set's recorded runs, with the versions that served them), or another version.
+		 */
+		baseline: {
+			kind: "recorded";
+			versions: Array<{
+				agentId?: string;
+				flowId?: string;
+				version: string;
+				cases: number;
+			}>;
+		} | {
+			kind: "version";
+			agentId: string;
+			version: string;
+			via: "explicit" | "live";
+			liveScope?: Record<string, unknown>;
+		};
+		/**
+		 * Where the test set's judgments came from.
+		 */
+		scope: Partial<{
+			projectId: string;
+		}>;
+		cases: number;
+		/**
+		 * Cases where a read with no recording ran live under `reads: 'recorded'`.
+		 */
+		diverged: number;
+		/**
+		 * Tool calls refused across the cases (what the candidate would have done).
+		 */
+		refusedWrites: number;
+		/**
+		 * Cases none of whose repetitions ran.
+		 */
+		errors: number;
+		/**
+		 * Flow cases that stopped at a write the replay refused: no output to score, so they're left out of the metrics.
+		 */
+		stopped: number;
+		reads: "recorded" | "live";
+		sampling: {
+			/**
+			 * The models that answered the candidate's replays, and how many replays each.
+			 */
+			models: Array<{
+				providerId: string;
+				model: string;
+				runs: number;
+			}>;
+		};
+		repetitions: number;
+		metrics: {
+			weightedYesShare: ComparisonMetric;
+			judgedCoverage: ComparisonMetric;
+			weightedPrecisionAtK: ComparisonMetric;
+		};
+	};
+	/**
+	 * One case of a comparison: its replay runs, the scores, the items kept, dropped and new, the tool calls, and why it didn't run when it didn't.
+	 */
+	export type ComparisonCaseResult = {
+		caseId: string;
+		/**
+		 * The candidate's replay runs, one per repetition.
+		 */
+		runIds: Array<string>;
+		/**
+		 * An output's score: Σ yesWeight and Σ totalWeight over its judged items, and over those among the first `k` ranked items.
+		 */
+		baseline: {
+			yesWeight: number;
+			totalWeight: number;
+			items: number;
+			judgedItems: number;
+			topK: {
+				yesWeight: number;
+				totalWeight: number;
+			};
+		};
+		/**
+		 * One per repetition that ran.
+		 */
+		candidate: Array<{
+			yesWeight: number;
+			totalWeight: number;
+			items: number;
+			judgedItems: number;
+			topK: {
+				yesWeight: number;
+				totalWeight: number;
+			};
+		}>;
+		/**
+		 * The first repetition's items against the judged ones.
+		 */
+		changes?: {
+			kept: Array<{
+				key: string;
+				rankBefore?: number;
+				rank?: number;
+			}>;
+			dropped: Array<{
+				key: string;
+				rankBefore?: number;
+			}>;
+			new: Array<{
+				key: string;
+				pointer: string;
+				rank?: number;
+			}>;
+		};
+		/**
+		 * The first repetition's tool calls, and what happened to each.
+		 */
+		tools?: Array<{
+			step: number;
+			callId: string;
+			toolId: string;
+			toolVersion: string;
+			arguments: unknown;
+			source: "live" | "recorded" | "refused";
+			reason?: string;
+		}>;
+		diverged: boolean;
+		refusedWrites: number;
+		noContext: boolean;
+		approvalSkipped: boolean;
+		error?: string;
+		/**
+		 * Set when the replay stopped at a refused write: what it would have done.
+		 */
+		stopped?: {
+			toolId: string;
+			arguments: unknown;
+			reason?: string;
+		};
+	};
+	/**
+	 * A comparison's `result` (a `judged` eval run's): the summary and each case. `EvalRun.result` stays an open object, since each kind has its own; the clients read it as this (TS `comparisonOf(run)`, Python `comparison_of(run)`).
+	 */
+	export type JudgedComparisonResult = {
+		summary: JudgedComparisonSummary;
+		perCase: Array<ComparisonCaseResult>;
 	};
 	export type EvalRun = {
 		runId: string;
@@ -7129,7 +7337,7 @@ declare namespace Schemas {
 		startedAt: string;
 		completedAt?: string;
 		/**
-		 * Kind-specific opaque JSON. For `accuracy`, contains `{ passCount, totalCount, meanScore, perCase[] }`. For `judged` (a comparison), `{ summary, perCase[] }`: the summary has the baseline (the versions behind the recorded runs) and the candidate (`{ kind: "agent", agentId, version }` or `{ kind: "flow", flowId, version }`), the case counts (`cases`, `diverged`, `refusedWrites`, `errors`, and `stopped`: flow cases that stopped at a write the replay refused, left out of the metrics), the models that answered, and `metrics` (`weightedYesShare`, `judgedCoverage`, `weightedPrecisionAtK`, each `{ baseline, candidate, delta, n, weight, baselineN, baselineWeight, direction, k?, spread? }`); each case has its replay runs, the scores, the items kept, dropped and new, the tool calls with what happened to each, and `stopped` (what it would have done) when it stopped. Other kinds define their own shapes as their dispatchers ship.
+		 * Kind-specific opaque JSON. For `accuracy`, contains `{ passCount, totalCount, meanScore, perCase[] }`. For `judged` (a comparison), `{ summary, perCase[] }`: the summary has the baseline (the versions behind the recorded runs) and the candidate (`{ kind: "agent", agentId, version }` or `{ kind: "flow", flowId, version, versions? }`), the case counts (`cases`, `diverged`, `refusedWrites`, `errors`, and `stopped`: flow cases that stopped at a write the replay refused, left out of the metrics), the models that answered, and `metrics` (`weightedYesShare`, `judgedCoverage`, `weightedPrecisionAtK`, each `{ baseline, candidate, delta, n, weight, baselineN, baselineWeight, direction, k?, spread? }`); each case has its replay runs, the scores, the items kept, dropped and new, the tool calls with what happened to each, and `stopped` (what it would have done) when it stopped. Other kinds define their own shapes as their dispatchers ship.
 		 */
 		result?: Record<string, unknown>;
 		error?: string;
@@ -7142,7 +7350,7 @@ declare namespace Schemas {
 		hasMore: boolean;
 	};
 	/**
-	 * Exactly one of `agentRef` or `flowRef` MUST be supplied. `dryRun: true` returns a plan preview without invoking the subject. For a `judged` suite (a test set), the run is a comparison: `agentRef` or `flowRef` with its `version` is the candidate, replayed on each case without doing anything the past run didn't (a flow stops at a write the replay refuses); `baseline` (default `'recorded'`), `reads` (default `recorded`), `repetitions` (default 1) and `k` (default 10) set how.
+	 * Exactly one of `agentRef` or `flowRef` MUST be supplied. `dryRun: true` returns a plan preview without invoking the subject. For a `judged` suite (a test set), the run is a comparison: `agentRef` or `flowRef` with its `version` is the candidate, replayed on each case without doing anything the past run didn't (a flow stops at a write the replay refuses); `baseline` (default `'recorded'`), `reads` (default `recorded`), `repetitions` (default 1) and `k` (default 10) set how. With `flowRef`, `versions` runs the flow with some of its agents or tools at other versions; an id the flow doesn't use, or a version that isn't published, is refused (`400 validation-failed`, each under `details.issues`).
 	 */
 	export type StartEvalRunBody = {
 		/**
@@ -7157,6 +7365,7 @@ declare namespace Schemas {
 		reads?: "recorded" | "live";
 		repetitions?: number;
 		k?: number;
+		versions?: FlowVersionOverrides;
 	};
 	export type StartEvalRunResult = {
 		runId: string;
@@ -9019,6 +9228,383 @@ declare const EvalRunStatus: z.ZodEnum<{
 	completed: "completed";
 	cancelled: "cancelled";
 }>;
+export type ComparisonMetric = Schemas.ComparisonMetric;
+export declare const ComparisonMetric: z.ZodObject<{
+	baseline: z.ZodNullable<z.ZodNumber>;
+	candidate: z.ZodNullable<z.ZodNumber>;
+	delta: z.ZodNullable<z.ZodNumber>;
+	n: z.ZodNumber;
+	weight: z.ZodNumber;
+	baselineN: z.ZodNumber;
+	baselineWeight: z.ZodNumber;
+	direction: z.ZodLiteral<"higher">;
+	k: z.ZodOptional<z.ZodNumber>;
+	spread: z.ZodOptional<z.ZodNumber>;
+}, z.core.$strict>;
+export type ComparisonCandidate = Schemas.ComparisonCandidate;
+export declare const ComparisonCandidate: z.ZodUnion<readonly [
+	z.ZodObject<{
+		kind: z.ZodLiteral<"agent">;
+		agentId: z.ZodString;
+		version: z.ZodString;
+	}, z.core.$strict>,
+	z.ZodObject<{
+		kind: z.ZodLiteral<"flow">;
+		flowId: z.ZodString;
+		version: z.ZodString;
+		versions: z.ZodOptional<z.ZodObject<{
+			tools: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+			agents: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+		}, z.core.$strict>>;
+	}, z.core.$strict>
+]>;
+export type JudgedComparisonSummary = Schemas.JudgedComparisonSummary;
+export declare const JudgedComparisonSummary: z.ZodObject<{
+	evalRunId: z.ZodString;
+	status: z.ZodEnum<{
+		failed: "failed";
+		completed: "completed";
+		partial: "partial";
+	}>;
+	completedAt: z.ZodISODateTime;
+	suite: z.ZodObject<{
+		id: z.ZodString;
+		version: z.ZodString;
+	}, z.core.$strict>;
+	candidate: z.ZodUnion<readonly [
+		z.ZodObject<{
+			kind: z.ZodLiteral<"agent">;
+			agentId: z.ZodString;
+			version: z.ZodString;
+		}, z.core.$strict>,
+		z.ZodObject<{
+			kind: z.ZodLiteral<"flow">;
+			flowId: z.ZodString;
+			version: z.ZodString;
+			versions: z.ZodOptional<z.ZodObject<{
+				tools: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+				agents: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+			}, z.core.$strict>>;
+		}, z.core.$strict>
+	]>;
+	baseline: z.ZodUnion<readonly [
+		z.ZodObject<{
+			kind: z.ZodLiteral<"recorded">;
+			versions: z.ZodArray<z.ZodObject<{
+				agentId: z.ZodOptional<z.ZodString>;
+				flowId: z.ZodOptional<z.ZodString>;
+				version: z.ZodString;
+				cases: z.ZodNumber;
+			}, z.core.$strict>>;
+		}, z.core.$strict>,
+		z.ZodObject<{
+			kind: z.ZodLiteral<"version">;
+			agentId: z.ZodString;
+			version: z.ZodString;
+			via: z.ZodEnum<{
+				live: "live";
+				explicit: "explicit";
+			}>;
+			liveScope: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+		}, z.core.$strict>
+	]>;
+	scope: z.ZodObject<{
+		projectId: z.ZodOptional<z.ZodString>;
+	}, z.core.$strict>;
+	cases: z.ZodNumber;
+	diverged: z.ZodNumber;
+	refusedWrites: z.ZodNumber;
+	errors: z.ZodNumber;
+	stopped: z.ZodNumber;
+	reads: z.ZodEnum<{
+		recorded: "recorded";
+		live: "live";
+	}>;
+	sampling: z.ZodObject<{
+		models: z.ZodArray<z.ZodObject<{
+			providerId: z.ZodString;
+			model: z.ZodString;
+			runs: z.ZodNumber;
+		}, z.core.$strict>>;
+	}, z.core.$strict>;
+	repetitions: z.ZodNumber;
+	metrics: z.ZodObject<{
+		weightedYesShare: z.ZodObject<{
+			baseline: z.ZodNullable<z.ZodNumber>;
+			candidate: z.ZodNullable<z.ZodNumber>;
+			delta: z.ZodNullable<z.ZodNumber>;
+			n: z.ZodNumber;
+			weight: z.ZodNumber;
+			baselineN: z.ZodNumber;
+			baselineWeight: z.ZodNumber;
+			direction: z.ZodLiteral<"higher">;
+			k: z.ZodOptional<z.ZodNumber>;
+			spread: z.ZodOptional<z.ZodNumber>;
+		}, z.core.$strict>;
+		judgedCoverage: z.ZodObject<{
+			baseline: z.ZodNullable<z.ZodNumber>;
+			candidate: z.ZodNullable<z.ZodNumber>;
+			delta: z.ZodNullable<z.ZodNumber>;
+			n: z.ZodNumber;
+			weight: z.ZodNumber;
+			baselineN: z.ZodNumber;
+			baselineWeight: z.ZodNumber;
+			direction: z.ZodLiteral<"higher">;
+			k: z.ZodOptional<z.ZodNumber>;
+			spread: z.ZodOptional<z.ZodNumber>;
+		}, z.core.$strict>;
+		weightedPrecisionAtK: z.ZodObject<{
+			baseline: z.ZodNullable<z.ZodNumber>;
+			candidate: z.ZodNullable<z.ZodNumber>;
+			delta: z.ZodNullable<z.ZodNumber>;
+			n: z.ZodNumber;
+			weight: z.ZodNumber;
+			baselineN: z.ZodNumber;
+			baselineWeight: z.ZodNumber;
+			direction: z.ZodLiteral<"higher">;
+			k: z.ZodOptional<z.ZodNumber>;
+			spread: z.ZodOptional<z.ZodNumber>;
+		}, z.core.$strict>;
+	}, z.core.$strict>;
+}, z.core.$strict>;
+export type ComparisonCaseResult = Schemas.ComparisonCaseResult;
+export declare const ComparisonCaseResult: z.ZodObject<{
+	caseId: z.ZodString;
+	runIds: z.ZodArray<z.ZodString>;
+	baseline: z.ZodObject<{
+		yesWeight: z.ZodNumber;
+		totalWeight: z.ZodNumber;
+		items: z.ZodNumber;
+		judgedItems: z.ZodNumber;
+		topK: z.ZodObject<{
+			yesWeight: z.ZodNumber;
+			totalWeight: z.ZodNumber;
+		}, z.core.$strict>;
+	}, z.core.$strict>;
+	candidate: z.ZodArray<z.ZodObject<{
+		yesWeight: z.ZodNumber;
+		totalWeight: z.ZodNumber;
+		items: z.ZodNumber;
+		judgedItems: z.ZodNumber;
+		topK: z.ZodObject<{
+			yesWeight: z.ZodNumber;
+			totalWeight: z.ZodNumber;
+		}, z.core.$strict>;
+	}, z.core.$strict>>;
+	changes: z.ZodOptional<z.ZodObject<{
+		kept: z.ZodArray<z.ZodObject<{
+			key: z.ZodString;
+			rankBefore: z.ZodOptional<z.ZodNumber>;
+			rank: z.ZodOptional<z.ZodNumber>;
+		}, z.core.$strict>>;
+		dropped: z.ZodArray<z.ZodObject<{
+			key: z.ZodString;
+			rankBefore: z.ZodOptional<z.ZodNumber>;
+		}, z.core.$strict>>;
+		new: z.ZodArray<z.ZodObject<{
+			key: z.ZodString;
+			pointer: z.ZodString;
+			rank: z.ZodOptional<z.ZodNumber>;
+		}, z.core.$strict>>;
+	}, z.core.$strict>>;
+	tools: z.ZodOptional<z.ZodArray<z.ZodObject<{
+		step: z.ZodNumber;
+		callId: z.ZodString;
+		toolId: z.ZodString;
+		toolVersion: z.ZodString;
+		arguments: z.ZodUnknown;
+		source: z.ZodEnum<{
+			recorded: "recorded";
+			live: "live";
+			refused: "refused";
+		}>;
+		reason: z.ZodOptional<z.ZodString>;
+	}, z.core.$strict>>>;
+	diverged: z.ZodBoolean;
+	refusedWrites: z.ZodNumber;
+	noContext: z.ZodBoolean;
+	approvalSkipped: z.ZodBoolean;
+	error: z.ZodOptional<z.ZodString>;
+	stopped: z.ZodOptional<z.ZodObject<{
+		toolId: z.ZodString;
+		arguments: z.ZodUnknown;
+		reason: z.ZodOptional<z.ZodString>;
+	}, z.core.$strict>>;
+}, z.core.$strict>;
+export type JudgedComparisonResult = Schemas.JudgedComparisonResult;
+export declare const JudgedComparisonResult: z.ZodObject<{
+	summary: z.ZodObject<{
+		evalRunId: z.ZodString;
+		status: z.ZodEnum<{
+			failed: "failed";
+			completed: "completed";
+			partial: "partial";
+		}>;
+		completedAt: z.ZodISODateTime;
+		suite: z.ZodObject<{
+			id: z.ZodString;
+			version: z.ZodString;
+		}, z.core.$strict>;
+		candidate: z.ZodUnion<readonly [
+			z.ZodObject<{
+				kind: z.ZodLiteral<"agent">;
+				agentId: z.ZodString;
+				version: z.ZodString;
+			}, z.core.$strict>,
+			z.ZodObject<{
+				kind: z.ZodLiteral<"flow">;
+				flowId: z.ZodString;
+				version: z.ZodString;
+				versions: z.ZodOptional<z.ZodObject<{
+					tools: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+					agents: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+				}, z.core.$strict>>;
+			}, z.core.$strict>
+		]>;
+		baseline: z.ZodUnion<readonly [
+			z.ZodObject<{
+				kind: z.ZodLiteral<"recorded">;
+				versions: z.ZodArray<z.ZodObject<{
+					agentId: z.ZodOptional<z.ZodString>;
+					flowId: z.ZodOptional<z.ZodString>;
+					version: z.ZodString;
+					cases: z.ZodNumber;
+				}, z.core.$strict>>;
+			}, z.core.$strict>,
+			z.ZodObject<{
+				kind: z.ZodLiteral<"version">;
+				agentId: z.ZodString;
+				version: z.ZodString;
+				via: z.ZodEnum<{
+					live: "live";
+					explicit: "explicit";
+				}>;
+				liveScope: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+			}, z.core.$strict>
+		]>;
+		scope: z.ZodObject<{
+			projectId: z.ZodOptional<z.ZodString>;
+		}, z.core.$strict>;
+		cases: z.ZodNumber;
+		diverged: z.ZodNumber;
+		refusedWrites: z.ZodNumber;
+		errors: z.ZodNumber;
+		stopped: z.ZodNumber;
+		reads: z.ZodEnum<{
+			recorded: "recorded";
+			live: "live";
+		}>;
+		sampling: z.ZodObject<{
+			models: z.ZodArray<z.ZodObject<{
+				providerId: z.ZodString;
+				model: z.ZodString;
+				runs: z.ZodNumber;
+			}, z.core.$strict>>;
+		}, z.core.$strict>;
+		repetitions: z.ZodNumber;
+		metrics: z.ZodObject<{
+			weightedYesShare: z.ZodObject<{
+				baseline: z.ZodNullable<z.ZodNumber>;
+				candidate: z.ZodNullable<z.ZodNumber>;
+				delta: z.ZodNullable<z.ZodNumber>;
+				n: z.ZodNumber;
+				weight: z.ZodNumber;
+				baselineN: z.ZodNumber;
+				baselineWeight: z.ZodNumber;
+				direction: z.ZodLiteral<"higher">;
+				k: z.ZodOptional<z.ZodNumber>;
+				spread: z.ZodOptional<z.ZodNumber>;
+			}, z.core.$strict>;
+			judgedCoverage: z.ZodObject<{
+				baseline: z.ZodNullable<z.ZodNumber>;
+				candidate: z.ZodNullable<z.ZodNumber>;
+				delta: z.ZodNullable<z.ZodNumber>;
+				n: z.ZodNumber;
+				weight: z.ZodNumber;
+				baselineN: z.ZodNumber;
+				baselineWeight: z.ZodNumber;
+				direction: z.ZodLiteral<"higher">;
+				k: z.ZodOptional<z.ZodNumber>;
+				spread: z.ZodOptional<z.ZodNumber>;
+			}, z.core.$strict>;
+			weightedPrecisionAtK: z.ZodObject<{
+				baseline: z.ZodNullable<z.ZodNumber>;
+				candidate: z.ZodNullable<z.ZodNumber>;
+				delta: z.ZodNullable<z.ZodNumber>;
+				n: z.ZodNumber;
+				weight: z.ZodNumber;
+				baselineN: z.ZodNumber;
+				baselineWeight: z.ZodNumber;
+				direction: z.ZodLiteral<"higher">;
+				k: z.ZodOptional<z.ZodNumber>;
+				spread: z.ZodOptional<z.ZodNumber>;
+			}, z.core.$strict>;
+		}, z.core.$strict>;
+	}, z.core.$strict>;
+	perCase: z.ZodArray<z.ZodObject<{
+		caseId: z.ZodString;
+		runIds: z.ZodArray<z.ZodString>;
+		baseline: z.ZodObject<{
+			yesWeight: z.ZodNumber;
+			totalWeight: z.ZodNumber;
+			items: z.ZodNumber;
+			judgedItems: z.ZodNumber;
+			topK: z.ZodObject<{
+				yesWeight: z.ZodNumber;
+				totalWeight: z.ZodNumber;
+			}, z.core.$strict>;
+		}, z.core.$strict>;
+		candidate: z.ZodArray<z.ZodObject<{
+			yesWeight: z.ZodNumber;
+			totalWeight: z.ZodNumber;
+			items: z.ZodNumber;
+			judgedItems: z.ZodNumber;
+			topK: z.ZodObject<{
+				yesWeight: z.ZodNumber;
+				totalWeight: z.ZodNumber;
+			}, z.core.$strict>;
+		}, z.core.$strict>>;
+		changes: z.ZodOptional<z.ZodObject<{
+			kept: z.ZodArray<z.ZodObject<{
+				key: z.ZodString;
+				rankBefore: z.ZodOptional<z.ZodNumber>;
+				rank: z.ZodOptional<z.ZodNumber>;
+			}, z.core.$strict>>;
+			dropped: z.ZodArray<z.ZodObject<{
+				key: z.ZodString;
+				rankBefore: z.ZodOptional<z.ZodNumber>;
+			}, z.core.$strict>>;
+			new: z.ZodArray<z.ZodObject<{
+				key: z.ZodString;
+				pointer: z.ZodString;
+				rank: z.ZodOptional<z.ZodNumber>;
+			}, z.core.$strict>>;
+		}, z.core.$strict>>;
+		tools: z.ZodOptional<z.ZodArray<z.ZodObject<{
+			step: z.ZodNumber;
+			callId: z.ZodString;
+			toolId: z.ZodString;
+			toolVersion: z.ZodString;
+			arguments: z.ZodUnknown;
+			source: z.ZodEnum<{
+				recorded: "recorded";
+				live: "live";
+				refused: "refused";
+			}>;
+			reason: z.ZodOptional<z.ZodString>;
+		}, z.core.$strict>>>;
+		diverged: z.ZodBoolean;
+		refusedWrites: z.ZodNumber;
+		noContext: z.ZodBoolean;
+		approvalSkipped: z.ZodBoolean;
+		error: z.ZodOptional<z.ZodString>;
+		stopped: z.ZodOptional<z.ZodObject<{
+			toolId: z.ZodString;
+			arguments: z.ZodUnknown;
+			reason: z.ZodOptional<z.ZodString>;
+		}, z.core.$strict>>;
+	}, z.core.$strict>>;
+}, z.core.$strict>;
 export type EvalRun = Schemas.EvalRun;
 declare const EvalRun: z.ZodObject<{
 	runId: z.ZodUUID;
@@ -9075,6 +9661,10 @@ declare const EvalRun: z.ZodObject<{
 		}>;
 		repetitions: z.ZodNumber;
 		k: z.ZodNumber;
+		versions: z.ZodOptional<z.ZodObject<{
+			tools: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+			agents: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+		}, z.core.$strict>>;
 	}, z.core.$strict>>;
 }, z.core.$strict>;
 export type EvalRunCollectionPage = Schemas.EvalRunCollectionPage;
@@ -9134,6 +9724,10 @@ declare const EvalRunCollectionPage: z.ZodObject<{
 			}>;
 			repetitions: z.ZodNumber;
 			k: z.ZodNumber;
+			versions: z.ZodOptional<z.ZodObject<{
+				tools: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+				agents: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+			}, z.core.$strict>>;
 		}, z.core.$strict>>;
 	}, z.core.$strict>>;
 	nextCursor: z.ZodOptional<z.ZodString>;
@@ -9171,6 +9765,10 @@ declare const StartEvalRunBody: z.ZodObject<{
 	}>>;
 	repetitions: z.ZodOptional<z.ZodNumber>;
 	k: z.ZodOptional<z.ZodNumber>;
+	versions: z.ZodOptional<z.ZodObject<{
+		tools: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+		agents: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+	}, z.core.$strict>>;
 }, z.core.$strict>;
 export type StartEvalRunResult = Schemas.StartEvalRunResult;
 declare const StartEvalRunResult: z.ZodObject<{
@@ -11338,6 +11936,13 @@ export type EvalRunPage = EvalRunCollectionPage;
  */
 export type StartEvalRunInput = Omit<StartEvalRunBody, "projectId">;
 export type StartEvalRunOutcome = StartEvalRunResult;
+/**
+ * A comparison's result, typed from the OpenAPI `JudgedComparisonResult`:
+ * its summary and each case. `run` is a `judged` eval run (a test set
+ * compared with a version); `undefined` for another kind of eval run, a
+ * dry run, or one that hasn't finished.
+ */
+export declare function comparisonOf(run: EvalRunRecord): JudgedComparisonResult | undefined;
 export interface StartEvalRunOptions {
 	/** Project the eval run belongs to. The route requires it. */
 	readonly projectId: string;