@tangle-network/agent-eval 0.220.2 → 0.221.0

package/dist/meta-eval/index.js

@@ -3,204 +3,14 @@ import { a as hashCanonical, i as compareCodeUnits } from "../canonical-Dq2hOQ8L
 import { t as mulberry32 } from "../random-DDDcC7YU.js";
 import { a as selfPreference, i as positionalBias, n as calibrateJudgeContinuous, o as verbosityBias, r as continuousAgreement, t as calibrateJudge } from "../judge-calibration-BkXMIWA4.js";
 import { s as runMetricExtractor } from "../query-B_RZ36an.js";
+import { a as validateObservationOptions, i as reduceOutcomeMetric, n as correlationSummary, r as hasVariation, t as assertUniqueObservationIds } from "../outcome-observations-FLmRpgYa.js";
+import { a as rankedProbabilityScore, i as multiclassBrierScore, n as calibrationCurve, r as calibrationFromPairs, t as brierScore } from "../calibration-CTwxpJ-t.js";
 import { a as judgeGateDecision, c as snapshotFromJudgeAgreement, d as judgeSentinelReport, f as snapshotFromAgreement, h as validateSentinelSnapshot, i as assertJudgeMayGate, l as evalHealthStamp, m as snapshotFromSentinelSet, n as JudgeGateRefusedError, o as measureJudgeAgreement, p as snapshotFromCalibration, r as addCalibrationVerdicts, s as registerCalibrationSet, t as DEFAULT_JUDGE_GATE_POLICY, u as fileSentinelStore } from "../judge-gate-Co5tuh57.js";
 import { t as computeInterval } from "../interval-Dq9gt7jx.js";
 import { t as clusterBootstrapMean } from "../cluster-bootstrap-DEwwhl0C.js";
-import { a as reduceOutcomeMetric, i as hasVariation, n as assertUniqueObservationIds, o as validateObservationOptions, r as correlationSummary, t as rubricPredictiveValidity } from "../rubric-predictive-validity-D3_oI72N.js";
+import { t as rubricPredictiveValidity } from "../rubric-predictive-validity-CeKW2GQN.js";
 import { n as InMemoryOutcomeStore, r as OutcomeStoreError, t as FileSystemOutcomeStore } from "../outcome-store-DUcMj-3D.js";
 import { z } from "zod";
-//#region src/meta-eval/calibration.ts
-/**
-* Calibration curve — binned "if eval says X, what does reality show?"
-*
-* Companion to correlationStudy. Raw correlation is a single number;
-* the calibration curve shows *where* the eval is well-calibrated vs
-* overconfident / underconfident. Buckets the eval metric, computes
-* mean outcome per bucket, reports expected-calibration-error (ECE).
-*/
-async function calibrationCurve(traceStore, outcomeStore, evalMetric, outcomeMetric, options = {}) {
-	const settings = {
-		...options,
-		range: options.range === void 0 ? void 0 : { ...options.range }
-	};
-	validateCalibrationRequest(evalMetric.id, outcomeMetric, settings);
-	const extract = evalMetric.extract ?? runMetricExtractor(evalMetric.id);
-	const metricId = evalMetric.id;
-	const runs = await traceStore.listRuns();
-	assertUniqueObservationIds(runs.map((run) => run.runId), "runId");
-	const outcomes = await outcomeStore.list();
-	const byRun = /* @__PURE__ */ new Map();
-	for (const o of outcomes) {
-		const arr = byRun.get(o.runId) ?? [];
-		arr.push(o);
-		byRun.set(o.runId, arr);
-	}
-	const pairs = [];
-	for (const run of runs) {
-		const os = byRun.get(run.runId);
-		if (!os?.length) continue;
-		const x = await extract(run, traceStore);
-		if (x === null || !Number.isFinite(x)) continue;
-		const y = reduceOutcomeMetric(os, outcomeMetric, "latest");
-		if (y === null) continue;
-		pairs.push({
-			x,
-			y
-		});
-	}
-	if (pairs.length < 2) return null;
-	return calibrationFromPairs(pairs.map((p) => ({
-		evalScore: p.x,
-		outcome: p.y
-	})), metricId, outcomeMetric, settings);
-}
-/** Measure already joined observations without constructing trace and outcome stores. */
-function calibrationFromPairs(inputPairs, evalMetric, outcomeMetric, options = {}) {
-	validateCalibrationRequest(evalMetric, outcomeMetric, options);
-	for (const [index, pair] of inputPairs.entries()) if (pair === null || typeof pair !== "object" || !Number.isFinite(pair.evalScore) || !Number.isFinite(pair.outcome)) throw new Error(`calibration pair ${index} must contain finite evalScore and outcome values`);
-	const pairs = inputPairs;
-	if (pairs.length < 2) return null;
-	const numBins = options.bins ?? 10;
-	const binning = options.binning ?? "equal-width";
-	const xs = pairs.map((p) => p.evalScore);
-	const lo = options.range?.lo ?? Math.min(...xs);
-	const hi = options.range?.hi ?? Math.max(...xs);
-	const span = hi - lo;
-	if (!Number.isFinite(span)) throw new Error("calibration range span must be finite");
-	const clipped = pairs.map((pair) => ({
-		...pair,
-		evalScore: Math.min(hi, Math.max(lo, pair.evalScore))
-	}));
-	const bins = [];
-	if (span === 0 || clipped.every((pair) => pair.evalScore === clipped[0].evalScore)) bins.push(toBin(clipped));
-	else if (binning === "equal-frequency") {
-		const sorted = [...clipped].sort((a, b) => a.evalScore - b.evalScore);
-		const count = Math.min(numBins, sorted.length);
-		for (let i = 0; i < count; i++) {
-			const start = Math.floor(i * sorted.length / count);
-			const end = Math.floor((i + 1) * sorted.length / count);
-			bins.push(toBin(sorted.slice(start, end)));
-		}
-	} else {
-		const groups = /* @__PURE__ */ new Map();
-		for (const pair of clipped) {
-			const index = Math.min(numBins - 1, Math.floor((pair.evalScore - lo) / span * numBins));
-			const group = groups.get(index) ?? [];
-			group.push(pair);
-			groups.set(index, group);
-		}
-		for (const [index, chunk] of [...groups].sort(([a], [b]) => a - b)) bins.push(toBin(chunk, lo + span * (index / numBins), lo + span * ((index + 1) / numBins)));
-	}
-	const total = bins.reduce((a, b) => a + b.n, 0);
-	const ece = bins.reduce((a, b) => a + b.n / total * b.gap, 0);
-	const maxGap = bins.reduce((a, b) => Math.max(a, b.gap), 0);
-	return {
-		evalMetric,
-		outcomeMetric,
-		n: pairs.length,
-		bins,
-		ece,
-		maxGap
-	};
-}
-/**
-* Mean squared error of probability forecasts against binary outcomes, Σ(p − y)²/N; always
-* forecasting 0.5 scores 0.25. For the reliability table and ECE use `calibrationFromPairs`
-* with `range: { lo: 0, hi: 1 }` (`binning: 'equal-frequency'` gives equal-mass bins).
-*/
-function brierScore(pairs) {
-	if (!pairs.length) throw new Error("Brier score requires at least one pair");
-	let total = 0;
-	for (const [index, pair] of pairs.entries()) {
-		if (pair === null || typeof pair !== "object" || !(pair.evalScore >= 0 && pair.evalScore <= 1) || pair.outcome !== 0 && pair.outcome !== 1) throw new Error(`Brier pair ${index} needs a probability in [0, 1] and an outcome of 0 or 1`);
-		total += (pair.evalScore - pair.outcome) ** 2;
-	}
-	return total / pairs.length;
-}
-/**
-* Multiclass Brier score in its sum form, mean over forecasts of Σₖ(pₖ − yₖ)², range [0, 2].
-* With two outcomes it is twice the binary `brierScore`. A uniform forecast over K outcomes
-* scores 1 − 1/K. The distribution is not renormalized here: repair provider rounding before
-* scoring (`normalizeSystemOneResult` in `/systemone/protocol`), so a defective answer stays
-* visible instead of being scored as if it were valid.
-*/
-function multiclassBrierScore(forecasts) {
-	if (!forecasts.length) throw new Error("multiclass Brier score requires at least one forecast");
-	let total = 0;
-	for (const [index, forecast] of forecasts.entries()) {
-		const entries = categoricalEntries(forecast, index, "multiclass Brier");
-		for (const [name, probability] of entries) total += (probability - (name === forecast.outcome ? 1 : 0)) ** 2;
-	}
-	return total / forecasts.length;
-}
-/**
-* Ranked probability score, mean over forecasts of Σₖ₌₁ᴷ⁻¹(Fₖ − Oₖ)² / (K − 1), where F and O are
-* the cumulative forecast and observation, range [0, 1]. Unlike multiclass Brier it charges
-* less for mass on a level next to the outcome than for mass far from it, which is the right
-* penalty for score questions. With two levels it equals the binary Brier score.
-*/
-function rankedProbabilityScore(forecasts) {
-	if (!forecasts.length) throw new Error("ranked probability score requires at least one forecast");
-	let total = 0;
-	for (const [index, forecast] of forecasts.entries()) {
-		const probabilities = forecast?.probabilities;
-		if (!Array.isArray(probabilities) || probabilities.length < 2) throw new Error(`RPS forecast ${index} needs at least two level probabilities`);
-		assertDistribution(probabilities, `RPS forecast ${index}`);
-		if (!Number.isInteger(forecast.outcome) || forecast.outcome < 0 || forecast.outcome >= probabilities.length) throw new Error(`RPS forecast ${index} outcome must index one of its ${probabilities.length} levels`);
-		let cumulative = 0;
-		let sum = 0;
-		for (let level = 0; level < probabilities.length - 1; level++) {
-			cumulative += probabilities[level];
-			sum += (cumulative - (level >= forecast.outcome ? 1 : 0)) ** 2;
-		}
-		total += sum / (probabilities.length - 1);
-	}
-	return total / forecasts.length;
-}
-function categoricalEntries(forecast, index, label) {
-	if (forecast === null || typeof forecast !== "object" || typeof forecast.outcome !== "string") throw new Error(`${label} forecast ${index} needs probabilities and a string outcome`);
-	const probabilities = forecast.probabilities;
-	if (probabilities === null || typeof probabilities !== "object" || Array.isArray(probabilities)) throw new Error(`${label} forecast ${index} probabilities must be an object keyed by outcome`);
-	const entries = Object.entries(probabilities);
-	if (entries.length < 2) throw new Error(`${label} forecast ${index} needs at least two outcomes`);
-	if (!entries.some(([name]) => name === forecast.outcome)) throw new Error(`${label} forecast ${index} has no probability for outcome ${JSON.stringify(forecast.outcome)}`);
-	assertDistribution(entries.map(([, probability]) => probability), `${label} forecast ${index}`);
-	return entries;
-}
-function assertDistribution(probabilities, label) {
-	let sum = 0;
-	for (const probability of probabilities) {
-		if (!(probability >= 0 && probability <= 1)) throw new Error(`${label} has a probability outside [0, 1]: ${probability}`);
-		sum += probability;
-	}
-	if (Math.abs(sum - 1) > 1e-6) throw new Error(`${label} probabilities sum to ${sum}, not 1`);
-}
-function toBin(chunk, lower, upper) {
-	const xs = chunk.map((c) => c.evalScore);
-	const ys = chunk.map((c) => c.outcome);
-	const evalMean = mean(xs);
-	const outcomeMean = mean(ys);
-	return {
-		lower: lower ?? Math.min(...xs),
-		upper: upper ?? Math.max(...xs),
-		n: chunk.length,
-		evalMean,
-		outcomeMean,
-		gap: Math.abs(outcomeMean - evalMean)
-	};
-}
-function mean(xs) {
-	return xs.reduce((sum, value) => sum + value / xs.length, 0);
-}
-function validateCalibrationRequest(evalMetric, outcomeMetric, options) {
-	assertUniqueObservationIds([evalMetric], "eval metric");
-	assertUniqueObservationIds([outcomeMetric], "outcome metric");
-	if (evalMetric.trim() !== evalMetric || outcomeMetric.trim() !== outcomeMetric) throw new Error("calibration metric identities must not have surrounding whitespace");
-	if (options.bins !== void 0 && (!Number.isSafeInteger(options.bins) || options.bins < 1)) throw new Error("calibration bins must be a positive safe integer");
-	if (options.binning !== void 0 && !["equal-width", "equal-frequency"].includes(options.binning)) throw new Error("calibration binning must be equal-width or equal-frequency");
-	if (options.range !== void 0 && (!Number.isFinite(options.range.lo) || !Number.isFinite(options.range.hi) || !Number.isFinite(options.range.hi - options.range.lo) || options.range.hi < options.range.lo)) throw new Error("calibration range must have finite ordered bounds");
-}
-//#endregion
 //#region src/meta-eval/correlation-study.ts
 /**
 * Correlation study — "does our eval score predict real-world outcomes?"

package/dist/meta-eval/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","names":[],"sources":["../../src/meta-eval/calibration.ts","../../src/meta-eval/correlation-study.ts","../../src/meta-eval/evaluator-admission.ts","../../src/meta-eval/plants.ts"],"sourcesContent":["/**\n * Calibration curve — binned \"if eval says X, what does reality show?\"\n *\n * Companion to correlationStudy. Raw correlation is a single number;\n * the calibration curve shows *where* the eval is well-calibrated vs\n * overconfident / underconfident. Buckets the eval metric, computes\n * mean outcome per bucket, reports expected-calibration-error (ECE).\n */\n\nimport { runMetricExtractor } from '../trace/query'\nimport type { TraceStore } from '../trace/store'\nimport type { EvalMetricSpec } from './correlation-study'\nimport { assertUniqueObservationIds, reduceOutcomeMetric } from './outcome-observations'\nimport type { DeploymentOutcome, OutcomeStore } from './outcome-store'\n\nexport interface CalibrationBin {\n  lower: number\n  upper: number\n  n: number\n  evalMean: number\n  outcomeMean: number\n  /** |outcomeMean − evalMean|; contributes to ECE weighted by n/total. */\n  gap: number\n}\n\nexport interface CalibrationReport {\n  evalMetric: string\n  outcomeMetric: string\n  n: number\n  bins: CalibrationBin[]\n  /** Expected Calibration Error — Σ (n_i/N) × |outcomeMean_i − evalMean_i|. */\n  ece: number\n  /** Largest observed difference between a bin's mean score and mean outcome. */\n  maxGap: number\n}\n\nexport interface CalibrationOptions {\n  /** Positive integer; empty bins are omitted. Default 10. */\n  bins?: number\n  /** Equal-width (fixed bin edges) or equal-frequency (quantile bins). */\n  binning?: 'equal-width' | 'equal-frequency'\n  /** Clip eval values to [lo, hi] before binning. */\n  range?: { lo: number; hi: number }\n}\n\nexport interface CalibrationPair {\n  evalScore: number\n  outcome: number\n}\n\nexport async function calibrationCurve(\n  traceStore: TraceStore,\n  outcomeStore: OutcomeStore,\n  evalMetric: EvalMetricSpec,\n  outcomeMetric: string,\n  options: CalibrationOptions = {},\n): Promise<CalibrationReport | null> {\n  const settings = {\n    ...options,\n    range: options.range === undefined ? undefined : { ...options.range },\n  }\n  validateCalibrationRequest(evalMetric.id, outcomeMetric, settings)\n  const extract = evalMetric.extract ?? runMetricExtractor(evalMetric.id)\n  const metricId = evalMetric.id\n  const runs = await traceStore.listRuns()\n  assertUniqueObservationIds(\n    runs.map((run) => run.runId),\n    'runId',\n  )\n  const outcomes = await outcomeStore.list()\n  const byRun = new Map<string, DeploymentOutcome[]>()\n  for (const o of outcomes) {\n    const arr = byRun.get(o.runId) ?? []\n    arr.push(o)\n    byRun.set(o.runId, arr)\n  }\n\n  const pairs: Array<{ x: number; y: number }> = []\n  for (const run of runs) {\n    const os = byRun.get(run.runId)\n    if (!os?.length) continue\n    const x = await extract(run, traceStore)\n    if (x === null || !Number.isFinite(x)) continue\n    const y = reduceOutcomeMetric(os, outcomeMetric, 'latest')\n    if (y === null) continue\n    pairs.push({ x, y })\n  }\n  if (pairs.length < 2) return null\n\n  return calibrationFromPairs(\n    pairs.map((p) => ({ evalScore: p.x, outcome: p.y })),\n    metricId,\n    outcomeMetric,\n    settings,\n  )\n}\n\n/** Measure already joined observations without constructing trace and outcome stores. */\nexport function calibrationFromPairs(\n  inputPairs: readonly CalibrationPair[],\n  evalMetric: string,\n  outcomeMetric: string,\n  options: CalibrationOptions = {},\n): CalibrationReport | null {\n  validateCalibrationRequest(evalMetric, outcomeMetric, options)\n  for (const [index, pair] of inputPairs.entries()) {\n    if (\n      pair === null ||\n      typeof pair !== 'object' ||\n      !Number.isFinite(pair.evalScore) ||\n      !Number.isFinite(pair.outcome)\n    ) {\n      throw new Error(`calibration pair ${index} must contain finite evalScore and outcome values`)\n    }\n  }\n  const pairs = inputPairs\n  if (pairs.length < 2) return null\n\n  const numBins = options.bins ?? 10\n  const binning = options.binning ?? 'equal-width'\n  const xs = pairs.map((p) => p.evalScore)\n  const lo = options.range?.lo ?? Math.min(...xs)\n  const hi = options.range?.hi ?? Math.max(...xs)\n  const span = hi - lo\n  if (!Number.isFinite(span)) throw new Error('calibration range span must be finite')\n  const clipped = pairs.map((pair) => ({\n    ...pair,\n    evalScore: Math.min(hi, Math.max(lo, pair.evalScore)),\n  }))\n\n  const bins: CalibrationBin[] = []\n  if (span === 0 || clipped.every((pair) => pair.evalScore === clipped[0]!.evalScore)) {\n    bins.push(toBin(clipped))\n  } else if (binning === 'equal-frequency') {\n    const sorted = [...clipped].sort((a, b) => a.evalScore - b.evalScore)\n    const count = Math.min(numBins, sorted.length)\n    for (let i = 0; i < count; i++) {\n      const start = Math.floor((i * sorted.length) / count)\n      const end = Math.floor(((i + 1) * sorted.length) / count)\n      bins.push(toBin(sorted.slice(start, end)))\n    }\n  } else {\n    const groups = new Map<number, CalibrationPair[]>()\n    for (const pair of clipped) {\n      const index = Math.min(numBins - 1, Math.floor(((pair.evalScore - lo) / span) * numBins))\n      const group = groups.get(index) ?? []\n      group.push(pair)\n      groups.set(index, group)\n    }\n    for (const [index, chunk] of [...groups].sort(([a], [b]) => a - b)) {\n      bins.push(toBin(chunk, lo + span * (index / numBins), lo + span * ((index + 1) / numBins)))\n    }\n  }\n\n  const total = bins.reduce((a, b) => a + b.n, 0)\n  const ece = bins.reduce((a, b) => a + (b.n / total) * b.gap, 0)\n  const maxGap = bins.reduce((a, b) => Math.max(a, b.gap), 0)\n\n  return { evalMetric, outcomeMetric, n: pairs.length, bins, ece, maxGap }\n}\n\n/**\n * Mean squared error of probability forecasts against binary outcomes, Σ(p − y)²/N; always\n * forecasting 0.5 scores 0.25. For the reliability table and ECE use `calibrationFromPairs`\n * with `range: { lo: 0, hi: 1 }` (`binning: 'equal-frequency'` gives equal-mass bins).\n */\nexport function brierScore(pairs: readonly CalibrationPair[]): number {\n  if (!pairs.length) throw new Error('Brier score requires at least one pair')\n  let total = 0\n  for (const [index, pair] of pairs.entries()) {\n    if (\n      pair === null ||\n      typeof pair !== 'object' ||\n      !(pair.evalScore >= 0 && pair.evalScore <= 1) ||\n      (pair.outcome !== 0 && pair.outcome !== 1)\n    ) {\n      throw new Error(`Brier pair ${index} needs a probability in [0, 1] and an outcome of 0 or 1`)\n    }\n    total += (pair.evalScore - pair.outcome) ** 2\n  }\n  return total / pairs.length\n}\n\n/** One categorical forecast: a distribution over named outcomes and the outcome that occurred. */\nexport interface CategoricalForecast {\n  /** Probability per outcome name. Must sum to 1 within 1e-6 and include `outcome`. */\n  probabilities: Readonly<Record<string, number>>\n  outcome: string\n}\n\n/**\n * Multiclass Brier score in its sum form, mean over forecasts of Σₖ(pₖ − yₖ)², range [0, 2].\n * With two outcomes it is twice the binary `brierScore`. A uniform forecast over K outcomes\n * scores 1 − 1/K. The distribution is not renormalized here: repair provider rounding before\n * scoring (`normalizeSystemOneResult` in `/systemone/protocol`), so a defective answer stays\n * visible instead of being scored as if it were valid.\n */\nexport function multiclassBrierScore(forecasts: readonly CategoricalForecast[]): number {\n  if (!forecasts.length) throw new Error('multiclass Brier score requires at least one forecast')\n  let total = 0\n  for (const [index, forecast] of forecasts.entries()) {\n    const entries = categoricalEntries(forecast, index, 'multiclass Brier')\n    for (const [name, probability] of entries) {\n      total += (probability - (name === forecast.outcome ? 1 : 0)) ** 2\n    }\n  }\n  return total / forecasts.length\n}\n\n/** One ordinal forecast: probabilities in level order, lowest level first. */\nexport interface OrdinalForecast {\n  probabilities: readonly number[]\n  /** Index into `probabilities` of the level that occurred. */\n  outcome: number\n}\n\n/**\n * Ranked probability score, mean over forecasts of Σₖ₌₁ᴷ⁻¹(Fₖ − Oₖ)² / (K − 1), where F and O are\n * the cumulative forecast and observation, range [0, 1]. Unlike multiclass Brier it charges\n * less for mass on a level next to the outcome than for mass far from it, which is the right\n * penalty for score questions. With two levels it equals the binary Brier score.\n */\nexport function rankedProbabilityScore(forecasts: readonly OrdinalForecast[]): number {\n  if (!forecasts.length) throw new Error('ranked probability score requires at least one forecast')\n  let total = 0\n  for (const [index, forecast] of forecasts.entries()) {\n    const probabilities = forecast?.probabilities\n    if (!Array.isArray(probabilities) || probabilities.length < 2) {\n      throw new Error(`RPS forecast ${index} needs at least two level probabilities`)\n    }\n    assertDistribution(probabilities, `RPS forecast ${index}`)\n    if (\n      !Number.isInteger(forecast.outcome) ||\n      forecast.outcome < 0 ||\n      forecast.outcome >= probabilities.length\n    ) {\n      throw new Error(\n        `RPS forecast ${index} outcome must index one of its ${probabilities.length} levels`,\n      )\n    }\n    let cumulative = 0\n    let sum = 0\n    for (let level = 0; level < probabilities.length - 1; level++) {\n      cumulative += probabilities[level]!\n      sum += (cumulative - (level >= forecast.outcome ? 1 : 0)) ** 2\n    }\n    total += sum / (probabilities.length - 1)\n  }\n  return total / forecasts.length\n}\n\nfunction categoricalEntries(\n  forecast: CategoricalForecast,\n  index: number,\n  label: string,\n): [string, number][] {\n  if (forecast === null || typeof forecast !== 'object' || typeof forecast.outcome !== 'string') {\n    throw new Error(`${label} forecast ${index} needs probabilities and a string outcome`)\n  }\n  const probabilities = forecast.probabilities\n  if (probabilities === null || typeof probabilities !== 'object' || Array.isArray(probabilities)) {\n    throw new Error(`${label} forecast ${index} probabilities must be an object keyed by outcome`)\n  }\n  const entries = Object.entries(probabilities)\n  if (entries.length < 2) throw new Error(`${label} forecast ${index} needs at least two outcomes`)\n  if (!entries.some(([name]) => name === forecast.outcome)) {\n    throw new Error(\n      `${label} forecast ${index} has no probability for outcome ${JSON.stringify(forecast.outcome)}`,\n    )\n  }\n  assertDistribution(\n    entries.map(([, probability]) => probability),\n    `${label} forecast ${index}`,\n  )\n  return entries\n}\n\nfunction assertDistribution(probabilities: readonly number[], label: string): void {\n  let sum = 0\n  for (const probability of probabilities) {\n    if (!(probability >= 0 && probability <= 1)) {\n      throw new Error(`${label} has a probability outside [0, 1]: ${probability}`)\n    }\n    sum += probability\n  }\n  if (Math.abs(sum - 1) > 1e-6) throw new Error(`${label} probabilities sum to ${sum}, not 1`)\n}\n\nfunction toBin(chunk: CalibrationPair[], lower?: number, upper?: number): CalibrationBin {\n  const xs = chunk.map((c) => c.evalScore)\n  const ys = chunk.map((c) => c.outcome)\n  const evalMean = mean(xs)\n  const outcomeMean = mean(ys)\n  return {\n    lower: lower ?? Math.min(...xs),\n    upper: upper ?? Math.max(...xs),\n    n: chunk.length,\n    evalMean,\n    outcomeMean,\n    gap: Math.abs(outcomeMean - evalMean),\n  }\n}\n\nfunction mean(xs: number[]): number {\n  return xs.reduce((sum, value) => sum + value / xs.length, 0)\n}\n\nfunction validateCalibrationRequest(\n  evalMetric: string,\n  outcomeMetric: string,\n  options: CalibrationOptions,\n): void {\n  assertUniqueObservationIds([evalMetric], 'eval metric')\n  assertUniqueObservationIds([outcomeMetric], 'outcome metric')\n  if (evalMetric.trim() !== evalMetric || outcomeMetric.trim() !== outcomeMetric) {\n    throw new Error('calibration metric identities must not have surrounding whitespace')\n  }\n  if (options.bins !== undefined && (!Number.isSafeInteger(options.bins) || options.bins < 1)) {\n    throw new Error('calibration bins must be a positive safe integer')\n  }\n  if (\n    options.binning !== undefined &&\n    !['equal-width', 'equal-frequency'].includes(options.binning)\n  ) {\n    throw new Error('calibration binning must be equal-width or equal-frequency')\n  }\n  if (\n    options.range !== undefined &&\n    (!Number.isFinite(options.range.lo) ||\n      !Number.isFinite(options.range.hi) ||\n      !Number.isFinite(options.range.hi - options.range.lo) ||\n      options.range.hi < options.range.lo)\n  ) {\n    throw new Error('calibration range must have finite ordered bounds')\n  }\n}\n","/**\n * Correlation study — \"does our eval score predict real-world outcomes?\"\n *\n * Joins traces and outcomes by runId and reports descriptive correlations.\n * Independent runs are the bootstrap observation unit.\n * Association alone does not establish causation or held-out predictive performance.\n */\n\nimport { runMetricExtractor } from '../trace/query'\nimport type { Run } from '../trace/schema'\nimport type { TraceStore } from '../trace/store'\nimport {\n  assertUniqueObservationIds,\n  correlationSummary,\n  hasVariation,\n  reduceOutcomeMetric,\n  validateObservationOptions,\n} from './outcome-observations'\nimport type { DeploymentOutcome, OutcomeFilter, OutcomeStore } from './outcome-store'\n\nexport interface EvalMetricSpec {\n  id: string\n  /** Extract a scalar from a run. Omit it and `id` must name one of\n   *  `RUN_METRICS`; any other `id` is refused. */\n  extract?: (run: Run, store: TraceStore) => Promise<number | null>\n}\n\nexport interface OutcomePair {\n  evalMetric: string\n  outcomeMetric: string\n}\n\nexport interface CorrelationResult {\n  evalMetric: string\n  outcomeMetric: string\n  n: number\n  pearson: number\n  spearman: number\n  /** 95% bootstrap CI for Pearson. */\n  pearsonCi95: { lower: number; upper: number } | null\n  /** 95% bootstrap CI for Spearman; null when no resample is estimable. */\n  spearmanCi95: { lower: number; upper: number } | null\n  /** Rough verdict: 'strong' ≥ 0.7, 'moderate' ≥ 0.4, else 'weak'. */\n  verdict: 'strong' | 'moderate' | 'weak'\n}\n\nexport interface CorrelationStudyResult {\n  pairs: CorrelationResult[]\n  /** Declared pairs without an estimable correlation, including their usable sample count. */\n  excludedPairs: Array<\n    OutcomePair & {\n      n: number\n      reason: 'insufficient_samples' | 'constant_eval_metric' | 'constant_outcome'\n    }\n  >\n  joinedSamples: number\n  skippedRuns: number\n}\n\nexport interface CorrelationStudyOptions {\n  /** Only join outcomes captured within this window after run.startedAt. */\n  maxCaptureLagMs?: number\n  /** Restrict to a subset of outcomes (cohort, region, source). */\n  outcomeFilter?: OutcomeFilter\n  /** Which outcome per run to use when multiple exist. Default 'latest'. */\n  reduction?: 'latest' | 'mean' | 'max'\n  /** Bootstrap iterations for the CI. Default 500. */\n  bootstrapIterations?: number\n  /** Seed for the bootstrap resampler. Absent, the seed is derived from the\n   *  paired observations, so the same study reproduces the same interval. */\n  seed?: number\n}\n\nexport async function correlationStudy(\n  traceStore: TraceStore,\n  outcomeStore: OutcomeStore,\n  evalMetrics: EvalMetricSpec[],\n  outcomeMetricNames: string[],\n  options: CorrelationStudyOptions = {},\n): Promise<CorrelationStudyResult> {\n  const reduction = options.reduction ?? 'latest'\n  const iterations = options.bootstrapIterations ?? 500\n  const seed = options.seed\n  validateObservationOptions(reduction, iterations, seed)\n  assertUniqueObservationIds(\n    evalMetrics.map((metric) => metric.id),\n    'eval metric',\n  )\n  assertUniqueObservationIds(outcomeMetricNames, 'outcome metric')\n  const maxLag = options.maxCaptureLagMs ?? Infinity\n  if (maxLag < 0 || Number.isNaN(maxLag)) {\n    throw new Error('maxCaptureLagMs must be nonnegative')\n  }\n  const extractors = evalMetrics.map((metric) => ({\n    ...metric,\n    extract: metric.extract ?? runMetricExtractor(metric.id),\n  }))\n  const metricNames = [...outcomeMetricNames]\n  const runs = await traceStore.listRuns()\n  assertUniqueObservationIds(\n    runs.map((run) => run.runId),\n    'runId',\n  )\n  const outcomes = await outcomeStore.list(options.outcomeFilter)\n  const outcomesByRun = new Map<string, DeploymentOutcome[]>()\n  for (const o of outcomes) {\n    const arr = outcomesByRun.get(o.runId) ?? []\n    arr.push(o)\n    outcomesByRun.set(o.runId, arr)\n  }\n\n  const pairs: Array<{ evalMetric: string; outcomeMetric: string; xs: number[]; ys: number[] }> = []\n  for (const em of extractors) {\n    for (const om of metricNames) {\n      pairs.push({ evalMetric: em.id, outcomeMetric: om, xs: [], ys: [] })\n    }\n  }\n\n  let joined = 0\n  let skipped = 0\n  for (const run of runs) {\n    const os = outcomesByRun.get(run.runId)\n    if (!os || os.length === 0) {\n      skipped++\n      continue\n    }\n    const eligible = os.filter((o) => {\n      const lag = o.capturedAt - run.startedAt\n      return lag >= 0 && lag <= maxLag\n    })\n    if (eligible.length === 0) {\n      skipped++\n      continue\n    }\n\n    let joinedThisRun = false\n    for (const em of extractors) {\n      const x = await em.extract(run, traceStore)\n      if (x === null || !Number.isFinite(x)) continue\n\n      for (const om of metricNames) {\n        const y = reduceOutcomeMetric(eligible, om, reduction)\n        if (y === null) continue\n        const pair = pairs.find((p) => p.evalMetric === em.id && p.outcomeMetric === om)!\n        pair.xs.push(x)\n        pair.ys.push(y)\n        joinedThisRun = true\n      }\n    }\n    if (joinedThisRun) joined++\n    else skipped++\n  }\n\n  const excludedPairs: CorrelationStudyResult['excludedPairs'] = []\n  const results: CorrelationResult[] = []\n  for (const p of pairs) {\n    const reason =\n      p.xs.length < 3\n        ? 'insufficient_samples'\n        : !hasVariation(p.xs)\n          ? 'constant_eval_metric'\n          : !hasVariation(p.ys)\n            ? 'constant_outcome'\n            : null\n    if (reason !== null) {\n      excludedPairs.push({\n        evalMetric: p.evalMetric,\n        outcomeMetric: p.outcomeMetric,\n        n: p.xs.length,\n        reason,\n      })\n      continue\n    }\n    const summary = correlationSummary(p.xs, p.ys, iterations, seed)\n    const verdict: CorrelationResult['verdict'] =\n      Math.abs(summary.pearson) >= 0.7\n        ? 'strong'\n        : Math.abs(summary.pearson) >= 0.4\n          ? 'moderate'\n          : 'weak'\n    results.push({\n      evalMetric: p.evalMetric,\n      outcomeMetric: p.outcomeMetric,\n      n: p.xs.length,\n      ...summary,\n      verdict,\n    })\n  }\n  return { pairs: results, excludedPairs, joinedSamples: joined, skippedRuns: skipped }\n}\n","import { z } from 'zod'\nimport { ValidationError } from '../errors'\nimport { type ComputedInterval, computeInterval } from '../experiment/ast/interval'\nimport { compareCodeUnits, hashCanonical, type LedgerHash } from '../ledger-core/canonical'\n\nconst identity = z\n  .string()\n  .min(1)\n  .refine((value) => value.trim() === value)\nexport const evaluatorAdmissionPolicySchema = z\n  .object({\n    confidence: z.number().finite().gt(0).lt(1),\n    maxFalseAcceptanceRate: z.number().finite().min(0).lt(1),\n    maxFalseRejectionRate: z.number().finite().min(0).lt(1),\n  })\n  .strict()\n\n/** Register both error limits before inspecting audit judgments. */\nexport type EvaluatorAdmissionPolicy = z.infer<typeof evaluatorAdmissionPolicySchema>\n\nexport const evaluatorAuditObservationSchema = z\n  .object({\n    id: identity,\n    independentUnitId: identity,\n    evidenceRef: identity,\n    expected: z.enum(['accept', 'reject']),\n    observed: z.enum(['accept', 'reject', 'unknown']),\n    exposure: z.enum(['fresh', 'development']),\n  })\n  .strict()\n\n/** Actual control judgments. Variants from one source retain one independent identity. */\nexport type EvaluatorAuditObservation = z.infer<typeof evaluatorAuditObservationSchema>\n\nconst inputSchema = z\n  .object({\n    evaluatorDigest: z.string().regex(/^sha256:[a-f0-9]{64}$/),\n    population: identity,\n    samplingFrame: identity,\n    authority: z\n      .object({\n        evaluatorAuthorId: identity,\n        auditorId: identity,\n        independenceEvidenceRef: identity,\n      })\n      .strict(),\n    policy: evaluatorAdmissionPolicySchema,\n    observations: z.array(evaluatorAuditObservationSchema),\n  })\n  .strict()\n\nexport type EvaluatorAuditInput = z.infer<typeof inputSchema>\n\nexport interface EvaluatorErrorRate {\n  /** Eligible control judgments in this class, including unknown judgments. */\n  cases: number\n  independentUnits: number\n  /** Units with at least one observed mistake. */\n  errorUnits: number\n  /** Units with unknown judgments and no observed mistake yet. */\n  unresolvedUnits: number\n  unknownCases: number\n  /** Missing judgments do not become measured zeros. */\n  errorRate: number | null\n  /** Exact bounds conservatively include every possible outcome of unknown judgments. */\n  interval: ComputedInterval | null\n  limit: number\n  verdict: 'pass' | 'fail' | 'inconclusive'\n}\n\nexport interface EvaluatorAdmissionReport {\n  evaluatorDigest: LedgerHash\n  policyDigest: LedgerHash\n  inputDigest: LedgerHash\n  reportDigest: LedgerHash\n  population: string\n  samplingFrame: string\n  authority: EvaluatorAuditInput['authority']\n  policy: EvaluatorAdmissionPolicy\n  /** Joint coverage for the two reported error-rate intervals. */\n  confidence: number\n  intervalConfidence: number\n  verdict: 'admit' | 'reject' | 'inconclusive'\n  reasons: string[]\n  observations: EvaluatorAuditObservation[]\n  coverage: {\n    cases: number\n    independentUnits: number\n    eligibleCases: number\n    eligibleIndependentUnits: number\n    excludedCases: number\n    unknownCases: number\n  }\n  exclusions: Array<{ id: string; independentUnitId: string; reason: 'development-exposure' }>\n  falseAcceptance: EvaluatorErrorRate\n  falseRejection: EvaluatorErrorRate\n}\n\nfunction errorRate(\n  rows: readonly EvaluatorAuditObservation[],\n  expected: 'accept' | 'reject',\n  limit: number,\n  level: number,\n): EvaluatorErrorRate {\n  const cases = rows.filter((row) => row.expected === expected)\n  const units = new Map<string, { error: boolean; unknown: boolean }>()\n  for (const row of cases) {\n    const unit = units.get(row.independentUnitId) ?? { error: false, unknown: false }\n    unit.unknown ||= row.observed === 'unknown'\n    unit.error ||= row.observed !== 'unknown' && row.observed !== expected\n    units.set(row.independentUnitId, unit)\n  }\n  const n = units.size\n  const errorUnits = [...units.values()].filter((unit) => unit.error).length\n  const unresolvedUnits = [...units.values()].filter((unit) => unit.unknown && !unit.error).length\n  const unknownCases = cases.filter((row) => row.observed === 'unknown').length\n  // Unknown judgments contribute to n; their most favorable and adverse outcomes bound the rate.\n  const interval =\n    n === 0\n      ? null\n      : {\n          lower: computeInterval(\n            { kind: 'clopper-pearson', level },\n            {\n              kind: 'binomial',\n              successes: errorUnits,\n              trials: n,\n            },\n          ).lower,\n          upper: computeInterval(\n            { kind: 'clopper-pearson', level },\n            {\n              kind: 'binomial',\n              successes: errorUnits + unresolvedUnits,\n              trials: n,\n            },\n          ).upper,\n          level,\n        }\n  const verdict =\n    interval && interval.lower > limit\n      ? 'fail'\n      : interval && interval.upper <= limit\n        ? 'pass'\n        : 'inconclusive'\n  return {\n    cases: cases.length,\n    independentUnits: n,\n    errorUnits,\n    unresolvedUnits,\n    unknownCases,\n    errorRate: n === 0 || unresolvedUnits > 0 ? null : errorUnits / n,\n    interval,\n    limit,\n    verdict,\n  }\n}\n\nfunction eligibleControls<\n  T extends Pick<EvaluatorAuditObservation, 'id' | 'independentUnitId' | 'exposure'>,\n>(observations: T[]) {\n  const ids = new Set<string>()\n  for (const row of observations) {\n    if (ids.has(row.id))\n      throw new ValidationError(`duplicate evaluator audit observation '${row.id}'`)\n    ids.add(row.id)\n  }\n  observations.sort((a, b) => compareCodeUnits(a.id, b.id))\n  const developmentUnits = new Set(\n    observations\n      .filter((row) => row.exposure === 'development')\n      .map((row) => row.independentUnitId),\n  )\n  const eligible = observations.filter((row) => !developmentUnits.has(row.independentUnitId))\n  const exclusions = observations\n    .filter((row) => developmentUnits.has(row.independentUnitId))\n    .map((row) => ({\n      id: row.id,\n      independentUnitId: row.independentUnitId,\n      reason: 'development-exposure' as const,\n    }))\n  return { eligible, exclusions }\n}\n\nfunction simultaneousIntervalConfidence(confidence: number): number {\n  const level = 1 - (1 - confidence) / 2\n  if (level >= 1)\n    throw new ValidationError(\n      'evaluator audit confidence is too close to one for simultaneous intervals',\n    )\n  return level\n}\n\nconst capacitySchema = z\n  .object({\n    policy: evaluatorAdmissionPolicySchema,\n    controls: z\n      .array(evaluatorAuditObservationSchema.omit({ observed: true, evidenceRef: true }))\n      .default([]),\n  })\n  .strict()\n\nexport type EvaluatorAuditPlanInput = z.input<typeof capacitySchema>\n\n/** The zero-error eligibility floor, not a statistical power calculation. */\nfunction zeroErrorUpperBound(trials: number, level: number): number {\n  return -Math.expm1(Math.log((1 - level) / 2) / trials)\n}\n\nfunction minimumAuditUnits(limit: number, level: number): number | null {\n  if (limit === 0) return null\n  const estimate = Math.ceil(Math.log((1 - level) / 2) / Math.log1p(-limit))\n  if (!Number.isFinite(estimate) || estimate > Number.MAX_SAFE_INTEGER) return null\n  const clears = (trials: number) => zeroErrorUpperBound(trials, level) <= limit\n  let minimum = Math.max(1, estimate)\n  while (minimum > 1 && clears(minimum - 1)) minimum -= 1\n  while (!clears(minimum)) {\n    if (minimum === Number.MAX_SAFE_INTEGER) return null\n    minimum += 1\n  }\n  return minimum\n}\n\n/**\n * Can the fixed control roster meet the policy even with zero errors?\n * Planning uses the admission rule's exact bounds and source-unit exclusions.\n * It accepts no judgments or authority and returns no admission report.\n */\nexport function planEvaluatorAudit(input: EvaluatorAuditPlanInput) {\n  const parsed = capacitySchema.safeParse(input)\n  if (!parsed.success)\n    throw new ValidationError(`invalid evaluator audit plan: ${parsed.error.message}`)\n  const { policy, controls } = parsed.data\n  const { eligible } = eligibleControls(controls)\n  const level = simultaneousIntervalConfidence(policy.confidence)\n  const capacity = (expected: EvaluatorAuditObservation['expected'], limit: number) => {\n    const independentUnits = new Set(\n      eligible.filter((row) => row.expected === expected).map((row) => row.independentUnitId),\n    ).size\n    const bestPossibleUpperBound =\n      independentUnits === 0 ? null : zeroErrorUpperBound(independentUnits, level)\n    const minimumIndependentUnits = minimumAuditUnits(limit, level)\n    return {\n      independentUnits,\n      bestPossibleUpperBound,\n      limit,\n      minimumIndependentUnits,\n      additionalIndependentUnits:\n        minimumIndependentUnits === null\n          ? null\n          : Math.max(0, minimumIndependentUnits - independentUnits),\n    }\n  }\n  const falseAcceptance = capacity('reject', policy.maxFalseAcceptanceRate)\n  const falseRejection = capacity('accept', policy.maxFalseRejectionRate)\n  const sufficient = [falseAcceptance, falseRejection].every(\n    (rate) => rate.bestPossibleUpperBound !== null && rate.bestPossibleUpperBound <= rate.limit,\n  )\n  return { sufficient, falseAcceptance, falseRejection }\n}\n\nexport type EvaluatorAuditPlan = ReturnType<typeof planEvaluatorAudit>\n\n/**\n * Audit frozen judgments using independent source units and exact binomial bounds.\n * A unit fails a class when any control in that class is misjudged.\n * The execution owner enforces auditor separation, fresh sampling, and evidence authenticity.\n */\nexport function auditEvaluator(input: EvaluatorAuditInput): EvaluatorAdmissionReport {\n  const parsed = inputSchema.safeParse(input)\n  if (!parsed.success) throw new ValidationError(`invalid evaluator audit: ${parsed.error.message}`)\n  const audit = parsed.data\n  if (audit.authority.evaluatorAuthorId === audit.authority.auditorId) {\n    throw new ValidationError('evaluator admission requires a separate declared audit authority')\n  }\n  const { eligible, exclusions } = eligibleControls(audit.observations)\n  const intervalConfidence = simultaneousIntervalConfidence(audit.policy.confidence)\n  const falseAcceptance = errorRate(\n    eligible,\n    'reject',\n    audit.policy.maxFalseAcceptanceRate,\n    intervalConfidence,\n  )\n  const falseRejection = errorRate(\n    eligible,\n    'accept',\n    audit.policy.maxFalseRejectionRate,\n    intervalConfidence,\n  )\n  const rates = [falseAcceptance, falseRejection]\n  const verdict = rates.some((rate) => rate.verdict === 'fail')\n    ? 'reject'\n    : rates.every((rate) => rate.verdict === 'pass')\n      ? 'admit'\n      : 'inconclusive'\n  const reasons: string[] = []\n  for (const [name, rate] of [\n    ['false acceptance', falseAcceptance],\n    ['false rejection', falseRejection],\n  ] as const) {\n    if (rate.independentUnits === 0) reasons.push(`${name}: no eligible independent units`)\n    else if (rate.unknownCases > 0)\n      reasons.push(`${name}: ${rate.unknownCases} judgments are unknown`)\n    if (rate.verdict === 'fail') reasons.push(`${name}: lower error bound exceeds ${rate.limit}`)\n    else if (rate.verdict === 'inconclusive' && rate.interval)\n      reasons.push(`${name}: upper error bound does not establish the required limit`)\n  }\n  if (verdict === 'admit')\n    reasons.push(\n      'both error bounds meet the registered limits, including the worst case for unknown judgments',\n    )\n  const body: Omit<EvaluatorAdmissionReport, 'reportDigest'> = {\n    evaluatorDigest: audit.evaluatorDigest as LedgerHash,\n    policyDigest: hashCanonical(audit.policy),\n    inputDigest: hashCanonical(audit),\n    population: audit.population,\n    samplingFrame: audit.samplingFrame,\n    authority: audit.authority,\n    policy: audit.policy,\n    confidence: audit.policy.confidence,\n    intervalConfidence,\n    verdict,\n    reasons,\n    observations: audit.observations,\n    coverage: {\n      cases: audit.observations.length,\n      independentUnits: new Set(audit.observations.map((row) => row.independentUnitId)).size,\n      eligibleCases: eligible.length,\n      eligibleIndependentUnits: new Set(eligible.map((row) => row.independentUnitId)).size,\n      excludedCases: exclusions.length,\n      unknownCases: eligible.filter((row) => row.observed === 'unknown').length,\n    },\n    exclusions,\n    falseAcceptance,\n    falseRejection,\n  }\n  return { ...body, reportDigest: hashCanonical(body) }\n}\n\nconst probabilityThresholdsSchema = z\n  .object({\n    rejectAtOrBelow: z.number().finite().min(0).max(1),\n    acceptAtOrAbove: z.number().finite().min(0).max(1),\n  })\n  .strict()\n  .refine((value) => value.rejectAtOrBelow < value.acceptAtOrAbove, {\n    message: 'rejection and acceptance thresholds must not overlap',\n  })\nconst probabilityObservationSchema = evaluatorAuditObservationSchema\n  .omit({ observed: true })\n  .extend({ acceptProbability: z.number().finite().min(0).max(1).nullable() })\n  .strict()\nconst probabilityAuditSchema = inputSchema.extend({\n  thresholds: probabilityThresholdsSchema,\n  observations: z.array(probabilityObservationSchema),\n})\n\n/** A frozen caller-owned rule over P(accept), not the provider's entropy/confidence statistic. */\nexport type ProbabilityPolicyAuditInput = z.infer<typeof probabilityAuditSchema>\n\nexport interface ProbabilityPolicyAuditReport extends EvaluatorAdmissionReport {\n  probabilityPolicy: {\n    sourceEvaluatorDigest: LedgerHash\n    thresholds: ProbabilityPolicyAuditInput['thresholds']\n    /** Retain probabilities by observation id so the reported mapping can be reproduced. */\n    observations: Array<{ id: string; acceptProbability: number | null }>\n  }\n}\n\n/**\n * Audit a fixed probability-to-decision rule with the existing independent-unit error bounds.\n * No threshold search, model calls, promotion, or assumption of calibrated probabilities.\n * Null scores and the open interval between thresholds remain unknown, never correct by default.\n */\nexport function auditProbabilityPolicy(\n  input: ProbabilityPolicyAuditInput,\n): ProbabilityPolicyAuditReport {\n  const parsed = probabilityAuditSchema.safeParse(input)\n  if (!parsed.success) {\n    throw new ValidationError(`invalid probability policy audit: ${parsed.error.message}`)\n  }\n  const { thresholds, observations, ...source } = parsed.data\n  observations.sort((a, b) => compareCodeUnits(a.id, b.id))\n  const probabilityPolicy = {\n    sourceEvaluatorDigest: source.evaluatorDigest as LedgerHash,\n    thresholds,\n    observations: observations.map(({ id, acceptProbability }) => ({ id, acceptProbability })),\n  }\n  const audit = auditEvaluator({\n    ...source,\n    evaluatorDigest: hashCanonical({\n      kind: 'probability-policy-v1',\n      sourceEvaluatorDigest: source.evaluatorDigest,\n      thresholds,\n    }),\n    observations: observations.map(({ acceptProbability, ...observation }) => ({\n      ...observation,\n      observed:\n        acceptProbability === null\n          ? 'unknown'\n          : acceptProbability >= thresholds.acceptAtOrAbove\n            ? 'accept'\n            : acceptProbability <= thresholds.rejectAtOrBelow\n              ? 'reject'\n              : 'unknown',\n    })),\n  })\n  const { reportDigest: _baseDigest, ...baseReport } = audit\n  const body = {\n    ...baseReport,\n    // Bind the actual probabilities too, including changes that select the same action.\n    inputDigest: hashCanonical(parsed.data),\n    probabilityPolicy,\n  }\n  return { ...body, reportDigest: hashCanonical(body) }\n}\n","/**\n * Plants — seeded known-wrong items that measure the grader, not the work.\n *\n * A grading run reports how the work scored. It cannot report whether the\n * grader would have noticed a wrong answer, because every item it saw was\n * authored in good faith. A plant closes that hole: an item authored wrong by\n * construction is mixed into the live set, graded by the same path as\n * everything else, and the share of plants the grader refused is the catch\n * rate.\n *\n * Measured motive: a sibling lab ran a deliverable gate that accepted any\n * non-empty submission. It produced six false certifications in seventeen\n * deliveries, and no agent lied — the gate never asked a question the format\n * could fail. A catch rate is the number that would have shown it on day one.\n *\n * This module composes existing primitives rather than adding parallel ones:\n *\n * - A plant IS a {@link GoldenItem} from `../judge-calibration`. Its\n *   `humanScore` is the grade a working grader owes the item, so the same\n *   array feeds `calibrateJudge` unchanged.\n * - The grader's output is `CandidateScore[]`, the array `calibrateJudge` and\n *   `snapshotFromSentinelSet` already consume.\n * - \"Caught\" is `snapshotFromSentinelSet`'s join with the labels inverted:\n *   the grade lands on the side of `acceptThreshold` the seed demands.\n * - The manifest is sealed with `hashCanonical` from `../ledger-core/canonical`,\n *   the digest the sealed-experiment path uses, so the answer key cannot be\n *   revised once the results are in.\n *\n * Authoring the wrong item is the step before all of that, and it is where the\n * measurement silently breaks. {@link plantByPerturbation} takes a claim a\n * grader verified and alters exactly one load-bearing value in its evidence,\n * so the grader is asked a question it must fail. Hand-rolled perturbation\n * bumps a number that is part of an identity — `python3`, `file42.txt`, `1.5`,\n * `utf-8` — and the check stops EXECUTING; the plant then measures the\n * environment rather than the grader. {@link perturbEvidence} refuses those\n * digit runs by construction.\n *\n * Blindness has two halves, and this module owns one. It never puts a plant\n * flag on a graded item: `seedPlants` returns the mixed set and a manifest,\n * and only the manifest knows which ids are seeded. Keeping the manifest out\n * of the graded workspace and publishing its `seal` before grading is the\n * caller's half; {@link catchRate} refuses a manifest whose contents no longer\n * match its seal.\n *\n * Refusals, because a catch rate that cannot refuse is not a measurement:\n *\n * - a seeded id with no result makes the report `incomplete`, never a rate\n *   over the results that did come back;\n * - zero seeded plants makes it `not_evaluated`, never 1.0;\n * - a result for an id the manifest never handed out is refused outright.\n */\n\nimport { CaptureIntegrityError, ValidationError } from '../errors'\nimport type { GoldenItem } from '../judge-calibration'\nimport { hashCanonical, type LedgerHash } from '../ledger-core/canonical'\nimport { mulberry32 } from '../statistics/random'\n\n/**\n * How a plant item was authored wrong. The class is reported separately in\n * {@link CatchRateReport.byKind} because a grader is routinely sharp on one\n * and blind to another.\n */\nexport type PlantKind =\n  /** A load-bearing value is altered: a number off by one, a comparison flipped. */\n  | 'wrong-value'\n  /** The item carries its own check, and that check passes without testing the claim. */\n  | 'self-certifying'\n  /** The check names an input that does not exist, so it cannot run at all. */\n  | 'unreachable-input'\n  /** A copy of an item already in the set, which is owed a duplicate flag rather than a second grade. */\n  | 'duplicate'\n\nconst PLANT_KINDS: readonly PlantKind[] = [\n  'wrong-value',\n  'self-certifying',\n  'unreachable-input',\n  'duplicate',\n]\n\n/** What a working grader owes a seeded item. */\nexport type PlantExpectation = 'reject' | 'accept'\n\nconst PLANT_EXPECTATIONS: readonly PlantExpectation[] = ['reject', 'accept']\n\n/** The label boundary a plant record is checked against at definition time. */\nconst RECORD_LABEL_BOUNDARY = 0.5\n\nexport interface Plant {\n  /** Name of the plant record. Reported in `missedIds` and `missingIds`. */\n  id: string\n  kind: PlantKind\n  /** The seeded item, indistinguishable from a real one once mixed. */\n  item: GoldenItem\n  /** The verdict a working grader owes this item. */\n  expectedVerdict: PlantExpectation\n}\n\n/**\n * Build one plant record and refuse an incoherent one.\n *\n * The refusal that matters is the last: a record whose `expectedVerdict`\n * disagrees with `item.humanScore` inverts the measurement silently, because\n * the same item then reads as wrong here and as correct to every calibration\n * instrument that joins on the id.\n *\n * `item` is copied field by field so a later mutation of the caller's object\n * cannot change what the manifest sealed, and `group` is dropped when it is\n * absent so the record always has a canonical JSON form.\n */\nexport function definePlant(input: {\n  id: string\n  kind: PlantKind\n  item: GoldenItem\n  expectedVerdict: PlantExpectation\n}): Plant {\n  const { id, kind, item, expectedVerdict } = input\n  if (typeof id !== 'string' || id.trim() === '') {\n    throw new ValidationError('definePlant: id must be a non-empty string')\n  }\n  if (!PLANT_KINDS.includes(kind)) {\n    throw new ValidationError(\n      `definePlant: plant \"${id}\" has kind ${JSON.stringify(kind)}; expected one of ${PLANT_KINDS.join(', ')}`,\n    )\n  }\n  if (!PLANT_EXPECTATIONS.includes(expectedVerdict)) {\n    throw new ValidationError(\n      `definePlant: plant \"${id}\" has expectedVerdict ${JSON.stringify(expectedVerdict)}; expected one of ${PLANT_EXPECTATIONS.join(', ')}`,\n    )\n  }\n  if (typeof item.itemId !== 'string' || item.itemId.trim() === '') {\n    throw new ValidationError(`definePlant: plant \"${id}\" has an empty item.itemId`)\n  }\n  if (!Number.isFinite(item.humanScore) || item.humanScore < 0 || item.humanScore > 1) {\n    throw new ValidationError(\n      `definePlant: plant \"${id}\" has humanScore ${item.humanScore}; expected a finite number in [0, 1]`,\n    )\n  }\n  if (item.group !== undefined && typeof item.group !== 'string') {\n    throw new ValidationError(`definePlant: plant \"${id}\" has a non-string item.group`)\n  }\n  if (item.humanScore === RECORD_LABEL_BOUNDARY) {\n    throw new ValidationError(\n      `definePlant: plant \"${id}\" has humanScore ${RECORD_LABEL_BOUNDARY}, which states neither a rejection nor an acceptance`,\n    )\n  }\n  const labelSays: PlantExpectation = item.humanScore < RECORD_LABEL_BOUNDARY ? 'reject' : 'accept'\n  if (labelSays !== expectedVerdict) {\n    throw new ValidationError(\n      `definePlant: plant \"${id}\" expects the grader to ${expectedVerdict} it, but humanScore ${item.humanScore} says ${labelSays}`,\n    )\n  }\n  return {\n    id,\n    kind,\n    item: {\n      itemId: item.itemId,\n      humanScore: item.humanScore,\n      ...(item.group === undefined ? {} : { group: item.group }),\n    },\n    expectedVerdict,\n  }\n}\n\n/**\n * A claim's executable evidence: the command a grader runs and the output it\n * must produce.\n */\nexport interface PlantEvidence {\n  /** The command a grader runs to test the claim. */\n  check?: string\n  /** The output that command must produce for the claim to stand. */\n  expect?: string\n}\n\n/** Which evidence field {@link perturbEvidence} altered. */\nexport type PerturbedField = 'check' | 'expect'\n\n/** How {@link perturbEvidence} altered it. */\nexport type PerturbationKind = 'number-off-by-one' | 'comparison-flipped'\n\nexport interface EvidencePerturbation {\n  /** The evidence with exactly one value altered. */\n  evidence: PlantEvidence\n  /** Which field changed. */\n  field: PerturbedField\n  /** How it changed. */\n  how: PerturbationKind\n  /**\n   * The exact substring that was replaced. A comparison token carries its\n   * surrounding spaces, because the spaces are what make ` -ge ` a comparison\n   * and not part of a word.\n   */\n  original: string\n  /** What replaced it. */\n  perturbed: string\n}\n\n/**\n * Characters that make a digit run part of something larger than a number.\n *\n * A run that touches one of these is an identity — `python3`, `file42.txt`,\n * `1.5`, `utf-8` — and bumping it renames a program, a file, a version, or an\n * encoding instead of falsifying the claim. The check then stops EXECUTING,\n * and a plant that cannot run measures the environment rather than the grader:\n * it reads as caught, for a reason that has nothing to do with the seeded\n * defect.\n */\nconst NUMBER_GLUE = /[A-Za-z0-9_./-]/\n\n/**\n * Comparison tokens that mean one thing only, each with its inversion.\n *\n * A bare `>` or `<` is absent on purpose: in a shell check it is a\n * redirection, so flipping it rewrites where output goes rather than what the\n * test asks, and the check stops testing the claim.\n */\nconst COMPARISON_FLIPS: readonly (readonly [string, string])[] = [\n  [' -ge ', ' -lt '],\n  [' -gt ', ' -le '],\n  [' -le ', ' -gt '],\n  [' -lt ', ' -ge '],\n  [' -eq ', ' -ne '],\n  [' -ne ', ' -eq '],\n  ['>=', '<'],\n  ['<=', '>'],\n  ['==', '!='],\n  ['!=', '=='],\n]\n\ninterface Span {\n  start: number\n  end: number\n}\n\n/**\n * The last digit run in `text` that is a number and nothing else, or null.\n *\n * The last is taken because a measured value trails its label — `count=42`,\n * `n>=5 OK cells=8` — and because the choice has to be deterministic: the same\n * claim must always yield the same plant, or two runs of the same authoring\n * step seed two different answer keys for one item.\n */\nfunction lastStandaloneNumber(text: string): Span | null {\n  const digits = /\\d+/g\n  let found: Span | null = null\n  let match = digits.exec(text)\n  while (match !== null) {\n    const start = match.index\n    const end = start + match[0].length\n    // charAt returns '' outside the string, and NUMBER_GLUE never matches ''.\n    if (!NUMBER_GLUE.test(text.charAt(start - 1)) && !NUMBER_GLUE.test(text.charAt(end))) {\n      found = { start, end }\n    }\n    match = digits.exec(text)\n  }\n  return found\n}\n\n/**\n * The first flippable comparison in `text`, or null.\n *\n * Any single flip inverts the test wherever it sits, so unlike the number rule\n * there is no safety ordering among the matches. The first is taken so the\n * result is fixed by the head of the command and does not move when a later\n * comparison is added.\n */\nfunction firstComparison(text: string): { span: Span; flipped: string } | null {\n  for (let index = 0; index < text.length; index += 1) {\n    for (const [token, flipped] of COMPARISON_FLIPS) {\n      if (text.startsWith(token, index)) {\n        return { span: { start: index, end: index + token.length }, flipped }\n      }\n    }\n  }\n  return null\n}\n\nfunction replaceSpan(text: string, span: Span, value: string): string {\n  return text.slice(0, span.start) + value + text.slice(span.end)\n}\n\n/**\n * Rebuild the evidence with one field replaced. An absent field is dropped\n * rather than set to undefined, so the record always has a canonical JSON form.\n */\nfunction withField(\n  check: string | undefined,\n  expect: string | undefined,\n  field: PerturbedField,\n  value: string,\n): PlantEvidence {\n  const nextCheck = field === 'check' ? value : check\n  const nextExpect = field === 'expect' ? value : expect\n  return {\n    ...(nextCheck === undefined ? {} : { check: nextCheck }),\n    ...(nextExpect === undefined ? {} : { expect: nextExpect }),\n  }\n}\n\nfunction offByOne(\n  check: string | undefined,\n  expect: string | undefined,\n  field: PerturbedField,\n  text: string,\n  span: Span,\n): EvidencePerturbation {\n  const original = text.slice(span.start, span.end)\n  const perturbed = (BigInt(original) + 1n).toString()\n  return {\n    evidence: withField(check, expect, field, replaceSpan(text, span, perturbed)),\n    field,\n    how: 'number-off-by-one',\n    original,\n    perturbed,\n  }\n}\n\n/**\n * Derive a known-wrong version of a claim's evidence by altering exactly one\n * value, or return null when no value can be altered safely.\n *\n * Exactly one value changes. A perturbation that moves two does not name the\n * defect it seeded, so a grader that catches it says nothing about which defect\n * the grader can see.\n *\n * The preference order IS the safety order:\n *\n * 1. the last standalone number in `expect` — it falsifies the claim while the\n *    command still runs exactly as it did;\n * 2. a flipped comparison in `check` — it inverts a test that was passing;\n * 3. the last standalone number in `check` — last because a number in a command\n *    can name an input (a port, a width, a version) rather than a threshold,\n *    and renaming an input breaks execution, not the claim.\n *\n * \"Standalone\" excludes a digit run glued to a word, a dot, a slash, or a\n * hyphen: `python3`, `file42.txt`, `1.5`, `utf-8`. That exclusion is why this\n * helper exists. Hand-rolled perturbation bumps one of those, the check stops\n * executing, and the plant measures the environment instead of the grader.\n *\n * A number is bumped with BigInt, so a value wider than\n * `Number.MAX_SAFE_INTEGER` moves by exactly one. Bumped as a `number` it\n * rounds back to itself, and the plant is then a correct claim the grader is\n * right to verify.\n */\nexport function perturbEvidence(evidence: PlantEvidence): EvidencePerturbation | null {\n  const check = typeof evidence.check === 'string' ? evidence.check : undefined\n  const expect = typeof evidence.expect === 'string' ? evidence.expect : undefined\n\n  if (expect !== undefined) {\n    const span = lastStandaloneNumber(expect)\n    if (span !== null) return offByOne(check, expect, 'expect', expect, span)\n  }\n  if (check !== undefined) {\n    const comparison = firstComparison(check)\n    if (comparison !== null) {\n      return {\n        evidence: withField(\n          check,\n          expect,\n          'check',\n          replaceSpan(check, comparison.span, comparison.flipped),\n        ),\n        field: 'check',\n        how: 'comparison-flipped',\n        original: check.slice(comparison.span.start, comparison.span.end),\n        perturbed: comparison.flipped,\n      }\n    }\n    const span = lastStandaloneNumber(check)\n    if (span !== null) return offByOne(check, expect, 'check', check, span)\n  }\n  return null\n}\n\nexport interface PerturbedPlant {\n  plant: Plant\n  /** The perturbed evidence to write into the item the grader is handed. */\n  evidence: PlantEvidence\n  field: PerturbedField\n  how: PerturbationKind\n  original: string\n  perturbed: string\n}\n\n/**\n * Derive a reject-direction plant from a claim a grader verified, by perturbing\n * one value.\n *\n * {@link definePlant} takes a {@link GoldenItem} and never authors a wrong item\n * from a right one, so until now every caller hand-rolled the perturbation.\n * This is that step, done once and the same way every time: the claim's\n * evidence is altered by {@link perturbEvidence}, the result is a `wrong-value`\n * plant the grader owes a `reject`, and the record itself is built by\n * `definePlant` so every refusal there still applies — in particular the one\n * that catches an `expectedVerdict` its `humanScore` contradicts.\n *\n * Returns null when nothing in the evidence can be perturbed. That is not a\n * failure: a claim with no executable evidence cannot be made wrong in a way a\n * grader could execute, and seeding it would measure prose.\n *\n * Refuses an empty `id` or `itemId`. Both name the record — one in the manifest\n * and in `missedIds`, the other in the join with the results — and a miss\n * nobody can look up is not a measurement.\n */\nexport function plantByPerturbation(input: {\n  id: string\n  itemId: string\n  evidence: PlantEvidence\n  /** Score for the perturbed item. Must sit below the run's accept threshold. Default 0. */\n  humanScore?: number\n  group?: string\n}): PerturbedPlant | null {\n  const { id, itemId, evidence, humanScore = 0, group } = input\n  if (typeof id !== 'string' || id.trim() === '') {\n    throw new ValidationError('plantByPerturbation: id must be a non-empty string')\n  }\n  if (typeof itemId !== 'string' || itemId.trim() === '') {\n    throw new ValidationError(`plantByPerturbation: plant \"${id}\" has an empty itemId`)\n  }\n  const perturbation = perturbEvidence(evidence)\n  if (perturbation === null) return null\n  const plant = definePlant({\n    id,\n    kind: 'wrong-value',\n    item: { itemId, humanScore, ...(group === undefined ? {} : { group }) },\n    expectedVerdict: 'reject',\n  })\n  return {\n    plant,\n    evidence: perturbation.evidence,\n    field: perturbation.field,\n    how: perturbation.how,\n    original: perturbation.original,\n    perturbed: perturbation.perturbed,\n  }\n}\n\nexport interface PlantManifest {\n  /**\n   * Digest over the seeded order, the plants, and the threshold. Publish it\n   * before the grading run: a manifest edited afterwards to match the results\n   * no longer matches its seal, and {@link catchRate} refuses it.\n   */\n  seal: LedgerHash\n  /** The seed that fixed the mix order. */\n  seed: number\n  /** The grade at or above which the graded policy's own gate accepts an item. */\n  acceptThreshold: number\n  /** Every item id in the seeded set, in the order handed out. */\n  itemIds: string[]\n  /** The seeded plants. This is the answer key; keep it out of the graded workspace. */\n  plants: Plant[]\n}\n\nexport interface SeededGradingSet {\n  /**\n   * The mixed set in seeded order. Operator-side: it still carries every\n   * item's `humanScore`, so hand the graded policy the payload each `itemId`\n   * names, never this array.\n   */\n  items: GoldenItem[]\n  manifest: PlantManifest\n}\n\nexport interface SeedPlantsOptions {\n  /**\n   * Fixes the mix order. The same dataset, plants, threshold, and seed always\n   * produce the same seeded order and the same seal.\n   */\n  seed?: number\n  /**\n   * The grade at or above which the graded policy's own gate accepts an item.\n   * Supply the threshold your gate uses; the default suits a judge scoring in\n   * [0, 1] with a pass at the midpoint.\n   */\n  acceptThreshold?: number\n}\n\n/**\n * Mix plants into a grading set and seal which items they are.\n *\n * Refusals: a duplicate id anywhere in the mixed set (the join is by id, so a\n * repeat makes one of the two unscoreable), a plant whose item id collides\n * with a dataset item (it would shadow real work and grade it as a plant), a\n * dataset item with no usable label, and a plant whose expectation the run's\n * `acceptThreshold` contradicts.\n */\nexport function seedPlants(\n  dataset: readonly GoldenItem[],\n  plants: readonly Plant[],\n  options: SeedPlantsOptions = {},\n): SeededGradingSet {\n  const seed = options.seed ?? 7\n  const acceptThreshold = options.acceptThreshold ?? 0.5\n  if (!Number.isFinite(seed)) {\n    throw new ValidationError(`seedPlants: seed must be a finite number, got ${seed}`)\n  }\n  if (!Number.isFinite(acceptThreshold) || acceptThreshold <= 0 || acceptThreshold > 1) {\n    throw new ValidationError(\n      `seedPlants: acceptThreshold must be a finite number in (0, 1], got ${acceptThreshold}`,\n    )\n  }\n\n  const datasetItems: GoldenItem[] = []\n  const seen = new Set<string>()\n  for (const item of dataset) {\n    if (typeof item.itemId !== 'string' || item.itemId.trim() === '') {\n      throw new ValidationError('seedPlants: a dataset item has an empty itemId')\n    }\n    if (!Number.isFinite(item.humanScore)) {\n      throw new ValidationError(\n        `seedPlants: dataset item \"${item.itemId}\" has a non-finite humanScore`,\n      )\n    }\n    if (seen.has(item.itemId)) {\n      throw new ValidationError(`seedPlants: duplicate dataset itemId \"${item.itemId}\"`)\n    }\n    seen.add(item.itemId)\n    datasetItems.push({\n      itemId: item.itemId,\n      humanScore: item.humanScore,\n      ...(item.group === undefined ? {} : { group: item.group }),\n    })\n  }\n\n  const sealedPlants: Plant[] = []\n  const plantIds = new Set<string>()\n  for (const plant of plants) {\n    const record = definePlant(plant)\n    if (plantIds.has(record.id)) {\n      throw new ValidationError(`seedPlants: duplicate plant id \"${record.id}\"`)\n    }\n    if (seen.has(record.item.itemId)) {\n      throw new ValidationError(\n        `seedPlants: plant \"${record.id}\" reuses itemId \"${record.item.itemId}\", which is already in the set`,\n      )\n    }\n    const thresholdSays: PlantExpectation =\n      record.item.humanScore >= acceptThreshold ? 'accept' : 'reject'\n    if (thresholdSays !== record.expectedVerdict) {\n      throw new ValidationError(\n        `seedPlants: plant \"${record.id}\" expects the grader to ${record.expectedVerdict} it, but humanScore ${record.item.humanScore} is on the ${thresholdSays} side of acceptThreshold ${acceptThreshold}`,\n      )\n    }\n    plantIds.add(record.id)\n    seen.add(record.item.itemId)\n    sealedPlants.push(record)\n  }\n\n  const items = shuffled(\n    [...datasetItems, ...sealedPlants.map((plant) => plant.item)],\n    mulberry32(seed),\n  )\n  const itemIds = items.map((item) => item.itemId)\n  const manifest: PlantManifest = {\n    seal: sealManifest({ seed, acceptThreshold, itemIds, plants: sealedPlants }),\n    seed,\n    acceptThreshold,\n    itemIds,\n    plants: sealedPlants,\n  }\n  return { items, manifest }\n}\n\n/**\n * Order by one independent uniform key per item, which is a uniform\n * permutation and a pure function of the seed. A comparator that returns a\n * fresh random sign instead is neither: it is not a consistent ordering, so\n * the permutation it produces is biased and depends on the sort algorithm.\n */\nfunction shuffled<T>(items: readonly T[], random: () => number): T[] {\n  return items\n    .map((item) => ({ item, key: random() }))\n    .sort((left, right) => left.key - right.key)\n    .map((entry) => entry.item)\n}\n\nfunction sealManifest(contents: Omit<PlantManifest, 'seal'>): LedgerHash {\n  return hashCanonical({\n    scheme: 'agent-eval.plant-manifest.v1',\n    seed: contents.seed,\n    acceptThreshold: contents.acceptThreshold,\n    itemIds: contents.itemIds,\n    plants: contents.plants,\n  })\n}\n\n/**\n * One grader outcome for one item. `score: null` says the grader ran and\n * declined to decide — the check never tested the item's defect. Any\n * `CandidateScore` from a judge run is already a valid outcome.\n */\nexport interface PlantOutcome {\n  itemId: string\n  score: number | null\n}\n\n/**\n * `evaluated` — a rate stands. `incomplete` — a seeded id had no result.\n * `not_evaluated` — nothing was seeded, or nothing seeded was decided.\n */\nexport type CatchRateStatus = 'evaluated' | 'incomplete' | 'not_evaluated'\n\nexport interface PlantKindCounts {\n  seeded: number\n  caught: number\n  missed: number\n  indecisive: number\n  /** caught / (caught + missed), or null when the report is not `evaluated`. */\n  rate: number | null\n}\n\n/**\n * How the same grader treated the unseeded items of the same set. No labels\n * are needed to read it: a grader that refuses everything scores `rate` 1.0\n * on reject-plants, and a `rejectionRate` of 1.0 here is what separates that\n * reflex from discrimination.\n */\nexport interface UnseededRejection {\n  n: number\n  decided: number\n  rejected: number\n  /** rejected / decided, or null when nothing unseeded was decided. */\n  rejectionRate: number | null\n}\n\nexport interface CatchRateReport {\n  status: CatchRateStatus\n  /** Why the status is not `evaluated`. Absent when it is. */\n  reason?: string\n  seeded: number\n  caught: number\n  missed: number\n  /**\n   * The grader returned a result and declined to decide. Counted apart: it\n   * enters neither side of `rate`.\n   */\n  indecisive: number\n  /** caught / (caught + missed), or null unless the status is `evaluated`. */\n  rate: number | null\n  /** One entry per kind actually seeded. A kind nobody seeded is absent, never zero. */\n  byKind: Partial<Record<PlantKind, PlantKindCounts>>\n  /** Plant ids the grader graded as the seed says it must not. */\n  missedIds: string[]\n  /** Plant ids with no result at all — the reason a status is `incomplete`. */\n  missingIds: string[]\n  unseeded: UnseededRejection\n}\n\n/**\n * Score a grading run against its sealed manifest.\n *\n * Refusals: a manifest whose contents no longer match its seal, a result for\n * an id the manifest never handed out, and a repeated result id. Each says\n * the results and the manifest describe different runs, and a rate computed\n * across two runs is a fabrication.\n */\nexport function catchRate(\n  results: readonly PlantOutcome[],\n  manifest: PlantManifest,\n): CatchRateReport {\n  const expectedSeal = sealManifest({\n    seed: manifest.seed,\n    acceptThreshold: manifest.acceptThreshold,\n    itemIds: manifest.itemIds,\n    plants: manifest.plants,\n  })\n  if (expectedSeal !== manifest.seal) {\n    throw new CaptureIntegrityError(\n      `catchRate: manifest contents hash to ${expectedSeal} but the manifest carries seal ${manifest.seal} — the plant set changed after it was sealed`,\n    )\n  }\n\n  const handedOut = new Set(manifest.itemIds)\n  const scoreByItemId = new Map<string, number | null>()\n  for (const result of results) {\n    if (!handedOut.has(result.itemId)) {\n      throw new ValidationError(\n        `catchRate: result for \"${result.itemId}\", which this manifest never handed out`,\n      )\n    }\n    if (scoreByItemId.has(result.itemId)) {\n      throw new ValidationError(`catchRate: duplicate result for \"${result.itemId}\"`)\n    }\n    if (result.score !== null && !Number.isFinite(result.score)) {\n      throw new ValidationError(\n        `catchRate: result for \"${result.itemId}\" has score ${result.score}; expected a finite number or null`,\n      )\n    }\n    scoreByItemId.set(result.itemId, result.score)\n  }\n\n  const byKind: Partial<Record<PlantKind, PlantKindCounts>> = {}\n  const missedIds: string[] = []\n  const missingIds: string[] = []\n  let caught = 0\n  let missed = 0\n  let indecisive = 0\n\n  for (const plant of manifest.plants) {\n    let counts = byKind[plant.kind]\n    if (counts === undefined) {\n      counts = { seeded: 0, caught: 0, missed: 0, indecisive: 0, rate: null }\n      byKind[plant.kind] = counts\n    }\n    counts.seeded += 1\n    if (!scoreByItemId.has(plant.item.itemId)) {\n      missingIds.push(plant.id)\n      continue\n    }\n    const score = scoreByItemId.get(plant.item.itemId) ?? null\n    if (score === null) {\n      indecisive += 1\n      counts.indecisive += 1\n      continue\n    }\n    const graderSaid: PlantExpectation = score >= manifest.acceptThreshold ? 'accept' : 'reject'\n    if (graderSaid === plant.expectedVerdict) {\n      caught += 1\n      counts.caught += 1\n    } else {\n      missed += 1\n      counts.missed += 1\n      missedIds.push(plant.id)\n    }\n  }\n\n  const seeded = manifest.plants.length\n  const decided = caught + missed\n  const report: CatchRateReport = {\n    status: 'evaluated',\n    seeded,\n    caught,\n    missed,\n    indecisive,\n    rate: null,\n    byKind,\n    missedIds,\n    missingIds,\n    unseeded: unseededRejection(manifest, scoreByItemId),\n  }\n\n  if (seeded === 0) {\n    return {\n      ...report,\n      status: 'not_evaluated',\n      reason: 'no plant was seeded, so the grader was never asked a question it could fail',\n    }\n  }\n  if (missingIds.length > 0) {\n    return {\n      ...report,\n      status: 'incomplete',\n      reason: `${missingIds.length} of ${seeded} seeded plants have no result: ${missingIds.join(', ')}`,\n    }\n  }\n  if (decided === 0) {\n    return {\n      ...report,\n      status: 'not_evaluated',\n      reason: `all ${seeded} seeded plants are indecisive: no check tested the seeded defect`,\n    }\n  }\n\n  for (const counts of Object.values(byKind)) {\n    const kindDecided = counts.caught + counts.missed\n    counts.rate = kindDecided === 0 ? null : counts.caught / kindDecided\n  }\n  report.rate = caught / decided\n  return report\n}\n\nfunction unseededRejection(\n  manifest: PlantManifest,\n  scoreByItemId: ReadonlyMap<string, number | null>,\n): UnseededRejection {\n  const plantItemIds = new Set(manifest.plants.map((plant) => plant.item.itemId))\n  let n = 0\n  let decided = 0\n  let rejected = 0\n  for (const itemId of manifest.itemIds) {\n    if (plantItemIds.has(itemId)) continue\n    n += 1\n    const score = scoreByItemId.get(itemId)\n    if (score === undefined || score === null) continue\n    decided += 1\n    if (score < manifest.acceptThreshold) rejected += 1\n  }\n  return { n, decided, rejected, rejectionRate: decided === 0 ? null : rejected / decided }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAkDA,eAAsB,iBACpB,YACA,cACA,YACA,eACA,UAA8B,CAAC,GACI;CACnC,MAAM,WAAW;EACf,GAAG;EACH,OAAO,QAAQ,UAAU,KAAA,IAAY,KAAA,IAAY,EAAE,GAAG,QAAQ,MAAM;CACtE;CACA,2BAA2B,WAAW,IAAI,eAAe,QAAQ;CACjE,MAAM,UAAU,WAAW,WAAW,mBAAmB,WAAW,EAAE;CACtE,MAAM,WAAW,WAAW;CAC5B,MAAM,OAAO,MAAM,WAAW,SAAS;CACvC,2BACE,KAAK,KAAK,QAAQ,IAAI,KAAK,GAC3B,OACF;CACA,MAAM,WAAW,MAAM,aAAa,KAAK;CACzC,MAAM,wBAAQ,IAAI,IAAiC;CACnD,KAAK,MAAM,KAAK,UAAU;EACxB,MAAM,MAAM,MAAM,IAAI,EAAE,KAAK,KAAK,CAAC;EACnC,IAAI,KAAK,CAAC;EACV,MAAM,IAAI,EAAE,OAAO,GAAG;CACxB;CAEA,MAAM,QAAyC,CAAC;CAChD,KAAK,MAAM,OAAO,MAAM;EACtB,MAAM,KAAK,MAAM,IAAI,IAAI,KAAK;EAC9B,IAAI,CAAC,IAAI,QAAQ;EACjB,MAAM,IAAI,MAAM,QAAQ,KAAK,UAAU;EACvC,IAAI,MAAM,QAAQ,CAAC,OAAO,SAAS,CAAC,GAAG;EACvC,MAAM,IAAI,oBAAoB,IAAI,eAAe,QAAQ;EACzD,IAAI,MAAM,MAAM;EAChB,MAAM,KAAK;GAAE;GAAG;EAAE,CAAC;CACrB;CACA,IAAI,MAAM,SAAS,GAAG,OAAO;CAE7B,OAAO,qBACL,MAAM,KAAK,OAAO;EAAE,WAAW,EAAE;EAAG,SAAS,EAAE;CAAE,EAAE,GACnD,UACA,eACA,QACF;AACF;;AAGA,SAAgB,qBACd,YACA,YACA,eACA,UAA8B,CAAC,GACL;CAC1B,2BAA2B,YAAY,eAAe,OAAO;CAC7D,KAAK,MAAM,CAAC,OAAO,SAAS,WAAW,QAAQ,GAC7C,IACE,SAAS,QACT,OAAO,SAAS,YAChB,CAAC,OAAO,SAAS,KAAK,SAAS,KAC/B,CAAC,OAAO,SAAS,KAAK,OAAO,GAE7B,MAAM,IAAI,MAAM,oBAAoB,MAAM,kDAAkD;CAGhG,MAAM,QAAQ;CACd,IAAI,MAAM,SAAS,GAAG,OAAO;CAE7B,MAAM,UAAU,QAAQ,QAAQ;CAChC,MAAM,UAAU,QAAQ,WAAW;CACnC,MAAM,KAAK,MAAM,KAAK,MAAM,EAAE,SAAS;CACvC,MAAM,KAAK,QAAQ,OAAO,MAAM,KAAK,IAAI,GAAG,EAAE;CAC9C,MAAM,KAAK,QAAQ,OAAO,MAAM,KAAK,IAAI,GAAG,EAAE;CAC9C,MAAM,OAAO,KAAK;CAClB,IAAI,CAAC,OAAO,SAAS,IAAI,GAAG,MAAM,IAAI,MAAM,uCAAuC;CACnF,MAAM,UAAU,MAAM,KAAK,UAAU;EACnC,GAAG;EACH,WAAW,KAAK,IAAI,IAAI,KAAK,IAAI,IAAI,KAAK,SAAS,CAAC;CACtD,EAAE;CAEF,MAAM,OAAyB,CAAC;CAChC,IAAI,SAAS,KAAK,QAAQ,OAAO,SAAS,KAAK,cAAc,QAAQ,EAAE,CAAE,SAAS,GAChF,KAAK,KAAK,MAAM,OAAO,CAAC;MACnB,IAAI,YAAY,mBAAmB;EACxC,MAAM,SAAS,CAAC,GAAG,OAAO,CAAC,CAAC,MAAM,GAAG,MAAM,EAAE,YAAY,EAAE,SAAS;EACpE,MAAM,QAAQ,KAAK,IAAI,SAAS,OAAO,MAAM;EAC7C,KAAK,IAAI,IAAI,GAAG,IAAI,OAAO,KAAK;GAC9B,MAAM,QAAQ,KAAK,MAAO,IAAI,OAAO,SAAU,KAAK;GACpD,MAAM,MAAM,KAAK,OAAQ,IAAI,KAAK,OAAO,SAAU,KAAK;GACxD,KAAK,KAAK,MAAM,OAAO,MAAM,OAAO,GAAG,CAAC,CAAC;EAC3C;CACF,OAAO;EACL,MAAM,yBAAS,IAAI,IAA+B;EAClD,KAAK,MAAM,QAAQ,SAAS;GAC1B,MAAM,QAAQ,KAAK,IAAI,UAAU,GAAG,KAAK,OAAQ,KAAK,YAAY,MAAM,OAAQ,OAAO,CAAC;GACxF,MAAM,QAAQ,OAAO,IAAI,KAAK,KAAK,CAAC;GACpC,MAAM,KAAK,IAAI;GACf,OAAO,IAAI,OAAO,KAAK;EACzB;EACA,KAAK,MAAM,CAAC,OAAO,UAAU,CAAC,GAAG,MAAM,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,IAAI,CAAC,GAC/D,KAAK,KAAK,MAAM,OAAO,KAAK,QAAQ,QAAQ,UAAU,KAAK,SAAS,QAAQ,KAAK,QAAQ,CAAC;CAE9F;CAEA,MAAM,QAAQ,KAAK,QAAQ,GAAG,MAAM,IAAI,EAAE,GAAG,CAAC;CAC9C,MAAM,MAAM,KAAK,QAAQ,GAAG,MAAM,IAAK,EAAE,IAAI,QAAS,EAAE,KAAK,CAAC;CAC9D,MAAM,SAAS,KAAK,QAAQ,GAAG,MAAM,KAAK,IAAI,GAAG,EAAE,GAAG,GAAG,CAAC;CAE1D,OAAO;EAAE;EAAY;EAAe,GAAG,MAAM;EAAQ;EAAM;EAAK;CAAO;AACzE;;;;;;AAOA,SAAgB,WAAW,OAA2C;CACpE,IAAI,CAAC,MAAM,QAAQ,MAAM,IAAI,MAAM,wCAAwC;CAC3E,IAAI,QAAQ;CACZ,KAAK,MAAM,CAAC,OAAO,SAAS,MAAM,QAAQ,GAAG;EAC3C,IACE,SAAS,QACT,OAAO,SAAS,YAChB,EAAE,KAAK,aAAa,KAAK,KAAK,aAAa,MAC1C,KAAK,YAAY,KAAK,KAAK,YAAY,GAExC,MAAM,IAAI,MAAM,cAAc,MAAM,wDAAwD;EAE9F,UAAU,KAAK,YAAY,KAAK,YAAY;CAC9C;CACA,OAAO,QAAQ,MAAM;AACvB;;;;;;;;AAgBA,SAAgB,qBAAqB,WAAmD;CACtF,IAAI,CAAC,UAAU,QAAQ,MAAM,IAAI,MAAM,uDAAuD;CAC9F,IAAI,QAAQ;CACZ,KAAK,MAAM,CAAC,OAAO,aAAa,UAAU,QAAQ,GAAG;EACnD,MAAM,UAAU,mBAAmB,UAAU,OAAO,kBAAkB;EACtE,KAAK,MAAM,CAAC,MAAM,gBAAgB,SAChC,UAAU,eAAe,SAAS,SAAS,UAAU,IAAI,OAAO;CAEpE;CACA,OAAO,QAAQ,UAAU;AAC3B;;;;;;;AAeA,SAAgB,uBAAuB,WAA+C;CACpF,IAAI,CAAC,UAAU,QAAQ,MAAM,IAAI,MAAM,yDAAyD;CAChG,IAAI,QAAQ;CACZ,KAAK,MAAM,CAAC,OAAO,aAAa,UAAU,QAAQ,GAAG;EACnD,MAAM,gBAAgB,UAAU;EAChC,IAAI,CAAC,MAAM,QAAQ,aAAa,KAAK,cAAc,SAAS,GAC1D,MAAM,IAAI,MAAM,gBAAgB,MAAM,wCAAwC;EAEhF,mBAAmB,eAAe,gBAAgB,OAAO;EACzD,IACE,CAAC,OAAO,UAAU,SAAS,OAAO,KAClC,SAAS,UAAU,KACnB,SAAS,WAAW,cAAc,QAElC,MAAM,IAAI,MACR,gBAAgB,MAAM,iCAAiC,cAAc,OAAO,QAC9E;EAEF,IAAI,aAAa;EACjB,IAAI,MAAM;EACV,KAAK,IAAI,QAAQ,GAAG,QAAQ,cAAc,SAAS,GAAG,SAAS;GAC7D,cAAc,cAAc;GAC5B,QAAQ,cAAc,SAAS,SAAS,UAAU,IAAI,OAAO;EAC/D;EACA,SAAS,OAAO,cAAc,SAAS;CACzC;CACA,OAAO,QAAQ,UAAU;AAC3B;AAEA,SAAS,mBACP,UACA,OACA,OACoB;CACpB,IAAI,aAAa,QAAQ,OAAO,aAAa,YAAY,OAAO,SAAS,YAAY,UACnF,MAAM,IAAI,MAAM,GAAG,MAAM,YAAY,MAAM,0CAA0C;CAEvF,MAAM,gBAAgB,SAAS;CAC/B,IAAI,kBAAkB,QAAQ,OAAO,kBAAkB,YAAY,MAAM,QAAQ,aAAa,GAC5F,MAAM,IAAI,MAAM,GAAG,MAAM,YAAY,MAAM,kDAAkD;CAE/F,MAAM,UAAU,OAAO,QAAQ,aAAa;CAC5C,IAAI,QAAQ,SAAS,GAAG,MAAM,IAAI,MAAM,GAAG,MAAM,YAAY,MAAM,6BAA6B;CAChG,IAAI,CAAC,QAAQ,MAAM,CAAC,UAAU,SAAS,SAAS,OAAO,GACrD,MAAM,IAAI,MACR,GAAG,MAAM,YAAY,MAAM,kCAAkC,KAAK,UAAU,SAAS,OAAO,GAC9F;CAEF,mBACE,QAAQ,KAAK,GAAG,iBAAiB,WAAW,GAC5C,GAAG,MAAM,YAAY,OACvB;CACA,OAAO;AACT;AAEA,SAAS,mBAAmB,eAAkC,OAAqB;CACjF,IAAI,MAAM;CACV,KAAK,MAAM,eAAe,eAAe;EACvC,IAAI,EAAE,eAAe,KAAK,eAAe,IACvC,MAAM,IAAI,MAAM,GAAG,MAAM,qCAAqC,aAAa;EAE7E,OAAO;CACT;CACA,IAAI,KAAK,IAAI,MAAM,CAAC,IAAI,MAAM,MAAM,IAAI,MAAM,GAAG,MAAM,wBAAwB,IAAI,QAAQ;AAC7F;AAEA,SAAS,MAAM,OAA0B,OAAgB,OAAgC;CACvF,MAAM,KAAK,MAAM,KAAK,MAAM,EAAE,SAAS;CACvC,MAAM,KAAK,MAAM,KAAK,MAAM,EAAE,OAAO;CACrC,MAAM,WAAW,KAAK,EAAE;CACxB,MAAM,cAAc,KAAK,EAAE;CAC3B,OAAO;EACL,OAAO,SAAS,KAAK,IAAI,GAAG,EAAE;EAC9B,OAAO,SAAS,KAAK,IAAI,GAAG,EAAE;EAC9B,GAAG,MAAM;EACT;EACA;EACA,KAAK,KAAK,IAAI,cAAc,QAAQ;CACtC;AACF;AAEA,SAAS,KAAK,IAAsB;CAClC,OAAO,GAAG,QAAQ,KAAK,UAAU,MAAM,QAAQ,GAAG,QAAQ,CAAC;AAC7D;AAEA,SAAS,2BACP,YACA,eACA,SACM;CACN,2BAA2B,CAAC,UAAU,GAAG,aAAa;CACtD,2BAA2B,CAAC,aAAa,GAAG,gBAAgB;CAC5D,IAAI,WAAW,KAAK,MAAM,cAAc,cAAc,KAAK,MAAM,eAC/D,MAAM,IAAI,MAAM,oEAAoE;CAEtF,IAAI,QAAQ,SAAS,KAAA,MAAc,CAAC,OAAO,cAAc,QAAQ,IAAI,KAAK,QAAQ,OAAO,IACvF,MAAM,IAAI,MAAM,kDAAkD;CAEpE,IACE,QAAQ,YAAY,KAAA,KACpB,CAAC,CAAC,eAAe,iBAAiB,CAAC,CAAC,SAAS,QAAQ,OAAO,GAE5D,MAAM,IAAI,MAAM,4DAA4D;CAE9E,IACE,QAAQ,UAAU,KAAA,MACjB,CAAC,OAAO,SAAS,QAAQ,MAAM,EAAE,KAChC,CAAC,OAAO,SAAS,QAAQ,MAAM,EAAE,KACjC,CAAC,OAAO,SAAS,QAAQ,MAAM,KAAK,QAAQ,MAAM,EAAE,KACpD,QAAQ,MAAM,KAAK,QAAQ,MAAM,KAEnC,MAAM,IAAI,MAAM,mDAAmD;AAEvE;;;;;;;;;;ACtQA,eAAsB,iBACpB,YACA,cACA,aACA,oBACA,UAAmC,CAAC,GACH;CACjC,MAAM,YAAY,QAAQ,aAAa;CACvC,MAAM,aAAa,QAAQ,uBAAuB;CAClD,MAAM,OAAO,QAAQ;CACrB,2BAA2B,WAAW,YAAY,IAAI;CACtD,2BACE,YAAY,KAAK,WAAW,OAAO,EAAE,GACrC,aACF;CACA,2BAA2B,oBAAoB,gBAAgB;CAC/D,MAAM,SAAS,QAAQ,mBAAmB;CAC1C,IAAI,SAAS,KAAK,OAAO,MAAM,MAAM,GACnC,MAAM,IAAI,MAAM,qCAAqC;CAEvD,MAAM,aAAa,YAAY,KAAK,YAAY;EAC9C,GAAG;EACH,SAAS,OAAO,WAAW,mBAAmB,OAAO,EAAE;CACzD,EAAE;CACF,MAAM,cAAc,CAAC,GAAG,kBAAkB;CAC1C,MAAM,OAAO,MAAM,WAAW,SAAS;CACvC,2BACE,KAAK,KAAK,QAAQ,IAAI,KAAK,GAC3B,OACF;CACA,MAAM,WAAW,MAAM,aAAa,KAAK,QAAQ,aAAa;CAC9D,MAAM,gCAAgB,IAAI,IAAiC;CAC3D,KAAK,MAAM,KAAK,UAAU;EACxB,MAAM,MAAM,cAAc,IAAI,EAAE,KAAK,KAAK,CAAC;EAC3C,IAAI,KAAK,CAAC;EACV,cAAc,IAAI,EAAE,OAAO,GAAG;CAChC;CAEA,MAAM,QAA0F,CAAC;CACjG,KAAK,MAAM,MAAM,YACf,KAAK,MAAM,MAAM,aACf,MAAM,KAAK;EAAE,YAAY,GAAG;EAAI,eAAe;EAAI,IAAI,CAAC;EAAG,IAAI,CAAC;CAAE,CAAC;CAIvE,IAAI,SAAS;CACb,IAAI,UAAU;CACd,KAAK,MAAM,OAAO,MAAM;EACtB,MAAM,KAAK,cAAc,IAAI,IAAI,KAAK;EACtC,IAAI,CAAC,MAAM,GAAG,WAAW,GAAG;GAC1B;GACA;EACF;EACA,MAAM,WAAW,GAAG,QAAQ,MAAM;GAChC,MAAM,MAAM,EAAE,aAAa,IAAI;GAC/B,OAAO,OAAO,KAAK,OAAO;EAC5B,CAAC;EACD,IAAI,SAAS,WAAW,GAAG;GACzB;GACA;EACF;EAEA,IAAI,gBAAgB;EACpB,KAAK,MAAM,MAAM,YAAY;GAC3B,MAAM,IAAI,MAAM,GAAG,QAAQ,KAAK,UAAU;GAC1C,IAAI,MAAM,QAAQ,CAAC,OAAO,SAAS,CAAC,GAAG;GAEvC,KAAK,MAAM,MAAM,aAAa;IAC5B,MAAM,IAAI,oBAAoB,UAAU,IAAI,SAAS;IACrD,IAAI,MAAM,MAAM;IAChB,MAAM,OAAO,MAAM,MAAM,MAAM,EAAE,eAAe,GAAG,MAAM,EAAE,kBAAkB,EAAE;IAC/E,KAAK,GAAG,KAAK,CAAC;IACd,KAAK,GAAG,KAAK,CAAC;IACd,gBAAgB;GAClB;EACF;EACA,IAAI,eAAe;OACd;CACP;CAEA,MAAM,gBAAyD,CAAC;CAChE,MAAM,UAA+B,CAAC;CACtC,KAAK,MAAM,KAAK,OAAO;EACrB,MAAM,SACJ,EAAE,GAAG,SAAS,IACV,yBACA,CAAC,aAAa,EAAE,EAAE,IAChB,yBACA,CAAC,aAAa,EAAE,EAAE,IAChB,qBACA;EACV,IAAI,WAAW,MAAM;GACnB,cAAc,KAAK;IACjB,YAAY,EAAE;IACd,eAAe,EAAE;IACjB,GAAG,EAAE,GAAG;IACR;GACF,CAAC;GACD;EACF;EACA,MAAM,UAAU,mBAAmB,EAAE,IAAI,EAAE,IAAI,YAAY,IAAI;EAC/D,MAAM,UACJ,KAAK,IAAI,QAAQ,OAAO,KAAK,KACzB,WACA,KAAK,IAAI,QAAQ,OAAO,KAAK,KAC3B,aACA;EACR,QAAQ,KAAK;GACX,YAAY,EAAE;GACd,eAAe,EAAE;GACjB,GAAG,EAAE,GAAG;GACR,GAAG;GACH;EACF,CAAC;CACH;CACA,OAAO;EAAE,OAAO;EAAS;EAAe,eAAe;EAAQ,aAAa;CAAQ;AACtF;;;ACxLA,MAAM,WAAW,EACd,OAAO,CAAC,CACR,IAAI,CAAC,CAAC,CACN,QAAQ,UAAU,MAAM,KAAK,MAAM,KAAK;AAC3C,MAAa,iCAAiC,EAC3C,OAAO;CACN,YAAY,EAAE,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;CAC1C,wBAAwB,EAAE,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC;CACvD,uBAAuB,EAAE,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC;AACxD,CAAC,CAAC,CACD,OAAO;AAKV,MAAa,kCAAkC,EAC5C,OAAO;CACN,IAAI;CACJ,mBAAmB;CACnB,aAAa;CACb,UAAU,EAAE,KAAK,CAAC,UAAU,QAAQ,CAAC;CACrC,UAAU,EAAE,KAAK;EAAC;EAAU;EAAU;CAAS,CAAC;CAChD,UAAU,EAAE,KAAK,CAAC,SAAS,aAAa,CAAC;AAC3C,CAAC,CAAC,CACD,OAAO;AAKV,MAAM,cAAc,EACjB,OAAO;CACN,iBAAiB,EAAE,OAAO,CAAC,CAAC,MAAM,uBAAuB;CACzD,YAAY;CACZ,eAAe;CACf,WAAW,EACR,OAAO;EACN,mBAAmB;EACnB,WAAW;EACX,yBAAyB;CAC3B,CAAC,CAAC,CACD,OAAO;CACV,QAAQ;CACR,cAAc,EAAE,MAAM,+BAA+B;AACvD,CAAC,CAAC,CACD,OAAO;AAiDV,SAAS,UACP,MACA,UACA,OACA,OACoB;CACpB,MAAM,QAAQ,KAAK,QAAQ,QAAQ,IAAI,aAAa,QAAQ;CAC5D,MAAM,wBAAQ,IAAI,IAAkD;CACpE,KAAK,MAAM,OAAO,OAAO;EACvB,MAAM,OAAO,MAAM,IAAI,IAAI,iBAAiB,KAAK;GAAE,OAAO;GAAO,SAAS;EAAM;EAChF,KAAK,YAAY,IAAI,aAAa;EAClC,KAAK,UAAU,IAAI,aAAa,aAAa,IAAI,aAAa;EAC9D,MAAM,IAAI,IAAI,mBAAmB,IAAI;CACvC;CACA,MAAM,IAAI,MAAM;CAChB,MAAM,aAAa,CAAC,GAAG,MAAM,OAAO,CAAC,CAAC,CAAC,QAAQ,SAAS,KAAK,KAAK,CAAC,CAAC;CACpE,MAAM,kBAAkB,CAAC,GAAG,MAAM,OAAO,CAAC,CAAC,CAAC,QAAQ,SAAS,KAAK,WAAW,CAAC,KAAK,KAAK,CAAC,CAAC;CAC1F,MAAM,eAAe,MAAM,QAAQ,QAAQ,IAAI,aAAa,SAAS,CAAC,CAAC;CAEvE,MAAM,WACJ,MAAM,IACF,OACA;EACE,OAAO,gBACL;GAAE,MAAM;GAAmB;EAAM,GACjC;GACE,MAAM;GACN,WAAW;GACX,QAAQ;EACV,CACF,CAAC,CAAC;EACF,OAAO,gBACL;GAAE,MAAM;GAAmB;EAAM,GACjC;GACE,MAAM;GACN,WAAW,aAAa;GACxB,QAAQ;EACV,CACF,CAAC,CAAC;EACF;CACF;CACN,MAAM,UACJ,YAAY,SAAS,QAAQ,QACzB,SACA,YAAY,SAAS,SAAS,QAC5B,SACA;CACR,OAAO;EACL,OAAO,MAAM;EACb,kBAAkB;EAClB;EACA;EACA;EACA,WAAW,MAAM,KAAK,kBAAkB,IAAI,OAAO,aAAa;EAChE;EACA;EACA;CACF;AACF;AAEA,SAAS,iBAEP,cAAmB;CACnB,MAAM,sBAAM,IAAI,IAAY;CAC5B,KAAK,MAAM,OAAO,cAAc;EAC9B,IAAI,IAAI,IAAI,IAAI,EAAE,GAChB,MAAM,IAAI,gBAAgB,0CAA0C,IAAI,GAAG,EAAE;EAC/E,IAAI,IAAI,IAAI,EAAE;CAChB;CACA,aAAa,MAAM,GAAG,MAAM,iBAAiB,EAAE,IAAI,EAAE,EAAE,CAAC;CACxD,MAAM,mBAAmB,IAAI,IAC3B,aACG,QAAQ,QAAQ,IAAI,aAAa,aAAa,CAAC,CAC/C,KAAK,QAAQ,IAAI,iBAAiB,CACvC;CASA,OAAO;EAAE,UARQ,aAAa,QAAQ,QAAQ,CAAC,iBAAiB,IAAI,IAAI,iBAAiB,CAQzE;EAAG,YAPA,aAChB,QAAQ,QAAQ,iBAAiB,IAAI,IAAI,iBAAiB,CAAC,CAAC,CAC5D,KAAK,SAAS;GACb,IAAI,IAAI;GACR,mBAAmB,IAAI;GACvB,QAAQ;EACV,EAC0B;CAAE;AAChC;AAEA,SAAS,+BAA+B,YAA4B;CAClE,MAAM,QAAQ,KAAK,IAAI,cAAc;CACrC,IAAI,SAAS,GACX,MAAM,IAAI,gBACR,2EACF;CACF,OAAO;AACT;AAEA,MAAM,iBAAiB,EACpB,OAAO;CACN,QAAQ;CACR,UAAU,EACP,MAAM,gCAAgC,KAAK;EAAE,UAAU;EAAM,aAAa;CAAK,CAAC,CAAC,CAAC,CAClF,QAAQ,CAAC,CAAC;AACf,CAAC,CAAC,CACD,OAAO;;AAKV,SAAS,oBAAoB,QAAgB,OAAuB;CAClE,OAAO,CAAC,KAAK,MAAM,KAAK,KAAK,IAAI,SAAS,CAAC,IAAI,MAAM;AACvD;AAEA,SAAS,kBAAkB,OAAe,OAA8B;CACtE,IAAI,UAAU,GAAG,OAAO;CACxB,MAAM,WAAW,KAAK,KAAK,KAAK,KAAK,IAAI,SAAS,CAAC,IAAI,KAAK,MAAM,CAAC,KAAK,CAAC;CACzE,IAAI,CAAC,OAAO,SAAS,QAAQ,KAAK,WAAW,OAAO,kBAAkB,OAAO;CAC7E,MAAM,UAAU,WAAmB,oBAAoB,QAAQ,KAAK,KAAK;CACzE,IAAI,UAAU,KAAK,IAAI,GAAG,QAAQ;CAClC,OAAO,UAAU,KAAK,OAAO,UAAU,CAAC,GAAG,WAAW;CACtD,OAAO,CAAC,OAAO,OAAO,GAAG;EACvB,IAAI,YAAY,OAAO,kBAAkB,OAAO;EAChD,WAAW;CACb;CACA,OAAO;AACT;;;;;;AAOA,SAAgB,mBAAmB,OAAgC;CACjE,MAAM,SAAS,eAAe,UAAU,KAAK;CAC7C,IAAI,CAAC,OAAO,SACV,MAAM,IAAI,gBAAgB,iCAAiC,OAAO,MAAM,SAAS;CACnF,MAAM,EAAE,QAAQ,aAAa,OAAO;CACpC,MAAM,EAAE,aAAa,iBAAiB,QAAQ;CAC9C,MAAM,QAAQ,+BAA+B,OAAO,UAAU;CAC9D,MAAM,YAAY,UAAiD,UAAkB;EACnF,MAAM,mBAAmB,IAAI,IAC3B,SAAS,QAAQ,QAAQ,IAAI,aAAa,QAAQ,CAAC,CAAC,KAAK,QAAQ,IAAI,iBAAiB,CACxF,CAAC,CAAC;EACF,MAAM,yBACJ,qBAAqB,IAAI,OAAO,oBAAoB,kBAAkB,KAAK;EAC7E,MAAM,0BAA0B,kBAAkB,OAAO,KAAK;EAC9D,OAAO;GACL;GACA;GACA;GACA;GACA,4BACE,4BAA4B,OACxB,OACA,KAAK,IAAI,GAAG,0BAA0B,gBAAgB;EAC9D;CACF;CACA,MAAM,kBAAkB,SAAS,UAAU,OAAO,sBAAsB;CACxE,MAAM,iBAAiB,SAAS,UAAU,OAAO,qBAAqB;CAItE,OAAO;EAAE,YAHU,CAAC,iBAAiB,cAAc,CAAC,CAAC,OAClD,SAAS,KAAK,2BAA2B,QAAQ,KAAK,0BAA0B,KAAK,KAEtE;EAAG;EAAiB;CAAe;AACvD;;;;;;AASA,SAAgB,eAAe,OAAsD;CACnF,MAAM,SAAS,YAAY,UAAU,KAAK;CAC1C,IAAI,CAAC,OAAO,SAAS,MAAM,IAAI,gBAAgB,4BAA4B,OAAO,MAAM,SAAS;CACjG,MAAM,QAAQ,OAAO;CACrB,IAAI,MAAM,UAAU,sBAAsB,MAAM,UAAU,WACxD,MAAM,IAAI,gBAAgB,kEAAkE;CAE9F,MAAM,EAAE,UAAU,eAAe,iBAAiB,MAAM,YAAY;CACpE,MAAM,qBAAqB,+BAA+B,MAAM,OAAO,UAAU;CACjF,MAAM,kBAAkB,UACtB,UACA,UACA,MAAM,OAAO,wBACb,kBACF;CACA,MAAM,iBAAiB,UACrB,UACA,UACA,MAAM,OAAO,uBACb,kBACF;CACA,MAAM,QAAQ,CAAC,iBAAiB,cAAc;CAC9C,MAAM,UAAU,MAAM,MAAM,SAAS,KAAK,YAAY,MAAM,IACxD,WACA,MAAM,OAAO,SAAS,KAAK,YAAY,MAAM,IAC3C,UACA;CACN,MAAM,UAAoB,CAAC;CAC3B,KAAK,MAAM,CAAC,MAAM,SAAS,CACzB,CAAC,oBAAoB,eAAe,GACpC,CAAC,mBAAmB,cAAc,CACpC,GAAY;EACV,IAAI,KAAK,qBAAqB,GAAG,QAAQ,KAAK,GAAG,KAAK,gCAAgC;OACjF,IAAI,KAAK,eAAe,GAC3B,QAAQ,KAAK,GAAG,KAAK,IAAI,KAAK,aAAa,uBAAuB;EACpE,IAAI,KAAK,YAAY,QAAQ,QAAQ,KAAK,GAAG,KAAK,8BAA8B,KAAK,OAAO;OACvF,IAAI,KAAK,YAAY,kBAAkB,KAAK,UAC/C,QAAQ,KAAK,GAAG,KAAK,0DAA0D;CACnF;CACA,IAAI,YAAY,SACd,QAAQ,KACN,8FACF;CACF,MAAM,OAAuD;EAC3D,iBAAiB,MAAM;EACvB,cAAc,cAAc,MAAM,MAAM;EACxC,aAAa,cAAc,KAAK;EAChC,YAAY,MAAM;EAClB,eAAe,MAAM;EACrB,WAAW,MAAM;EACjB,QAAQ,MAAM;EACd,YAAY,MAAM,OAAO;EACzB;EACA;EACA;EACA,cAAc,MAAM;EACpB,UAAU;GACR,OAAO,MAAM,aAAa;GAC1B,kBAAkB,IAAI,IAAI,MAAM,aAAa,KAAK,QAAQ,IAAI,iBAAiB,CAAC,CAAC,CAAC;GAClF,eAAe,SAAS;GACxB,0BAA0B,IAAI,IAAI,SAAS,KAAK,QAAQ,IAAI,iBAAiB,CAAC,CAAC,CAAC;GAChF,eAAe,WAAW;GAC1B,cAAc,SAAS,QAAQ,QAAQ,IAAI,aAAa,SAAS,CAAC,CAAC;EACrE;EACA;EACA;EACA;CACF;CACA,OAAO;EAAE,GAAG;EAAM,cAAc,cAAc,IAAI;CAAE;AACtD;AAEA,MAAM,8BAA8B,EACjC,OAAO;CACN,iBAAiB,EAAE,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;CACjD,iBAAiB,EAAE,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;AACnD,CAAC,CAAC,CACD,OAAO,CAAC,CACR,QAAQ,UAAU,MAAM,kBAAkB,MAAM,iBAAiB,EAChE,SAAS,uDACX,CAAC;AACH,MAAM,+BAA+B,gCAClC,KAAK,EAAE,UAAU,KAAK,CAAC,CAAC,CACxB,OAAO,EAAE,mBAAmB,EAAE,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAC3E,OAAO;AACV,MAAM,yBAAyB,YAAY,OAAO;CAChD,YAAY;CACZ,cAAc,EAAE,MAAM,4BAA4B;AACpD,CAAC;;;;;;AAmBD,SAAgB,uBACd,OAC8B;CAC9B,MAAM,SAAS,uBAAuB,UAAU,KAAK;CACrD,IAAI,CAAC,OAAO,SACV,MAAM,IAAI,gBAAgB,qCAAqC,OAAO,MAAM,SAAS;CAEvF,MAAM,EAAE,YAAY,cAAc,GAAG,WAAW,OAAO;CACvD,aAAa,MAAM,GAAG,MAAM,iBAAiB,EAAE,IAAI,EAAE,EAAE,CAAC;CACxD,MAAM,oBAAoB;EACxB,uBAAuB,OAAO;EAC9B;EACA,cAAc,aAAa,KAAK,EAAE,IAAI,yBAAyB;GAAE;GAAI;EAAkB,EAAE;CAC3F;CAoBA,MAAM,EAAE,cAAc,aAAa,GAAG,eAnBxB,eAAe;EAC3B,GAAG;EACH,iBAAiB,cAAc;GAC7B,MAAM;GACN,uBAAuB,OAAO;GAC9B;EACF,CAAC;EACD,cAAc,aAAa,KAAK,EAAE,mBAAmB,GAAG,mBAAmB;GACzE,GAAG;GACH,UACE,sBAAsB,OAClB,YACA,qBAAqB,WAAW,kBAC9B,WACA,qBAAqB,WAAW,kBAC9B,WACA;EACZ,EAAE;CACJ,CACyD;CACzD,MAAM,OAAO;EACX,GAAG;EAEH,aAAa,cAAc,OAAO,IAAI;EACtC;CACF;CACA,OAAO;EAAE,GAAG;EAAM,cAAc,cAAc,IAAI;CAAE;AACtD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;ACvVA,MAAM,cAAoC;CACxC;CACA;CACA;CACA;AACF;AAKA,MAAM,qBAAkD,CAAC,UAAU,QAAQ;;AAG3E,MAAM,wBAAwB;;;;;;;;;;;;;AAwB9B,SAAgB,YAAY,OAKlB;CACR,MAAM,EAAE,IAAI,MAAM,MAAM,oBAAoB;CAC5C,IAAI,OAAO,OAAO,YAAY,GAAG,KAAK,MAAM,IAC1C,MAAM,IAAI,gBAAgB,4CAA4C;CAExE,IAAI,CAAC,YAAY,SAAS,IAAI,GAC5B,MAAM,IAAI,gBACR,uBAAuB,GAAG,aAAa,KAAK,UAAU,IAAI,EAAE,oBAAoB,YAAY,KAAK,IAAI,GACvG;CAEF,IAAI,CAAC,mBAAmB,SAAS,eAAe,GAC9C,MAAM,IAAI,gBACR,uBAAuB,GAAG,wBAAwB,KAAK,UAAU,eAAe,EAAE,oBAAoB,mBAAmB,KAAK,IAAI,GACpI;CAEF,IAAI,OAAO,KAAK,WAAW,YAAY,KAAK,OAAO,KAAK,MAAM,IAC5D,MAAM,IAAI,gBAAgB,uBAAuB,GAAG,2BAA2B;CAEjF,IAAI,CAAC,OAAO,SAAS,KAAK,UAAU,KAAK,KAAK,aAAa,KAAK,KAAK,aAAa,GAChF,MAAM,IAAI,gBACR,uBAAuB,GAAG,mBAAmB,KAAK,WAAW,qCAC/D;CAEF,IAAI,KAAK,UAAU,KAAA,KAAa,OAAO,KAAK,UAAU,UACpD,MAAM,IAAI,gBAAgB,uBAAuB,GAAG,8BAA8B;CAEpF,IAAI,KAAK,eAAe,uBACtB,MAAM,IAAI,gBACR,uBAAuB,GAAG,mBAAmB,sBAAsB,qDACrE;CAEF,MAAM,YAA8B,KAAK,aAAa,wBAAwB,WAAW;CACzF,IAAI,cAAc,iBAChB,MAAM,IAAI,gBACR,uBAAuB,GAAG,0BAA0B,gBAAgB,sBAAsB,KAAK,WAAW,QAAQ,WACpH;CAEF,OAAO;EACL;EACA;EACA,MAAM;GACJ,QAAQ,KAAK;GACb,YAAY,KAAK;GACjB,GAAI,KAAK,UAAU,KAAA,IAAY,CAAC,IAAI,EAAE,OAAO,KAAK,MAAM;EAC1D;EACA;CACF;AACF;;;;;;;;;;;AA8CA,MAAM,cAAc;;;;;;;;AASpB,MAAM,mBAA2D;CAC/D,CAAC,SAAS,OAAO;CACjB,CAAC,SAAS,OAAO;CACjB,CAAC,SAAS,OAAO;CACjB,CAAC,SAAS,OAAO;CACjB,CAAC,SAAS,OAAO;CACjB,CAAC,SAAS,OAAO;CACjB,CAAC,MAAM,GAAG;CACV,CAAC,MAAM,GAAG;CACV,CAAC,MAAM,IAAI;CACX,CAAC,MAAM,IAAI;AACb;;;;;;;;;AAeA,SAAS,qBAAqB,MAA2B;CACvD,MAAM,SAAS;CACf,IAAI,QAAqB;CACzB,IAAI,QAAQ,OAAO,KAAK,IAAI;CAC5B,OAAO,UAAU,MAAM;EACrB,MAAM,QAAQ,MAAM;EACpB,MAAM,MAAM,QAAQ,MAAM,EAAE,CAAC;EAE7B,IAAI,CAAC,YAAY,KAAK,KAAK,OAAO,QAAQ,CAAC,CAAC,KAAK,CAAC,YAAY,KAAK,KAAK,OAAO,GAAG,CAAC,GACjF,QAAQ;GAAE;GAAO;EAAI;EAEvB,QAAQ,OAAO,KAAK,IAAI;CAC1B;CACA,OAAO;AACT;;;;;;;;;AAUA,SAAS,gBAAgB,MAAsD;CAC7E,KAAK,IAAI,QAAQ,GAAG,QAAQ,KAAK,QAAQ,SAAS,GAChD,KAAK,MAAM,CAAC,OAAO,YAAY,kBAC7B,IAAI,KAAK,WAAW,OAAO,KAAK,GAC9B,OAAO;EAAE,MAAM;GAAE,OAAO;GAAO,KAAK,QAAQ,MAAM;EAAO;EAAG;CAAQ;CAI1E,OAAO;AACT;AAEA,SAAS,YAAY,MAAc,MAAY,OAAuB;CACpE,OAAO,KAAK,MAAM,GAAG,KAAK,KAAK,IAAI,QAAQ,KAAK,MAAM,KAAK,GAAG;AAChE;;;;;AAMA,SAAS,UACP,OACA,QACA,OACA,OACe;CACf,MAAM,YAAY,UAAU,UAAU,QAAQ;CAC9C,MAAM,aAAa,UAAU,WAAW,QAAQ;CAChD,OAAO;EACL,GAAI,cAAc,KAAA,IAAY,CAAC,IAAI,EAAE,OAAO,UAAU;EACtD,GAAI,eAAe,KAAA,IAAY,CAAC,IAAI,EAAE,QAAQ,WAAW;CAC3D;AACF;AAEA,SAAS,SACP,OACA,QACA,OACA,MACA,MACsB;CACtB,MAAM,WAAW,KAAK,MAAM,KAAK,OAAO,KAAK,GAAG;CAChD,MAAM,aAAa,OAAO,QAAQ,IAAI,GAAA,CAAI,SAAS;CACnD,OAAO;EACL,UAAU,UAAU,OAAO,QAAQ,OAAO,YAAY,MAAM,MAAM,SAAS,CAAC;EAC5E;EACA,KAAK;EACL;EACA;CACF;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA6BA,SAAgB,gBAAgB,UAAsD;CACpF,MAAM,QAAQ,OAAO,SAAS,UAAU,WAAW,SAAS,QAAQ,KAAA;CACpE,MAAM,SAAS,OAAO,SAAS,WAAW,WAAW,SAAS,SAAS,KAAA;CAEvE,IAAI,WAAW,KAAA,GAAW;EACxB,MAAM,OAAO,qBAAqB,MAAM;EACxC,IAAI,SAAS,MAAM,OAAO,SAAS,OAAO,QAAQ,UAAU,QAAQ,IAAI;CAC1E;CACA,IAAI,UAAU,KAAA,GAAW;EACvB,MAAM,aAAa,gBAAgB,KAAK;EACxC,IAAI,eAAe,MACjB,OAAO;GACL,UAAU,UACR,OACA,QACA,SACA,YAAY,OAAO,WAAW,MAAM,WAAW,OAAO,CACxD;GACA,OAAO;GACP,KAAK;GACL,UAAU,MAAM,MAAM,WAAW,KAAK,OAAO,WAAW,KAAK,GAAG;GAChE,WAAW,WAAW;EACxB;EAEF,MAAM,OAAO,qBAAqB,KAAK;EACvC,IAAI,SAAS,MAAM,OAAO,SAAS,OAAO,QAAQ,SAAS,OAAO,IAAI;CACxE;CACA,OAAO;AACT;;;;;;;;;;;;;;;;;;;;;AAgCA,SAAgB,oBAAoB,OAOV;CACxB,MAAM,EAAE,IAAI,QAAQ,UAAU,aAAa,GAAG,UAAU;CACxD,IAAI,OAAO,OAAO,YAAY,GAAG,KAAK,MAAM,IAC1C,MAAM,IAAI,gBAAgB,oDAAoD;CAEhF,IAAI,OAAO,WAAW,YAAY,OAAO,KAAK,MAAM,IAClD,MAAM,IAAI,gBAAgB,+BAA+B,GAAG,sBAAsB;CAEpF,MAAM,eAAe,gBAAgB,QAAQ;CAC7C,IAAI,iBAAiB,MAAM,OAAO;CAOlC,OAAO;EACL,OAPY,YAAY;GACxB;GACA,MAAM;GACN,MAAM;IAAE;IAAQ;IAAY,GAAI,UAAU,KAAA,IAAY,CAAC,IAAI,EAAE,MAAM;GAAG;GACtE,iBAAiB;EACnB,CAEM;EACJ,UAAU,aAAa;EACvB,OAAO,aAAa;EACpB,KAAK,aAAa;EAClB,UAAU,aAAa;EACvB,WAAW,aAAa;CAC1B;AACF;;;;;;;;;;AAoDA,SAAgB,WACd,SACA,QACA,UAA6B,CAAC,GACZ;CAClB,MAAM,OAAO,QAAQ,QAAQ;CAC7B,MAAM,kBAAkB,QAAQ,mBAAmB;CACnD,IAAI,CAAC,OAAO,SAAS,IAAI,GACvB,MAAM,IAAI,gBAAgB,iDAAiD,MAAM;CAEnF,IAAI,CAAC,OAAO,SAAS,eAAe,KAAK,mBAAmB,KAAK,kBAAkB,GACjF,MAAM,IAAI,gBACR,sEAAsE,iBACxE;CAGF,MAAM,eAA6B,CAAC;CACpC,MAAM,uBAAO,IAAI,IAAY;CAC7B,KAAK,MAAM,QAAQ,SAAS;EAC1B,IAAI,OAAO,KAAK,WAAW,YAAY,KAAK,OAAO,KAAK,MAAM,IAC5D,MAAM,IAAI,gBAAgB,gDAAgD;EAE5E,IAAI,CAAC,OAAO,SAAS,KAAK,UAAU,GAClC,MAAM,IAAI,gBACR,6BAA6B,KAAK,OAAO,8BAC3C;EAEF,IAAI,KAAK,IAAI,KAAK,MAAM,GACtB,MAAM,IAAI,gBAAgB,yCAAyC,KAAK,OAAO,EAAE;EAEnF,KAAK,IAAI,KAAK,MAAM;EACpB,aAAa,KAAK;GAChB,QAAQ,KAAK;GACb,YAAY,KAAK;GACjB,GAAI,KAAK,UAAU,KAAA,IAAY,CAAC,IAAI,EAAE,OAAO,KAAK,MAAM;EAC1D,CAAC;CACH;CAEA,MAAM,eAAwB,CAAC;CAC/B,MAAM,2BAAW,IAAI,IAAY;CACjC,KAAK,MAAM,SAAS,QAAQ;EAC1B,MAAM,SAAS,YAAY,KAAK;EAChC,IAAI,SAAS,IAAI,OAAO,EAAE,GACxB,MAAM,IAAI,gBAAgB,mCAAmC,OAAO,GAAG,EAAE;EAE3E,IAAI,KAAK,IAAI,OAAO,KAAK,MAAM,GAC7B,MAAM,IAAI,gBACR,sBAAsB,OAAO,GAAG,mBAAmB,OAAO,KAAK,OAAO,+BACxE;EAEF,MAAM,gBACJ,OAAO,KAAK,cAAc,kBAAkB,WAAW;EACzD,IAAI,kBAAkB,OAAO,iBAC3B,MAAM,IAAI,gBACR,sBAAsB,OAAO,GAAG,0BAA0B,OAAO,gBAAgB,sBAAsB,OAAO,KAAK,WAAW,aAAa,cAAc,2BAA2B,iBACtL;EAEF,SAAS,IAAI,OAAO,EAAE;EACtB,KAAK,IAAI,OAAO,KAAK,MAAM;EAC3B,aAAa,KAAK,MAAM;CAC1B;CAEA,MAAM,QAAQ,SACZ,CAAC,GAAG,cAAc,GAAG,aAAa,KAAK,UAAU,MAAM,IAAI,CAAC,GAC5D,WAAW,IAAI,CACjB;CACA,MAAM,UAAU,MAAM,KAAK,SAAS,KAAK,MAAM;CAQ/C,OAAO;EAAE;EAAO,UAAA;GANd,MAAM,aAAa;IAAE;IAAM;IAAiB;IAAS,QAAQ;GAAa,CAAC;GAC3E;GACA;GACA;GACA,QAAQ;EAEa;CAAE;AAC3B;;;;;;;AAQA,SAAS,SAAY,OAAqB,QAA2B;CACnE,OAAO,MACJ,KAAK,UAAU;EAAE;EAAM,KAAK,OAAO;CAAE,EAAE,CAAC,CACxC,MAAM,MAAM,UAAU,KAAK,MAAM,MAAM,GAAG,CAAC,CAC3C,KAAK,UAAU,MAAM,IAAI;AAC9B;AAEA,SAAS,aAAa,UAAmD;CACvE,OAAO,cAAc;EACnB,QAAQ;EACR,MAAM,SAAS;EACf,iBAAiB,SAAS;EAC1B,SAAS,SAAS;EAClB,QAAQ,SAAS;CACnB,CAAC;AACH;;;;;;;;;AAwEA,SAAgB,UACd,SACA,UACiB;CACjB,MAAM,eAAe,aAAa;EAChC,MAAM,SAAS;EACf,iBAAiB,SAAS;EAC1B,SAAS,SAAS;EAClB,QAAQ,SAAS;CACnB,CAAC;CACD,IAAI,iBAAiB,SAAS,MAC5B,MAAM,IAAI,sBACR,wCAAwC,aAAa,iCAAiC,SAAS,KAAK,6CACtG;CAGF,MAAM,YAAY,IAAI,IAAI,SAAS,OAAO;CAC1C,MAAM,gCAAgB,IAAI,IAA2B;CACrD,KAAK,MAAM,UAAU,SAAS;EAC5B,IAAI,CAAC,UAAU,IAAI,OAAO,MAAM,GAC9B,MAAM,IAAI,gBACR,0BAA0B,OAAO,OAAO,wCAC1C;EAEF,IAAI,cAAc,IAAI,OAAO,MAAM,GACjC,MAAM,IAAI,gBAAgB,oCAAoC,OAAO,OAAO,EAAE;EAEhF,IAAI,OAAO,UAAU,QAAQ,CAAC,OAAO,SAAS,OAAO,KAAK,GACxD,MAAM,IAAI,gBACR,0BAA0B,OAAO,OAAO,cAAc,OAAO,MAAM,mCACrE;EAEF,cAAc,IAAI,OAAO,QAAQ,OAAO,KAAK;CAC/C;CAEA,MAAM,SAAsD,CAAC;CAC7D,MAAM,YAAsB,CAAC;CAC7B,MAAM,aAAuB,CAAC;CAC9B,IAAI,SAAS;CACb,IAAI,SAAS;CACb,IAAI,aAAa;CAEjB,KAAK,MAAM,SAAS,SAAS,QAAQ;EACnC,IAAI,SAAS,OAAO,MAAM;EAC1B,IAAI,WAAW,KAAA,GAAW;GACxB,SAAS;IAAE,QAAQ;IAAG,QAAQ;IAAG,QAAQ;IAAG,YAAY;IAAG,MAAM;GAAK;GACtE,OAAO,MAAM,QAAQ;EACvB;EACA,OAAO,UAAU;EACjB,IAAI,CAAC,cAAc,IAAI,MAAM,KAAK,MAAM,GAAG;GACzC,WAAW,KAAK,MAAM,EAAE;GACxB;EACF;EACA,MAAM,QAAQ,cAAc,IAAI,MAAM,KAAK,MAAM,KAAK;EACtD,IAAI,UAAU,MAAM;GAClB,cAAc;GACd,OAAO,cAAc;GACrB;EACF;EAEA,KADqC,SAAS,SAAS,kBAAkB,WAAW,cACjE,MAAM,iBAAiB;GACxC,UAAU;GACV,OAAO,UAAU;EACnB,OAAO;GACL,UAAU;GACV,OAAO,UAAU;GACjB,UAAU,KAAK,MAAM,EAAE;EACzB;CACF;CAEA,MAAM,SAAS,SAAS,OAAO;CAC/B,MAAM,UAAU,SAAS;CACzB,MAAM,SAA0B;EAC9B,QAAQ;EACR;EACA;EACA;EACA;EACA,MAAM;EACN;EACA;EACA;EACA,UAAU,kBAAkB,UAAU,aAAa;CACrD;CAEA,IAAI,WAAW,GACb,OAAO;EACL,GAAG;EACH,QAAQ;EACR,QAAQ;CACV;CAEF,IAAI,WAAW,SAAS,GACtB,OAAO;EACL,GAAG;EACH,QAAQ;EACR,QAAQ,GAAG,WAAW,OAAO,MAAM,OAAO,iCAAiC,WAAW,KAAK,IAAI;CACjG;CAEF,IAAI,YAAY,GACd,OAAO;EACL,GAAG;EACH,QAAQ;EACR,QAAQ,OAAO,OAAO;CACxB;CAGF,KAAK,MAAM,UAAU,OAAO,OAAO,MAAM,GAAG;EAC1C,MAAM,cAAc,OAAO,SAAS,OAAO;EAC3C,OAAO,OAAO,gBAAgB,IAAI,OAAO,OAAO,SAAS;CAC3D;CACA,OAAO,OAAO,SAAS;CACvB,OAAO;AACT;AAEA,SAAS,kBACP,UACA,eACmB;CACnB,MAAM,eAAe,IAAI,IAAI,SAAS,OAAO,KAAK,UAAU,MAAM,KAAK,MAAM,CAAC;CAC9E,IAAI,IAAI;CACR,IAAI,UAAU;CACd,IAAI,WAAW;CACf,KAAK,MAAM,UAAU,SAAS,SAAS;EACrC,IAAI,aAAa,IAAI,MAAM,GAAG;EAC9B,KAAK;EACL,MAAM,QAAQ,cAAc,IAAI,MAAM;EACtC,IAAI,UAAU,KAAA,KAAa,UAAU,MAAM;EAC3C,WAAW;EACX,IAAI,QAAQ,SAAS,iBAAiB,YAAY;CACpD;CACA,OAAO;EAAE;EAAG;EAAS;EAAU,eAAe,YAAY,IAAI,OAAO,WAAW;CAAQ;AAC1F"}
+{"version":3,"file":"index.js","names":[],"sources":["../../src/meta-eval/correlation-study.ts","../../src/meta-eval/evaluator-admission.ts","../../src/meta-eval/plants.ts"],"sourcesContent":["/**\n * Correlation study — \"does our eval score predict real-world outcomes?\"\n *\n * Joins traces and outcomes by runId and reports descriptive correlations.\n * Independent runs are the bootstrap observation unit.\n * Association alone does not establish causation or held-out predictive performance.\n */\n\nimport { runMetricExtractor } from '../trace/query'\nimport type { Run } from '../trace/schema'\nimport type { TraceStore } from '../trace/store'\nimport {\n  assertUniqueObservationIds,\n  correlationSummary,\n  hasVariation,\n  reduceOutcomeMetric,\n  validateObservationOptions,\n} from './outcome-observations'\nimport type { DeploymentOutcome, OutcomeFilter, OutcomeStore } from './outcome-store'\n\nexport interface EvalMetricSpec {\n  id: string\n  /** Extract a scalar from a run. Omit it and `id` must name one of\n   *  `RUN_METRICS`; any other `id` is refused. */\n  extract?: (run: Run, store: TraceStore) => Promise<number | null>\n}\n\nexport interface OutcomePair {\n  evalMetric: string\n  outcomeMetric: string\n}\n\nexport interface CorrelationResult {\n  evalMetric: string\n  outcomeMetric: string\n  n: number\n  pearson: number\n  spearman: number\n  /** 95% bootstrap CI for Pearson. */\n  pearsonCi95: { lower: number; upper: number } | null\n  /** 95% bootstrap CI for Spearman; null when no resample is estimable. */\n  spearmanCi95: { lower: number; upper: number } | null\n  /** Rough verdict: 'strong' ≥ 0.7, 'moderate' ≥ 0.4, else 'weak'. */\n  verdict: 'strong' | 'moderate' | 'weak'\n}\n\nexport interface CorrelationStudyResult {\n  pairs: CorrelationResult[]\n  /** Declared pairs without an estimable correlation, including their usable sample count. */\n  excludedPairs: Array<\n    OutcomePair & {\n      n: number\n      reason: 'insufficient_samples' | 'constant_eval_metric' | 'constant_outcome'\n    }\n  >\n  joinedSamples: number\n  skippedRuns: number\n}\n\nexport interface CorrelationStudyOptions {\n  /** Only join outcomes captured within this window after run.startedAt. */\n  maxCaptureLagMs?: number\n  /** Restrict to a subset of outcomes (cohort, region, source). */\n  outcomeFilter?: OutcomeFilter\n  /** Which outcome per run to use when multiple exist. Default 'latest'. */\n  reduction?: 'latest' | 'mean' | 'max'\n  /** Bootstrap iterations for the CI. Default 500. */\n  bootstrapIterations?: number\n  /** Seed for the bootstrap resampler. Absent, the seed is derived from the\n   *  paired observations, so the same study reproduces the same interval. */\n  seed?: number\n}\n\nexport async function correlationStudy(\n  traceStore: TraceStore,\n  outcomeStore: OutcomeStore,\n  evalMetrics: EvalMetricSpec[],\n  outcomeMetricNames: string[],\n  options: CorrelationStudyOptions = {},\n): Promise<CorrelationStudyResult> {\n  const reduction = options.reduction ?? 'latest'\n  const iterations = options.bootstrapIterations ?? 500\n  const seed = options.seed\n  validateObservationOptions(reduction, iterations, seed)\n  assertUniqueObservationIds(\n    evalMetrics.map((metric) => metric.id),\n    'eval metric',\n  )\n  assertUniqueObservationIds(outcomeMetricNames, 'outcome metric')\n  const maxLag = options.maxCaptureLagMs ?? Infinity\n  if (maxLag < 0 || Number.isNaN(maxLag)) {\n    throw new Error('maxCaptureLagMs must be nonnegative')\n  }\n  const extractors = evalMetrics.map((metric) => ({\n    ...metric,\n    extract: metric.extract ?? runMetricExtractor(metric.id),\n  }))\n  const metricNames = [...outcomeMetricNames]\n  const runs = await traceStore.listRuns()\n  assertUniqueObservationIds(\n    runs.map((run) => run.runId),\n    'runId',\n  )\n  const outcomes = await outcomeStore.list(options.outcomeFilter)\n  const outcomesByRun = new Map<string, DeploymentOutcome[]>()\n  for (const o of outcomes) {\n    const arr = outcomesByRun.get(o.runId) ?? []\n    arr.push(o)\n    outcomesByRun.set(o.runId, arr)\n  }\n\n  const pairs: Array<{ evalMetric: string; outcomeMetric: string; xs: number[]; ys: number[] }> = []\n  for (const em of extractors) {\n    for (const om of metricNames) {\n      pairs.push({ evalMetric: em.id, outcomeMetric: om, xs: [], ys: [] })\n    }\n  }\n\n  let joined = 0\n  let skipped = 0\n  for (const run of runs) {\n    const os = outcomesByRun.get(run.runId)\n    if (!os || os.length === 0) {\n      skipped++\n      continue\n    }\n    const eligible = os.filter((o) => {\n      const lag = o.capturedAt - run.startedAt\n      return lag >= 0 && lag <= maxLag\n    })\n    if (eligible.length === 0) {\n      skipped++\n      continue\n    }\n\n    let joinedThisRun = false\n    for (const em of extractors) {\n      const x = await em.extract(run, traceStore)\n      if (x === null || !Number.isFinite(x)) continue\n\n      for (const om of metricNames) {\n        const y = reduceOutcomeMetric(eligible, om, reduction)\n        if (y === null) continue\n        const pair = pairs.find((p) => p.evalMetric === em.id && p.outcomeMetric === om)!\n        pair.xs.push(x)\n        pair.ys.push(y)\n        joinedThisRun = true\n      }\n    }\n    if (joinedThisRun) joined++\n    else skipped++\n  }\n\n  const excludedPairs: CorrelationStudyResult['excludedPairs'] = []\n  const results: CorrelationResult[] = []\n  for (const p of pairs) {\n    const reason =\n      p.xs.length < 3\n        ? 'insufficient_samples'\n        : !hasVariation(p.xs)\n          ? 'constant_eval_metric'\n          : !hasVariation(p.ys)\n            ? 'constant_outcome'\n            : null\n    if (reason !== null) {\n      excludedPairs.push({\n        evalMetric: p.evalMetric,\n        outcomeMetric: p.outcomeMetric,\n        n: p.xs.length,\n        reason,\n      })\n      continue\n    }\n    const summary = correlationSummary(p.xs, p.ys, iterations, seed)\n    const verdict: CorrelationResult['verdict'] =\n      Math.abs(summary.pearson) >= 0.7\n        ? 'strong'\n        : Math.abs(summary.pearson) >= 0.4\n          ? 'moderate'\n          : 'weak'\n    results.push({\n      evalMetric: p.evalMetric,\n      outcomeMetric: p.outcomeMetric,\n      n: p.xs.length,\n      ...summary,\n      verdict,\n    })\n  }\n  return { pairs: results, excludedPairs, joinedSamples: joined, skippedRuns: skipped }\n}\n","import { z } from 'zod'\nimport { ValidationError } from '../errors'\nimport { type ComputedInterval, computeInterval } from '../experiment/ast/interval'\nimport { compareCodeUnits, hashCanonical, type LedgerHash } from '../ledger-core/canonical'\n\nconst identity = z\n  .string()\n  .min(1)\n  .refine((value) => value.trim() === value)\nexport const evaluatorAdmissionPolicySchema = z\n  .object({\n    confidence: z.number().finite().gt(0).lt(1),\n    maxFalseAcceptanceRate: z.number().finite().min(0).lt(1),\n    maxFalseRejectionRate: z.number().finite().min(0).lt(1),\n  })\n  .strict()\n\n/** Register both error limits before inspecting audit judgments. */\nexport type EvaluatorAdmissionPolicy = z.infer<typeof evaluatorAdmissionPolicySchema>\n\nexport const evaluatorAuditObservationSchema = z\n  .object({\n    id: identity,\n    independentUnitId: identity,\n    evidenceRef: identity,\n    expected: z.enum(['accept', 'reject']),\n    observed: z.enum(['accept', 'reject', 'unknown']),\n    exposure: z.enum(['fresh', 'development']),\n  })\n  .strict()\n\n/** Actual control judgments. Variants from one source retain one independent identity. */\nexport type EvaluatorAuditObservation = z.infer<typeof evaluatorAuditObservationSchema>\n\nconst inputSchema = z\n  .object({\n    evaluatorDigest: z.string().regex(/^sha256:[a-f0-9]{64}$/),\n    population: identity,\n    samplingFrame: identity,\n    authority: z\n      .object({\n        evaluatorAuthorId: identity,\n        auditorId: identity,\n        independenceEvidenceRef: identity,\n      })\n      .strict(),\n    policy: evaluatorAdmissionPolicySchema,\n    observations: z.array(evaluatorAuditObservationSchema),\n  })\n  .strict()\n\nexport type EvaluatorAuditInput = z.infer<typeof inputSchema>\n\nexport interface EvaluatorErrorRate {\n  /** Eligible control judgments in this class, including unknown judgments. */\n  cases: number\n  independentUnits: number\n  /** Units with at least one observed mistake. */\n  errorUnits: number\n  /** Units with unknown judgments and no observed mistake yet. */\n  unresolvedUnits: number\n  unknownCases: number\n  /** Missing judgments do not become measured zeros. */\n  errorRate: number | null\n  /** Exact bounds conservatively include every possible outcome of unknown judgments. */\n  interval: ComputedInterval | null\n  limit: number\n  verdict: 'pass' | 'fail' | 'inconclusive'\n}\n\nexport interface EvaluatorAdmissionReport {\n  evaluatorDigest: LedgerHash\n  policyDigest: LedgerHash\n  inputDigest: LedgerHash\n  reportDigest: LedgerHash\n  population: string\n  samplingFrame: string\n  authority: EvaluatorAuditInput['authority']\n  policy: EvaluatorAdmissionPolicy\n  /** Joint coverage for the two reported error-rate intervals. */\n  confidence: number\n  intervalConfidence: number\n  verdict: 'admit' | 'reject' | 'inconclusive'\n  reasons: string[]\n  observations: EvaluatorAuditObservation[]\n  coverage: {\n    cases: number\n    independentUnits: number\n    eligibleCases: number\n    eligibleIndependentUnits: number\n    excludedCases: number\n    unknownCases: number\n  }\n  exclusions: Array<{ id: string; independentUnitId: string; reason: 'development-exposure' }>\n  falseAcceptance: EvaluatorErrorRate\n  falseRejection: EvaluatorErrorRate\n}\n\nfunction errorRate(\n  rows: readonly EvaluatorAuditObservation[],\n  expected: 'accept' | 'reject',\n  limit: number,\n  level: number,\n): EvaluatorErrorRate {\n  const cases = rows.filter((row) => row.expected === expected)\n  const units = new Map<string, { error: boolean; unknown: boolean }>()\n  for (const row of cases) {\n    const unit = units.get(row.independentUnitId) ?? { error: false, unknown: false }\n    unit.unknown ||= row.observed === 'unknown'\n    unit.error ||= row.observed !== 'unknown' && row.observed !== expected\n    units.set(row.independentUnitId, unit)\n  }\n  const n = units.size\n  const errorUnits = [...units.values()].filter((unit) => unit.error).length\n  const unresolvedUnits = [...units.values()].filter((unit) => unit.unknown && !unit.error).length\n  const unknownCases = cases.filter((row) => row.observed === 'unknown').length\n  // Unknown judgments contribute to n; their most favorable and adverse outcomes bound the rate.\n  const interval =\n    n === 0\n      ? null\n      : {\n          lower: computeInterval(\n            { kind: 'clopper-pearson', level },\n            {\n              kind: 'binomial',\n              successes: errorUnits,\n              trials: n,\n            },\n          ).lower,\n          upper: computeInterval(\n            { kind: 'clopper-pearson', level },\n            {\n              kind: 'binomial',\n              successes: errorUnits + unresolvedUnits,\n              trials: n,\n            },\n          ).upper,\n          level,\n        }\n  const verdict =\n    interval && interval.lower > limit\n      ? 'fail'\n      : interval && interval.upper <= limit\n        ? 'pass'\n        : 'inconclusive'\n  return {\n    cases: cases.length,\n    independentUnits: n,\n    errorUnits,\n    unresolvedUnits,\n    unknownCases,\n    errorRate: n === 0 || unresolvedUnits > 0 ? null : errorUnits / n,\n    interval,\n    limit,\n    verdict,\n  }\n}\n\nfunction eligibleControls<\n  T extends Pick<EvaluatorAuditObservation, 'id' | 'independentUnitId' | 'exposure'>,\n>(observations: T[]) {\n  const ids = new Set<string>()\n  for (const row of observations) {\n    if (ids.has(row.id))\n      throw new ValidationError(`duplicate evaluator audit observation '${row.id}'`)\n    ids.add(row.id)\n  }\n  observations.sort((a, b) => compareCodeUnits(a.id, b.id))\n  const developmentUnits = new Set(\n    observations\n      .filter((row) => row.exposure === 'development')\n      .map((row) => row.independentUnitId),\n  )\n  const eligible = observations.filter((row) => !developmentUnits.has(row.independentUnitId))\n  const exclusions = observations\n    .filter((row) => developmentUnits.has(row.independentUnitId))\n    .map((row) => ({\n      id: row.id,\n      independentUnitId: row.independentUnitId,\n      reason: 'development-exposure' as const,\n    }))\n  return { eligible, exclusions }\n}\n\nfunction simultaneousIntervalConfidence(confidence: number): number {\n  const level = 1 - (1 - confidence) / 2\n  if (level >= 1)\n    throw new ValidationError(\n      'evaluator audit confidence is too close to one for simultaneous intervals',\n    )\n  return level\n}\n\nconst capacitySchema = z\n  .object({\n    policy: evaluatorAdmissionPolicySchema,\n    controls: z\n      .array(evaluatorAuditObservationSchema.omit({ observed: true, evidenceRef: true }))\n      .default([]),\n  })\n  .strict()\n\nexport type EvaluatorAuditPlanInput = z.input<typeof capacitySchema>\n\n/** The zero-error eligibility floor, not a statistical power calculation. */\nfunction zeroErrorUpperBound(trials: number, level: number): number {\n  return -Math.expm1(Math.log((1 - level) / 2) / trials)\n}\n\nfunction minimumAuditUnits(limit: number, level: number): number | null {\n  if (limit === 0) return null\n  const estimate = Math.ceil(Math.log((1 - level) / 2) / Math.log1p(-limit))\n  if (!Number.isFinite(estimate) || estimate > Number.MAX_SAFE_INTEGER) return null\n  const clears = (trials: number) => zeroErrorUpperBound(trials, level) <= limit\n  let minimum = Math.max(1, estimate)\n  while (minimum > 1 && clears(minimum - 1)) minimum -= 1\n  while (!clears(minimum)) {\n    if (minimum === Number.MAX_SAFE_INTEGER) return null\n    minimum += 1\n  }\n  return minimum\n}\n\n/**\n * Can the fixed control roster meet the policy even with zero errors?\n * Planning uses the admission rule's exact bounds and source-unit exclusions.\n * It accepts no judgments or authority and returns no admission report.\n */\nexport function planEvaluatorAudit(input: EvaluatorAuditPlanInput) {\n  const parsed = capacitySchema.safeParse(input)\n  if (!parsed.success)\n    throw new ValidationError(`invalid evaluator audit plan: ${parsed.error.message}`)\n  const { policy, controls } = parsed.data\n  const { eligible } = eligibleControls(controls)\n  const level = simultaneousIntervalConfidence(policy.confidence)\n  const capacity = (expected: EvaluatorAuditObservation['expected'], limit: number) => {\n    const independentUnits = new Set(\n      eligible.filter((row) => row.expected === expected).map((row) => row.independentUnitId),\n    ).size\n    const bestPossibleUpperBound =\n      independentUnits === 0 ? null : zeroErrorUpperBound(independentUnits, level)\n    const minimumIndependentUnits = minimumAuditUnits(limit, level)\n    return {\n      independentUnits,\n      bestPossibleUpperBound,\n      limit,\n      minimumIndependentUnits,\n      additionalIndependentUnits:\n        minimumIndependentUnits === null\n          ? null\n          : Math.max(0, minimumIndependentUnits - independentUnits),\n    }\n  }\n  const falseAcceptance = capacity('reject', policy.maxFalseAcceptanceRate)\n  const falseRejection = capacity('accept', policy.maxFalseRejectionRate)\n  const sufficient = [falseAcceptance, falseRejection].every(\n    (rate) => rate.bestPossibleUpperBound !== null && rate.bestPossibleUpperBound <= rate.limit,\n  )\n  return { sufficient, falseAcceptance, falseRejection }\n}\n\nexport type EvaluatorAuditPlan = ReturnType<typeof planEvaluatorAudit>\n\n/**\n * Audit frozen judgments using independent source units and exact binomial bounds.\n * A unit fails a class when any control in that class is misjudged.\n * The execution owner enforces auditor separation, fresh sampling, and evidence authenticity.\n */\nexport function auditEvaluator(input: EvaluatorAuditInput): EvaluatorAdmissionReport {\n  const parsed = inputSchema.safeParse(input)\n  if (!parsed.success) throw new ValidationError(`invalid evaluator audit: ${parsed.error.message}`)\n  const audit = parsed.data\n  if (audit.authority.evaluatorAuthorId === audit.authority.auditorId) {\n    throw new ValidationError('evaluator admission requires a separate declared audit authority')\n  }\n  const { eligible, exclusions } = eligibleControls(audit.observations)\n  const intervalConfidence = simultaneousIntervalConfidence(audit.policy.confidence)\n  const falseAcceptance = errorRate(\n    eligible,\n    'reject',\n    audit.policy.maxFalseAcceptanceRate,\n    intervalConfidence,\n  )\n  const falseRejection = errorRate(\n    eligible,\n    'accept',\n    audit.policy.maxFalseRejectionRate,\n    intervalConfidence,\n  )\n  const rates = [falseAcceptance, falseRejection]\n  const verdict = rates.some((rate) => rate.verdict === 'fail')\n    ? 'reject'\n    : rates.every((rate) => rate.verdict === 'pass')\n      ? 'admit'\n      : 'inconclusive'\n  const reasons: string[] = []\n  for (const [name, rate] of [\n    ['false acceptance', falseAcceptance],\n    ['false rejection', falseRejection],\n  ] as const) {\n    if (rate.independentUnits === 0) reasons.push(`${name}: no eligible independent units`)\n    else if (rate.unknownCases > 0)\n      reasons.push(`${name}: ${rate.unknownCases} judgments are unknown`)\n    if (rate.verdict === 'fail') reasons.push(`${name}: lower error bound exceeds ${rate.limit}`)\n    else if (rate.verdict === 'inconclusive' && rate.interval)\n      reasons.push(`${name}: upper error bound does not establish the required limit`)\n  }\n  if (verdict === 'admit')\n    reasons.push(\n      'both error bounds meet the registered limits, including the worst case for unknown judgments',\n    )\n  const body: Omit<EvaluatorAdmissionReport, 'reportDigest'> = {\n    evaluatorDigest: audit.evaluatorDigest as LedgerHash,\n    policyDigest: hashCanonical(audit.policy),\n    inputDigest: hashCanonical(audit),\n    population: audit.population,\n    samplingFrame: audit.samplingFrame,\n    authority: audit.authority,\n    policy: audit.policy,\n    confidence: audit.policy.confidence,\n    intervalConfidence,\n    verdict,\n    reasons,\n    observations: audit.observations,\n    coverage: {\n      cases: audit.observations.length,\n      independentUnits: new Set(audit.observations.map((row) => row.independentUnitId)).size,\n      eligibleCases: eligible.length,\n      eligibleIndependentUnits: new Set(eligible.map((row) => row.independentUnitId)).size,\n      excludedCases: exclusions.length,\n      unknownCases: eligible.filter((row) => row.observed === 'unknown').length,\n    },\n    exclusions,\n    falseAcceptance,\n    falseRejection,\n  }\n  return { ...body, reportDigest: hashCanonical(body) }\n}\n\nconst probabilityThresholdsSchema = z\n  .object({\n    rejectAtOrBelow: z.number().finite().min(0).max(1),\n    acceptAtOrAbove: z.number().finite().min(0).max(1),\n  })\n  .strict()\n  .refine((value) => value.rejectAtOrBelow < value.acceptAtOrAbove, {\n    message: 'rejection and acceptance thresholds must not overlap',\n  })\nconst probabilityObservationSchema = evaluatorAuditObservationSchema\n  .omit({ observed: true })\n  .extend({ acceptProbability: z.number().finite().min(0).max(1).nullable() })\n  .strict()\nconst probabilityAuditSchema = inputSchema.extend({\n  thresholds: probabilityThresholdsSchema,\n  observations: z.array(probabilityObservationSchema),\n})\n\n/** A frozen caller-owned rule over P(accept), not the provider's entropy/confidence statistic. */\nexport type ProbabilityPolicyAuditInput = z.infer<typeof probabilityAuditSchema>\n\nexport interface ProbabilityPolicyAuditReport extends EvaluatorAdmissionReport {\n  probabilityPolicy: {\n    sourceEvaluatorDigest: LedgerHash\n    thresholds: ProbabilityPolicyAuditInput['thresholds']\n    /** Retain probabilities by observation id so the reported mapping can be reproduced. */\n    observations: Array<{ id: string; acceptProbability: number | null }>\n  }\n}\n\n/**\n * Audit a fixed probability-to-decision rule with the existing independent-unit error bounds.\n * No threshold search, model calls, promotion, or assumption of calibrated probabilities.\n * Null scores and the open interval between thresholds remain unknown, never correct by default.\n */\nexport function auditProbabilityPolicy(\n  input: ProbabilityPolicyAuditInput,\n): ProbabilityPolicyAuditReport {\n  const parsed = probabilityAuditSchema.safeParse(input)\n  if (!parsed.success) {\n    throw new ValidationError(`invalid probability policy audit: ${parsed.error.message}`)\n  }\n  const { thresholds, observations, ...source } = parsed.data\n  observations.sort((a, b) => compareCodeUnits(a.id, b.id))\n  const probabilityPolicy = {\n    sourceEvaluatorDigest: source.evaluatorDigest as LedgerHash,\n    thresholds,\n    observations: observations.map(({ id, acceptProbability }) => ({ id, acceptProbability })),\n  }\n  const audit = auditEvaluator({\n    ...source,\n    evaluatorDigest: hashCanonical({\n      kind: 'probability-policy-v1',\n      sourceEvaluatorDigest: source.evaluatorDigest,\n      thresholds,\n    }),\n    observations: observations.map(({ acceptProbability, ...observation }) => ({\n      ...observation,\n      observed:\n        acceptProbability === null\n          ? 'unknown'\n          : acceptProbability >= thresholds.acceptAtOrAbove\n            ? 'accept'\n            : acceptProbability <= thresholds.rejectAtOrBelow\n              ? 'reject'\n              : 'unknown',\n    })),\n  })\n  const { reportDigest: _baseDigest, ...baseReport } = audit\n  const body = {\n    ...baseReport,\n    // Bind the actual probabilities too, including changes that select the same action.\n    inputDigest: hashCanonical(parsed.data),\n    probabilityPolicy,\n  }\n  return { ...body, reportDigest: hashCanonical(body) }\n}\n","/**\n * Plants — seeded known-wrong items that measure the grader, not the work.\n *\n * A grading run reports how the work scored. It cannot report whether the\n * grader would have noticed a wrong answer, because every item it saw was\n * authored in good faith. A plant closes that hole: an item authored wrong by\n * construction is mixed into the live set, graded by the same path as\n * everything else, and the share of plants the grader refused is the catch\n * rate.\n *\n * Measured motive: a sibling lab ran a deliverable gate that accepted any\n * non-empty submission. It produced six false certifications in seventeen\n * deliveries, and no agent lied — the gate never asked a question the format\n * could fail. A catch rate is the number that would have shown it on day one.\n *\n * This module composes existing primitives rather than adding parallel ones:\n *\n * - A plant IS a {@link GoldenItem} from `../judge-calibration`. Its\n *   `humanScore` is the grade a working grader owes the item, so the same\n *   array feeds `calibrateJudge` unchanged.\n * - The grader's output is `CandidateScore[]`, the array `calibrateJudge` and\n *   `snapshotFromSentinelSet` already consume.\n * - \"Caught\" is `snapshotFromSentinelSet`'s join with the labels inverted:\n *   the grade lands on the side of `acceptThreshold` the seed demands.\n * - The manifest is sealed with `hashCanonical` from `../ledger-core/canonical`,\n *   the digest the sealed-experiment path uses, so the answer key cannot be\n *   revised once the results are in.\n *\n * Authoring the wrong item is the step before all of that, and it is where the\n * measurement silently breaks. {@link plantByPerturbation} takes a claim a\n * grader verified and alters exactly one load-bearing value in its evidence,\n * so the grader is asked a question it must fail. Hand-rolled perturbation\n * bumps a number that is part of an identity — `python3`, `file42.txt`, `1.5`,\n * `utf-8` — and the check stops EXECUTING; the plant then measures the\n * environment rather than the grader. {@link perturbEvidence} refuses those\n * digit runs by construction.\n *\n * Blindness has two halves, and this module owns one. It never puts a plant\n * flag on a graded item: `seedPlants` returns the mixed set and a manifest,\n * and only the manifest knows which ids are seeded. Keeping the manifest out\n * of the graded workspace and publishing its `seal` before grading is the\n * caller's half; {@link catchRate} refuses a manifest whose contents no longer\n * match its seal.\n *\n * Refusals, because a catch rate that cannot refuse is not a measurement:\n *\n * - a seeded id with no result makes the report `incomplete`, never a rate\n *   over the results that did come back;\n * - zero seeded plants makes it `not_evaluated`, never 1.0;\n * - a result for an id the manifest never handed out is refused outright.\n */\n\nimport { CaptureIntegrityError, ValidationError } from '../errors'\nimport type { GoldenItem } from '../judge-calibration'\nimport { hashCanonical, type LedgerHash } from '../ledger-core/canonical'\nimport { mulberry32 } from '../statistics/random'\n\n/**\n * How a plant item was authored wrong. The class is reported separately in\n * {@link CatchRateReport.byKind} because a grader is routinely sharp on one\n * and blind to another.\n */\nexport type PlantKind =\n  /** A load-bearing value is altered: a number off by one, a comparison flipped. */\n  | 'wrong-value'\n  /** The item carries its own check, and that check passes without testing the claim. */\n  | 'self-certifying'\n  /** The check names an input that does not exist, so it cannot run at all. */\n  | 'unreachable-input'\n  /** A copy of an item already in the set, which is owed a duplicate flag rather than a second grade. */\n  | 'duplicate'\n\nconst PLANT_KINDS: readonly PlantKind[] = [\n  'wrong-value',\n  'self-certifying',\n  'unreachable-input',\n  'duplicate',\n]\n\n/** What a working grader owes a seeded item. */\nexport type PlantExpectation = 'reject' | 'accept'\n\nconst PLANT_EXPECTATIONS: readonly PlantExpectation[] = ['reject', 'accept']\n\n/** The label boundary a plant record is checked against at definition time. */\nconst RECORD_LABEL_BOUNDARY = 0.5\n\nexport interface Plant {\n  /** Name of the plant record. Reported in `missedIds` and `missingIds`. */\n  id: string\n  kind: PlantKind\n  /** The seeded item, indistinguishable from a real one once mixed. */\n  item: GoldenItem\n  /** The verdict a working grader owes this item. */\n  expectedVerdict: PlantExpectation\n}\n\n/**\n * Build one plant record and refuse an incoherent one.\n *\n * The refusal that matters is the last: a record whose `expectedVerdict`\n * disagrees with `item.humanScore` inverts the measurement silently, because\n * the same item then reads as wrong here and as correct to every calibration\n * instrument that joins on the id.\n *\n * `item` is copied field by field so a later mutation of the caller's object\n * cannot change what the manifest sealed, and `group` is dropped when it is\n * absent so the record always has a canonical JSON form.\n */\nexport function definePlant(input: {\n  id: string\n  kind: PlantKind\n  item: GoldenItem\n  expectedVerdict: PlantExpectation\n}): Plant {\n  const { id, kind, item, expectedVerdict } = input\n  if (typeof id !== 'string' || id.trim() === '') {\n    throw new ValidationError('definePlant: id must be a non-empty string')\n  }\n  if (!PLANT_KINDS.includes(kind)) {\n    throw new ValidationError(\n      `definePlant: plant \"${id}\" has kind ${JSON.stringify(kind)}; expected one of ${PLANT_KINDS.join(', ')}`,\n    )\n  }\n  if (!PLANT_EXPECTATIONS.includes(expectedVerdict)) {\n    throw new ValidationError(\n      `definePlant: plant \"${id}\" has expectedVerdict ${JSON.stringify(expectedVerdict)}; expected one of ${PLANT_EXPECTATIONS.join(', ')}`,\n    )\n  }\n  if (typeof item.itemId !== 'string' || item.itemId.trim() === '') {\n    throw new ValidationError(`definePlant: plant \"${id}\" has an empty item.itemId`)\n  }\n  if (!Number.isFinite(item.humanScore) || item.humanScore < 0 || item.humanScore > 1) {\n    throw new ValidationError(\n      `definePlant: plant \"${id}\" has humanScore ${item.humanScore}; expected a finite number in [0, 1]`,\n    )\n  }\n  if (item.group !== undefined && typeof item.group !== 'string') {\n    throw new ValidationError(`definePlant: plant \"${id}\" has a non-string item.group`)\n  }\n  if (item.humanScore === RECORD_LABEL_BOUNDARY) {\n    throw new ValidationError(\n      `definePlant: plant \"${id}\" has humanScore ${RECORD_LABEL_BOUNDARY}, which states neither a rejection nor an acceptance`,\n    )\n  }\n  const labelSays: PlantExpectation = item.humanScore < RECORD_LABEL_BOUNDARY ? 'reject' : 'accept'\n  if (labelSays !== expectedVerdict) {\n    throw new ValidationError(\n      `definePlant: plant \"${id}\" expects the grader to ${expectedVerdict} it, but humanScore ${item.humanScore} says ${labelSays}`,\n    )\n  }\n  return {\n    id,\n    kind,\n    item: {\n      itemId: item.itemId,\n      humanScore: item.humanScore,\n      ...(item.group === undefined ? {} : { group: item.group }),\n    },\n    expectedVerdict,\n  }\n}\n\n/**\n * A claim's executable evidence: the command a grader runs and the output it\n * must produce.\n */\nexport interface PlantEvidence {\n  /** The command a grader runs to test the claim. */\n  check?: string\n  /** The output that command must produce for the claim to stand. */\n  expect?: string\n}\n\n/** Which evidence field {@link perturbEvidence} altered. */\nexport type PerturbedField = 'check' | 'expect'\n\n/** How {@link perturbEvidence} altered it. */\nexport type PerturbationKind = 'number-off-by-one' | 'comparison-flipped'\n\nexport interface EvidencePerturbation {\n  /** The evidence with exactly one value altered. */\n  evidence: PlantEvidence\n  /** Which field changed. */\n  field: PerturbedField\n  /** How it changed. */\n  how: PerturbationKind\n  /**\n   * The exact substring that was replaced. A comparison token carries its\n   * surrounding spaces, because the spaces are what make ` -ge ` a comparison\n   * and not part of a word.\n   */\n  original: string\n  /** What replaced it. */\n  perturbed: string\n}\n\n/**\n * Characters that make a digit run part of something larger than a number.\n *\n * A run that touches one of these is an identity — `python3`, `file42.txt`,\n * `1.5`, `utf-8` — and bumping it renames a program, a file, a version, or an\n * encoding instead of falsifying the claim. The check then stops EXECUTING,\n * and a plant that cannot run measures the environment rather than the grader:\n * it reads as caught, for a reason that has nothing to do with the seeded\n * defect.\n */\nconst NUMBER_GLUE = /[A-Za-z0-9_./-]/\n\n/**\n * Comparison tokens that mean one thing only, each with its inversion.\n *\n * A bare `>` or `<` is absent on purpose: in a shell check it is a\n * redirection, so flipping it rewrites where output goes rather than what the\n * test asks, and the check stops testing the claim.\n */\nconst COMPARISON_FLIPS: readonly (readonly [string, string])[] = [\n  [' -ge ', ' -lt '],\n  [' -gt ', ' -le '],\n  [' -le ', ' -gt '],\n  [' -lt ', ' -ge '],\n  [' -eq ', ' -ne '],\n  [' -ne ', ' -eq '],\n  ['>=', '<'],\n  ['<=', '>'],\n  ['==', '!='],\n  ['!=', '=='],\n]\n\ninterface Span {\n  start: number\n  end: number\n}\n\n/**\n * The last digit run in `text` that is a number and nothing else, or null.\n *\n * The last is taken because a measured value trails its label — `count=42`,\n * `n>=5 OK cells=8` — and because the choice has to be deterministic: the same\n * claim must always yield the same plant, or two runs of the same authoring\n * step seed two different answer keys for one item.\n */\nfunction lastStandaloneNumber(text: string): Span | null {\n  const digits = /\\d+/g\n  let found: Span | null = null\n  let match = digits.exec(text)\n  while (match !== null) {\n    const start = match.index\n    const end = start + match[0].length\n    // charAt returns '' outside the string, and NUMBER_GLUE never matches ''.\n    if (!NUMBER_GLUE.test(text.charAt(start - 1)) && !NUMBER_GLUE.test(text.charAt(end))) {\n      found = { start, end }\n    }\n    match = digits.exec(text)\n  }\n  return found\n}\n\n/**\n * The first flippable comparison in `text`, or null.\n *\n * Any single flip inverts the test wherever it sits, so unlike the number rule\n * there is no safety ordering among the matches. The first is taken so the\n * result is fixed by the head of the command and does not move when a later\n * comparison is added.\n */\nfunction firstComparison(text: string): { span: Span; flipped: string } | null {\n  for (let index = 0; index < text.length; index += 1) {\n    for (const [token, flipped] of COMPARISON_FLIPS) {\n      if (text.startsWith(token, index)) {\n        return { span: { start: index, end: index + token.length }, flipped }\n      }\n    }\n  }\n  return null\n}\n\nfunction replaceSpan(text: string, span: Span, value: string): string {\n  return text.slice(0, span.start) + value + text.slice(span.end)\n}\n\n/**\n * Rebuild the evidence with one field replaced. An absent field is dropped\n * rather than set to undefined, so the record always has a canonical JSON form.\n */\nfunction withField(\n  check: string | undefined,\n  expect: string | undefined,\n  field: PerturbedField,\n  value: string,\n): PlantEvidence {\n  const nextCheck = field === 'check' ? value : check\n  const nextExpect = field === 'expect' ? value : expect\n  return {\n    ...(nextCheck === undefined ? {} : { check: nextCheck }),\n    ...(nextExpect === undefined ? {} : { expect: nextExpect }),\n  }\n}\n\nfunction offByOne(\n  check: string | undefined,\n  expect: string | undefined,\n  field: PerturbedField,\n  text: string,\n  span: Span,\n): EvidencePerturbation {\n  const original = text.slice(span.start, span.end)\n  const perturbed = (BigInt(original) + 1n).toString()\n  return {\n    evidence: withField(check, expect, field, replaceSpan(text, span, perturbed)),\n    field,\n    how: 'number-off-by-one',\n    original,\n    perturbed,\n  }\n}\n\n/**\n * Derive a known-wrong version of a claim's evidence by altering exactly one\n * value, or return null when no value can be altered safely.\n *\n * Exactly one value changes. A perturbation that moves two does not name the\n * defect it seeded, so a grader that catches it says nothing about which defect\n * the grader can see.\n *\n * The preference order IS the safety order:\n *\n * 1. the last standalone number in `expect` — it falsifies the claim while the\n *    command still runs exactly as it did;\n * 2. a flipped comparison in `check` — it inverts a test that was passing;\n * 3. the last standalone number in `check` — last because a number in a command\n *    can name an input (a port, a width, a version) rather than a threshold,\n *    and renaming an input breaks execution, not the claim.\n *\n * \"Standalone\" excludes a digit run glued to a word, a dot, a slash, or a\n * hyphen: `python3`, `file42.txt`, `1.5`, `utf-8`. That exclusion is why this\n * helper exists. Hand-rolled perturbation bumps one of those, the check stops\n * executing, and the plant measures the environment instead of the grader.\n *\n * A number is bumped with BigInt, so a value wider than\n * `Number.MAX_SAFE_INTEGER` moves by exactly one. Bumped as a `number` it\n * rounds back to itself, and the plant is then a correct claim the grader is\n * right to verify.\n */\nexport function perturbEvidence(evidence: PlantEvidence): EvidencePerturbation | null {\n  const check = typeof evidence.check === 'string' ? evidence.check : undefined\n  const expect = typeof evidence.expect === 'string' ? evidence.expect : undefined\n\n  if (expect !== undefined) {\n    const span = lastStandaloneNumber(expect)\n    if (span !== null) return offByOne(check, expect, 'expect', expect, span)\n  }\n  if (check !== undefined) {\n    const comparison = firstComparison(check)\n    if (comparison !== null) {\n      return {\n        evidence: withField(\n          check,\n          expect,\n          'check',\n          replaceSpan(check, comparison.span, comparison.flipped),\n        ),\n        field: 'check',\n        how: 'comparison-flipped',\n        original: check.slice(comparison.span.start, comparison.span.end),\n        perturbed: comparison.flipped,\n      }\n    }\n    const span = lastStandaloneNumber(check)\n    if (span !== null) return offByOne(check, expect, 'check', check, span)\n  }\n  return null\n}\n\nexport interface PerturbedPlant {\n  plant: Plant\n  /** The perturbed evidence to write into the item the grader is handed. */\n  evidence: PlantEvidence\n  field: PerturbedField\n  how: PerturbationKind\n  original: string\n  perturbed: string\n}\n\n/**\n * Derive a reject-direction plant from a claim a grader verified, by perturbing\n * one value.\n *\n * {@link definePlant} takes a {@link GoldenItem} and never authors a wrong item\n * from a right one, so until now every caller hand-rolled the perturbation.\n * This is that step, done once and the same way every time: the claim's\n * evidence is altered by {@link perturbEvidence}, the result is a `wrong-value`\n * plant the grader owes a `reject`, and the record itself is built by\n * `definePlant` so every refusal there still applies — in particular the one\n * that catches an `expectedVerdict` its `humanScore` contradicts.\n *\n * Returns null when nothing in the evidence can be perturbed. That is not a\n * failure: a claim with no executable evidence cannot be made wrong in a way a\n * grader could execute, and seeding it would measure prose.\n *\n * Refuses an empty `id` or `itemId`. Both name the record — one in the manifest\n * and in `missedIds`, the other in the join with the results — and a miss\n * nobody can look up is not a measurement.\n */\nexport function plantByPerturbation(input: {\n  id: string\n  itemId: string\n  evidence: PlantEvidence\n  /** Score for the perturbed item. Must sit below the run's accept threshold. Default 0. */\n  humanScore?: number\n  group?: string\n}): PerturbedPlant | null {\n  const { id, itemId, evidence, humanScore = 0, group } = input\n  if (typeof id !== 'string' || id.trim() === '') {\n    throw new ValidationError('plantByPerturbation: id must be a non-empty string')\n  }\n  if (typeof itemId !== 'string' || itemId.trim() === '') {\n    throw new ValidationError(`plantByPerturbation: plant \"${id}\" has an empty itemId`)\n  }\n  const perturbation = perturbEvidence(evidence)\n  if (perturbation === null) return null\n  const plant = definePlant({\n    id,\n    kind: 'wrong-value',\n    item: { itemId, humanScore, ...(group === undefined ? {} : { group }) },\n    expectedVerdict: 'reject',\n  })\n  return {\n    plant,\n    evidence: perturbation.evidence,\n    field: perturbation.field,\n    how: perturbation.how,\n    original: perturbation.original,\n    perturbed: perturbation.perturbed,\n  }\n}\n\nexport interface PlantManifest {\n  /**\n   * Digest over the seeded order, the plants, and the threshold. Publish it\n   * before the grading run: a manifest edited afterwards to match the results\n   * no longer matches its seal, and {@link catchRate} refuses it.\n   */\n  seal: LedgerHash\n  /** The seed that fixed the mix order. */\n  seed: number\n  /** The grade at or above which the graded policy's own gate accepts an item. */\n  acceptThreshold: number\n  /** Every item id in the seeded set, in the order handed out. */\n  itemIds: string[]\n  /** The seeded plants. This is the answer key; keep it out of the graded workspace. */\n  plants: Plant[]\n}\n\nexport interface SeededGradingSet {\n  /**\n   * The mixed set in seeded order. Operator-side: it still carries every\n   * item's `humanScore`, so hand the graded policy the payload each `itemId`\n   * names, never this array.\n   */\n  items: GoldenItem[]\n  manifest: PlantManifest\n}\n\nexport interface SeedPlantsOptions {\n  /**\n   * Fixes the mix order. The same dataset, plants, threshold, and seed always\n   * produce the same seeded order and the same seal.\n   */\n  seed?: number\n  /**\n   * The grade at or above which the graded policy's own gate accepts an item.\n   * Supply the threshold your gate uses; the default suits a judge scoring in\n   * [0, 1] with a pass at the midpoint.\n   */\n  acceptThreshold?: number\n}\n\n/**\n * Mix plants into a grading set and seal which items they are.\n *\n * Refusals: a duplicate id anywhere in the mixed set (the join is by id, so a\n * repeat makes one of the two unscoreable), a plant whose item id collides\n * with a dataset item (it would shadow real work and grade it as a plant), a\n * dataset item with no usable label, and a plant whose expectation the run's\n * `acceptThreshold` contradicts.\n */\nexport function seedPlants(\n  dataset: readonly GoldenItem[],\n  plants: readonly Plant[],\n  options: SeedPlantsOptions = {},\n): SeededGradingSet {\n  const seed = options.seed ?? 7\n  const acceptThreshold = options.acceptThreshold ?? 0.5\n  if (!Number.isFinite(seed)) {\n    throw new ValidationError(`seedPlants: seed must be a finite number, got ${seed}`)\n  }\n  if (!Number.isFinite(acceptThreshold) || acceptThreshold <= 0 || acceptThreshold > 1) {\n    throw new ValidationError(\n      `seedPlants: acceptThreshold must be a finite number in (0, 1], got ${acceptThreshold}`,\n    )\n  }\n\n  const datasetItems: GoldenItem[] = []\n  const seen = new Set<string>()\n  for (const item of dataset) {\n    if (typeof item.itemId !== 'string' || item.itemId.trim() === '') {\n      throw new ValidationError('seedPlants: a dataset item has an empty itemId')\n    }\n    if (!Number.isFinite(item.humanScore)) {\n      throw new ValidationError(\n        `seedPlants: dataset item \"${item.itemId}\" has a non-finite humanScore`,\n      )\n    }\n    if (seen.has(item.itemId)) {\n      throw new ValidationError(`seedPlants: duplicate dataset itemId \"${item.itemId}\"`)\n    }\n    seen.add(item.itemId)\n    datasetItems.push({\n      itemId: item.itemId,\n      humanScore: item.humanScore,\n      ...(item.group === undefined ? {} : { group: item.group }),\n    })\n  }\n\n  const sealedPlants: Plant[] = []\n  const plantIds = new Set<string>()\n  for (const plant of plants) {\n    const record = definePlant(plant)\n    if (plantIds.has(record.id)) {\n      throw new ValidationError(`seedPlants: duplicate plant id \"${record.id}\"`)\n    }\n    if (seen.has(record.item.itemId)) {\n      throw new ValidationError(\n        `seedPlants: plant \"${record.id}\" reuses itemId \"${record.item.itemId}\", which is already in the set`,\n      )\n    }\n    const thresholdSays: PlantExpectation =\n      record.item.humanScore >= acceptThreshold ? 'accept' : 'reject'\n    if (thresholdSays !== record.expectedVerdict) {\n      throw new ValidationError(\n        `seedPlants: plant \"${record.id}\" expects the grader to ${record.expectedVerdict} it, but humanScore ${record.item.humanScore} is on the ${thresholdSays} side of acceptThreshold ${acceptThreshold}`,\n      )\n    }\n    plantIds.add(record.id)\n    seen.add(record.item.itemId)\n    sealedPlants.push(record)\n  }\n\n  const items = shuffled(\n    [...datasetItems, ...sealedPlants.map((plant) => plant.item)],\n    mulberry32(seed),\n  )\n  const itemIds = items.map((item) => item.itemId)\n  const manifest: PlantManifest = {\n    seal: sealManifest({ seed, acceptThreshold, itemIds, plants: sealedPlants }),\n    seed,\n    acceptThreshold,\n    itemIds,\n    plants: sealedPlants,\n  }\n  return { items, manifest }\n}\n\n/**\n * Order by one independent uniform key per item, which is a uniform\n * permutation and a pure function of the seed. A comparator that returns a\n * fresh random sign instead is neither: it is not a consistent ordering, so\n * the permutation it produces is biased and depends on the sort algorithm.\n */\nfunction shuffled<T>(items: readonly T[], random: () => number): T[] {\n  return items\n    .map((item) => ({ item, key: random() }))\n    .sort((left, right) => left.key - right.key)\n    .map((entry) => entry.item)\n}\n\nfunction sealManifest(contents: Omit<PlantManifest, 'seal'>): LedgerHash {\n  return hashCanonical({\n    scheme: 'agent-eval.plant-manifest.v1',\n    seed: contents.seed,\n    acceptThreshold: contents.acceptThreshold,\n    itemIds: contents.itemIds,\n    plants: contents.plants,\n  })\n}\n\n/**\n * One grader outcome for one item. `score: null` says the grader ran and\n * declined to decide — the check never tested the item's defect. Any\n * `CandidateScore` from a judge run is already a valid outcome.\n */\nexport interface PlantOutcome {\n  itemId: string\n  score: number | null\n}\n\n/**\n * `evaluated` — a rate stands. `incomplete` — a seeded id had no result.\n * `not_evaluated` — nothing was seeded, or nothing seeded was decided.\n */\nexport type CatchRateStatus = 'evaluated' | 'incomplete' | 'not_evaluated'\n\nexport interface PlantKindCounts {\n  seeded: number\n  caught: number\n  missed: number\n  indecisive: number\n  /** caught / (caught + missed), or null when the report is not `evaluated`. */\n  rate: number | null\n}\n\n/**\n * How the same grader treated the unseeded items of the same set. No labels\n * are needed to read it: a grader that refuses everything scores `rate` 1.0\n * on reject-plants, and a `rejectionRate` of 1.0 here is what separates that\n * reflex from discrimination.\n */\nexport interface UnseededRejection {\n  n: number\n  decided: number\n  rejected: number\n  /** rejected / decided, or null when nothing unseeded was decided. */\n  rejectionRate: number | null\n}\n\nexport interface CatchRateReport {\n  status: CatchRateStatus\n  /** Why the status is not `evaluated`. Absent when it is. */\n  reason?: string\n  seeded: number\n  caught: number\n  missed: number\n  /**\n   * The grader returned a result and declined to decide. Counted apart: it\n   * enters neither side of `rate`.\n   */\n  indecisive: number\n  /** caught / (caught + missed), or null unless the status is `evaluated`. */\n  rate: number | null\n  /** One entry per kind actually seeded. A kind nobody seeded is absent, never zero. */\n  byKind: Partial<Record<PlantKind, PlantKindCounts>>\n  /** Plant ids the grader graded as the seed says it must not. */\n  missedIds: string[]\n  /** Plant ids with no result at all — the reason a status is `incomplete`. */\n  missingIds: string[]\n  unseeded: UnseededRejection\n}\n\n/**\n * Score a grading run against its sealed manifest.\n *\n * Refusals: a manifest whose contents no longer match its seal, a result for\n * an id the manifest never handed out, and a repeated result id. Each says\n * the results and the manifest describe different runs, and a rate computed\n * across two runs is a fabrication.\n */\nexport function catchRate(\n  results: readonly PlantOutcome[],\n  manifest: PlantManifest,\n): CatchRateReport {\n  const expectedSeal = sealManifest({\n    seed: manifest.seed,\n    acceptThreshold: manifest.acceptThreshold,\n    itemIds: manifest.itemIds,\n    plants: manifest.plants,\n  })\n  if (expectedSeal !== manifest.seal) {\n    throw new CaptureIntegrityError(\n      `catchRate: manifest contents hash to ${expectedSeal} but the manifest carries seal ${manifest.seal} — the plant set changed after it was sealed`,\n    )\n  }\n\n  const handedOut = new Set(manifest.itemIds)\n  const scoreByItemId = new Map<string, number | null>()\n  for (const result of results) {\n    if (!handedOut.has(result.itemId)) {\n      throw new ValidationError(\n        `catchRate: result for \"${result.itemId}\", which this manifest never handed out`,\n      )\n    }\n    if (scoreByItemId.has(result.itemId)) {\n      throw new ValidationError(`catchRate: duplicate result for \"${result.itemId}\"`)\n    }\n    if (result.score !== null && !Number.isFinite(result.score)) {\n      throw new ValidationError(\n        `catchRate: result for \"${result.itemId}\" has score ${result.score}; expected a finite number or null`,\n      )\n    }\n    scoreByItemId.set(result.itemId, result.score)\n  }\n\n  const byKind: Partial<Record<PlantKind, PlantKindCounts>> = {}\n  const missedIds: string[] = []\n  const missingIds: string[] = []\n  let caught = 0\n  let missed = 0\n  let indecisive = 0\n\n  for (const plant of manifest.plants) {\n    let counts = byKind[plant.kind]\n    if (counts === undefined) {\n      counts = { seeded: 0, caught: 0, missed: 0, indecisive: 0, rate: null }\n      byKind[plant.kind] = counts\n    }\n    counts.seeded += 1\n    if (!scoreByItemId.has(plant.item.itemId)) {\n      missingIds.push(plant.id)\n      continue\n    }\n    const score = scoreByItemId.get(plant.item.itemId) ?? null\n    if (score === null) {\n      indecisive += 1\n      counts.indecisive += 1\n      continue\n    }\n    const graderSaid: PlantExpectation = score >= manifest.acceptThreshold ? 'accept' : 'reject'\n    if (graderSaid === plant.expectedVerdict) {\n      caught += 1\n      counts.caught += 1\n    } else {\n      missed += 1\n      counts.missed += 1\n      missedIds.push(plant.id)\n    }\n  }\n\n  const seeded = manifest.plants.length\n  const decided = caught + missed\n  const report: CatchRateReport = {\n    status: 'evaluated',\n    seeded,\n    caught,\n    missed,\n    indecisive,\n    rate: null,\n    byKind,\n    missedIds,\n    missingIds,\n    unseeded: unseededRejection(manifest, scoreByItemId),\n  }\n\n  if (seeded === 0) {\n    return {\n      ...report,\n      status: 'not_evaluated',\n      reason: 'no plant was seeded, so the grader was never asked a question it could fail',\n    }\n  }\n  if (missingIds.length > 0) {\n    return {\n      ...report,\n      status: 'incomplete',\n      reason: `${missingIds.length} of ${seeded} seeded plants have no result: ${missingIds.join(', ')}`,\n    }\n  }\n  if (decided === 0) {\n    return {\n      ...report,\n      status: 'not_evaluated',\n      reason: `all ${seeded} seeded plants are indecisive: no check tested the seeded defect`,\n    }\n  }\n\n  for (const counts of Object.values(byKind)) {\n    const kindDecided = counts.caught + counts.missed\n    counts.rate = kindDecided === 0 ? null : counts.caught / kindDecided\n  }\n  report.rate = caught / decided\n  return report\n}\n\nfunction unseededRejection(\n  manifest: PlantManifest,\n  scoreByItemId: ReadonlyMap<string, number | null>,\n): UnseededRejection {\n  const plantItemIds = new Set(manifest.plants.map((plant) => plant.item.itemId))\n  let n = 0\n  let decided = 0\n  let rejected = 0\n  for (const itemId of manifest.itemIds) {\n    if (plantItemIds.has(itemId)) continue\n    n += 1\n    const score = scoreByItemId.get(itemId)\n    if (score === undefined || score === null) continue\n    decided += 1\n    if (score < manifest.acceptThreshold) rejected += 1\n  }\n  return { n, decided, rejected, rejectionRate: decided === 0 ? null : rejected / decided }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAyEA,eAAsB,iBACpB,YACA,cACA,aACA,oBACA,UAAmC,CAAC,GACH;CACjC,MAAM,YAAY,QAAQ,aAAa;CACvC,MAAM,aAAa,QAAQ,uBAAuB;CAClD,MAAM,OAAO,QAAQ;CACrB,2BAA2B,WAAW,YAAY,IAAI;CACtD,2BACE,YAAY,KAAK,WAAW,OAAO,EAAE,GACrC,aACF;CACA,2BAA2B,oBAAoB,gBAAgB;CAC/D,MAAM,SAAS,QAAQ,mBAAmB;CAC1C,IAAI,SAAS,KAAK,OAAO,MAAM,MAAM,GACnC,MAAM,IAAI,MAAM,qCAAqC;CAEvD,MAAM,aAAa,YAAY,KAAK,YAAY;EAC9C,GAAG;EACH,SAAS,OAAO,WAAW,mBAAmB,OAAO,EAAE;CACzD,EAAE;CACF,MAAM,cAAc,CAAC,GAAG,kBAAkB;CAC1C,MAAM,OAAO,MAAM,WAAW,SAAS;CACvC,2BACE,KAAK,KAAK,QAAQ,IAAI,KAAK,GAC3B,OACF;CACA,MAAM,WAAW,MAAM,aAAa,KAAK,QAAQ,aAAa;CAC9D,MAAM,gCAAgB,IAAI,IAAiC;CAC3D,KAAK,MAAM,KAAK,UAAU;EACxB,MAAM,MAAM,cAAc,IAAI,EAAE,KAAK,KAAK,CAAC;EAC3C,IAAI,KAAK,CAAC;EACV,cAAc,IAAI,EAAE,OAAO,GAAG;CAChC;CAEA,MAAM,QAA0F,CAAC;CACjG,KAAK,MAAM,MAAM,YACf,KAAK,MAAM,MAAM,aACf,MAAM,KAAK;EAAE,YAAY,GAAG;EAAI,eAAe;EAAI,IAAI,CAAC;EAAG,IAAI,CAAC;CAAE,CAAC;CAIvE,IAAI,SAAS;CACb,IAAI,UAAU;CACd,KAAK,MAAM,OAAO,MAAM;EACtB,MAAM,KAAK,cAAc,IAAI,IAAI,KAAK;EACtC,IAAI,CAAC,MAAM,GAAG,WAAW,GAAG;GAC1B;GACA;EACF;EACA,MAAM,WAAW,GAAG,QAAQ,MAAM;GAChC,MAAM,MAAM,EAAE,aAAa,IAAI;GAC/B,OAAO,OAAO,KAAK,OAAO;EAC5B,CAAC;EACD,IAAI,SAAS,WAAW,GAAG;GACzB;GACA;EACF;EAEA,IAAI,gBAAgB;EACpB,KAAK,MAAM,MAAM,YAAY;GAC3B,MAAM,IAAI,MAAM,GAAG,QAAQ,KAAK,UAAU;GAC1C,IAAI,MAAM,QAAQ,CAAC,OAAO,SAAS,CAAC,GAAG;GAEvC,KAAK,MAAM,MAAM,aAAa;IAC5B,MAAM,IAAI,oBAAoB,UAAU,IAAI,SAAS;IACrD,IAAI,MAAM,MAAM;IAChB,MAAM,OAAO,MAAM,MAAM,MAAM,EAAE,eAAe,GAAG,MAAM,EAAE,kBAAkB,EAAE;IAC/E,KAAK,GAAG,KAAK,CAAC;IACd,KAAK,GAAG,KAAK,CAAC;IACd,gBAAgB;GAClB;EACF;EACA,IAAI,eAAe;OACd;CACP;CAEA,MAAM,gBAAyD,CAAC;CAChE,MAAM,UAA+B,CAAC;CACtC,KAAK,MAAM,KAAK,OAAO;EACrB,MAAM,SACJ,EAAE,GAAG,SAAS,IACV,yBACA,CAAC,aAAa,EAAE,EAAE,IAChB,yBACA,CAAC,aAAa,EAAE,EAAE,IAChB,qBACA;EACV,IAAI,WAAW,MAAM;GACnB,cAAc,KAAK;IACjB,YAAY,EAAE;IACd,eAAe,EAAE;IACjB,GAAG,EAAE,GAAG;IACR;GACF,CAAC;GACD;EACF;EACA,MAAM,UAAU,mBAAmB,EAAE,IAAI,EAAE,IAAI,YAAY,IAAI;EAC/D,MAAM,UACJ,KAAK,IAAI,QAAQ,OAAO,KAAK,KACzB,WACA,KAAK,IAAI,QAAQ,OAAO,KAAK,KAC3B,aACA;EACR,QAAQ,KAAK;GACX,YAAY,EAAE;GACd,eAAe,EAAE;GACjB,GAAG,EAAE,GAAG;GACR,GAAG;GACH;EACF,CAAC;CACH;CACA,OAAO;EAAE,OAAO;EAAS;EAAe,eAAe;EAAQ,aAAa;CAAQ;AACtF;;;ACxLA,MAAM,WAAW,EACd,OAAO,CAAC,CACR,IAAI,CAAC,CAAC,CACN,QAAQ,UAAU,MAAM,KAAK,MAAM,KAAK;AAC3C,MAAa,iCAAiC,EAC3C,OAAO;CACN,YAAY,EAAE,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;CAC1C,wBAAwB,EAAE,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC;CACvD,uBAAuB,EAAE,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC;AACxD,CAAC,CAAC,CACD,OAAO;AAKV,MAAa,kCAAkC,EAC5C,OAAO;CACN,IAAI;CACJ,mBAAmB;CACnB,aAAa;CACb,UAAU,EAAE,KAAK,CAAC,UAAU,QAAQ,CAAC;CACrC,UAAU,EAAE,KAAK;EAAC;EAAU;EAAU;CAAS,CAAC;CAChD,UAAU,EAAE,KAAK,CAAC,SAAS,aAAa,CAAC;AAC3C,CAAC,CAAC,CACD,OAAO;AAKV,MAAM,cAAc,EACjB,OAAO;CACN,iBAAiB,EAAE,OAAO,CAAC,CAAC,MAAM,uBAAuB;CACzD,YAAY;CACZ,eAAe;CACf,WAAW,EACR,OAAO;EACN,mBAAmB;EACnB,WAAW;EACX,yBAAyB;CAC3B,CAAC,CAAC,CACD,OAAO;CACV,QAAQ;CACR,cAAc,EAAE,MAAM,+BAA+B;AACvD,CAAC,CAAC,CACD,OAAO;AAiDV,SAAS,UACP,MACA,UACA,OACA,OACoB;CACpB,MAAM,QAAQ,KAAK,QAAQ,QAAQ,IAAI,aAAa,QAAQ;CAC5D,MAAM,wBAAQ,IAAI,IAAkD;CACpE,KAAK,MAAM,OAAO,OAAO;EACvB,MAAM,OAAO,MAAM,IAAI,IAAI,iBAAiB,KAAK;GAAE,OAAO;GAAO,SAAS;EAAM;EAChF,KAAK,YAAY,IAAI,aAAa;EAClC,KAAK,UAAU,IAAI,aAAa,aAAa,IAAI,aAAa;EAC9D,MAAM,IAAI,IAAI,mBAAmB,IAAI;CACvC;CACA,MAAM,IAAI,MAAM;CAChB,MAAM,aAAa,CAAC,GAAG,MAAM,OAAO,CAAC,CAAC,CAAC,QAAQ,SAAS,KAAK,KAAK,CAAC,CAAC;CACpE,MAAM,kBAAkB,CAAC,GAAG,MAAM,OAAO,CAAC,CAAC,CAAC,QAAQ,SAAS,KAAK,WAAW,CAAC,KAAK,KAAK,CAAC,CAAC;CAC1F,MAAM,eAAe,MAAM,QAAQ,QAAQ,IAAI,aAAa,SAAS,CAAC,CAAC;CAEvE,MAAM,WACJ,MAAM,IACF,OACA;EACE,OAAO,gBACL;GAAE,MAAM;GAAmB;EAAM,GACjC;GACE,MAAM;GACN,WAAW;GACX,QAAQ;EACV,CACF,CAAC,CAAC;EACF,OAAO,gBACL;GAAE,MAAM;GAAmB;EAAM,GACjC;GACE,MAAM;GACN,WAAW,aAAa;GACxB,QAAQ;EACV,CACF,CAAC,CAAC;EACF;CACF;CACN,MAAM,UACJ,YAAY,SAAS,QAAQ,QACzB,SACA,YAAY,SAAS,SAAS,QAC5B,SACA;CACR,OAAO;EACL,OAAO,MAAM;EACb,kBAAkB;EAClB;EACA;EACA;EACA,WAAW,MAAM,KAAK,kBAAkB,IAAI,OAAO,aAAa;EAChE;EACA;EACA;CACF;AACF;AAEA,SAAS,iBAEP,cAAmB;CACnB,MAAM,sBAAM,IAAI,IAAY;CAC5B,KAAK,MAAM,OAAO,cAAc;EAC9B,IAAI,IAAI,IAAI,IAAI,EAAE,GAChB,MAAM,IAAI,gBAAgB,0CAA0C,IAAI,GAAG,EAAE;EAC/E,IAAI,IAAI,IAAI,EAAE;CAChB;CACA,aAAa,MAAM,GAAG,MAAM,iBAAiB,EAAE,IAAI,EAAE,EAAE,CAAC;CACxD,MAAM,mBAAmB,IAAI,IAC3B,aACG,QAAQ,QAAQ,IAAI,aAAa,aAAa,CAAC,CAC/C,KAAK,QAAQ,IAAI,iBAAiB,CACvC;CASA,OAAO;EAAE,UARQ,aAAa,QAAQ,QAAQ,CAAC,iBAAiB,IAAI,IAAI,iBAAiB,CAQzE;EAAG,YAPA,aAChB,QAAQ,QAAQ,iBAAiB,IAAI,IAAI,iBAAiB,CAAC,CAAC,CAC5D,KAAK,SAAS;GACb,IAAI,IAAI;GACR,mBAAmB,IAAI;GACvB,QAAQ;EACV,EAC0B;CAAE;AAChC;AAEA,SAAS,+BAA+B,YAA4B;CAClE,MAAM,QAAQ,KAAK,IAAI,cAAc;CACrC,IAAI,SAAS,GACX,MAAM,IAAI,gBACR,2EACF;CACF,OAAO;AACT;AAEA,MAAM,iBAAiB,EACpB,OAAO;CACN,QAAQ;CACR,UAAU,EACP,MAAM,gCAAgC,KAAK;EAAE,UAAU;EAAM,aAAa;CAAK,CAAC,CAAC,CAAC,CAClF,QAAQ,CAAC,CAAC;AACf,CAAC,CAAC,CACD,OAAO;;AAKV,SAAS,oBAAoB,QAAgB,OAAuB;CAClE,OAAO,CAAC,KAAK,MAAM,KAAK,KAAK,IAAI,SAAS,CAAC,IAAI,MAAM;AACvD;AAEA,SAAS,kBAAkB,OAAe,OAA8B;CACtE,IAAI,UAAU,GAAG,OAAO;CACxB,MAAM,WAAW,KAAK,KAAK,KAAK,KAAK,IAAI,SAAS,CAAC,IAAI,KAAK,MAAM,CAAC,KAAK,CAAC;CACzE,IAAI,CAAC,OAAO,SAAS,QAAQ,KAAK,WAAW,OAAO,kBAAkB,OAAO;CAC7E,MAAM,UAAU,WAAmB,oBAAoB,QAAQ,KAAK,KAAK;CACzE,IAAI,UAAU,KAAK,IAAI,GAAG,QAAQ;CAClC,OAAO,UAAU,KAAK,OAAO,UAAU,CAAC,GAAG,WAAW;CACtD,OAAO,CAAC,OAAO,OAAO,GAAG;EACvB,IAAI,YAAY,OAAO,kBAAkB,OAAO;EAChD,WAAW;CACb;CACA,OAAO;AACT;;;;;;AAOA,SAAgB,mBAAmB,OAAgC;CACjE,MAAM,SAAS,eAAe,UAAU,KAAK;CAC7C,IAAI,CAAC,OAAO,SACV,MAAM,IAAI,gBAAgB,iCAAiC,OAAO,MAAM,SAAS;CACnF,MAAM,EAAE,QAAQ,aAAa,OAAO;CACpC,MAAM,EAAE,aAAa,iBAAiB,QAAQ;CAC9C,MAAM,QAAQ,+BAA+B,OAAO,UAAU;CAC9D,MAAM,YAAY,UAAiD,UAAkB;EACnF,MAAM,mBAAmB,IAAI,IAC3B,SAAS,QAAQ,QAAQ,IAAI,aAAa,QAAQ,CAAC,CAAC,KAAK,QAAQ,IAAI,iBAAiB,CACxF,CAAC,CAAC;EACF,MAAM,yBACJ,qBAAqB,IAAI,OAAO,oBAAoB,kBAAkB,KAAK;EAC7E,MAAM,0BAA0B,kBAAkB,OAAO,KAAK;EAC9D,OAAO;GACL;GACA;GACA;GACA;GACA,4BACE,4BAA4B,OACxB,OACA,KAAK,IAAI,GAAG,0BAA0B,gBAAgB;EAC9D;CACF;CACA,MAAM,kBAAkB,SAAS,UAAU,OAAO,sBAAsB;CACxE,MAAM,iBAAiB,SAAS,UAAU,OAAO,qBAAqB;CAItE,OAAO;EAAE,YAHU,CAAC,iBAAiB,cAAc,CAAC,CAAC,OAClD,SAAS,KAAK,2BAA2B,QAAQ,KAAK,0BAA0B,KAAK,KAEtE;EAAG;EAAiB;CAAe;AACvD;;;;;;AASA,SAAgB,eAAe,OAAsD;CACnF,MAAM,SAAS,YAAY,UAAU,KAAK;CAC1C,IAAI,CAAC,OAAO,SAAS,MAAM,IAAI,gBAAgB,4BAA4B,OAAO,MAAM,SAAS;CACjG,MAAM,QAAQ,OAAO;CACrB,IAAI,MAAM,UAAU,sBAAsB,MAAM,UAAU,WACxD,MAAM,IAAI,gBAAgB,kEAAkE;CAE9F,MAAM,EAAE,UAAU,eAAe,iBAAiB,MAAM,YAAY;CACpE,MAAM,qBAAqB,+BAA+B,MAAM,OAAO,UAAU;CACjF,MAAM,kBAAkB,UACtB,UACA,UACA,MAAM,OAAO,wBACb,kBACF;CACA,MAAM,iBAAiB,UACrB,UACA,UACA,MAAM,OAAO,uBACb,kBACF;CACA,MAAM,QAAQ,CAAC,iBAAiB,cAAc;CAC9C,MAAM,UAAU,MAAM,MAAM,SAAS,KAAK,YAAY,MAAM,IACxD,WACA,MAAM,OAAO,SAAS,KAAK,YAAY,MAAM,IAC3C,UACA;CACN,MAAM,UAAoB,CAAC;CAC3B,KAAK,MAAM,CAAC,MAAM,SAAS,CACzB,CAAC,oBAAoB,eAAe,GACpC,CAAC,mBAAmB,cAAc,CACpC,GAAY;EACV,IAAI,KAAK,qBAAqB,GAAG,QAAQ,KAAK,GAAG,KAAK,gCAAgC;OACjF,IAAI,KAAK,eAAe,GAC3B,QAAQ,KAAK,GAAG,KAAK,IAAI,KAAK,aAAa,uBAAuB;EACpE,IAAI,KAAK,YAAY,QAAQ,QAAQ,KAAK,GAAG,KAAK,8BAA8B,KAAK,OAAO;OACvF,IAAI,KAAK,YAAY,kBAAkB,KAAK,UAC/C,QAAQ,KAAK,GAAG,KAAK,0DAA0D;CACnF;CACA,IAAI,YAAY,SACd,QAAQ,KACN,8FACF;CACF,MAAM,OAAuD;EAC3D,iBAAiB,MAAM;EACvB,cAAc,cAAc,MAAM,MAAM;EACxC,aAAa,cAAc,KAAK;EAChC,YAAY,MAAM;EAClB,eAAe,MAAM;EACrB,WAAW,MAAM;EACjB,QAAQ,MAAM;EACd,YAAY,MAAM,OAAO;EACzB;EACA;EACA;EACA,cAAc,MAAM;EACpB,UAAU;GACR,OAAO,MAAM,aAAa;GAC1B,kBAAkB,IAAI,IAAI,MAAM,aAAa,KAAK,QAAQ,IAAI,iBAAiB,CAAC,CAAC,CAAC;GAClF,eAAe,SAAS;GACxB,0BAA0B,IAAI,IAAI,SAAS,KAAK,QAAQ,IAAI,iBAAiB,CAAC,CAAC,CAAC;GAChF,eAAe,WAAW;GAC1B,cAAc,SAAS,QAAQ,QAAQ,IAAI,aAAa,SAAS,CAAC,CAAC;EACrE;EACA;EACA;EACA;CACF;CACA,OAAO;EAAE,GAAG;EAAM,cAAc,cAAc,IAAI;CAAE;AACtD;AAEA,MAAM,8BAA8B,EACjC,OAAO;CACN,iBAAiB,EAAE,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;CACjD,iBAAiB,EAAE,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;AACnD,CAAC,CAAC,CACD,OAAO,CAAC,CACR,QAAQ,UAAU,MAAM,kBAAkB,MAAM,iBAAiB,EAChE,SAAS,uDACX,CAAC;AACH,MAAM,+BAA+B,gCAClC,KAAK,EAAE,UAAU,KAAK,CAAC,CAAC,CACxB,OAAO,EAAE,mBAAmB,EAAE,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAC3E,OAAO;AACV,MAAM,yBAAyB,YAAY,OAAO;CAChD,YAAY;CACZ,cAAc,EAAE,MAAM,4BAA4B;AACpD,CAAC;;;;;;AAmBD,SAAgB,uBACd,OAC8B;CAC9B,MAAM,SAAS,uBAAuB,UAAU,KAAK;CACrD,IAAI,CAAC,OAAO,SACV,MAAM,IAAI,gBAAgB,qCAAqC,OAAO,MAAM,SAAS;CAEvF,MAAM,EAAE,YAAY,cAAc,GAAG,WAAW,OAAO;CACvD,aAAa,MAAM,GAAG,MAAM,iBAAiB,EAAE,IAAI,EAAE,EAAE,CAAC;CACxD,MAAM,oBAAoB;EACxB,uBAAuB,OAAO;EAC9B;EACA,cAAc,aAAa,KAAK,EAAE,IAAI,yBAAyB;GAAE;GAAI;EAAkB,EAAE;CAC3F;CAoBA,MAAM,EAAE,cAAc,aAAa,GAAG,eAnBxB,eAAe;EAC3B,GAAG;EACH,iBAAiB,cAAc;GAC7B,MAAM;GACN,uBAAuB,OAAO;GAC9B;EACF,CAAC;EACD,cAAc,aAAa,KAAK,EAAE,mBAAmB,GAAG,mBAAmB;GACzE,GAAG;GACH,UACE,sBAAsB,OAClB,YACA,qBAAqB,WAAW,kBAC9B,WACA,qBAAqB,WAAW,kBAC9B,WACA;EACZ,EAAE;CACJ,CACyD;CACzD,MAAM,OAAO;EACX,GAAG;EAEH,aAAa,cAAc,OAAO,IAAI;EACtC;CACF;CACA,OAAO;EAAE,GAAG;EAAM,cAAc,cAAc,IAAI;CAAE;AACtD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;ACvVA,MAAM,cAAoC;CACxC;CACA;CACA;CACA;AACF;AAKA,MAAM,qBAAkD,CAAC,UAAU,QAAQ;;AAG3E,MAAM,wBAAwB;;;;;;;;;;;;;AAwB9B,SAAgB,YAAY,OAKlB;CACR,MAAM,EAAE,IAAI,MAAM,MAAM,oBAAoB;CAC5C,IAAI,OAAO,OAAO,YAAY,GAAG,KAAK,MAAM,IAC1C,MAAM,IAAI,gBAAgB,4CAA4C;CAExE,IAAI,CAAC,YAAY,SAAS,IAAI,GAC5B,MAAM,IAAI,gBACR,uBAAuB,GAAG,aAAa,KAAK,UAAU,IAAI,EAAE,oBAAoB,YAAY,KAAK,IAAI,GACvG;CAEF,IAAI,CAAC,mBAAmB,SAAS,eAAe,GAC9C,MAAM,IAAI,gBACR,uBAAuB,GAAG,wBAAwB,KAAK,UAAU,eAAe,EAAE,oBAAoB,mBAAmB,KAAK,IAAI,GACpI;CAEF,IAAI,OAAO,KAAK,WAAW,YAAY,KAAK,OAAO,KAAK,MAAM,IAC5D,MAAM,IAAI,gBAAgB,uBAAuB,GAAG,2BAA2B;CAEjF,IAAI,CAAC,OAAO,SAAS,KAAK,UAAU,KAAK,KAAK,aAAa,KAAK,KAAK,aAAa,GAChF,MAAM,IAAI,gBACR,uBAAuB,GAAG,mBAAmB,KAAK,WAAW,qCAC/D;CAEF,IAAI,KAAK,UAAU,KAAA,KAAa,OAAO,KAAK,UAAU,UACpD,MAAM,IAAI,gBAAgB,uBAAuB,GAAG,8BAA8B;CAEpF,IAAI,KAAK,eAAe,uBACtB,MAAM,IAAI,gBACR,uBAAuB,GAAG,mBAAmB,sBAAsB,qDACrE;CAEF,MAAM,YAA8B,KAAK,aAAa,wBAAwB,WAAW;CACzF,IAAI,cAAc,iBAChB,MAAM,IAAI,gBACR,uBAAuB,GAAG,0BAA0B,gBAAgB,sBAAsB,KAAK,WAAW,QAAQ,WACpH;CAEF,OAAO;EACL;EACA;EACA,MAAM;GACJ,QAAQ,KAAK;GACb,YAAY,KAAK;GACjB,GAAI,KAAK,UAAU,KAAA,IAAY,CAAC,IAAI,EAAE,OAAO,KAAK,MAAM;EAC1D;EACA;CACF;AACF;;;;;;;;;;;AA8CA,MAAM,cAAc;;;;;;;;AASpB,MAAM,mBAA2D;CAC/D,CAAC,SAAS,OAAO;CACjB,CAAC,SAAS,OAAO;CACjB,CAAC,SAAS,OAAO;CACjB,CAAC,SAAS,OAAO;CACjB,CAAC,SAAS,OAAO;CACjB,CAAC,SAAS,OAAO;CACjB,CAAC,MAAM,GAAG;CACV,CAAC,MAAM,GAAG;CACV,CAAC,MAAM,IAAI;CACX,CAAC,MAAM,IAAI;AACb;;;;;;;;;AAeA,SAAS,qBAAqB,MAA2B;CACvD,MAAM,SAAS;CACf,IAAI,QAAqB;CACzB,IAAI,QAAQ,OAAO,KAAK,IAAI;CAC5B,OAAO,UAAU,MAAM;EACrB,MAAM,QAAQ,MAAM;EACpB,MAAM,MAAM,QAAQ,MAAM,EAAE,CAAC;EAE7B,IAAI,CAAC,YAAY,KAAK,KAAK,OAAO,QAAQ,CAAC,CAAC,KAAK,CAAC,YAAY,KAAK,KAAK,OAAO,GAAG,CAAC,GACjF,QAAQ;GAAE;GAAO;EAAI;EAEvB,QAAQ,OAAO,KAAK,IAAI;CAC1B;CACA,OAAO;AACT;;;;;;;;;AAUA,SAAS,gBAAgB,MAAsD;CAC7E,KAAK,IAAI,QAAQ,GAAG,QAAQ,KAAK,QAAQ,SAAS,GAChD,KAAK,MAAM,CAAC,OAAO,YAAY,kBAC7B,IAAI,KAAK,WAAW,OAAO,KAAK,GAC9B,OAAO;EAAE,MAAM;GAAE,OAAO;GAAO,KAAK,QAAQ,MAAM;EAAO;EAAG;CAAQ;CAI1E,OAAO;AACT;AAEA,SAAS,YAAY,MAAc,MAAY,OAAuB;CACpE,OAAO,KAAK,MAAM,GAAG,KAAK,KAAK,IAAI,QAAQ,KAAK,MAAM,KAAK,GAAG;AAChE;;;;;AAMA,SAAS,UACP,OACA,QACA,OACA,OACe;CACf,MAAM,YAAY,UAAU,UAAU,QAAQ;CAC9C,MAAM,aAAa,UAAU,WAAW,QAAQ;CAChD,OAAO;EACL,GAAI,cAAc,KAAA,IAAY,CAAC,IAAI,EAAE,OAAO,UAAU;EACtD,GAAI,eAAe,KAAA,IAAY,CAAC,IAAI,EAAE,QAAQ,WAAW;CAC3D;AACF;AAEA,SAAS,SACP,OACA,QACA,OACA,MACA,MACsB;CACtB,MAAM,WAAW,KAAK,MAAM,KAAK,OAAO,KAAK,GAAG;CAChD,MAAM,aAAa,OAAO,QAAQ,IAAI,GAAA,CAAI,SAAS;CACnD,OAAO;EACL,UAAU,UAAU,OAAO,QAAQ,OAAO,YAAY,MAAM,MAAM,SAAS,CAAC;EAC5E;EACA,KAAK;EACL;EACA;CACF;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA6BA,SAAgB,gBAAgB,UAAsD;CACpF,MAAM,QAAQ,OAAO,SAAS,UAAU,WAAW,SAAS,QAAQ,KAAA;CACpE,MAAM,SAAS,OAAO,SAAS,WAAW,WAAW,SAAS,SAAS,KAAA;CAEvE,IAAI,WAAW,KAAA,GAAW;EACxB,MAAM,OAAO,qBAAqB,MAAM;EACxC,IAAI,SAAS,MAAM,OAAO,SAAS,OAAO,QAAQ,UAAU,QAAQ,IAAI;CAC1E;CACA,IAAI,UAAU,KAAA,GAAW;EACvB,MAAM,aAAa,gBAAgB,KAAK;EACxC,IAAI,eAAe,MACjB,OAAO;GACL,UAAU,UACR,OACA,QACA,SACA,YAAY,OAAO,WAAW,MAAM,WAAW,OAAO,CACxD;GACA,OAAO;GACP,KAAK;GACL,UAAU,MAAM,MAAM,WAAW,KAAK,OAAO,WAAW,KAAK,GAAG;GAChE,WAAW,WAAW;EACxB;EAEF,MAAM,OAAO,qBAAqB,KAAK;EACvC,IAAI,SAAS,MAAM,OAAO,SAAS,OAAO,QAAQ,SAAS,OAAO,IAAI;CACxE;CACA,OAAO;AACT;;;;;;;;;;;;;;;;;;;;;AAgCA,SAAgB,oBAAoB,OAOV;CACxB,MAAM,EAAE,IAAI,QAAQ,UAAU,aAAa,GAAG,UAAU;CACxD,IAAI,OAAO,OAAO,YAAY,GAAG,KAAK,MAAM,IAC1C,MAAM,IAAI,gBAAgB,oDAAoD;CAEhF,IAAI,OAAO,WAAW,YAAY,OAAO,KAAK,MAAM,IAClD,MAAM,IAAI,gBAAgB,+BAA+B,GAAG,sBAAsB;CAEpF,MAAM,eAAe,gBAAgB,QAAQ;CAC7C,IAAI,iBAAiB,MAAM,OAAO;CAOlC,OAAO;EACL,OAPY,YAAY;GACxB;GACA,MAAM;GACN,MAAM;IAAE;IAAQ;IAAY,GAAI,UAAU,KAAA,IAAY,CAAC,IAAI,EAAE,MAAM;GAAG;GACtE,iBAAiB;EACnB,CAEM;EACJ,UAAU,aAAa;EACvB,OAAO,aAAa;EACpB,KAAK,aAAa;EAClB,UAAU,aAAa;EACvB,WAAW,aAAa;CAC1B;AACF;;;;;;;;;;AAoDA,SAAgB,WACd,SACA,QACA,UAA6B,CAAC,GACZ;CAClB,MAAM,OAAO,QAAQ,QAAQ;CAC7B,MAAM,kBAAkB,QAAQ,mBAAmB;CACnD,IAAI,CAAC,OAAO,SAAS,IAAI,GACvB,MAAM,IAAI,gBAAgB,iDAAiD,MAAM;CAEnF,IAAI,CAAC,OAAO,SAAS,eAAe,KAAK,mBAAmB,KAAK,kBAAkB,GACjF,MAAM,IAAI,gBACR,sEAAsE,iBACxE;CAGF,MAAM,eAA6B,CAAC;CACpC,MAAM,uBAAO,IAAI,IAAY;CAC7B,KAAK,MAAM,QAAQ,SAAS;EAC1B,IAAI,OAAO,KAAK,WAAW,YAAY,KAAK,OAAO,KAAK,MAAM,IAC5D,MAAM,IAAI,gBAAgB,gDAAgD;EAE5E,IAAI,CAAC,OAAO,SAAS,KAAK,UAAU,GAClC,MAAM,IAAI,gBACR,6BAA6B,KAAK,OAAO,8BAC3C;EAEF,IAAI,KAAK,IAAI,KAAK,MAAM,GACtB,MAAM,IAAI,gBAAgB,yCAAyC,KAAK,OAAO,EAAE;EAEnF,KAAK,IAAI,KAAK,MAAM;EACpB,aAAa,KAAK;GAChB,QAAQ,KAAK;GACb,YAAY,KAAK;GACjB,GAAI,KAAK,UAAU,KAAA,IAAY,CAAC,IAAI,EAAE,OAAO,KAAK,MAAM;EAC1D,CAAC;CACH;CAEA,MAAM,eAAwB,CAAC;CAC/B,MAAM,2BAAW,IAAI,IAAY;CACjC,KAAK,MAAM,SAAS,QAAQ;EAC1B,MAAM,SAAS,YAAY,KAAK;EAChC,IAAI,SAAS,IAAI,OAAO,EAAE,GACxB,MAAM,IAAI,gBAAgB,mCAAmC,OAAO,GAAG,EAAE;EAE3E,IAAI,KAAK,IAAI,OAAO,KAAK,MAAM,GAC7B,MAAM,IAAI,gBACR,sBAAsB,OAAO,GAAG,mBAAmB,OAAO,KAAK,OAAO,+BACxE;EAEF,MAAM,gBACJ,OAAO,KAAK,cAAc,kBAAkB,WAAW;EACzD,IAAI,kBAAkB,OAAO,iBAC3B,MAAM,IAAI,gBACR,sBAAsB,OAAO,GAAG,0BAA0B,OAAO,gBAAgB,sBAAsB,OAAO,KAAK,WAAW,aAAa,cAAc,2BAA2B,iBACtL;EAEF,SAAS,IAAI,OAAO,EAAE;EACtB,KAAK,IAAI,OAAO,KAAK,MAAM;EAC3B,aAAa,KAAK,MAAM;CAC1B;CAEA,MAAM,QAAQ,SACZ,CAAC,GAAG,cAAc,GAAG,aAAa,KAAK,UAAU,MAAM,IAAI,CAAC,GAC5D,WAAW,IAAI,CACjB;CACA,MAAM,UAAU,MAAM,KAAK,SAAS,KAAK,MAAM;CAQ/C,OAAO;EAAE;EAAO,UAAA;GANd,MAAM,aAAa;IAAE;IAAM;IAAiB;IAAS,QAAQ;GAAa,CAAC;GAC3E;GACA;GACA;GACA,QAAQ;EAEa;CAAE;AAC3B;;;;;;;AAQA,SAAS,SAAY,OAAqB,QAA2B;CACnE,OAAO,MACJ,KAAK,UAAU;EAAE;EAAM,KAAK,OAAO;CAAE,EAAE,CAAC,CACxC,MAAM,MAAM,UAAU,KAAK,MAAM,MAAM,GAAG,CAAC,CAC3C,KAAK,UAAU,MAAM,IAAI;AAC9B;AAEA,SAAS,aAAa,UAAmD;CACvE,OAAO,cAAc;EACnB,QAAQ;EACR,MAAM,SAAS;EACf,iBAAiB,SAAS;EAC1B,SAAS,SAAS;EAClB,QAAQ,SAAS;CACnB,CAAC;AACH;;;;;;;;;AAwEA,SAAgB,UACd,SACA,UACiB;CACjB,MAAM,eAAe,aAAa;EAChC,MAAM,SAAS;EACf,iBAAiB,SAAS;EAC1B,SAAS,SAAS;EAClB,QAAQ,SAAS;CACnB,CAAC;CACD,IAAI,iBAAiB,SAAS,MAC5B,MAAM,IAAI,sBACR,wCAAwC,aAAa,iCAAiC,SAAS,KAAK,6CACtG;CAGF,MAAM,YAAY,IAAI,IAAI,SAAS,OAAO;CAC1C,MAAM,gCAAgB,IAAI,IAA2B;CACrD,KAAK,MAAM,UAAU,SAAS;EAC5B,IAAI,CAAC,UAAU,IAAI,OAAO,MAAM,GAC9B,MAAM,IAAI,gBACR,0BAA0B,OAAO,OAAO,wCAC1C;EAEF,IAAI,cAAc,IAAI,OAAO,MAAM,GACjC,MAAM,IAAI,gBAAgB,oCAAoC,OAAO,OAAO,EAAE;EAEhF,IAAI,OAAO,UAAU,QAAQ,CAAC,OAAO,SAAS,OAAO,KAAK,GACxD,MAAM,IAAI,gBACR,0BAA0B,OAAO,OAAO,cAAc,OAAO,MAAM,mCACrE;EAEF,cAAc,IAAI,OAAO,QAAQ,OAAO,KAAK;CAC/C;CAEA,MAAM,SAAsD,CAAC;CAC7D,MAAM,YAAsB,CAAC;CAC7B,MAAM,aAAuB,CAAC;CAC9B,IAAI,SAAS;CACb,IAAI,SAAS;CACb,IAAI,aAAa;CAEjB,KAAK,MAAM,SAAS,SAAS,QAAQ;EACnC,IAAI,SAAS,OAAO,MAAM;EAC1B,IAAI,WAAW,KAAA,GAAW;GACxB,SAAS;IAAE,QAAQ;IAAG,QAAQ;IAAG,QAAQ;IAAG,YAAY;IAAG,MAAM;GAAK;GACtE,OAAO,MAAM,QAAQ;EACvB;EACA,OAAO,UAAU;EACjB,IAAI,CAAC,cAAc,IAAI,MAAM,KAAK,MAAM,GAAG;GACzC,WAAW,KAAK,MAAM,EAAE;GACxB;EACF;EACA,MAAM,QAAQ,cAAc,IAAI,MAAM,KAAK,MAAM,KAAK;EACtD,IAAI,UAAU,MAAM;GAClB,cAAc;GACd,OAAO,cAAc;GACrB;EACF;EAEA,KADqC,SAAS,SAAS,kBAAkB,WAAW,cACjE,MAAM,iBAAiB;GACxC,UAAU;GACV,OAAO,UAAU;EACnB,OAAO;GACL,UAAU;GACV,OAAO,UAAU;GACjB,UAAU,KAAK,MAAM,EAAE;EACzB;CACF;CAEA,MAAM,SAAS,SAAS,OAAO;CAC/B,MAAM,UAAU,SAAS;CACzB,MAAM,SAA0B;EAC9B,QAAQ;EACR;EACA;EACA;EACA;EACA,MAAM;EACN;EACA;EACA;EACA,UAAU,kBAAkB,UAAU,aAAa;CACrD;CAEA,IAAI,WAAW,GACb,OAAO;EACL,GAAG;EACH,QAAQ;EACR,QAAQ;CACV;CAEF,IAAI,WAAW,SAAS,GACtB,OAAO;EACL,GAAG;EACH,QAAQ;EACR,QAAQ,GAAG,WAAW,OAAO,MAAM,OAAO,iCAAiC,WAAW,KAAK,IAAI;CACjG;CAEF,IAAI,YAAY,GACd,OAAO;EACL,GAAG;EACH,QAAQ;EACR,QAAQ,OAAO,OAAO;CACxB;CAGF,KAAK,MAAM,UAAU,OAAO,OAAO,MAAM,GAAG;EAC1C,MAAM,cAAc,OAAO,SAAS,OAAO;EAC3C,OAAO,OAAO,gBAAgB,IAAI,OAAO,OAAO,SAAS;CAC3D;CACA,OAAO,OAAO,SAAS;CACvB,OAAO;AACT;AAEA,SAAS,kBACP,UACA,eACmB;CACnB,MAAM,eAAe,IAAI,IAAI,SAAS,OAAO,KAAK,UAAU,MAAM,KAAK,MAAM,CAAC;CAC9E,IAAI,IAAI;CACR,IAAI,UAAU;CACd,IAAI,WAAW;CACf,KAAK,MAAM,UAAU,SAAS,SAAS;EACrC,IAAI,aAAa,IAAI,MAAM,GAAG;EAC9B,KAAK;EACL,MAAM,QAAQ,cAAc,IAAI,MAAM;EACtC,IAAI,UAAU,KAAA,KAAa,UAAU,MAAM;EAC3C,WAAW;EACX,IAAI,QAAQ,SAAS,iBAAiB,YAAY;CACpD;CACA,OAAO;EAAE;EAAG;EAAS;EAAU,eAAe,YAAY,IAAI,OAAO,WAAW;CAAQ;AAC1F"}

package/dist/multishot/golden/index.d.ts

@@ -1,4 +1,4 @@
-import { A as MultishotTransportRequest, D as MultishotToolDefinition, T as MultishotResult, c as RunMultishotMatrixResult, f as RunMultishotOptions, s as RunMultishotMatrixOptions, w as MultishotPersona } from "../../matrix-Cf-AGxGh.js";
+import { A as MultishotTransportRequest, D as MultishotToolDefinition, T as MultishotResult, c as RunMultishotMatrixResult, f as RunMultishotOptions, s as RunMultishotMatrixOptions, w as MultishotPersona } from "../../matrix-BXo3PSqM.js";
 //#region src/multishot/golden/compare.d.ts
 interface CompareOptions {
   /** Stop after this many mismatches. A structural divergence high in the

package/dist/multishot/index.d.ts

@@ -1,5 +1,5 @@
-import { C as JudgeDimension, w as JudgeScore } from "../types-CHfn1zg5.js";
-import { A as MultishotTransportRequest, C as MultishotMessage, D as MultishotToolDefinition, E as MultishotShape, M as MultishotTransportToolCall, N as assertMultishotShotResult, O as MultishotToolExecutor, S as MultishotFatalToolError, T as MultishotResult, _ as renderDimensions, a as MultishotCellOutput, b as MultishotArtifact, c as RunMultishotMatrixResult, d as MultishotShot, f as RunMultishotOptions, g as MultishotJudgeConfig, h as JudgeRunResult, i as ConversationJudgeInput, j as MultishotTransportResponse, k as MultishotTransport, l as computeCellComposite, m as DEFAULT_JUDGE_MODEL, n as CellCompositeInput, o as MultishotJudges, p as runMultishot, r as CellCompositeScore, s as RunMultishotMatrixOptions, t as ArtifactJudgeInput, u as runMultishotMatrix, v as renderJsonFooter, w as MultishotPersona, x as MultishotDriverEmptyError, y as runJudge } from "../matrix-Cf-AGxGh.js";
+import { C as JudgeDimension, w as JudgeScore } from "../types-DDKfH-N6.js";
+import { A as MultishotTransportRequest, C as MultishotMessage, D as MultishotToolDefinition, E as MultishotShape, M as MultishotTransportToolCall, N as assertMultishotShotResult, O as MultishotToolExecutor, S as MultishotFatalToolError, T as MultishotResult, _ as renderDimensions, a as MultishotCellOutput, b as MultishotArtifact, c as RunMultishotMatrixResult, d as MultishotShot, f as RunMultishotOptions, g as MultishotJudgeConfig, h as JudgeRunResult, i as ConversationJudgeInput, j as MultishotTransportResponse, k as MultishotTransport, l as computeCellComposite, m as DEFAULT_JUDGE_MODEL, n as CellCompositeInput, o as MultishotJudges, p as runMultishot, r as CellCompositeScore, s as RunMultishotMatrixOptions, t as ArtifactJudgeInput, u as runMultishotMatrix, v as renderJsonFooter, w as MultishotPersona, x as MultishotDriverEmptyError, y as runJudge } from "../matrix-BXo3PSqM.js";
 import { AgentProfile } from "@tangle-network/agent-interface";
 //#region src/multishot/cost.d.ts
 /**

package/dist/openapi.json

@@ -2,7 +2,7 @@
   "openapi": "3.1.0",
   "info": {
     "title": "@tangle-network/agent-eval — wire protocol",
-    "version": "0.220.2",
+    "version": "0.221.0",
     "description": "HTTP and stdio RPC interface to agent-eval. The TypeScript runtime is the source of truth; this spec is the contract that cross-language clients (Python, Rust, Go) generate from.\n\nWire-protocol version: 1.1.0. Bumps on breaking changes to request/response schemas.",
     "contact": {
       "name": "Tangle Network",