@tangle-network/agent-eval 0.179.0 → 0.181.0

package/dist/meta-eval/index.js

@@ -1,12 +1,13 @@
 import { n as CaptureIntegrityError, s as ValidationError } from "../errors-Dngq5h35.js";
-import { a as hashCanonical } from "../canonical-DPyQ_rpt.js";
-import { i as makeRng } from "../internal-BMFSR8Ns.js";
+import { a as hashCanonical, i as compareCodeUnits } from "../canonical-DPyQ_rpt.js";
 import { t as mulberry32 } from "../random-Dn5fPWkt.js";
-import { a as spearmanR, r as pearsonR } from "../descriptive-1V17A-qa.js";
+import { a as selfPreference, i as positionalBias, n as calibrateJudgeContinuous, o as verbosityBias, r as continuousAgreement, t as calibrateJudge } from "../judge-calibration-DYmaBtJr.js";
 import { u as runMetricExtractor } from "../query-D1nLIKt7.js";
+import { r as computeInterval } from "../ast-CP9ae9B0.js";
 import { t as analyzeSeries } from "../series-convergence-CjO2QdRW.js";
-import { t as rubricPredictiveValidity } from "../rubric-predictive-validity-2D5Gw9z9.js";
-import { n as InMemoryOutcomeStore, t as FileSystemOutcomeStore } from "../outcome-store-ChBKlTd_.js";
+import { a as reduceOutcomeMetric, i as hasVariation, n as assertUniqueObservationIds, o as validateObservationOptions, r as correlationSummary, t as rubricPredictiveValidity } from "../rubric-predictive-validity-CCK-1B7w.js";
+import { n as InMemoryOutcomeStore, r as OutcomeStoreError, t as FileSystemOutcomeStore } from "../outcome-store-BXlkwMPR.js";
+import { z } from "zod";
 //#region src/meta-eval/calibration.ts
 /**
 * Calibration curve — binned "if eval says X, what does reality show?"
@@ -17,7 +18,15 @@ import { n as InMemoryOutcomeStore, t as FileSystemOutcomeStore } from "../outco
 * mean outcome per bucket, reports expected-calibration-error (ECE).
 */
 async function calibrationCurve(traceStore, outcomeStore, evalMetric, outcomeMetric, options = {}) {
+	const settings = {
+		...options,
+		range: options.range === void 0 ? void 0 : { ...options.range }
+	};
+	validateCalibrationRequest(evalMetric.id, outcomeMetric, settings);
+	const extract = evalMetric.extract ?? runMetricExtractor(evalMetric.id);
+	const metricId = evalMetric.id;
 	const runs = await traceStore.listRuns();
+	assertUniqueObservationIds(runs.map((run) => run.runId), "runId");
 	const outcomes = await outcomeStore.list();
 	const byRun = /* @__PURE__ */ new Map();
 	for (const o of outcomes) {
@@ -25,15 +34,14 @@ async function calibrationCurve(traceStore, outcomeStore, evalMetric, outcomeMet
 		arr.push(o);
 		byRun.set(o.runId, arr);
 	}
-	const extract = evalMetric.extract ?? runMetricExtractor(evalMetric.id);
 	const pairs = [];
 	for (const run of runs) {
 		const os = byRun.get(run.runId);
 		if (!os?.length) continue;
 		const x = await extract(run, traceStore);
 		if (x === null || !Number.isFinite(x)) continue;
-		const y = [...os].sort((a, b) => b.capturedAt - a.capturedAt)[0].metrics[outcomeMetric];
-		if (typeof y !== "number" || !Number.isFinite(y)) continue;
+		const y = reduceOutcomeMetric(os, outcomeMetric, "latest");
+		if (y === null) continue;
 		pairs.push({
 			x,
 			y
@@ -43,35 +51,44 @@ async function calibrationCurve(traceStore, outcomeStore, evalMetric, outcomeMet
 	return calibrationFromPairs(pairs.map((p) => ({
 		evalScore: p.x,
 		outcome: p.y
-	})), evalMetric.id, outcomeMetric, options);
+	})), metricId, outcomeMetric, settings);
 }
+/** Measure already joined observations without constructing trace and outcome stores. */
 function calibrationFromPairs(inputPairs, evalMetric, outcomeMetric, options = {}) {
-	const pairs = inputPairs.filter((pair) => Number.isFinite(pair.evalScore) && Number.isFinite(pair.outcome));
+	validateCalibrationRequest(evalMetric, outcomeMetric, options);
+	for (const [index, pair] of inputPairs.entries()) if (pair === null || typeof pair !== "object" || !Number.isFinite(pair.evalScore) || !Number.isFinite(pair.outcome)) throw new Error(`calibration pair ${index} must contain finite evalScore and outcome values`);
+	const pairs = inputPairs;
 	if (pairs.length < 2) return null;
 	const numBins = options.bins ?? 10;
 	const binning = options.binning ?? "equal-width";
 	const xs = pairs.map((p) => p.evalScore);
 	const lo = options.range?.lo ?? Math.min(...xs);
 	const hi = options.range?.hi ?? Math.max(...xs);
+	const span = hi - lo;
+	if (!Number.isFinite(span)) throw new Error("calibration range span must be finite");
+	const clipped = pairs.map((pair) => ({
+		...pair,
+		evalScore: Math.min(hi, Math.max(lo, pair.evalScore))
+	}));
 	const bins = [];
-	if (binning === "equal-frequency") {
-		const sorted = [...pairs].sort((a, b) => a.evalScore - b.evalScore);
-		const perBin = Math.max(1, Math.floor(sorted.length / numBins));
-		for (let i = 0; i < sorted.length; i += perBin) {
-			const chunk = sorted.slice(i, i + perBin);
-			if (chunk.length === 0) continue;
-			bins.push(toBin(chunk));
+	if (span === 0 || clipped.every((pair) => pair.evalScore === clipped[0].evalScore)) bins.push(toBin(clipped));
+	else if (binning === "equal-frequency") {
+		const sorted = [...clipped].sort((a, b) => a.evalScore - b.evalScore);
+		const count = Math.min(numBins, sorted.length);
+		for (let i = 0; i < count; i++) {
+			const start = Math.floor(i * sorted.length / count);
+			const end = Math.floor((i + 1) * sorted.length / count);
+			bins.push(toBin(sorted.slice(start, end)));
 		}
 	} else {
-		const width = (hi - lo) / numBins;
-		if (width === 0) return null;
-		for (let i = 0; i < numBins; i++) {
-			const binLo = lo + i * width;
-			const binHi = i === numBins - 1 ? hi + 1e-9 : lo + (i + 1) * width;
-			const chunk = pairs.filter((p) => p.evalScore >= binLo && p.evalScore < binHi);
-			if (chunk.length === 0) continue;
-			bins.push(toBin(chunk, binLo, binHi));
+		const groups = /* @__PURE__ */ new Map();
+		for (const pair of clipped) {
+			const index = Math.min(numBins - 1, Math.floor((pair.evalScore - lo) / span * numBins));
+			const group = groups.get(index) ?? [];
+			group.push(pair);
+			groups.set(index, group);
 		}
+		for (const [index, chunk] of [...groups].sort(([a], [b]) => a - b)) bins.push(toBin(chunk, lo + span * (index / numBins), lo + span * ((index + 1) / numBins)));
 	}
 	const total = bins.reduce((a, b) => a + b.n, 0);
 	const ece = bins.reduce((a, b) => a + b.n / total * b.gap, 0);
@@ -100,22 +117,41 @@ function toBin(chunk, lower, upper) {
 	};
 }
 function mean(xs) {
-	return xs.reduce((a, b) => a + b, 0) / xs.length;
+	return xs.reduce((sum, value) => sum + value / xs.length, 0);
+}
+function validateCalibrationRequest(evalMetric, outcomeMetric, options) {
+	assertUniqueObservationIds([evalMetric], "eval metric");
+	assertUniqueObservationIds([outcomeMetric], "outcome metric");
+	if (evalMetric.trim() !== evalMetric || outcomeMetric.trim() !== outcomeMetric) throw new Error("calibration metric identities must not have surrounding whitespace");
+	if (options.bins !== void 0 && (!Number.isSafeInteger(options.bins) || options.bins < 1)) throw new Error("calibration bins must be a positive safe integer");
+	if (options.binning !== void 0 && !["equal-width", "equal-frequency"].includes(options.binning)) throw new Error("calibration binning must be equal-width or equal-frequency");
+	if (options.range !== void 0 && (!Number.isFinite(options.range.lo) || !Number.isFinite(options.range.hi) || !Number.isFinite(options.range.hi - options.range.lo) || options.range.hi < options.range.lo)) throw new Error("calibration range must have finite ordered bounds");
 }
 //#endregion
 //#region src/meta-eval/correlation-study.ts
 /**
 * Correlation study — "does our eval score predict real-world outcomes?"
 *
-* This is the load-bearing signal. Takes a TraceStore + OutcomeStore,
-* joins on runId, computes Pearson + Spearman + bootstrap CI for every
-* (evalMetric, outcomeMetric) pair the caller declares.
-*
-* Without this number the framework is ornamental. With it and r > 0.6
-* the framework is a moat — no other agent-eval tool publishes one.
+* Joins traces and outcomes by runId and reports descriptive correlations.
+* Independent runs are the bootstrap observation unit.
+* Association alone does not establish causation or held-out predictive performance.
 */
 async function correlationStudy(traceStore, outcomeStore, evalMetrics, outcomeMetricNames, options = {}) {
+	const reduction = options.reduction ?? "latest";
+	const iterations = options.bootstrapIterations ?? 500;
+	const seed = options.seed;
+	validateObservationOptions(reduction, iterations, seed);
+	assertUniqueObservationIds(evalMetrics.map((metric) => metric.id), "eval metric");
+	assertUniqueObservationIds(outcomeMetricNames, "outcome metric");
+	const maxLag = options.maxCaptureLagMs ?? Infinity;
+	if (maxLag < 0 || Number.isNaN(maxLag)) throw new Error("maxCaptureLagMs must be nonnegative");
+	const extractors = evalMetrics.map((metric) => ({
+		...metric,
+		extract: metric.extract ?? runMetricExtractor(metric.id)
+	}));
+	const metricNames = [...outcomeMetricNames];
 	const runs = await traceStore.listRuns();
+	assertUniqueObservationIds(runs.map((run) => run.runId), "runId");
 	const outcomes = await outcomeStore.list(options.outcomeFilter);
 	const outcomesByRun = /* @__PURE__ */ new Map();
 	for (const o of outcomes) {
@@ -123,10 +159,8 @@ async function correlationStudy(traceStore, outcomeStore, evalMetrics, outcomeMe
 		arr.push(o);
 		outcomesByRun.set(o.runId, arr);
 	}
-	const reduction = options.reduction ?? "latest";
-	const maxLag = options.maxCaptureLagMs ?? Infinity;
 	const pairs = [];
-	for (const em of evalMetrics) for (const om of outcomeMetricNames) pairs.push({
+	for (const em of extractors) for (const om of metricNames) pairs.push({
 		evalMetric: em.id,
 		outcomeMetric: om,
 		xs: [],
@@ -140,91 +174,205 @@ async function correlationStudy(traceStore, outcomeStore, evalMetrics, outcomeMe
 			skipped++;
 			continue;
 		}
-		const eligible = os.filter((o) => o.capturedAt - run.startedAt <= maxLag);
+		const eligible = os.filter((o) => {
+			const lag = o.capturedAt - run.startedAt;
+			return lag >= 0 && lag <= maxLag;
+		});
 		if (eligible.length === 0) {
 			skipped++;
 			continue;
 		}
-		for (const em of evalMetrics) {
-			const x = await (em.extract ?? runMetricExtractor(em.id))(run, traceStore);
+		let joinedThisRun = false;
+		for (const em of extractors) {
+			const x = await em.extract(run, traceStore);
 			if (x === null || !Number.isFinite(x)) continue;
-			for (const om of outcomeMetricNames) {
-				const values = eligible.map((o) => o.metrics[om]).filter((v) => typeof v === "number" && Number.isFinite(v));
-				if (values.length === 0) continue;
-				const y = reduce(values, reduction, eligible);
+			for (const om of metricNames) {
+				const y = reduceOutcomeMetric(eligible, om, reduction);
 				if (y === null) continue;
 				const pair = pairs.find((p) => p.evalMetric === em.id && p.outcomeMetric === om);
 				pair.xs.push(x);
 				pair.ys.push(y);
+				joinedThisRun = true;
 			}
 		}
-		joined++;
+		if (joinedThisRun) joined++;
+		else skipped++;
 	}
-	return {
-		pairs: pairs.filter((p) => p.xs.length >= 3).map((p) => {
-			const pearson = pearsonR(p.xs, p.ys);
-			const spearman = spearmanR(p.xs, p.ys);
-			const pearsonCi95 = bootstrapPearsonCi(p.xs, p.ys, options.bootstrapIterations ?? 500, options.seed);
-			const verdict = Math.abs(pearson) >= .7 ? "strong" : Math.abs(pearson) >= .4 ? "moderate" : "weak";
-			return {
+	const excludedPairs = [];
+	const results = [];
+	for (const p of pairs) {
+		const reason = p.xs.length < 3 ? "insufficient_samples" : !hasVariation(p.xs) ? "constant_eval_metric" : !hasVariation(p.ys) ? "constant_outcome" : null;
+		if (reason !== null) {
+			excludedPairs.push({
 				evalMetric: p.evalMetric,
 				outcomeMetric: p.outcomeMetric,
 				n: p.xs.length,
-				pearson,
-				spearman,
-				pearsonCi95,
-				verdict
-			};
-		}),
+				reason
+			});
+			continue;
+		}
+		const summary = correlationSummary(p.xs, p.ys, iterations, seed);
+		const verdict = Math.abs(summary.pearson) >= .7 ? "strong" : Math.abs(summary.pearson) >= .4 ? "moderate" : "weak";
+		results.push({
+			evalMetric: p.evalMetric,
+			outcomeMetric: p.outcomeMetric,
+			n: p.xs.length,
+			...summary,
+			verdict
+		});
+	}
+	return {
+		pairs: results,
+		excludedPairs,
 		joinedSamples: joined,
 		skippedRuns: skipped
 	};
 }
-function reduce(values, kind, outcomes) {
-	if (values.length === 0) return null;
-	if (kind === "mean") return values.reduce((a, b) => a + b, 0) / values.length;
-	if (kind === "max") return Math.max(...values);
-	const latest = [...outcomes].sort((a, b) => b.capturedAt - a.capturedAt)[0];
-	if (!latest) return null;
-	const latestKey = Object.keys(latest.metrics)[0];
-	const v = latestKey !== void 0 ? latest.metrics[latestKey] : void 0;
-	const paired = outcomes.map((o) => {
-		const k = Object.keys(o.metrics)[0];
-		return {
-			at: o.capturedAt,
-			v: k !== void 0 ? values.find((x) => o.metrics[k] === x) : void 0
+//#endregion
+//#region src/meta-eval/evaluator-admission.ts
+const identity = z.string().min(1).refine((value) => value.trim() === value);
+const policySchema = z.object({
+	confidence: z.number().finite().gt(0).lt(1),
+	maxFalseAcceptanceRate: z.number().finite().min(0).lt(1),
+	maxFalseRejectionRate: z.number().finite().min(0).lt(1)
+}).strict();
+const observationSchema = z.object({
+	id: identity,
+	independentUnitId: identity,
+	evidenceRef: identity,
+	expected: z.enum(["accept", "reject"]),
+	observed: z.enum([
+		"accept",
+		"reject",
+		"unknown"
+	]),
+	exposure: z.enum(["fresh", "development"])
+}).strict();
+const inputSchema = z.object({
+	evaluatorDigest: z.string().regex(/^sha256:[a-f0-9]{64}$/),
+	population: identity,
+	samplingFrame: identity,
+	authority: z.object({
+		evaluatorAuthorId: identity,
+		auditorId: identity,
+		independenceEvidenceRef: identity
+	}).strict(),
+	policy: policySchema,
+	observations: z.array(observationSchema)
+}).strict();
+function errorRate(rows, expected, limit, level) {
+	const cases = rows.filter((row) => row.expected === expected);
+	const units = /* @__PURE__ */ new Map();
+	for (const row of cases) {
+		const unit = units.get(row.independentUnitId) ?? {
+			error: false,
+			unknown: false
 		};
-	}).filter((p) => p.v !== void 0);
-	if (paired.length === 0) return v ?? null;
-	return paired.sort((a, b) => b.at - a.at)[0]?.v ?? null;
-}
-function bootstrapPearsonCi(xs, ys, iterations, seed) {
-	const n = xs.length;
-	if (n < 3) return {
-		lower: NaN,
-		upper: NaN
+		unit.unknown ||= row.observed === "unknown";
+		unit.error ||= row.observed !== "unknown" && row.observed !== expected;
+		units.set(row.independentUnitId, unit);
+	}
+	const n = units.size;
+	const errorUnits = [...units.values()].filter((unit) => unit.error).length;
+	const unresolvedUnits = [...units.values()].filter((unit) => unit.unknown && !unit.error).length;
+	const unknownCases = cases.filter((row) => row.observed === "unknown").length;
+	const interval = n === 0 ? null : {
+		lower: computeInterval({
+			kind: "clopper-pearson",
+			level
+		}, {
+			kind: "binomial",
+			successes: errorUnits,
+			trials: n
+		}).lower,
+		upper: computeInterval({
+			kind: "clopper-pearson",
+			level
+		}, {
+			kind: "binomial",
+			successes: errorUnits + unresolvedUnits,
+			trials: n
+		}).upper,
+		level
 	};
-	const rng = makeRng(seed, xs, ys);
-	const rs = [];
-	for (let b = 0; b < iterations; b++) {
-		const rx = new Array(n);
-		const ry = new Array(n);
-		for (let i = 0; i < n; i++) {
-			const idx = Math.floor(rng() * n);
-			rx[i] = xs[idx];
-			ry[i] = ys[idx];
-		}
-		const r = pearsonR(rx, ry);
-		if (Number.isFinite(r)) rs.push(r);
+	const verdict = interval && interval.lower > limit ? "fail" : interval && interval.upper <= limit ? "pass" : "inconclusive";
+	return {
+		cases: cases.length,
+		independentUnits: n,
+		errorUnits,
+		unresolvedUnits,
+		unknownCases,
+		errorRate: n === 0 || unresolvedUnits > 0 ? null : errorUnits / n,
+		interval,
+		limit,
+		verdict
+	};
+}
+/**
+* Audit frozen judgments using independent source units and exact binomial bounds.
+* A unit fails a class when any control in that class is misjudged.
+* The execution owner enforces auditor separation, fresh sampling, and evidence authenticity.
+*/
+function auditEvaluator(input) {
+	const parsed = inputSchema.safeParse(input);
+	if (!parsed.success) throw new ValidationError(`invalid evaluator audit: ${parsed.error.message}`);
+	const audit = parsed.data;
+	if (audit.authority.evaluatorAuthorId === audit.authority.auditorId) throw new ValidationError("evaluator admission requires a separate declared audit authority");
+	const ids = /* @__PURE__ */ new Set();
+	for (const row of audit.observations) {
+		if (ids.has(row.id)) throw new ValidationError(`duplicate evaluator audit observation '${row.id}'`);
+		ids.add(row.id);
+	}
+	audit.observations.sort((a, b) => compareCodeUnits(a.id, b.id));
+	const developmentUnits = new Set(audit.observations.filter((row) => row.exposure === "development").map((row) => row.independentUnitId));
+	const eligible = audit.observations.filter((row) => !developmentUnits.has(row.independentUnitId));
+	const exclusions = audit.observations.filter((row) => developmentUnits.has(row.independentUnitId)).map((row) => ({
+		id: row.id,
+		independentUnitId: row.independentUnitId,
+		reason: "development-exposure"
+	}));
+	const intervalConfidence = 1 - (1 - audit.policy.confidence) / 2;
+	if (intervalConfidence >= 1) throw new ValidationError("evaluator audit confidence is too close to one for simultaneous intervals");
+	const falseAcceptance = errorRate(eligible, "reject", audit.policy.maxFalseAcceptanceRate, intervalConfidence);
+	const falseRejection = errorRate(eligible, "accept", audit.policy.maxFalseRejectionRate, intervalConfidence);
+	const rates = [falseAcceptance, falseRejection];
+	const verdict = rates.some((rate) => rate.verdict === "fail") ? "reject" : rates.every((rate) => rate.verdict === "pass") ? "admit" : "inconclusive";
+	const reasons = [];
+	for (const [name, rate] of [["false acceptance", falseAcceptance], ["false rejection", falseRejection]]) {
+		if (rate.independentUnits === 0) reasons.push(`${name}: no eligible independent units`);
+		else if (rate.unknownCases > 0) reasons.push(`${name}: ${rate.unknownCases} judgments are unknown`);
+		if (rate.verdict === "fail") reasons.push(`${name}: lower error bound exceeds ${rate.limit}`);
+		else if (rate.verdict === "inconclusive" && rate.interval) reasons.push(`${name}: upper error bound does not establish the required limit`);
 	}
-	rs.sort((a, b) => a - b);
-	if (rs.length === 0) return {
-		lower: NaN,
-		upper: NaN
+	if (verdict === "admit") reasons.push("both error bounds meet the registered limits, including the worst case for unknown judgments");
+	const body = {
+		evaluatorDigest: audit.evaluatorDigest,
+		policyDigest: hashCanonical(audit.policy),
+		inputDigest: hashCanonical(audit),
+		population: audit.population,
+		samplingFrame: audit.samplingFrame,
+		authority: audit.authority,
+		policy: audit.policy,
+		confidence: audit.policy.confidence,
+		intervalConfidence,
+		verdict,
+		reasons,
+		observations: audit.observations,
+		coverage: {
+			cases: audit.observations.length,
+			independentUnits: new Set(audit.observations.map((row) => row.independentUnitId)).size,
+			eligibleCases: eligible.length,
+			eligibleIndependentUnits: new Set(eligible.map((row) => row.independentUnitId)).size,
+			excludedCases: exclusions.length,
+			unknownCases: eligible.filter((row) => row.observed === "unknown").length
+		},
+		exclusions,
+		falseAcceptance,
+		falseRejection
 	};
 	return {
-		lower: rs[Math.floor(.025 * rs.length)],
-		upper: rs[Math.min(rs.length - 1, Math.floor(.975 * rs.length))]
+		...body,
+		reportDigest: hashCanonical(body)
 	};
 }
 //#endregion
@@ -982,6 +1130,6 @@ function evalHealthStamp(report) {
 	};
 }
 //#endregion
-export { FileSystemOutcomeStore, InMemoryOutcomeStore, calibrationCurve, catchRate, correlationStudy, definePlant, evalHealthStamp, fileSentinelStore, inMemorySentinelStore, judgeSentinelReport, perturbEvidence, plantByPerturbation, rubricPredictiveValidity, seedPlants, snapshotFromAgreement, snapshotFromCalibration, snapshotFromSentinelSet, validateSentinelSnapshot };
+export { FileSystemOutcomeStore, InMemoryOutcomeStore, OutcomeStoreError, auditEvaluator, calibrateJudge, calibrateJudgeContinuous, calibrationCurve, calibrationFromPairs, catchRate, continuousAgreement, correlationStudy, definePlant, evalHealthStamp, fileSentinelStore, inMemorySentinelStore, judgeSentinelReport, perturbEvidence, plantByPerturbation, positionalBias, rubricPredictiveValidity, seedPlants, selfPreference, snapshotFromAgreement, snapshotFromCalibration, snapshotFromSentinelSet, validateSentinelSnapshot, verbosityBias };
 
 //# sourceMappingURL=index.js.map