@tangle-network/agent-eval 0.217.1 → 0.219.0

package/dist/experiment/index.js

@@ -9,7 +9,7 @@ import { c as studentTCdf, i as pairedBootstrap, l as studentTQuantile, n as BOO
 import { a as requiredSampleSize, i as requiredPairedSampleSize, n as mcnemarRequiredN, r as pairedMde, t as mcnemarPower } from "../power-and-mde-N5nU4_Ly.js";
 import { t as eProcess } from "../sequential-eprocess-DodI6RoF.js";
 import { a as heldoutSignificance, o as pairHoldout } from "../statistical-heldout-Bksv92Th.js";
-import { S as openFinalEvidenceLedger, b as FinalEvidenceConflictError, i as verifyEvidenceReceipt, n as createEvidenceReceipt, r as isIndependentEvidence, t as createCampaignEvidenceReceipt, x as FinalEvidenceError, y as powerPreflight } from "../campaign-evidence-CQYw8_0H.js";
+import { S as openFinalEvidenceLedger, b as FinalEvidenceConflictError, i as verifyEvidenceReceipt, n as createEvidenceReceipt, r as isIndependentEvidence, t as createCampaignEvidenceReceipt, x as FinalEvidenceError, y as powerPreflight } from "../campaign-evidence-DunzcJGv.js";
 import { n as evaluatePredicate, r as readField, t as compareValues } from "../predicate-FdKmElA_.js";
 import { r as summarizeEvaluationUnits, t as defineEvaluationClaim } from "../claim-BPVuyHuy.js";
 import { n as pairedEvalueSequence } from "../sequential-BVv1sjsw.js";
@@ -1261,8 +1261,8 @@ function sealRandomizedCanaryRule(rule) {
 		algo: "sha256-rfc8785"
 	};
 }
-/** Apply one registered analysis to the entire frozen assignment roster. */
-function decideRandomizedCanary(sealed, cohort, rows, verifyCohort) {
+/** Check the registration seal, the cohort receipt, the frozen roster and the independent verifier. */
+function verifyCohortEvidence(sealed, cohort, rows, verifyCohort) {
 	if (sealed.algo !== "sha256-rfc8785") throw new ValidationError("randomized canary: unsupported registration digest scheme");
 	validateRule(sealed.rule);
 	if (hashCanonical(sealed.rule) !== sealed.digest) throw new ValidationError("randomized canary: registration digest mismatch");
@@ -1287,57 +1287,83 @@ function decideRandomizedCanary(sealed, cohort, rows, verifyCohort) {
 	if (!Number.isSafeInteger(cohort.assignmentCount) || cohort.assignmentCount < 1 || rows.length !== cohort.assignmentCount) throw new ValidationError("randomized canary: assignment count differs from complete ledger receipt");
 	if (randomizedCanaryRosterDigest(rows) !== cohort.assignmentRosterDigest) throw new ValidationError("randomized canary: assignment roster digest mismatch");
 	if (randomizedCanaryObservationDigest(rows) !== cohort.observationSnapshotDigest) throw new ValidationError("randomized canary: frozen observation snapshot digest mismatch");
+	verifyCohortTimes(sealed, cohort);
+	if (typeof verifyCohort !== "function") throw new ValidationError("randomized canary: independent cohort verifier is required");
+	const verification = verifyCohort(cohort, sealed);
+	if (verification?.verified !== true || verification.authorityId !== cohort.authorityId || verification.protocolWitnessedBeforeTraffic !== true || verification.familySlotReservedBeforeTraffic !== true || verification.eligibilityAndDispositionVerified !== true || verification.randomizationVerified !== true || verification.rosterCompleteAtCutoff !== true || verification.outcomeAndBillingSnapshotFrozen !== true || verification.sourceJoinsVerified !== true || verification.armIsolationVerified !== true) throw new ValidationError(`randomized canary: cohort attestation or completeness unverified${verification?.verified === false ? `: ${verification.reason}` : ""}`);
+}
+/** Require the ledger closure, observation freeze and attestation times to match the registered cutoff. */
+function verifyCohortTimes(sealed, cohort) {
 	const cutoff = Date.parse(sealed.rule.analysisCutoff);
 	const closedAt = Date.parse(cohort.closedAt);
 	const frozenAt = Date.parse(cohort.observationFrozenAt);
 	const attestedAt = Date.parse(cohort.attestedAt);
 	const expectedFreezeAt = cutoff + sealed.rule.outcomeMaturityMs;
 	if (!Number.isFinite(closedAt) || closedAt < cutoff || !cohort.closedAt.endsWith("Z") || !Number.isFinite(frozenAt) || frozenAt !== expectedFreezeAt || closedAt > frozenAt || !cohort.observationFrozenAt.endsWith("Z") || !Number.isFinite(attestedAt) || attestedAt < frozenAt || !cohort.attestedAt.endsWith("Z")) throw new ValidationError("randomized canary: ledger closure or frozen observation time differs from registered cutoffs");
-	if (typeof verifyCohort !== "function") throw new ValidationError("randomized canary: independent cohort verifier is required");
-	const verification = verifyCohort(cohort, sealed);
-	if (verification?.verified !== true || verification.authorityId !== cohort.authorityId || verification.protocolWitnessedBeforeTraffic !== true || verification.familySlotReservedBeforeTraffic !== true || verification.eligibilityAndDispositionVerified !== true || verification.randomizationVerified !== true || verification.rosterCompleteAtCutoff !== true || verification.outcomeAndBillingSnapshotFrozen !== true || verification.sourceJoinsVerified !== true || verification.armIsolationVerified !== true) throw new ValidationError(`randomized canary: cohort attestation or completeness unverified${verification?.verified === false ? `: ${verification.reason}` : ""}`);
-	const clusters = /* @__PURE__ */ new Map();
-	const arms = {
-		control: [],
-		candidate: []
+}
+/** Validate one row's evidence fields and count its gaps; returns whether it is a recorded no-execution. */
+function checkObservationRow(row, index, tally) {
+	if (row.checkedOutcome !== 0 && row.checkedOutcome !== 1 && row.checkedOutcome !== null) throw new ValidationError(`randomized canary: assignment ${index} outcome must be 0, 1, or null`);
+	if (row.checkedOutcome === null) tally.missingOutcomes++;
+	if (row.outcomeSourceId === null) {
+		if (row.checkedOutcome !== null) throw new ValidationError(`randomized canary: assignment ${index} has an outcome without its source`);
+	} else nonEmpty(row.outcomeSourceId, `assignment ${index} outcome source id`);
+	if (row.billedCostUsd === null) tally.missingBilling++;
+	else if (!Number.isFinite(row.billedCostUsd) || row.billedCostUsd < 0) throw new ValidationError(`randomized canary: assignment ${index} billed cost must be nonnegative`);
+	if (row.billingSourceId === null) {
+		if (row.billedCostUsd !== null) throw new ValidationError(`randomized canary: assignment ${index} has billed cost without its source`);
+	} else nonEmpty(row.billingSourceId, `assignment ${index} billing source id`);
+	if (row.turns === null) tally.missingTurns++;
+	else if (!Number.isSafeInteger(row.turns) || row.turns < 0) throw new ValidationError(`randomized canary: assignment ${index} turns must be a nonnegative integer`);
+	if (row.traceSourceId !== null) nonEmpty(row.traceSourceId, `assignment ${index} trace source id`);
+	if (row.noExecutionSourceId !== null) nonEmpty(row.noExecutionSourceId, `assignment ${index} no-execution source id`);
+	if (row.traceSourceId !== null && row.noExecutionSourceId !== null) throw new ValidationError(`randomized canary: assignment ${index} has conflicting trace and no-execution receipts`);
+	const noExecution = row.noExecutionSourceId !== null && row.turns === 0 && row.billedCostUsd === 0 && row.checkedOutcome === 0;
+	if (row.traceSourceId === null && !noExecution) tally.missingTraces++;
+	for (const field of [
+		"servedProfileDigest",
+		"servedCodeRevisionDigest",
+		"outcomeCheckerDigest"
+	]) if (row[field] !== null) nonEmpty(row[field], `assignment ${index} ${field}`);
+	return noExecution;
+}
+/** Whether a row was served by, or checked with, anything other than the registered profile, code and checker. */
+function drifted(row, noExecution, rule) {
+	const expectedProfile = row.arm === "control" ? rule.controlProfileDigest : rule.candidateProfileDigest;
+	return row.traceSourceId !== null && (row.servedProfileDigest !== expectedProfile || row.servedCodeRevisionDigest !== rule.servedCodeRevisionDigest) || row.checkedOutcome !== null && row.outcomeCheckerDigest !== rule.outcomeCheckerDigest || noExecution && (row.servedProfileDigest !== null || row.servedCodeRevisionDigest !== null);
+}
+/** Validate every row against the registered rule, then group by arm and cluster. */
+function tallyObservations(rule, rows) {
+	const tally = {
+		clusters: /* @__PURE__ */ new Map(),
+		arms: {
+			control: [],
+			candidate: []
+		},
+		missingOutcomes: 0,
+		missingBilling: 0,
+		missingTurns: 0,
+		missingTraces: 0,
+		executionDrift: 0
 	};
-	let missingOutcomes = 0;
-	let missingBilling = 0;
-	let missingTurns = 0;
-	let missingTraces = 0;
-	let executionDrift = 0;
 	for (const [index, row] of rows.entries()) {
-		if (row.checkedOutcome !== 0 && row.checkedOutcome !== 1 && row.checkedOutcome !== null) throw new ValidationError(`randomized canary: assignment ${index} outcome must be 0, 1, or null`);
-		if (row.checkedOutcome === null) missingOutcomes++;
-		if (row.outcomeSourceId === null) {
-			if (row.checkedOutcome !== null) throw new ValidationError(`randomized canary: assignment ${index} has an outcome without its source`);
-		} else nonEmpty(row.outcomeSourceId, `assignment ${index} outcome source id`);
-		if (row.billedCostUsd === null) missingBilling++;
-		else if (!Number.isFinite(row.billedCostUsd) || row.billedCostUsd < 0) throw new ValidationError(`randomized canary: assignment ${index} billed cost must be nonnegative`);
-		if (row.billingSourceId === null) {
-			if (row.billedCostUsd !== null) throw new ValidationError(`randomized canary: assignment ${index} has billed cost without its source`);
-		} else nonEmpty(row.billingSourceId, `assignment ${index} billing source id`);
-		if (row.turns === null) missingTurns++;
-		else if (!Number.isSafeInteger(row.turns) || row.turns < 0) throw new ValidationError(`randomized canary: assignment ${index} turns must be a nonnegative integer`);
-		if (row.traceSourceId !== null) nonEmpty(row.traceSourceId, `assignment ${index} trace source id`);
-		if (row.noExecutionSourceId !== null) nonEmpty(row.noExecutionSourceId, `assignment ${index} no-execution source id`);
-		if (row.traceSourceId !== null && row.noExecutionSourceId !== null) throw new ValidationError(`randomized canary: assignment ${index} has conflicting trace and no-execution receipts`);
-		const noExecution = row.noExecutionSourceId !== null && row.turns === 0 && row.billedCostUsd === 0 && row.checkedOutcome === 0;
-		if (row.traceSourceId === null && !noExecution) missingTraces++;
-		for (const field of [
-			"servedProfileDigest",
-			"servedCodeRevisionDigest",
-			"outcomeCheckerDigest"
-		]) if (row[field] !== null) nonEmpty(row[field], `assignment ${index} ${field}`);
-		const expectedProfile = row.arm === "control" ? sealed.rule.controlProfileDigest : sealed.rule.candidateProfileDigest;
-		if (row.traceSourceId !== null && (row.servedProfileDigest !== expectedProfile || row.servedCodeRevisionDigest !== sealed.rule.servedCodeRevisionDigest) || row.checkedOutcome !== null && row.outcomeCheckerDigest !== sealed.rule.outcomeCheckerDigest || noExecution && (row.servedProfileDigest !== null || row.servedCodeRevisionDigest !== null)) executionDrift++;
-		arms[row.arm].push(row);
-		const bucket = clusters.get(row.clusterId) ?? [];
+		if (drifted(row, checkObservationRow(row, index, tally), rule)) tally.executionDrift++;
+		tally.arms[row.arm].push(row);
+		const bucket = tally.clusters.get(row.clusterId) ?? [];
 		bucket.push(row);
-		clusters.set(row.clusterId, bucket);
+		tally.clusters.set(row.clusterId, bucket);
 	}
-	const controlClusters = [...clusters.values()].filter((bucket) => bucket.some((r) => r.arm === "control")).length;
-	const candidateClusters = [...clusters.values()].filter((bucket) => bucket.some((r) => r.arm === "candidate")).length;
+	return tally;
+}
+/** Count the clusters that hold at least one row of the arm. */
+function clustersWithArm(tally, arm) {
+	return [...tally.clusters.values()].filter((bucket) => bucket.some((r) => r.arm === arm)).length;
+}
+/** The decision with every count and total filled in and no interval or pass yet. */
+function baseDecision(sealed, cohort, rowCount, tally) {
+	const { clusters, arms, missingOutcomes, missingBilling, missingTurns } = tally;
+	const controlClusters = clustersWithArm(tally, "control");
+	const candidateClusters = clustersWithArm(tally, "candidate");
 	if (sealed.rule.assignmentUnit === sealed.rule.clusterUnit) {
 		for (const [clusterId, bucket] of clusters) if (bucket.length !== 1) throw new ValidationError(`randomized canary: ${sealed.rule.assignmentUnit} cluster '${clusterId}' has multiple assignments`);
 	}
@@ -1358,7 +1384,7 @@ function decideRandomizedCanary(sealed, cohort, rows, verifyCohort) {
 	const p0 = outcomesComplete ? sum("control", "checkedOutcome") / n0 : null;
 	const p1 = outcomesComplete ? sum("candidate", "checkedOutcome") / n1 : null;
 	const delta = p0 === null || p1 === null ? null : p1 - p0;
-	const result = {
+	return {
 		registrationDigest: sealed.digest,
 		cohortReceiptSourceId: cohort.attestationSourceId,
 		assignmentRosterDigest: cohort.assignmentRosterDigest,
@@ -1366,7 +1392,7 @@ function decideRandomizedCanary(sealed, cohort, rows, verifyCohort) {
 		method: "cluster-robust-t",
 		confidence: .95,
 		familyAdjustedConfidence: 1 - .05 / sealed.rule.confirmatoryFamilySize,
-		assignments: rows.length,
+		assignments: rowCount,
 		controlAssignments: n0,
 		candidateAssignments: n1,
 		clusters: clusters.size,
@@ -1376,8 +1402,8 @@ function decideRandomizedCanary(sealed, cohort, rows, verifyCohort) {
 		missingOutcomes,
 		missingBilling,
 		missingTurns,
-		missingTraces,
-		executionDrift,
+		missingTraces: tally.missingTraces,
+		executionDrift: tally.executionDrift,
 		controlSuccessRate: p0,
 		candidateSuccessRate: p1,
 		delta,
@@ -1393,32 +1419,33 @@ function decideRandomizedCanary(sealed, cohort, rows, verifyCohort) {
 		refusal: null,
 		successCriterionMet: false
 	};
-	if (missingOutcomes > 0 || missingBilling > 0 || missingTurns > 0 || n0 === 0 || n1 === 0) {
-		result.refusal = "missing-evidence";
-		return result;
-	}
-	if (missingTraces > 0) {
-		result.refusal = "missing-trace";
-		return result;
-	}
-	if (executionDrift > 0) {
-		result.refusal = "execution-drift";
-		return result;
-	}
-	if (clusters.size < sealed.rule.minimumClusters || controlClusters < 20 || candidateClusters < 20) {
-		result.refusal = "insufficient-clusters";
-		return result;
-	}
-	for (const bucket of clusters.values()) if (bucket.filter((r) => r.arm === "control").length / n0 > .1 || bucket.filter((r) => r.arm === "candidate").length / n1 > .1) {
-		result.refusal = "dominant-cluster";
-		return result;
-	}
+}
+/** The first reason the evidence cannot support an estimate, or null when it can. */
+function refusalReason(rule, tally, result) {
+	const { clusters } = tally;
+	const n0 = result.controlAssignments;
+	const n1 = result.candidateAssignments;
+	if (tally.missingOutcomes > 0 || tally.missingBilling > 0 || tally.missingTurns > 0 || n0 === 0 || n1 === 0) return "missing-evidence";
+	if (tally.missingTraces > 0) return "missing-trace";
+	if (tally.executionDrift > 0) return "execution-drift";
+	if (clusters.size < rule.minimumClusters || result.controlClusters < 20 || result.candidateClusters < 20) return "insufficient-clusters";
+	for (const bucket of clusters.values()) if (bucket.filter((r) => r.arm === "control").length / n0 > .1 || bucket.filter((r) => r.arm === "candidate").length / n1 > .1) return "dominant-cluster";
+	return null;
+}
+/** Fill the intervals, p-value and passes, or refuse on zero variance. */
+function estimateRiskDifference(rule, tally, result) {
+	const { clusters } = tally;
+	const p0 = result.controlSuccessRate;
+	const p1 = result.candidateSuccessRate;
+	const delta = result.delta;
+	const n0 = result.controlAssignments;
+	const n1 = result.candidateAssignments;
 	const scores = [...clusters.values()].map((bucket) => bucket.reduce((score, row) => score + (row.arm === "candidate" ? (row.checkedOutcome - p1) / n1 : -(row.checkedOutcome - p0) / n0), 0));
 	const G = clusters.size;
-	const variance = G / (G - 1) * (rows.length - 1) / (rows.length - 2) * scores.reduce((total, score) => total + score * score, 0);
+	const variance = G / (G - 1) * (result.assignments - 1) / (result.assignments - 2) * scores.reduce((total, score) => total + score * score, 0);
 	if (!Number.isFinite(variance) || variance <= 0) {
 		result.refusal = "zero-variance";
-		return result;
+		return;
 	}
 	const se = Math.sqrt(variance);
 	const t = delta / se;
@@ -1434,8 +1461,20 @@ function decideRandomizedCanary(sealed, cohort, rows, verifyCohort) {
 	};
 	result.unadjustedPValue = Math.min(1, 2 * (1 - studentTCdf(Math.abs(t), G - 1)));
 	result.headline95Pass = result.nominal95Interval.low > 0;
-	result.familywisePass = result.familyAdjustedInterval.low > sealed.rule.minimumLift;
+	result.familywisePass = result.familyAdjustedInterval.low > rule.minimumLift;
 	result.successCriterionMet = result.headline95Pass && result.familywisePass;
+}
+/** Apply one registered analysis to the entire frozen assignment roster. */
+function decideRandomizedCanary(sealed, cohort, rows, verifyCohort) {
+	verifyCohortEvidence(sealed, cohort, rows, verifyCohort);
+	const tally = tallyObservations(sealed.rule, rows);
+	const result = baseDecision(sealed, cohort, rows.length, tally);
+	const refusal = refusalReason(sealed.rule, tally, result);
+	if (refusal !== null) {
+		result.refusal = refusal;
+		return result;
+	}
+	estimateRiskDifference(sealed.rule, tally, result);
 	return result;
 }
 //#endregion