@tangle-network/agent-eval 0.215.1 → 0.216.1

package/dist/experiment/index.js

@@ -1,22 +1,317 @@
-import { n as CaptureIntegrityError, s as ValidationError } from "../errors-Dngq5h35.js";
-import { a as hashCanonical, t as LEDGER_HASH_PATTERN } from "../canonical-BsZVgBhW.js";
-import { f as evaluateHypothesis, g as verifyManifest, h as signManifest, m as manifestContentDigest, p as hashJson } from "../agent-profile-cell-BYc0bxsL.js";
-import { n as studentTQuantile, t as studentTCdf } from "../student-t-BQeKJx2N.js";
-import { t as mulberry32 } from "../random-Dn5fPWkt.js";
-import { n as bonferroni, r as holm, t as benjaminiHochberg } from "../multiplicity-D9cjmStq.js";
-import { a as pairedRiskDifferenceExact, c as wilson, i as pairedRiskDifference, n as mcnemar, o as pairedRiskDifferenceScore } from "../paired-binary-Q36bhn2R.js";
-import { r as pairedBootstrap, t as BOOTSTRAP_GATE_MIN_N } from "../paired-tests-jbwPdSz5.js";
-import { a as requiredSampleSize, i as requiredPairedSampleSize, n as mcnemarRequiredN, r as pairedMde, t as mcnemarPower } from "../power-and-mde-fK5Qocwo.js";
-import { n as pairArms, r as pairRunRecords, t as comparePairedArms } from "../paired-arms-DzKn0TfY.js";
-import { t as eProcess } from "../sequential-eprocess-D1jKoihe.js";
-import { a as heldoutSignificance, o as pairHoldout } from "../statistical-heldout-1Gbmex7v.js";
-import { D as FinalEvidenceError, E as FinalEvidenceConflictError, O as openFinalEvidenceLedger, T as powerPreflight, a as verifyEvidenceReceipt, i as isIndependentEvidence, n as EVIDENCE_RECEIPT_VERSION, r as createEvidenceReceipt, t as createCampaignEvidenceReceipt } from "../campaign-evidence-OcUciD-N.js";
-import { _ as runSelectionRule, a as evaluateHaltRule, c as evaluatePopulationReproducibilityGate, d as evaluateProvenanceGate, f as executeDecisionRule, g as readField, h as projectNLadderBudget, i as evaluateCondition, l as evaluatePowerFloorGate, m as powerFloorProblems, n as computeEstimand, o as evaluateIdentityGate, p as intervalSpecProblems, r as computeInterval, s as evaluateOracleDeterminismGate, t as classifyReissue, u as evaluatePredicate, v as runUniformPassBudget } from "../ast-DJ6Bc7ab.js";
-import { r as summarizeEvaluationUnits, t as defineEvaluationClaim } from "../claim-BXcZw-jo.js";
+import { n as CaptureIntegrityError, s as ValidationError } from "../errors-DVZiqS7K.js";
+import { a as hashCanonical, t as LEDGER_HASH_PATTERN } from "../canonical-B2RqLofv.js";
+import { f as evaluateHypothesis, g as verifyManifest, h as signManifest, m as manifestContentDigest, p as hashJson } from "../agent-profile-cell-BK8mlqlq.js";
+import { t as mulberry32 } from "../random-ByUSnV9w.js";
+import { n as bonferroni, r as holm, t as benjaminiHochberg } from "../multiplicity-DQUFcbW4.js";
+import { a as pairedRiskDifferenceExact, c as wilson, i as pairedRiskDifference, n as mcnemar, o as pairedRiskDifferenceScore } from "../paired-binary-CvpFHfKZ.js";
+import { n as studentTQuantile, t as studentTCdf } from "../student-t-DVPW7CdL.js";
+import { i as pairedBootstrap, n as BOOTSTRAP_GATE_MIN_N } from "../mean-DHoMiudD.js";
+import { a as requiredSampleSize, i as requiredPairedSampleSize, n as mcnemarRequiredN, r as pairedMde, t as mcnemarPower } from "../power-and-mde-Hj2ubb4B.js";
+import { n as pairArms, r as pairRunRecords, t as comparePairedArms } from "../paired-arms-CwMGJsrS.js";
+import { t as eProcess } from "../sequential-eprocess-DM_qENCl.js";
+import { a as heldoutSignificance, o as pairHoldout } from "../statistical-heldout-42NlCNDe.js";
+import { D as FinalEvidenceError, E as FinalEvidenceConflictError, O as openFinalEvidenceLedger, T as powerPreflight, a as verifyEvidenceReceipt, i as isIndependentEvidence, n as EVIDENCE_RECEIPT_VERSION, r as createEvidenceReceipt, t as createCampaignEvidenceReceipt } from "../campaign-evidence-Clrn4hBV.js";
+import { n as evaluatePredicate, r as readField, t as compareValues } from "../predicate-FdKmElA_.js";
+import { r as summarizeEvaluationUnits, t as defineEvaluationClaim } from "../claim-q4HQhK1G.js";
 import { n as pairedEvalueSequence, r as sequentialCrossingHorizon } from "../sequential-CzK5DarL.js";
-import { n as paretoPolicy, r as paretoSignificanceGate, t as buildEvidenceVector } from "../promotion-policy-CWW35UWq.js";
-import { n as sequentialPairedGate, t as sequentialDecide } from "../sequential-D-1PPCmM.js";
+import { n as intervalSpecProblems, r as computeEstimand, t as computeInterval } from "../interval-CELpye1n.js";
+import { n as paretoPolicy, r as paretoSignificanceGate, t as buildEvidenceVector } from "../promotion-policy-KO57gbyV.js";
+import { n as sequentialPairedGate, t as sequentialDecide } from "../sequential-Bf_pbE50.js";
 import { z } from "zod";
+//#region src/experiment/ast/budget.ts
+function evaluateHaltRule(halt, gates) {
+	const seen = new Map(gates.map((g) => [g.id, g]));
+	const missing = halt.when.gates.filter((id) => !seen.has(id));
+	if (missing.length > 0) throw new ValidationError(`evaluateHaltRule: halt references gates that were not evaluated: [${missing.join(", ")}]`);
+	const failed = halt.when.gates.filter((id) => !seen.get(id).passed);
+	return failed.length > 0 ? {
+		fired: true,
+		action: halt.action,
+		failedGates: failed
+	} : {
+		fired: false,
+		action: null,
+		failedGates: []
+	};
+}
+/**
+* Execute the uniform-pass schedule against measured pass costs. Pass 1 always
+* runs; each later pass runs only when the cumulative spend plus the last
+* measured pass cost stays at or under the registered ceiling. The registered
+* ledger is the pre-spend the ceiling counts.
+*/
+function runUniformPassBudget(rule, measuredPassCosts) {
+	let cumulative = rule.ledger.reduce((sum, entry) => sum + entry.usd, 0);
+	const decisions = [];
+	let uniformN = 0;
+	for (let pass = 1; pass <= rule.maxPasses; pass++) {
+		if (pass === 1) {
+			if (measuredPassCosts[0] === void 0) break;
+			cumulative += measuredPassCosts[0];
+			uniformN = 1;
+			continue;
+		}
+		const projected = measuredPassCosts[pass - 2];
+		if (projected === void 0) break;
+		const go = cumulative + projected <= rule.ceilingUsd;
+		decisions.push({
+			pass,
+			cumulativeBefore: cumulative,
+			projected,
+			go
+		});
+		if (!go || measuredPassCosts[pass - 1] === void 0) break;
+		cumulative += measuredPassCosts[pass - 1];
+		uniformN = pass;
+	}
+	return {
+		decisions,
+		uniformN
+	};
+}
+/**
+* Walk the registered n-ladder and pick the first affordable step. When no
+* step fits the ceiling, the rule refuses and reports the projection instead
+* of shrinking the row set — "never subset rows" is the registered invariant.
+*/
+function projectNLadderBudget(rule, measured) {
+	const projections = rule.steps.map((n) => {
+		const projectedUsd = measured.unitCostUsd * measured.rows * n;
+		return {
+			n,
+			projectedUsd,
+			affordable: projectedUsd <= rule.ceilingUsd
+		};
+	});
+	const first = projections.find((p) => p.affordable);
+	if (first) return {
+		chosenN: first.n,
+		projections,
+		refusal: null
+	};
+	return {
+		chosenN: null,
+		projections,
+		refusal: {
+			onExhaust: rule.onExhaust,
+			reason: `no ladder step fits the ${rule.ceilingUsd} USD ceiling at ${measured.rows} rows x ${measured.unitCostUsd} USD per unit`
+		}
+	};
+}
+//#endregion
+//#region src/experiment/ast/decision.ts
+/** A decision rule's branches did not cover the evidence. */
+var DecisionTableNotTotalError = class extends ValidationError {};
+function evaluateCondition(condition, evidence) {
+	switch (condition.kind) {
+		case "interval-excludes-zero": {
+			const interval = evidence.intervals[condition.interval];
+			if (!interval) throw new ValidationError(`evaluateCondition: interval '${condition.interval}' is not in the evidence`);
+			return (interval.lower > 0 || interval.upper < 0) && (condition.sign === "positive" ? interval.lower > 0 : interval.upper < 0);
+		}
+		case "interval-includes-zero": {
+			const interval = evidence.intervals[condition.interval];
+			if (!interval) throw new ValidationError(`evaluateCondition: interval '${condition.interval}' is not in the evidence`);
+			return interval.lower <= 0 && interval.upper >= 0;
+		}
+		case "quantity-threshold": {
+			const value = evidence.quantities[condition.quantity];
+			if (value === void 0) throw new ValidationError(`evaluateCondition: quantity '${condition.quantity}' is not in the evidence`);
+			return compareValues(value, condition.op, condition.value);
+		}
+		case "obligation-met": return evidence.obligationsMet[condition.obligation] === true;
+		case "all": return condition.of.every((c) => evaluateCondition(c, evidence));
+		case "any": return condition.of.some((c) => evaluateCondition(c, evidence));
+		case "not": return !evaluateCondition(condition.of, evidence);
+	}
+}
+function executeDecisionRule(rule, evidence) {
+	if (rule.kind === "report-only") return {
+		verdict: "report-only",
+		report: [...rule.estimands, ...rule.intervals]
+	};
+	for (const branch of rule.branches) if (evaluateCondition(branch.when, evidence)) return {
+		verdict: branch.verdict,
+		report: branch.report
+	};
+	throw new DecisionTableNotTotalError("executeDecisionRule: decision table is not total — no branch matched the evidence");
+}
+//#endregion
+//#region src/experiment/ast/reissue.ts
+/**
+* Classify one rollout event under the registered reissue policy. A carrier
+* event within the issue budget is reissued; a model outcome always stands;
+* a carrier event past `maxIssues` is exhausted and reported, never retried.
+*/
+function classifyReissue(policy, event, issuesSoFar) {
+	if (!policy.carrierEvents.includes(event)) return "stands";
+	return issuesSoFar < policy.maxIssues ? "reissue" : "exhausted";
+}
+//#endregion
+//#region src/experiment/ast/selection.ts
+/**
+* Execute a selection rule.
+*
+* Round-robin walks groups in lexicographic order and takes ids in
+* within-group order until `take` ids are chosen. Filter-of keeps the ids of
+* `bases[rule.base]` whose record satisfies the predicate, in the registered
+* order. Ids absent from `records` are evaluated on their id alone (fields
+* derived from the id via `idFields`), so a sealed base outlives its source
+* records.
+*/
+function runSelectionRule(rule, records, options) {
+	if (rule.kind === "round-robin") {
+		const allowed = new Set(rule.reads);
+		for (const field of [rule.groupBy, rule.withinOrder.field]) if (!allowed.has(field)) throw new ValidationError(`runSelectionRule: round-robin reads '${field}' but its closed read set is [${rule.reads.join(", ")}]`);
+		const byGroup = /* @__PURE__ */ new Map();
+		for (const record of records) {
+			const group = String(readField(record, rule.groupBy));
+			const id = String(readField(record, rule.withinOrder.field));
+			const bucket = byGroup.get(group);
+			if (bucket) bucket.push(id);
+			else byGroup.set(group, [id]);
+		}
+		for (const ids of byGroup.values()) {
+			ids.sort();
+			if (rule.withinOrder.dir === "desc") ids.reverse();
+		}
+		const groups = [...byGroup.keys()].sort();
+		const chosen = [];
+		let cursor = 0;
+		while (chosen.length < rule.take && groups.some((g) => byGroup.get(g).length > 0)) {
+			const group = groups[cursor % groups.length];
+			const ids = byGroup.get(group);
+			if (ids.length > 0) chosen.push(ids.shift());
+			cursor += 1;
+		}
+		return chosen;
+	}
+	const base = options.bases?.[rule.base];
+	if (!base) throw new ValidationError(`runSelectionRule: filter-of base '${rule.base}' was not provided`);
+	const index = new Map(records.map((r) => [String(readField(r, options.idField)), r]));
+	const sorted = [...base.filter((id) => {
+		const record = index.get(id) ?? options.idFields?.(id);
+		if (!record) throw new ValidationError(`runSelectionRule: base id '${id}' has no record and no idFields derivation`);
+		return evaluatePredicate(rule.keep, record);
+	})].sort();
+	if (rule.order.dir === "desc") sorted.reverse();
+	return sorted;
+}
+//#endregion
+//#region src/experiment/ast/validity.ts
+/**
+* Replicate-flip counting over graded states. A state whose replicates split
+* between pass and fail is flipping; its flip rate is the minority share.
+*/
+function evaluateOracleDeterminismGate(id, gate, repsByState) {
+	const evidence = {};
+	let passed = true;
+	for (const [state, reps] of Object.entries(repsByState)) {
+		const passes = reps.filter(Boolean).length;
+		const flipRate = reps.length === 0 ? 0 : Math.min(passes, reps.length - passes) / reps.length;
+		evidence[state] = {
+			passes,
+			replicates: reps.length,
+			flipRate
+		};
+		if (flipRate > gate.maxFlipRate) passed = false;
+	}
+	return {
+		id,
+		passed,
+		evidence
+	};
+}
+/**
+* Join two population snapshots on `joinOn` and compare the registered fields.
+* Only rows present in both snapshots are compared; a presence change is a
+* different failure and needs its own gate.
+*/
+function evaluatePopulationReproducibilityGate(id, gate, populations) {
+	const rightByKey = new Map(populations.right.map((r) => [String(readField(r, gate.joinOn)), r]));
+	const changed = [];
+	for (const left of populations.left) {
+		const key = String(readField(left, gate.joinOn));
+		const right = rightByKey.get(key);
+		if (!right) continue;
+		const moved = gate.compare.filter((f) => readField(left, f) !== readField(right, f));
+		if (moved.length > 0) changed.push(`${key} ${moved.map((f) => `${f}:${String(readField(left, f))}->${String(readField(right, f))}`).join(" ")}`);
+	}
+	return {
+		id,
+		passed: changed.length <= gate.maxChangedRows,
+		evidence: changed
+	};
+}
+/** The registered claim about provenance must hold on the provenance record. */
+function evaluateProvenanceGate(id, gate, provenance) {
+	const passed = evaluatePredicate(gate.claim, provenance);
+	return {
+		id,
+		passed,
+		evidence: { claimHolds: passed }
+	};
+}
+/** Final path segment equality between the pinned and the served identity. */
+function evaluateIdentityGate(id, _gate, identities) {
+	const basename = (s) => s.split("/").pop() ?? s;
+	const passed = basename(identities.pinned) === basename(identities.served);
+	return {
+		id,
+		passed,
+		evidence: {
+			pinned: identities.pinned,
+			served: identities.served,
+			matched: passed
+		}
+	};
+}
+/**
+* The design's power at minimumEffect must reach the registered target.
+* The curve must cover the registered effect grid exactly — a curve
+* computed on a different grid is different evidence and is refused.
+*/
+function evaluatePowerFloorGate(id, gate, curve) {
+	const problems = powerFloorProblems(gate);
+	if (problems.length > 0) throw new ValidationError(`evaluatePowerFloorGate: ${problems.join("; ")}`);
+	if (curve.some((point) => !Number.isFinite(point.power) || point.power < 0 || point.power > 1)) throw new ValidationError("evaluatePowerFloorGate: powers must be finite and in [0,1]");
+	const byEffect = new Map(curve.map((point) => [point.effect, point.power]));
+	if (byEffect.size !== curve.length) throw new ValidationError("evaluatePowerFloorGate: curve contains duplicate effects");
+	const missing = gate.effectGrid.filter((effect) => !byEffect.has(effect));
+	if (missing.length > 0) throw new ValidationError(`evaluatePowerFloorGate: curve does not cover registered effects [${missing.join(", ")}]`);
+	const extra = curve.filter((point) => !gate.effectGrid.includes(point.effect));
+	if (extra.length > 0) throw new ValidationError(`evaluatePowerFloorGate: curve contains unregistered effects [${extra.map((point) => point.effect).join(", ")}]`);
+	const powers = gate.effectGrid.map((effect) => byEffect.get(effect));
+	const maxPower = Math.max(...powers);
+	const powerAtMinimumEffect = byEffect.get(gate.minimumEffect);
+	return {
+		id,
+		passed: powerAtMinimumEffect >= gate.target,
+		evidence: {
+			target: gate.target,
+			minimumEffect: gate.minimumEffect,
+			powerAtMinimumEffect,
+			maxPower,
+			curve: gate.effectGrid.map((effect) => ({
+				effect,
+				power: byEffect.get(effect)
+			}))
+		}
+	};
+}
+/** Shared by seal validation and direct execution; malformed designs never pass. */
+function powerFloorProblems(gate) {
+	const problems = [];
+	if (!Number.isFinite(gate.target) || gate.target <= 0 || gate.target > 1) problems.push("power target must be in (0,1]");
+	if (!Number.isFinite(gate.minimumEffect) || gate.minimumEffect <= 0) problems.push("minimumEffect must be positive and finite");
+	if (gate.effectGrid.length === 0 || gate.effectGrid.some((effect) => !Number.isFinite(effect))) problems.push("effectGrid must contain finite effects");
+	if (new Set(gate.effectGrid).size !== gate.effectGrid.length) problems.push("effectGrid contains duplicate effects");
+	if (!gate.effectGrid.includes(gate.minimumEffect)) problems.push("effectGrid must contain minimumEffect exactly; no interpolation is assumed");
+	for (const field of ["trials", "resamples"]) if (!Number.isInteger(gate.sim[field]) || gate.sim[field] <= 0) problems.push(`power simulation ${field} must be a positive integer`);
+	if (!Number.isInteger(gate.sim.seed)) problems.push("power simulation seed must be an integer");
+	return problems;
+}
+//#endregion
 //#region src/experiment/budget.ts
 /**
 * Matched-budget verification between arms, as a refusal object.
@@ -450,8 +745,7 @@ async function openSealedExperiment(sealed) {
 		decide: (evidence) => executeDecisionRule(spec.decision, evidence),
 		admit: (records) => executeAdmissionRule(need(spec.admission, "admission rule"), records),
 		select: (name, records, options) => {
-			const rule = need(spec.selections?.[name], `selection '${name}'`);
-			return runSelectionRule(rule, records, {
+			return runSelectionRule(need(spec.selections?.[name], `selection '${name}'`), records, {
 				idField: options.idField,
 				bases: spec.sealedSubsets
 			});