@tangle-network/agent-eval 0.132.0 → 0.133.1

package/dist/rl.js

@@ -1,17 +1,16 @@
 import { s as ValidationError } from "./errors-8YnH8WlF.js";
-import { N as wilcoxonSignedRank, t as benjaminiHochberg } from "./statistics-CnnxdpOg.js";
+import { N as wilcoxonSignedRank, t as benjaminiHochberg } from "./statistics-DWM_AyLe.js";
 import { r as observedSplitScore, s as trainingScore } from "./reward-nw2xZGZG.js";
-import { o as runTaskScore } from "./run-record-CN8Zd21B.js";
+import { o as runTaskScore } from "./run-record-BIwU2wdV.js";
 import { l as assertRewardGate } from "./schema-C6DW4ZHR.js";
 import { t as isSplitEligible } from "./exporters-q9iL-2Jf.js";
-import { t as mintRolloutRows } from "./mint-D5_87M5L.js";
+import { t as mintRolloutRows } from "./mint-BvkwcYZU.js";
 import { a as InMemoryTraceStore } from "./integrity-BzRbCHzi.js";
-import { c as campaignCellToRunRecord, i as filterDeterministicallyRewarded, n as extractVerifiableReward, r as extractVerifiableRewardsFromRecords, t as detectRewardHacking } from "./reward-hacking-BEvjdUtD.js";
-import { t as runEvalCampaign } from "./eval-campaign-mDKhkdUq.js";
+import { c as campaignCellToRunRecord, i as filterDeterministicallyRewarded, n as extractVerifiableReward, r as extractVerifiableRewardsFromRecords, t as detectRewardHacking } from "./reward-hacking-Dl2UBzej.js";
+import { t as runEvalCampaign } from "./eval-campaign-DXhpZghy.js";
 import { t as evaluateInterimReleaseConfidence } from "./sequential-Br0mAPHA.js";
-import { t as rubricPredictiveValidity } from "./rubric-predictive-validity-B3xmbmS1.js";
+import { t as rubricPredictiveValidity } from "./rubric-predictive-validity-QG7ydk0s.js";
 import { n as thompsonCurriculum, r as varianceBasedCurriculum, t as observationsFromRunRecords } from "./active-curriculum-C4mk67HP.js";
-import { i as selfNormalizedImportanceWeighting, n as inverseProbabilityWeighting, r as offPolicyEstimateAll, t as doublyRobust } from "./off-policy-DvgzvtIx.js";
 import { n as InMemoryOutcomeStore, t as FileSystemOutcomeStore } from "./outcome-store-ChBKlTd_.js";
 import { appendFileSync, existsSync, mkdirSync, readFileSync } from "node:fs";
 import { dirname } from "node:path";
@@ -1242,6 +1241,222 @@ async function buildDatasetFromCorpus(corpusPath, config, opts = {}) {
 	return buildRlDataset(rows, lookups, config);
 }
 //#endregion
+//#region src/rl/off-policy.ts
+/**
+* Off-policy evaluation primitives.
+*
+* Standard inverse-probability-weighted (IPS), self-normalized
+* importance-weighted (SNIPS), and doubly-robust (DR) estimators for the
+* value of a *target* policy given trajectories collected under a
+* *behavior* policy. This is the canonical RL eval task: "we have last
+* week's runs, we changed the policy — how would the new one do without
+* re-running?"
+*
+* The math here is textbook (Dudík, Langford, Li 2011 for DR; Swaminathan
+* & Joachims 2015 for SNIPS) but the *application* to LLM-agent
+* evaluation needs care:
+*
+*   - The "policy" is the (prompt, tool config, model snapshot) triple.
+*     Two policies have the same probability over an action *iff* their
+*     LLM call would emit the same token with the same probability —
+*     which is generally unknowable without the model log-probs.
+*   - For LLM agents, propensity scores must be supplied by the caller
+*     (logged in the trace, recovered from token log-probs, or estimated
+*     via a learned propensity model). We do NOT estimate propensity here.
+*   - Doubly-robust requires two outputs from a Q-function: its prediction
+*     for the logged action and its expectation under the target policy.
+*     Consumers compute these with a tabular estimate, regression fit, or
+*     learned reward model before constructing the trajectories.
+*
+* Bias / variance tradeoffs:
+*   - IPS: unbiased; high variance for small overlap, infinite variance
+*     when target has support outside behavior.
+*   - SNIPS: lower variance, slight bias; usually preferred in practice.
+*   - DR: doubly-robust — unbiased if either propensity OR Q-function is
+*     correct. Lowest practical variance when Q is decent. Use this.
+*
+* Caveat the panel will land: on the LLM-agent setting, propensity scores
+* recovered from token log-probs are noisy, the action space is enormous,
+* and overlap is often poor. These estimators are useful but not magic;
+* complement with `replayCampaign` (exact replay where the request hashes
+* match) for high-confidence answers and OPE for the gap.
+*/
+/**
+* Inverse Probability Weighting (Horvitz-Thompson). Unbiased estimator
+* of E[reward under target policy]. Variance scales with the spread of
+* target/behavior ratios.
+*/
+function inverseProbabilityWeighting(trajectories, opts = {}) {
+	const cap = opts.weightCap ?? Infinity;
+	const clip = opts.rewardClip ?? {
+		low: 0,
+		high: 1
+	};
+	if (trajectories.length === 0) return zeroEstimate();
+	const weights = [];
+	const weightedRewards = [];
+	let maxW = 0;
+	for (const t of trajectories) {
+		if (t.behaviorProb <= 0) throw new ValidationError(`inverseProbabilityWeighting: behaviorProb must be > 0 (runId=${t.runId})`);
+		const w = Math.min(cap, t.targetProb / t.behaviorProb);
+		const r = clamp(t.reward, clip.low, clip.high);
+		weights.push(w);
+		weightedRewards.push(w * r);
+		if (w > maxW) maxW = w;
+	}
+	const n = weights.length;
+	const value = weightedRewards.reduce((s, x) => s + x, 0) / n;
+	const variance = weightedRewards.reduce((s, x) => s + (x - value) ** 2, 0) / Math.max(1, n - 1);
+	const sumW = weights.reduce((s, w) => s + w, 0);
+	const sumW2 = weights.reduce((s, w) => s + w * w, 0);
+	const effN = sumW === 0 ? 0 : sumW * sumW / sumW2;
+	return {
+		value,
+		standardError: Math.sqrt(variance / n),
+		effectiveSampleSize: effN,
+		n,
+		maxImportanceWeight: maxW
+	};
+}
+/**
+* Self-Normalized Importance Sampling. Lower variance than vanilla IPS at
+* the cost of small bias (vanishing as N grows). The right default for
+* LLM-agent evaluation where overlap is often poor.
+*/
+function selfNormalizedImportanceWeighting(trajectories, opts = {}) {
+	const cap = opts.weightCap ?? Infinity;
+	const clip = opts.rewardClip ?? {
+		low: 0,
+		high: 1
+	};
+	if (trajectories.length === 0) return zeroEstimate();
+	const weights = [];
+	const rewards = [];
+	let maxW = 0;
+	for (const t of trajectories) {
+		if (t.behaviorProb <= 0) throw new ValidationError(`selfNormalizedImportanceWeighting: behaviorProb must be > 0 (runId=${t.runId})`);
+		const w = Math.min(cap, t.targetProb / t.behaviorProb);
+		weights.push(w);
+		rewards.push(clamp(t.reward, clip.low, clip.high));
+		if (w > maxW) maxW = w;
+	}
+	const sumW = weights.reduce((s, w) => s + w, 0);
+	const sumWR = weights.reduce((s, w, i) => s + w * rewards[i], 0);
+	const value = sumW === 0 ? 0 : sumWR / sumW;
+	const sumW2 = weights.reduce((s, w) => s + w * w, 0);
+	const effN = sumW === 0 ? 0 : sumW * sumW / sumW2;
+	const variance = weights.map((w, i) => w * (rewards[i] - value)).reduce((s, x) => s + x * x, 0) / Math.max(1, sumW * sumW);
+	return {
+		value,
+		standardError: Math.sqrt(variance),
+		effectiveSampleSize: effN,
+		n: trajectories.length,
+		maxImportanceWeight: maxW
+	};
+}
+/**
+* Doubly-robust off-policy estimator (Dudík, Langford, Li 2011).
+*
+*     V_DR = (1/N) * sum_i [ v_hat_target_i
+*              + (target_prob_i / behavior_prob_i) * (r_i - q_hat_chosen_i) ]
+*
+* Unbiased if EITHER:
+*   - the importance ratios are correct (IPS-style validity), OR
+*   - the Q-hat function is correct (model-based validity).
+*
+* In practice both are imperfect, but the residual bias is the *product*
+* of both errors — much smaller than either alone. This is why DR is the
+* default in production OPE pipelines.
+*
+* `qHatChosen` and `vHatTarget` must be supplied together. Rows with neither
+* use the exact IPS contribution. `contributionCounts` makes the mix explicit
+* in the result.
+* Callers must cross-fit the Q-function or train it on independent rows;
+* fitting and evaluating Q on the same outcomes leaks the answer.
+*/
+function doublyRobust(trajectories, opts = {}) {
+	const cap = opts.weightCap ?? Infinity;
+	const clip = opts.rewardClip ?? {
+		low: 0,
+		high: 1
+	};
+	if (trajectories.length === 0) return {
+		...zeroEstimate(),
+		contributionCounts: {
+			dr: 0,
+			ipsFallback: 0
+		}
+	};
+	const contributions = [];
+	const contributionCounts = {
+		dr: 0,
+		ipsFallback: 0
+	};
+	let maxW = 0;
+	let sumW = 0;
+	let sumW2 = 0;
+	for (const t of trajectories) {
+		if (t.behaviorProb <= 0) throw new ValidationError(`doublyRobust: behaviorProb must be > 0 (runId=${t.runId})`);
+		const w = Math.min(cap, t.targetProb / t.behaviorProb);
+		const r = clamp(t.reward, clip.low, clip.high);
+		const rawQHatChosen = t.qHatChosen;
+		const rawVHatTarget = t.vHatTarget;
+		const hasQHatChosen = rawQHatChosen !== null && rawQHatChosen !== void 0;
+		const hasVHatTarget = rawVHatTarget !== null && rawVHatTarget !== void 0;
+		if (hasQHatChosen !== hasVHatTarget) throw new ValidationError(`doublyRobust: qHatChosen and vHatTarget must be supplied together (runId=${t.runId})`);
+		if (hasQHatChosen && hasVHatTarget) {
+			if (!Number.isFinite(rawQHatChosen) || !Number.isFinite(rawVHatTarget)) throw new ValidationError(`doublyRobust: qHatChosen and vHatTarget must be finite (runId=${t.runId})`);
+			const qHatChosen = clamp(rawQHatChosen, clip.low, clip.high);
+			const vHatTarget = clamp(rawVHatTarget, clip.low, clip.high);
+			contributions.push(vHatTarget + w * (r - qHatChosen));
+			contributionCounts.dr += 1;
+		} else {
+			contributions.push(w * r);
+			contributionCounts.ipsFallback += 1;
+		}
+		if (w > maxW) maxW = w;
+		sumW += w;
+		sumW2 += w * w;
+	}
+	const n = contributions.length;
+	const value = contributions.reduce((s, x) => s + x, 0) / n;
+	const variance = contributions.reduce((s, x) => s + (x - value) ** 2, 0) / Math.max(1, n - 1);
+	const effN = sumW === 0 ? 0 : sumW * sumW / sumW2;
+	return {
+		value,
+		standardError: Math.sqrt(variance / n),
+		effectiveSampleSize: effN,
+		n,
+		maxImportanceWeight: maxW,
+		contributionCounts
+	};
+}
+/**
+* Convenience: run all three estimators and return them side-by-side.
+* The recommended diagnostic — agreement across estimators is a much
+* stronger signal than any single one.
+*/
+function offPolicyEstimateAll(trajectories, opts = {}) {
+	return {
+		ips: inverseProbabilityWeighting(trajectories, opts),
+		snips: selfNormalizedImportanceWeighting(trajectories, opts),
+		dr: doublyRobust(trajectories, opts)
+	};
+}
+function zeroEstimate() {
+	return {
+		value: 0,
+		standardError: 0,
+		effectiveSampleSize: 0,
+		n: 0,
+		maxImportanceWeight: 0
+	};
+}
+function clamp(x, lo, hi) {
+	if (!Number.isFinite(x)) return lo;
+	return Math.max(lo, Math.min(hi, x));
+}
+//#endregion
 //#region src/rl/predictive-validity-researcher.ts
 /**
 * Concrete `Researcher` driven by `rubricPredictiveValidity`. The brain: