@tangle-network/agent-eval 0.128.1 → 0.129.0

package/dist/belief-state/index.js

@@ -14,12 +14,15 @@ import {
 } from "../chunk-T4SQEITX.js";
 import {
   offPolicyEstimateAll
-} from "../chunk-VGRCHJON.js";
+} from "../chunk-T6RLYGAD.js";
 import {
   confidenceInterval
-} from "../chunk-MHELPNRP.js";
+} from "../chunk-ZHTZ4EYI.js";
 import "../chunk-VI2UW6B6.js";
 import "../chunk-PXE2VKMX.js";
+import {
+  trainingScore
+} from "../chunk-OIUOT4QD.js";
 import {
   ValidationError
 } from "../chunk-ONWEPEDO.js";
@@ -73,9 +76,6 @@ function embeddedBeliefOpeTargetPolicy(id = "embedded-target-prob") {
     },
     vHatTargetOf(point) {
       return point.vHatTarget;
-    },
-    qHatOf(point) {
-      return point.qHat;
     }
   };
 }
@@ -94,12 +94,10 @@ function beliefDecisionsToOffPolicyTrajectories(points, targetPolicy, options =
     let targetProb;
     let qHatChosen;
     let vHatTarget;
-    let qHat;
     try {
       targetProb = targetPolicy.targetProbOf(point);
       qHatChosen = targetPolicy.qHatChosenOf?.(point);
       vHatTarget = targetPolicy.vHatTargetOf?.(point);
-      qHat = targetPolicy.qHatOf?.(point);
     } catch (error) {
       diagnostics.push(
         `${point.id}: target policy ${targetPolicy.id} threw (${errorMessage(error)})`
@@ -122,18 +120,13 @@ function beliefDecisionsToOffPolicyTrajectories(points, targetPolicy, options =
       );
       continue;
     }
-    if (!hasQHatChosen && !hasVHatTarget && qHat !== null && qHat !== void 0 && !isTargetProbability(qHat)) {
-      diagnostics.push(`${point.id}: invalid qHat ${formatProbability(qHat)}; ignoring qHat`);
-      qHat = null;
-    }
     trajectories.push({
       runId: point.id,
       reward: rewardOf(point),
       behaviorProb: point.behaviorProb,
       targetProb,
       ...qHatChosen !== void 0 ? { qHatChosen } : {},
-      ...vHatTarget !== void 0 ? { vHatTarget } : {},
-      qHat
+      ...vHatTarget !== void 0 ? { vHatTarget } : {}
     });
   }
   return {
@@ -151,8 +144,7 @@ function evaluateBeliefOffPolicy(points, targetPolicy, options = {}) {
     minEffectiveSampleSize: options.minEffectiveSampleSize ?? 30,
     minEffectiveSampleRatio: options.minEffectiveSampleRatio ?? 0.25,
     dropped: trajectoryReport.dropped,
-    diagnostics: trajectoryReport.diagnostics,
-    legacyScalarContributions: estimates.dr.contributionCounts?.legacyScalar ?? 0
+    diagnostics: trajectoryReport.diagnostics
   });
   return { targetPolicyId: targetPolicy.id, ...estimates, support };
 }
@@ -165,11 +157,6 @@ function supportDiagnostics(estimate, options) {
   if (options.dropped > 0) {
     reasons.push(`dropped ${options.dropped} unsupported decision(s)`);
   }
-  if (options.legacyScalarContributions > 0) {
-    reasons.push(
-      `${options.legacyScalarContributions} decision(s) used deprecated scalar qHat; supply qHatChosen and vHatTarget for contextual doubly robust estimation`
-    );
-  }
   if (estimate.effectiveSampleSize < options.minEffectiveSampleSize) {
     reasons.push(
       `effective sample size ${estimate.effectiveSampleSize.toFixed(2)} below ${options.minEffectiveSampleSize}`
@@ -248,7 +235,6 @@ function thresholdSelectivePolicy(options) {
         targetProb: point.targetProb,
         qHatChosen: point.qHatChosen,
         vHatTarget: point.vHatTarget,
-        qHat: point.qHat,
         reason: confidence >= threshold ? "confidence threshold passed" : "confidence threshold failed"
       };
     }
@@ -708,9 +694,7 @@ function outcomeScore2(outcome) {
   return null;
 }
 function scoreFromRun(run) {
-  if (typeof run.outcome.holdoutScore === "number") return run.outcome.holdoutScore;
-  if (typeof run.outcome.searchScore === "number") return run.outcome.searchScore;
-  return null;
+  return trainingScore(run) ?? null;
 }
 function sortBuckets(a, b) {
   return b.n - a.n || a.id.localeCompare(b.id);
@@ -1075,7 +1059,6 @@ function parseDecisionEvent(event, context) {
       targetProb: numberField(payload, "targetProb"),
       qHatChosen: finiteUnitField(payload, "qHatChosen"),
       vHatTarget: finiteUnitField(payload, "vHatTarget"),
-      qHat: finiteUnitField(payload, "qHat"),
       costUsd: nonNegativeNumberField(payload, "costUsd"),
       evidence,
       outcome: parseOutcome(payload),
@@ -1191,7 +1174,6 @@ function runtimeDecisionPointToBeliefDecisionPoint(point, options) {
       targetProb: finiteNumberOrUndefined(options.targetProb),
       qHatChosen: options.qHatChosen === null ? null : unitProbabilityOrUndefined(options.qHatChosen),
       vHatTarget: options.vHatTarget === null ? null : unitProbabilityOrUndefined(options.vHatTarget),
-      qHat: options.qHat === null ? null : unitProbabilityOrUndefined(options.qHat),
       costUsd: nonNegativeNumberOrUndefined(options.costUsd),
       evidence: evidence.map((ref) => runtimeEvidenceToBeliefEvidence(ref, point)),
       outcome: options.outcome,
@@ -1422,7 +1404,6 @@ function buildRuntimeBeliefPhase0Measurement(options) {
         targetProb: label.targetProb,
         qHatChosen: label.qHatChosen,
         vHatTarget: label.vHatTarget,
-        qHat: label.qHat,
         costUsd: label.costUsd,
         outcome: label.outcome,
         lifecycleEvents: options.events,
@@ -1718,7 +1699,8 @@ ${input.context}` : "",
       evidenceRefs: ["evidence id"],
       wouldChangeMindIf: ["observable evidence"],
       targetProb: "optional number in [0,1]",
-      qHat: "optional number in [0,1]"
+      qHatChosen: "optional number in [0,1], paired with vHatTarget",
+      vHatTarget: "optional number in [0,1], paired with qHatChosen"
     })
   ].filter(Boolean).join("\n");
 }
@@ -1817,11 +1799,27 @@ function normalizeProbeResponse(response, options) {
       reason: `invalid targetProb ${String(response.targetProb)}`
     });
   }
-  if (response.qHat !== void 0 && response.qHat !== null && !isUnitProbability(response.qHat)) {
+  const hasQHatChosen = response.qHatChosen !== void 0 && response.qHatChosen !== null;
+  const hasVHatTarget = response.vHatTarget !== void 0 && response.vHatTarget !== null;
+  if (hasQHatChosen !== hasVHatTarget) {
+    diagnostics.push({
+      decisionId: options.point.id,
+      severity: "error",
+      reason: "qHatChosen and vHatTarget must be supplied together"
+    });
+  }
+  if (hasQHatChosen && !isUnitProbability(response.qHatChosen)) {
+    diagnostics.push({
+      decisionId: options.point.id,
+      severity: "error",
+      reason: `invalid qHatChosen ${String(response.qHatChosen)}`
+    });
+  }
+  if (hasVHatTarget && !isUnitProbability(response.vHatTarget)) {
     diagnostics.push({
       decisionId: options.point.id,
       severity: "error",
-      reason: `invalid qHat ${String(response.qHat)}`
+      reason: `invalid vHatTarget ${String(response.vHatTarget)}`
     });
   }
   if (diagnostics.length > 0 || !predictedAction) return { diagnostics };
@@ -1834,7 +1832,8 @@ function normalizeProbeResponse(response, options) {
       evidenceRefs: compactStrings(response.evidenceRefs),
       wouldChangeMindIf: compactStrings(response.wouldChangeMindIf),
       ...response.targetProb !== void 0 ? { targetProb: response.targetProb } : {},
-      ...response.qHat !== void 0 ? { qHat: response.qHat } : {},
+      ...response.qHatChosen !== void 0 ? { qHatChosen: response.qHatChosen } : {},
+      ...response.vHatTarget !== void 0 ? { vHatTarget: response.vHatTarget } : {},
       ...response.metadata ? { metadata: response.metadata } : {}
     },
     diagnostics