@tangle-network/agent-eval 0.77.0 → 0.80.0

package/dist/belief-state/index.js

@@ -0,0 +1,486 @@
+import {
+  calibrationFromPairs
+} from "../chunk-NPCTHQIO.js";
+import {
+  offPolicyEstimateAll
+} from "../chunk-4DIJWVUT.js";
+import {
+  confidenceInterval
+} from "../chunk-ITBRCT73.js";
+import {
+  ValidationError
+} from "../chunk-3BFEG2F6.js";
+import "../chunk-PZ5AY32C.js";
+
+// src/belief-state/calibration.ts
+function calibrateBeliefDecisions(points, options = {}) {
+  const filtered = filterCalibrationRegion(points, options);
+  const pairs = filtered.filter((point) => typeof point.confidence === "number" && point.outcome).map((point) => ({
+    evalScore: point.confidence,
+    outcome: outcomeScore(point)
+  })).filter((pair) => Number.isFinite(pair.outcome));
+  const minPairs = options.minPairs ?? 10;
+  if (pairs.length < minPairs) return null;
+  return calibrationFromPairs(pairs, "belief-confidence", "decision-outcome", {
+    bins: options.bins ?? 5,
+    range: { lo: 0, hi: 1 }
+  });
+}
+function filterCalibrationRegion(points, options) {
+  const region = options.region ?? "all";
+  if (region === "all") return points;
+  const policy = options.policy;
+  if (!policy) {
+    throw new ValidationError(
+      `calibrateBeliefDecisions: policy is required when region is "${region}"`
+    );
+  }
+  return points.filter((point) => {
+    const accepted = policy.decide(point).action === "accept";
+    return region === "accepted" ? accepted : !accepted;
+  });
+}
+function outcomeScore(point) {
+  if (typeof point.outcome?.reward === "number") return point.outcome.reward;
+  if (typeof point.outcome?.score === "number") return point.outcome.score;
+  if (point.outcome?.success === true) return 1;
+  if (point.outcome?.success === false) return 0;
+  return Number.NaN;
+}
+
+// src/belief-state/extract.ts
+var DECISION_MARKERS = /* @__PURE__ */ new Set(["belief_decision", "belief.decision", "decision_point"]);
+var DECISION_KINDS = /* @__PURE__ */ new Set([
+  "continue",
+  "verify",
+  "ask",
+  "retry",
+  "stop",
+  "memory-write",
+  "memory-read",
+  "tool-select",
+  "skill-select",
+  "workflow-select",
+  "surface-promote"
+]);
+async function extractBeliefDecisionPoints(store, options = {}) {
+  const runs = options.runIds ? (await Promise.all(options.runIds.map((runId) => store.getRun(runId)))).filter(Boolean) : await store.listRuns();
+  const decisions = [];
+  const diagnostics = [];
+  for (const run of runs) {
+    if (!run) continue;
+    const events = await store.events({ runId: run.runId });
+    const spans = await store.spans({ runId: run.runId });
+    const spanIds = new Set(spans.map((span) => span.spanId));
+    let stepIndex = 0;
+    for (const event of [...events].sort((a, b) => a.timestamp - b.timestamp)) {
+      const parsed = parseDecisionEvent(event, {
+        scenarioId: run.scenarioId,
+        stepIndex,
+        spanExists: event.spanId ? spanIds.has(event.spanId) : false
+      });
+      if (!parsed) continue;
+      if ("diagnostic" in parsed) {
+        diagnostics.push(parsed.diagnostic);
+        continue;
+      }
+      decisions.push(parsed.decision);
+      stepIndex++;
+    }
+  }
+  return { decisions, diagnostics };
+}
+function parseDecisionEvent(event, context) {
+  const payload = event.payload;
+  const marker = stringField(payload, "kind") ?? stringField(payload, "type");
+  if (!marker || !DECISION_MARKERS.has(marker)) return null;
+  const decisionKind = stringField(payload, "decisionKind");
+  if (!decisionKind || !DECISION_KINDS.has(decisionKind)) {
+    return {
+      diagnostic: {
+        runId: event.runId,
+        eventId: event.eventId,
+        severity: "warning",
+        reason: `belief decision event has unsupported decisionKind "${decisionKind ?? ""}"`
+      }
+    };
+  }
+  const chosenAction = stringField(payload, "chosenAction");
+  if (!chosenAction) {
+    return {
+      diagnostic: {
+        runId: event.runId,
+        eventId: event.eventId,
+        severity: "warning",
+        reason: "belief decision event is missing chosenAction"
+      }
+    };
+  }
+  const evidence = [
+    {
+      source: "event",
+      id: event.eventId,
+      runId: event.runId,
+      eventId: event.eventId
+    }
+  ];
+  if (event.spanId && context.spanExists) {
+    evidence.push({ source: "span", id: event.spanId, runId: event.runId, spanId: event.spanId });
+  }
+  return {
+    decision: {
+      id: stringField(payload, "id") ?? event.eventId,
+      runId: event.runId,
+      scenarioId: stringField(payload, "scenarioId") ?? context.scenarioId,
+      stepIndex: numberField(payload, "stepIndex") ?? context.stepIndex,
+      kind: decisionKind,
+      chosenAction,
+      candidateActions: stringArrayField(payload, "candidateActions"),
+      confidence: finiteUnitField(payload, "confidence"),
+      behaviorProb: numberField(payload, "behaviorProb"),
+      targetProb: numberField(payload, "targetProb"),
+      qHat: finiteUnitField(payload, "qHat"),
+      costUsd: nonNegativeNumberField(payload, "costUsd"),
+      evidence,
+      outcome: parseOutcome(payload),
+      metadata: recordField(payload, "metadata")
+    }
+  };
+}
+function parseOutcome(payload) {
+  const value = recordField(payload, "outcome");
+  if (!value) return void 0;
+  return {
+    success: typeof value.success === "boolean" ? value.success : void 0,
+    score: finiteUnitField(value, "score"),
+    reward: finiteUnitField(value, "reward"),
+    costUsd: nonNegativeNumberField(value, "costUsd"),
+    observedAt: stringField(value, "observedAt"),
+    metadata: recordField(value, "metadata")
+  };
+}
+function stringField(obj, key) {
+  const value = obj[key];
+  return typeof value === "string" && value.length > 0 ? value : void 0;
+}
+function numberField(obj, key) {
+  const value = obj[key];
+  return typeof value === "number" && Number.isFinite(value) ? value : void 0;
+}
+function finiteUnitField(obj, key) {
+  const value = numberField(obj, key);
+  return value === void 0 ? void 0 : Math.max(0, Math.min(1, value));
+}
+function nonNegativeNumberField(obj, key) {
+  const value = numberField(obj, key);
+  return value === void 0 ? void 0 : Math.max(0, value);
+}
+function stringArrayField(obj, key) {
+  const value = obj[key];
+  if (!Array.isArray(value)) return void 0;
+  const strings = value.filter(
+    (item) => typeof item === "string" && item.length > 0
+  );
+  return strings.length > 0 ? strings : void 0;
+}
+function recordField(obj, key) {
+  const value = obj[key];
+  if (!value || typeof value !== "object" || Array.isArray(value)) return void 0;
+  return value;
+}
+
+// src/belief-state/ope.ts
+function embeddedBeliefOpeTargetPolicy(id = "embedded-target-prob") {
+  return {
+    id,
+    targetProbOf(point) {
+      return point.targetProb;
+    },
+    qHatOf(point) {
+      return point.qHat;
+    }
+  };
+}
+function beliefDecisionsToOffPolicyTrajectories(points, targetPolicy, options = {}) {
+  const trajectories = [];
+  const diagnostics = [];
+  for (const point of points) {
+    if (!point.outcome) {
+      diagnostics.push(`${point.id}: missing outcome`);
+      continue;
+    }
+    if (!isBehaviorProbability(point.behaviorProb)) {
+      diagnostics.push(`${point.id}: invalid behaviorProb ${formatProbability(point.behaviorProb)}`);
+      continue;
+    }
+    let targetProb;
+    let qHat;
+    try {
+      targetProb = targetPolicy.targetProbOf(point);
+      qHat = targetPolicy.qHatOf?.(point);
+    } catch (error) {
+      diagnostics.push(
+        `${point.id}: target policy ${targetPolicy.id} threw (${errorMessage(error)})`
+      );
+      continue;
+    }
+    if (!isTargetProbability(targetProb)) {
+      diagnostics.push(`${point.id}: invalid targetProb ${formatProbability(targetProb)}`);
+      continue;
+    }
+    if (qHat !== null && qHat !== void 0 && !isTargetProbability(qHat)) {
+      diagnostics.push(`${point.id}: invalid qHat ${formatProbability(qHat)}; ignoring qHat`);
+      qHat = null;
+    }
+    trajectories.push({
+      runId: point.id,
+      reward: rewardOf(point),
+      behaviorProb: point.behaviorProb,
+      targetProb,
+      qHat
+    });
+  }
+  return {
+    targetPolicyId: targetPolicy.id,
+    trajectories,
+    dropped: points.length - trajectories.length,
+    diagnostics: compactDiagnostics(diagnostics, options.maxDiagnostics ?? 20)
+  };
+}
+function evaluateBeliefOffPolicy(points, targetPolicy, options = {}) {
+  const trajectoryReport = beliefDecisionsToOffPolicyTrajectories(points, targetPolicy, options);
+  const { trajectories } = trajectoryReport;
+  const estimates = offPolicyEstimateAll(trajectories, options);
+  const support = supportDiagnostics(estimates.dr, {
+    minEffectiveSampleSize: options.minEffectiveSampleSize ?? 30,
+    minEffectiveSampleRatio: options.minEffectiveSampleRatio ?? 0.25,
+    dropped: trajectoryReport.dropped,
+    diagnostics: trajectoryReport.diagnostics
+  });
+  return { targetPolicyId: targetPolicy.id, ...estimates, support };
+}
+function supportDiagnostics(estimate, options) {
+  const ratio = estimate.n > 0 ? estimate.effectiveSampleSize / estimate.n : 0;
+  const reasons = [...options.diagnostics];
+  if (estimate.n === 0) {
+    reasons.push("no valid OPE trajectories");
+  }
+  if (options.dropped > 0) {
+    reasons.push(`dropped ${options.dropped} unsupported decision(s)`);
+  }
+  if (estimate.effectiveSampleSize < options.minEffectiveSampleSize) {
+    reasons.push(
+      `effective sample size ${estimate.effectiveSampleSize.toFixed(2)} below ${options.minEffectiveSampleSize}`
+    );
+  }
+  if (ratio < options.minEffectiveSampleRatio) {
+    reasons.push(
+      `effective sample ratio ${ratio.toFixed(2)} below ${options.minEffectiveSampleRatio}`
+    );
+  }
+  if (estimate.maxImportanceWeight > 10) {
+    reasons.push(`max importance weight ${estimate.maxImportanceWeight.toFixed(2)} is high`);
+  }
+  return {
+    supported: reasons.length === 0,
+    n: estimate.n,
+    dropped: options.dropped,
+    effectiveSampleSize: estimate.effectiveSampleSize,
+    effectiveSampleRatio: ratio,
+    maxImportanceWeight: estimate.maxImportanceWeight,
+    reasons
+  };
+}
+function rewardOf(point) {
+  if (typeof point.outcome?.reward === "number") return point.outcome.reward;
+  if (typeof point.outcome?.score === "number") return point.outcome.score;
+  if (point.outcome?.success === true) return 1;
+  return 0;
+}
+function isBehaviorProbability(value) {
+  return typeof value === "number" && Number.isFinite(value) && value > 0 && value <= 1;
+}
+function isTargetProbability(value) {
+  return typeof value === "number" && Number.isFinite(value) && value >= 0 && value <= 1;
+}
+function formatProbability(value) {
+  return typeof value === "number" ? String(value) : String(value ?? "missing");
+}
+function errorMessage(error) {
+  return error instanceof Error ? error.message : String(error);
+}
+function compactDiagnostics(diagnostics, maxDiagnostics) {
+  if (diagnostics.length <= maxDiagnostics) return diagnostics;
+  return [
+    ...diagnostics.slice(0, maxDiagnostics),
+    `${diagnostics.length - maxDiagnostics} additional OPE diagnostic(s) omitted`
+  ];
+}
+
+// src/belief-state/selective.ts
+var DEFAULT_UTILITY = {
+  successUtility: 1,
+  failureUtility: -1,
+  deferUtility: 0,
+  verifyCost: 0.05,
+  askCost: 0.05,
+  retryCost: 0.1,
+  stopUtility: 0,
+  costWeight: 1
+};
+function thresholdSelectivePolicy(options) {
+  const threshold = options.confidenceThreshold;
+  if (!Number.isFinite(threshold) || threshold < 0 || threshold > 1) {
+    throw new ValidationError(
+      `thresholdSelectivePolicy: confidenceThreshold must be in [0, 1], got ${threshold}`
+    );
+  }
+  const belowThresholdAction = options.belowThresholdAction ?? "verify";
+  return {
+    id: options.id ?? `confidence>=${threshold}`,
+    decide(point) {
+      const confidence = point.confidence ?? 0;
+      return {
+        action: confidence >= threshold ? "accept" : belowThresholdAction,
+        confidence,
+        targetProb: point.targetProb,
+        qHat: point.qHat,
+        reason: confidence >= threshold ? "confidence threshold passed" : "confidence threshold failed"
+      };
+    }
+  };
+}
+function evaluateBeliefSelectivePolicy(points, policy, options = {}) {
+  const utility = { ...DEFAULT_UTILITY, ...options.utility ?? {} };
+  const scored = points.filter((point) => point.outcome);
+  const minN = options.minN ?? 30;
+  const minAccepted = options.minAccepted ?? 5;
+  const minUtilityDelta = options.minUtilityDelta ?? 0;
+  const deltas = [];
+  const acceptedRewards = [];
+  const rejectedRewards = [];
+  let baselineUtility = 0;
+  let policyUtility = 0;
+  let accepted = 0;
+  let acceptedErrors = 0;
+  for (const point of scored) {
+    const baseline = acceptUtility(point, utility);
+    const decision = policy.decide(point);
+    const candidate = policyDecisionUtility(point, decision.action, utility);
+    const reward = rewardOf2(point, utility);
+    baselineUtility += baseline;
+    policyUtility += candidate;
+    deltas.push(candidate - baseline);
+    if (decision.action === "accept") {
+      accepted++;
+      acceptedRewards.push(reward);
+      if (reward < 0) acceptedErrors++;
+    } else {
+      rejectedRewards.push(reward);
+    }
+  }
+  const n = scored.length;
+  const rejected = Math.max(0, n - accepted);
+  const ci = confidenceInterval(deltas, 0.95, { seed: options.seed ?? 17 });
+  const reasons = [];
+  if (n < minN) reasons.push(`need at least ${minN} scored decisions, got ${n}`);
+  if (accepted < minAccepted)
+    reasons.push(`need at least ${minAccepted} accepted decisions, got ${accepted}`);
+  if (ci.lower <= minUtilityDelta) {
+    reasons.push(`utility CI lower bound ${ci.lower.toFixed(4)} does not clear ${minUtilityDelta}`);
+  }
+  const recommendation = n < minN || accepted < minAccepted ? "need_more_data" : ci.lower > minUtilityDelta ? "ship" : "hold";
+  return {
+    policyId: policy.id,
+    n,
+    accepted,
+    rejected,
+    coverage: n > 0 ? accepted / n : 0,
+    acceptedErrorRate: accepted > 0 ? acceptedErrors / accepted : 0,
+    baselineUtility,
+    policyUtility,
+    utilityDelta: policyUtility - baselineUtility,
+    utilityCi95: ci,
+    rejectedMeanReward: rejectedRewards.length > 0 ? mean(rejectedRewards) : null,
+    recommendation,
+    reasons
+  };
+}
+function acceptUtility(point, utility) {
+  return rewardOf2(point, utility) - utility.costWeight * (point.costUsd ?? point.outcome?.costUsd ?? 0);
+}
+function policyDecisionUtility(point, action, utility) {
+  if (action === "accept") return acceptUtility(point, utility);
+  if (action === "verify") return utility.deferUtility - utility.verifyCost;
+  if (action === "ask") return utility.deferUtility - utility.askCost;
+  if (action === "retry") return utility.deferUtility - utility.retryCost;
+  if (action === "stop") return utility.stopUtility;
+  return utility.deferUtility;
+}
+function rewardOf2(point, utility) {
+  const outcome = point.outcome;
+  if (!outcome) return utility.failureUtility;
+  if (typeof outcome.reward === "number") return 2 * outcome.reward - 1;
+  if (typeof outcome.score === "number") return 2 * outcome.score - 1;
+  if (outcome.success === true) return utility.successUtility;
+  if (outcome.success === false) return utility.failureUtility;
+  return utility.failureUtility;
+}
+function mean(values) {
+  return values.reduce((sum, value) => sum + value, 0) / values.length;
+}
+
+// src/belief-state/report.ts
+function analyzeBeliefPolicy(options) {
+  const selective = evaluateBeliefSelectivePolicy(options.points, options.policy, options.selective);
+  const calibration = calibrateBeliefDecisions(options.points, options.calibration);
+  const opeTargetPolicy = options.ope?.targetPolicy;
+  const ope = opeTargetPolicy ? evaluateBeliefOffPolicy(options.points, opeTargetPolicy, options.ope) : null;
+  const diagnostics = [];
+  const selectiveStatus = selective.recommendation;
+  const calibrationStatus = calibration ? "supported" : "unsupported";
+  const opeRequested = options.requireOpe === true || options.ope !== void 0;
+  const opeStatus = ope ? ope.support.supported ? "supported" : "unsupported" : opeRequested ? "unsupported" : "not_requested";
+  if (!calibration) diagnostics.push("calibration unsupported: not enough confidence/outcome pairs");
+  if (opeRequested && !opeTargetPolicy) diagnostics.push("OPE unsupported: missing target policy");
+  else if (ope && !ope.support.supported)
+    diagnostics.push(...ope.support.reasons.map((reason) => `OPE unsupported: ${reason}`));
+  const status = overallStatus({
+    selectiveStatus,
+    hasCalibration: calibration !== null,
+    opeStatus,
+    opeRequested
+  });
+  return {
+    policyId: options.policy.id,
+    n: options.points.length,
+    status,
+    selectiveStatus,
+    calibrationStatus,
+    opeStatus,
+    ...ope ? { opeTargetPolicyId: ope.targetPolicyId } : {},
+    selective,
+    ...calibration ? { calibration } : {},
+    ...ope ? { ope } : {},
+    diagnostics
+  };
+}
+function overallStatus(options) {
+  if (options.selectiveStatus === "need_more_data" || !options.hasCalibration) {
+    return "need_more_data";
+  }
+  if (options.selectiveStatus === "hold") return "hold";
+  if (options.opeRequested && options.opeStatus !== "supported") return "hold";
+  return "ship";
+}
+export {
+  analyzeBeliefPolicy,
+  beliefDecisionsToOffPolicyTrajectories,
+  calibrateBeliefDecisions,
+  embeddedBeliefOpeTargetPolicy,
+  evaluateBeliefOffPolicy,
+  evaluateBeliefSelectivePolicy,
+  extractBeliefDecisionPoints,
+  thresholdSelectivePolicy
+};
+//# sourceMappingURL=index.js.map

package/dist/belief-state/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/belief-state/calibration.ts","../../src/belief-state/extract.ts","../../src/belief-state/ope.ts","../../src/belief-state/selective.ts","../../src/belief-state/report.ts"],"sourcesContent":["import { ValidationError } from '../errors'\nimport { calibrationFromPairs } from '../meta-eval/calibration'\nimport type { BeliefDecisionPoint, BeliefSelectivePolicy } from './types'\n\nexport type BeliefCalibrationRegion = 'all' | 'accepted' | 'rejected'\n\nexport interface BeliefCalibrationOptions {\n  bins?: number\n  minPairs?: number\n  policy?: BeliefSelectivePolicy\n  region?: BeliefCalibrationRegion\n}\n\nexport function calibrateBeliefDecisions(\n  points: BeliefDecisionPoint[],\n  options: BeliefCalibrationOptions = {},\n) {\n  const filtered = filterCalibrationRegion(points, options)\n  const pairs = filtered\n    .filter((point) => typeof point.confidence === 'number' && point.outcome)\n    .map((point) => ({\n      evalScore: point.confidence!,\n      outcome: outcomeScore(point),\n    }))\n    .filter((pair) => Number.isFinite(pair.outcome))\n  const minPairs = options.minPairs ?? 10\n  if (pairs.length < minPairs) return null\n  return calibrationFromPairs(pairs, 'belief-confidence', 'decision-outcome', {\n    bins: options.bins ?? 5,\n    range: { lo: 0, hi: 1 },\n  })\n}\n\nfunction filterCalibrationRegion(\n  points: BeliefDecisionPoint[],\n  options: BeliefCalibrationOptions,\n): BeliefDecisionPoint[] {\n  const region = options.region ?? 'all'\n  if (region === 'all') return points\n  const policy = options.policy\n  if (!policy) {\n    throw new ValidationError(\n      `calibrateBeliefDecisions: policy is required when region is \"${region}\"`,\n    )\n  }\n  return points.filter((point) => {\n    const accepted = policy.decide(point).action === 'accept'\n    return region === 'accepted' ? accepted : !accepted\n  })\n}\n\nfunction outcomeScore(point: BeliefDecisionPoint): number {\n  if (typeof point.outcome?.reward === 'number') return point.outcome.reward\n  if (typeof point.outcome?.score === 'number') return point.outcome.score\n  if (point.outcome?.success === true) return 1\n  if (point.outcome?.success === false) return 0\n  return Number.NaN\n}\n","import type { TraceEvent } from '../trace/schema'\nimport type { TraceStore } from '../trace/store'\nimport type {\n  BeliefDecisionExtractionDiagnostic,\n  BeliefDecisionExtractionReport,\n  BeliefDecisionKind,\n  BeliefDecisionOutcome,\n  BeliefDecisionPoint,\n  BeliefEvidenceRef,\n} from './types'\n\nexport interface ExtractBeliefDecisionPointsOptions {\n  runIds?: string[]\n}\n\nconst DECISION_MARKERS = new Set(['belief_decision', 'belief.decision', 'decision_point'])\nconst DECISION_KINDS: ReadonlySet<string> = new Set([\n  'continue',\n  'verify',\n  'ask',\n  'retry',\n  'stop',\n  'memory-write',\n  'memory-read',\n  'tool-select',\n  'skill-select',\n  'workflow-select',\n  'surface-promote',\n])\n\nexport async function extractBeliefDecisionPoints(\n  store: TraceStore,\n  options: ExtractBeliefDecisionPointsOptions = {},\n): Promise<BeliefDecisionExtractionReport> {\n  const runs = options.runIds\n    ? (await Promise.all(options.runIds.map((runId) => store.getRun(runId)))).filter(Boolean)\n    : await store.listRuns()\n  const decisions: BeliefDecisionPoint[] = []\n  const diagnostics: BeliefDecisionExtractionDiagnostic[] = []\n\n  for (const run of runs) {\n    if (!run) continue\n    const events = await store.events({ runId: run.runId })\n    const spans = await store.spans({ runId: run.runId })\n    const spanIds = new Set(spans.map((span) => span.spanId))\n    let stepIndex = 0\n    for (const event of [...events].sort((a, b) => a.timestamp - b.timestamp)) {\n      const parsed = parseDecisionEvent(event, {\n        scenarioId: run.scenarioId,\n        stepIndex,\n        spanExists: event.spanId ? spanIds.has(event.spanId) : false,\n      })\n      if (!parsed) continue\n      if ('diagnostic' in parsed) {\n        diagnostics.push(parsed.diagnostic)\n        continue\n      }\n      decisions.push(parsed.decision)\n      stepIndex++\n    }\n  }\n\n  return { decisions, diagnostics }\n}\n\nfunction parseDecisionEvent(\n  event: TraceEvent,\n  context: { scenarioId?: string; stepIndex: number; spanExists: boolean },\n): { decision: BeliefDecisionPoint } | { diagnostic: BeliefDecisionExtractionDiagnostic } | null {\n  const payload = event.payload\n  const marker = stringField(payload, 'kind') ?? stringField(payload, 'type')\n  if (!marker || !DECISION_MARKERS.has(marker)) return null\n\n  const decisionKind = stringField(payload, 'decisionKind')\n  if (!decisionKind || !DECISION_KINDS.has(decisionKind)) {\n    return {\n      diagnostic: {\n        runId: event.runId,\n        eventId: event.eventId,\n        severity: 'warning',\n        reason: `belief decision event has unsupported decisionKind \"${decisionKind ?? ''}\"`,\n      },\n    }\n  }\n\n  const chosenAction = stringField(payload, 'chosenAction')\n  if (!chosenAction) {\n    return {\n      diagnostic: {\n        runId: event.runId,\n        eventId: event.eventId,\n        severity: 'warning',\n        reason: 'belief decision event is missing chosenAction',\n      },\n    }\n  }\n\n  const evidence: BeliefEvidenceRef[] = [\n    {\n      source: 'event',\n      id: event.eventId,\n      runId: event.runId,\n      eventId: event.eventId,\n    },\n  ]\n  if (event.spanId && context.spanExists) {\n    evidence.push({ source: 'span', id: event.spanId, runId: event.runId, spanId: event.spanId })\n  }\n\n  return {\n    decision: {\n      id: stringField(payload, 'id') ?? event.eventId,\n      runId: event.runId,\n      scenarioId: stringField(payload, 'scenarioId') ?? context.scenarioId,\n      stepIndex: numberField(payload, 'stepIndex') ?? context.stepIndex,\n      kind: decisionKind as BeliefDecisionKind,\n      chosenAction,\n      candidateActions: stringArrayField(payload, 'candidateActions'),\n      confidence: finiteUnitField(payload, 'confidence'),\n      behaviorProb: numberField(payload, 'behaviorProb'),\n      targetProb: numberField(payload, 'targetProb'),\n      qHat: finiteUnitField(payload, 'qHat'),\n      costUsd: nonNegativeNumberField(payload, 'costUsd'),\n      evidence,\n      outcome: parseOutcome(payload),\n      metadata: recordField(payload, 'metadata'),\n    },\n  }\n}\n\nfunction parseOutcome(payload: Record<string, unknown>): BeliefDecisionOutcome | undefined {\n  const value = recordField(payload, 'outcome')\n  if (!value) return undefined\n  return {\n    success: typeof value.success === 'boolean' ? value.success : undefined,\n    score: finiteUnitField(value, 'score'),\n    reward: finiteUnitField(value, 'reward'),\n    costUsd: nonNegativeNumberField(value, 'costUsd'),\n    observedAt: stringField(value, 'observedAt'),\n    metadata: recordField(value, 'metadata'),\n  }\n}\n\nfunction stringField(obj: Record<string, unknown>, key: string): string | undefined {\n  const value = obj[key]\n  return typeof value === 'string' && value.length > 0 ? value : undefined\n}\n\nfunction numberField(obj: Record<string, unknown>, key: string): number | undefined {\n  const value = obj[key]\n  return typeof value === 'number' && Number.isFinite(value) ? value : undefined\n}\n\nfunction finiteUnitField(obj: Record<string, unknown>, key: string): number | undefined {\n  const value = numberField(obj, key)\n  return value === undefined ? undefined : Math.max(0, Math.min(1, value))\n}\n\nfunction nonNegativeNumberField(obj: Record<string, unknown>, key: string): number | undefined {\n  const value = numberField(obj, key)\n  return value === undefined ? undefined : Math.max(0, value)\n}\n\nfunction stringArrayField(obj: Record<string, unknown>, key: string): string[] | undefined {\n  const value = obj[key]\n  if (!Array.isArray(value)) return undefined\n  const strings = value.filter(\n    (item): item is string => typeof item === 'string' && item.length > 0,\n  )\n  return strings.length > 0 ? strings : undefined\n}\n\nfunction recordField(\n  obj: Record<string, unknown>,\n  key: string,\n): Record<string, unknown> | undefined {\n  const value = obj[key]\n  if (!value || typeof value !== 'object' || Array.isArray(value)) return undefined\n  return value as Record<string, unknown>\n}\n","import {\n  type OffPolicyOptions,\n  type OffPolicyTrajectory,\n  offPolicyEstimateAll,\n} from '../rl/off-policy'\nimport type {\n  BeliefDecisionPoint,\n  BeliefOpeReport,\n  BeliefOpeSupportDiagnostics,\n  BeliefOpeTargetPolicy,\n} from './types'\n\nexport interface BeliefOpeOptions extends OffPolicyOptions {\n  minEffectiveSampleSize?: number\n  minEffectiveSampleRatio?: number\n  maxDiagnostics?: number\n}\n\nexport interface BeliefOffPolicyTrajectoryReport {\n  targetPolicyId: string\n  trajectories: OffPolicyTrajectory[]\n  dropped: number\n  diagnostics: string[]\n}\n\nexport function embeddedBeliefOpeTargetPolicy(id = 'embedded-target-prob'): BeliefOpeTargetPolicy {\n  return {\n    id,\n    targetProbOf(point) {\n      return point.targetProb\n    },\n    qHatOf(point) {\n      return point.qHat\n    },\n  }\n}\n\nexport function beliefDecisionsToOffPolicyTrajectories(\n  points: BeliefDecisionPoint[],\n  targetPolicy: BeliefOpeTargetPolicy,\n  options: Pick<BeliefOpeOptions, 'maxDiagnostics'> = {},\n): BeliefOffPolicyTrajectoryReport {\n  const trajectories: OffPolicyTrajectory[] = []\n  const diagnostics: string[] = []\n  for (const point of points) {\n    if (!point.outcome) {\n      diagnostics.push(`${point.id}: missing outcome`)\n      continue\n    }\n    if (!isBehaviorProbability(point.behaviorProb)) {\n      diagnostics.push(`${point.id}: invalid behaviorProb ${formatProbability(point.behaviorProb)}`)\n      continue\n    }\n\n    let targetProb: number | null | undefined\n    let qHat: number | null | undefined\n    try {\n      targetProb = targetPolicy.targetProbOf(point)\n      qHat = targetPolicy.qHatOf?.(point)\n    } catch (error) {\n      diagnostics.push(\n        `${point.id}: target policy ${targetPolicy.id} threw (${errorMessage(error)})`,\n      )\n      continue\n    }\n    if (!isTargetProbability(targetProb)) {\n      diagnostics.push(`${point.id}: invalid targetProb ${formatProbability(targetProb)}`)\n      continue\n    }\n    if (qHat !== null && qHat !== undefined && !isTargetProbability(qHat)) {\n      diagnostics.push(`${point.id}: invalid qHat ${formatProbability(qHat)}; ignoring qHat`)\n      qHat = null\n    }\n\n    trajectories.push({\n      runId: point.id,\n      reward: rewardOf(point),\n      behaviorProb: point.behaviorProb,\n      targetProb,\n      qHat,\n    })\n  }\n  return {\n    targetPolicyId: targetPolicy.id,\n    trajectories,\n    dropped: points.length - trajectories.length,\n    diagnostics: compactDiagnostics(diagnostics, options.maxDiagnostics ?? 20),\n  }\n}\n\nexport function evaluateBeliefOffPolicy(\n  points: BeliefDecisionPoint[],\n  targetPolicy: BeliefOpeTargetPolicy,\n  options: BeliefOpeOptions = {},\n): BeliefOpeReport {\n  const trajectoryReport = beliefDecisionsToOffPolicyTrajectories(points, targetPolicy, options)\n  const { trajectories } = trajectoryReport\n  const estimates = offPolicyEstimateAll(trajectories, options)\n  const support = supportDiagnostics(estimates.dr, {\n    minEffectiveSampleSize: options.minEffectiveSampleSize ?? 30,\n    minEffectiveSampleRatio: options.minEffectiveSampleRatio ?? 0.25,\n    dropped: trajectoryReport.dropped,\n    diagnostics: trajectoryReport.diagnostics,\n  })\n  return { targetPolicyId: targetPolicy.id, ...estimates, support }\n}\n\nfunction supportDiagnostics(\n  estimate: { n: number; effectiveSampleSize: number; maxImportanceWeight: number },\n  options: {\n    minEffectiveSampleSize: number\n    minEffectiveSampleRatio: number\n    dropped: number\n    diagnostics: string[]\n  },\n): BeliefOpeSupportDiagnostics {\n  const ratio = estimate.n > 0 ? estimate.effectiveSampleSize / estimate.n : 0\n  const reasons: string[] = [...options.diagnostics]\n  if (estimate.n === 0) {\n    reasons.push('no valid OPE trajectories')\n  }\n  if (options.dropped > 0) {\n    reasons.push(`dropped ${options.dropped} unsupported decision(s)`)\n  }\n  if (estimate.effectiveSampleSize < options.minEffectiveSampleSize) {\n    reasons.push(\n      `effective sample size ${estimate.effectiveSampleSize.toFixed(2)} below ${options.minEffectiveSampleSize}`,\n    )\n  }\n  if (ratio < options.minEffectiveSampleRatio) {\n    reasons.push(\n      `effective sample ratio ${ratio.toFixed(2)} below ${options.minEffectiveSampleRatio}`,\n    )\n  }\n  if (estimate.maxImportanceWeight > 10) {\n    reasons.push(`max importance weight ${estimate.maxImportanceWeight.toFixed(2)} is high`)\n  }\n  return {\n    supported: reasons.length === 0,\n    n: estimate.n,\n    dropped: options.dropped,\n    effectiveSampleSize: estimate.effectiveSampleSize,\n    effectiveSampleRatio: ratio,\n    maxImportanceWeight: estimate.maxImportanceWeight,\n    reasons,\n  }\n}\n\nfunction rewardOf(point: BeliefDecisionPoint): number {\n  if (typeof point.outcome?.reward === 'number') return point.outcome.reward\n  if (typeof point.outcome?.score === 'number') return point.outcome.score\n  if (point.outcome?.success === true) return 1\n  return 0\n}\n\nfunction isBehaviorProbability(value: unknown): value is number {\n  return typeof value === 'number' && Number.isFinite(value) && value > 0 && value <= 1\n}\n\nfunction isTargetProbability(value: unknown): value is number {\n  return typeof value === 'number' && Number.isFinite(value) && value >= 0 && value <= 1\n}\n\nfunction formatProbability(value: unknown): string {\n  return typeof value === 'number' ? String(value) : String(value ?? 'missing')\n}\n\nfunction errorMessage(error: unknown): string {\n  return error instanceof Error ? error.message : String(error)\n}\n\nfunction compactDiagnostics(diagnostics: string[], maxDiagnostics: number): string[] {\n  if (diagnostics.length <= maxDiagnostics) return diagnostics\n  return [\n    ...diagnostics.slice(0, maxDiagnostics),\n    `${diagnostics.length - maxDiagnostics} additional OPE diagnostic(s) omitted`,\n  ]\n}\n","import { ValidationError } from '../errors'\nimport { confidenceInterval } from '../statistics'\nimport type {\n  BeliefDecisionPoint,\n  BeliefPolicyAction,\n  BeliefSelectivePolicy,\n  BeliefSelectivePolicyMetrics,\n  BeliefUtilityOptions,\n} from './types'\n\nexport interface EvaluateBeliefSelectivePolicyOptions {\n  utility?: BeliefUtilityOptions\n  minN?: number\n  minAccepted?: number\n  minUtilityDelta?: number\n  seed?: number\n}\n\nconst DEFAULT_UTILITY: Required<BeliefUtilityOptions> = {\n  successUtility: 1,\n  failureUtility: -1,\n  deferUtility: 0,\n  verifyCost: 0.05,\n  askCost: 0.05,\n  retryCost: 0.1,\n  stopUtility: 0,\n  costWeight: 1,\n}\n\nexport function thresholdSelectivePolicy(options: {\n  id?: string\n  confidenceThreshold: number\n  belowThresholdAction?: Exclude<BeliefPolicyAction, 'accept'>\n}): BeliefSelectivePolicy {\n  const threshold = options.confidenceThreshold\n  if (!Number.isFinite(threshold) || threshold < 0 || threshold > 1) {\n    throw new ValidationError(\n      `thresholdSelectivePolicy: confidenceThreshold must be in [0, 1], got ${threshold}`,\n    )\n  }\n  const belowThresholdAction = options.belowThresholdAction ?? 'verify'\n  return {\n    id: options.id ?? `confidence>=${threshold}`,\n    decide(point) {\n      const confidence = point.confidence ?? 0\n      return {\n        action: confidence >= threshold ? 'accept' : belowThresholdAction,\n        confidence,\n        targetProb: point.targetProb,\n        qHat: point.qHat,\n        reason:\n          confidence >= threshold ? 'confidence threshold passed' : 'confidence threshold failed',\n      }\n    },\n  }\n}\n\nexport function evaluateBeliefSelectivePolicy(\n  points: BeliefDecisionPoint[],\n  policy: BeliefSelectivePolicy,\n  options: EvaluateBeliefSelectivePolicyOptions = {},\n): BeliefSelectivePolicyMetrics {\n  const utility = { ...DEFAULT_UTILITY, ...(options.utility ?? {}) }\n  const scored = points.filter((point) => point.outcome)\n  const minN = options.minN ?? 30\n  const minAccepted = options.minAccepted ?? 5\n  const minUtilityDelta = options.minUtilityDelta ?? 0\n  const deltas: number[] = []\n  const acceptedRewards: number[] = []\n  const rejectedRewards: number[] = []\n  let baselineUtility = 0\n  let policyUtility = 0\n  let accepted = 0\n  let acceptedErrors = 0\n\n  for (const point of scored) {\n    const baseline = acceptUtility(point, utility)\n    const decision = policy.decide(point)\n    const candidate = policyDecisionUtility(point, decision.action, utility)\n    const reward = rewardOf(point, utility)\n    baselineUtility += baseline\n    policyUtility += candidate\n    deltas.push(candidate - baseline)\n    if (decision.action === 'accept') {\n      accepted++\n      acceptedRewards.push(reward)\n      if (reward < 0) acceptedErrors++\n    } else {\n      rejectedRewards.push(reward)\n    }\n  }\n\n  const n = scored.length\n  const rejected = Math.max(0, n - accepted)\n  const ci = confidenceInterval(deltas, 0.95, { seed: options.seed ?? 17 })\n  const reasons: string[] = []\n  if (n < minN) reasons.push(`need at least ${minN} scored decisions, got ${n}`)\n  if (accepted < minAccepted)\n    reasons.push(`need at least ${minAccepted} accepted decisions, got ${accepted}`)\n  if (ci.lower <= minUtilityDelta) {\n    reasons.push(`utility CI lower bound ${ci.lower.toFixed(4)} does not clear ${minUtilityDelta}`)\n  }\n  const recommendation =\n    n < minN || accepted < minAccepted\n      ? 'need_more_data'\n      : ci.lower > minUtilityDelta\n        ? 'ship'\n        : 'hold'\n\n  return {\n    policyId: policy.id,\n    n,\n    accepted,\n    rejected,\n    coverage: n > 0 ? accepted / n : 0,\n    acceptedErrorRate: accepted > 0 ? acceptedErrors / accepted : 0,\n    baselineUtility,\n    policyUtility,\n    utilityDelta: policyUtility - baselineUtility,\n    utilityCi95: ci,\n    rejectedMeanReward: rejectedRewards.length > 0 ? mean(rejectedRewards) : null,\n    recommendation,\n    reasons,\n  }\n}\n\nfunction acceptUtility(\n  point: BeliefDecisionPoint,\n  utility: Required<BeliefUtilityOptions>,\n): number {\n  return (\n    rewardOf(point, utility) - utility.costWeight * (point.costUsd ?? point.outcome?.costUsd ?? 0)\n  )\n}\n\nfunction policyDecisionUtility(\n  point: BeliefDecisionPoint,\n  action: BeliefPolicyAction,\n  utility: Required<BeliefUtilityOptions>,\n): number {\n  if (action === 'accept') return acceptUtility(point, utility)\n  if (action === 'verify') return utility.deferUtility - utility.verifyCost\n  if (action === 'ask') return utility.deferUtility - utility.askCost\n  if (action === 'retry') return utility.deferUtility - utility.retryCost\n  if (action === 'stop') return utility.stopUtility\n  return utility.deferUtility\n}\n\nfunction rewardOf(point: BeliefDecisionPoint, utility: Required<BeliefUtilityOptions>): number {\n  const outcome = point.outcome\n  if (!outcome) return utility.failureUtility\n  if (typeof outcome.reward === 'number') return 2 * outcome.reward - 1\n  if (typeof outcome.score === 'number') return 2 * outcome.score - 1\n  if (outcome.success === true) return utility.successUtility\n  if (outcome.success === false) return utility.failureUtility\n  return utility.failureUtility\n}\n\nfunction mean(values: number[]): number {\n  return values.reduce((sum, value) => sum + value, 0) / values.length\n}\n","import { type BeliefCalibrationOptions, calibrateBeliefDecisions } from './calibration'\nimport { type BeliefOpeOptions, evaluateBeliefOffPolicy } from './ope'\nimport {\n  type EvaluateBeliefSelectivePolicyOptions,\n  evaluateBeliefSelectivePolicy,\n} from './selective'\nimport type {\n  BeliefDecisionPoint,\n  BeliefEvaluationStatus,\n  BeliefOpeStatus,\n  BeliefOpeTargetPolicy,\n  BeliefPolicyEvaluationReport,\n  BeliefSelectivePolicy,\n} from './types'\n\nexport interface AnalyzeBeliefPolicyOpeOptions extends BeliefOpeOptions {\n  targetPolicy?: BeliefOpeTargetPolicy\n}\n\nexport interface AnalyzeBeliefPolicyOptions {\n  points: BeliefDecisionPoint[]\n  policy: BeliefSelectivePolicy\n  selective?: EvaluateBeliefSelectivePolicyOptions\n  calibration?: BeliefCalibrationOptions\n  ope?: AnalyzeBeliefPolicyOpeOptions\n  requireOpe?: boolean\n}\n\nexport function analyzeBeliefPolicy(\n  options: AnalyzeBeliefPolicyOptions,\n): BeliefPolicyEvaluationReport {\n  const selective = evaluateBeliefSelectivePolicy(options.points, options.policy, options.selective)\n  const calibration = calibrateBeliefDecisions(options.points, options.calibration)\n  const opeTargetPolicy = options.ope?.targetPolicy\n  const ope = opeTargetPolicy\n    ? evaluateBeliefOffPolicy(options.points, opeTargetPolicy, options.ope)\n    : null\n  const diagnostics: string[] = []\n  const selectiveStatus = selective.recommendation\n  const calibrationStatus = calibration ? 'supported' : 'unsupported'\n  const opeRequested = options.requireOpe === true || options.ope !== undefined\n  const opeStatus: BeliefOpeStatus = ope\n    ? ope.support.supported\n      ? 'supported'\n      : 'unsupported'\n    : opeRequested\n      ? 'unsupported'\n      : 'not_requested'\n\n  if (!calibration) diagnostics.push('calibration unsupported: not enough confidence/outcome pairs')\n  if (opeRequested && !opeTargetPolicy) diagnostics.push('OPE unsupported: missing target policy')\n  else if (ope && !ope.support.supported)\n    diagnostics.push(...ope.support.reasons.map((reason) => `OPE unsupported: ${reason}`))\n\n  const status = overallStatus({\n    selectiveStatus,\n    hasCalibration: calibration !== null,\n    opeStatus,\n    opeRequested,\n  })\n\n  return {\n    policyId: options.policy.id,\n    n: options.points.length,\n    status,\n    selectiveStatus,\n    calibrationStatus,\n    opeStatus,\n    ...(ope ? { opeTargetPolicyId: ope.targetPolicyId } : {}),\n    selective,\n    ...(calibration ? { calibration } : {}),\n    ...(ope ? { ope } : {}),\n    diagnostics,\n  }\n}\n\nfunction overallStatus(options: {\n  selectiveStatus: BeliefEvaluationStatus\n  hasCalibration: boolean\n  opeStatus: BeliefOpeStatus\n  opeRequested: boolean\n}): BeliefEvaluationStatus {\n  if (options.selectiveStatus === 'need_more_data' || !options.hasCalibration) {\n    return 'need_more_data'\n  }\n  if (options.selectiveStatus === 'hold') return 'hold'\n  if (options.opeRequested && options.opeStatus !== 'supported') return 'hold'\n  return 'ship'\n}\n"],"mappings":";;;;;;;;;;;;;;;AAaO,SAAS,yBACd,QACA,UAAoC,CAAC,GACrC;AACA,QAAM,WAAW,wBAAwB,QAAQ,OAAO;AACxD,QAAM,QAAQ,SACX,OAAO,CAAC,UAAU,OAAO,MAAM,eAAe,YAAY,MAAM,OAAO,EACvE,IAAI,CAAC,WAAW;AAAA,IACf,WAAW,MAAM;AAAA,IACjB,SAAS,aAAa,KAAK;AAAA,EAC7B,EAAE,EACD,OAAO,CAAC,SAAS,OAAO,SAAS,KAAK,OAAO,CAAC;AACjD,QAAM,WAAW,QAAQ,YAAY;AACrC,MAAI,MAAM,SAAS,SAAU,QAAO;AACpC,SAAO,qBAAqB,OAAO,qBAAqB,oBAAoB;AAAA,IAC1E,MAAM,QAAQ,QAAQ;AAAA,IACtB,OAAO,EAAE,IAAI,GAAG,IAAI,EAAE;AAAA,EACxB,CAAC;AACH;AAEA,SAAS,wBACP,QACA,SACuB;AACvB,QAAM,SAAS,QAAQ,UAAU;AACjC,MAAI,WAAW,MAAO,QAAO;AAC7B,QAAM,SAAS,QAAQ;AACvB,MAAI,CAAC,QAAQ;AACX,UAAM,IAAI;AAAA,MACR,gEAAgE,MAAM;AAAA,IACxE;AAAA,EACF;AACA,SAAO,OAAO,OAAO,CAAC,UAAU;AAC9B,UAAM,WAAW,OAAO,OAAO,KAAK,EAAE,WAAW;AACjD,WAAO,WAAW,aAAa,WAAW,CAAC;AAAA,EAC7C,CAAC;AACH;AAEA,SAAS,aAAa,OAAoC;AACxD,MAAI,OAAO,MAAM,SAAS,WAAW,SAAU,QAAO,MAAM,QAAQ;AACpE,MAAI,OAAO,MAAM,SAAS,UAAU,SAAU,QAAO,MAAM,QAAQ;AACnE,MAAI,MAAM,SAAS,YAAY,KAAM,QAAO;AAC5C,MAAI,MAAM,SAAS,YAAY,MAAO,QAAO;AAC7C,SAAO,OAAO;AAChB;;;AC1CA,IAAM,mBAAmB,oBAAI,IAAI,CAAC,mBAAmB,mBAAmB,gBAAgB,CAAC;AACzF,IAAM,iBAAsC,oBAAI,IAAI;AAAA,EAClD;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAED,eAAsB,4BACpB,OACA,UAA8C,CAAC,GACN;AACzC,QAAM,OAAO,QAAQ,UAChB,MAAM,QAAQ,IAAI,QAAQ,OAAO,IAAI,CAAC,UAAU,MAAM,OAAO,KAAK,CAAC,CAAC,GAAG,OAAO,OAAO,IACtF,MAAM,MAAM,SAAS;AACzB,QAAM,YAAmC,CAAC;AAC1C,QAAM,cAAoD,CAAC;AAE3D,aAAW,OAAO,MAAM;AACtB,QAAI,CAAC,IAAK;AACV,UAAM,SAAS,MAAM,MAAM,OAAO,EAAE,OAAO,IAAI,MAAM,CAAC;AACtD,UAAM,QAAQ,MAAM,MAAM,MAAM,EAAE,OAAO,IAAI,MAAM,CAAC;AACpD,UAAM,UAAU,IAAI,IAAI,MAAM,IAAI,CAAC,SAAS,KAAK,MAAM,CAAC;AACxD,QAAI,YAAY;AAChB,eAAW,SAAS,CAAC,GAAG,MAAM,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,YAAY,EAAE,SAAS,GAAG;AACzE,YAAM,SAAS,mBAAmB,OAAO;AAAA,QACvC,YAAY,IAAI;AAAA,QAChB;AAAA,QACA,YAAY,MAAM,SAAS,QAAQ,IAAI,MAAM,MAAM,IAAI;AAAA,MACzD,CAAC;AACD,UAAI,CAAC,OAAQ;AACb,UAAI,gBAAgB,QAAQ;AAC1B,oBAAY,KAAK,OAAO,UAAU;AAClC;AAAA,MACF;AACA,gBAAU,KAAK,OAAO,QAAQ;AAC9B;AAAA,IACF;AAAA,EACF;AAEA,SAAO,EAAE,WAAW,YAAY;AAClC;AAEA,SAAS,mBACP,OACA,SAC+F;AAC/F,QAAM,UAAU,MAAM;AACtB,QAAM,SAAS,YAAY,SAAS,MAAM,KAAK,YAAY,SAAS,MAAM;AAC1E,MAAI,CAAC,UAAU,CAAC,iBAAiB,IAAI,MAAM,EAAG,QAAO;AAErD,QAAM,eAAe,YAAY,SAAS,cAAc;AACxD,MAAI,CAAC,gBAAgB,CAAC,eAAe,IAAI,YAAY,GAAG;AACtD,WAAO;AAAA,MACL,YAAY;AAAA,QACV,OAAO,MAAM;AAAA,QACb,SAAS,MAAM;AAAA,QACf,UAAU;AAAA,QACV,QAAQ,uDAAuD,gBAAgB,EAAE;AAAA,MACnF;AAAA,IACF;AAAA,EACF;AAEA,QAAM,eAAe,YAAY,SAAS,cAAc;AACxD,MAAI,CAAC,cAAc;AACjB,WAAO;AAAA,MACL,YAAY;AAAA,QACV,OAAO,MAAM;AAAA,QACb,SAAS,MAAM;AAAA,QACf,UAAU;AAAA,QACV,QAAQ;AAAA,MACV;AAAA,IACF;AAAA,EACF;AAEA,QAAM,WAAgC;AAAA,IACpC;AAAA,MACE,QAAQ;AAAA,MACR,IAAI,MAAM;AAAA,MACV,OAAO,MAAM;AAAA,MACb,SAAS,MAAM;AAAA,IACjB;AAAA,EACF;AACA,MAAI,MAAM,UAAU,QAAQ,YAAY;AACtC,aAAS,KAAK,EAAE,QAAQ,QAAQ,IAAI,MAAM,QAAQ,OAAO,MAAM,OAAO,QAAQ,MAAM,OAAO,CAAC;AAAA,EAC9F;AAEA,SAAO;AAAA,IACL,UAAU;AAAA,MACR,IAAI,YAAY,SAAS,IAAI,KAAK,MAAM;AAAA,MACxC,OAAO,MAAM;AAAA,MACb,YAAY,YAAY,SAAS,YAAY,KAAK,QAAQ;AAAA,MAC1D,WAAW,YAAY,SAAS,WAAW,KAAK,QAAQ;AAAA,MACxD,MAAM;AAAA,MACN;AAAA,MACA,kBAAkB,iBAAiB,SAAS,kBAAkB;AAAA,MAC9D,YAAY,gBAAgB,SAAS,YAAY;AAAA,MACjD,cAAc,YAAY,SAAS,cAAc;AAAA,MACjD,YAAY,YAAY,SAAS,YAAY;AAAA,MAC7C,MAAM,gBAAgB,SAAS,MAAM;AAAA,MACrC,SAAS,uBAAuB,SAAS,SAAS;AAAA,MAClD;AAAA,MACA,SAAS,aAAa,OAAO;AAAA,MAC7B,UAAU,YAAY,SAAS,UAAU;AAAA,IAC3C;AAAA,EACF;AACF;AAEA,SAAS,aAAa,SAAqE;AACzF,QAAM,QAAQ,YAAY,SAAS,SAAS;AAC5C,MAAI,CAAC,MAAO,QAAO;AACnB,SAAO;AAAA,IACL,SAAS,OAAO,MAAM,YAAY,YAAY,MAAM,UAAU;AAAA,IAC9D,OAAO,gBAAgB,OAAO,OAAO;AAAA,IACrC,QAAQ,gBAAgB,OAAO,QAAQ;AAAA,IACvC,SAAS,uBAAuB,OAAO,SAAS;AAAA,IAChD,YAAY,YAAY,OAAO,YAAY;AAAA,IAC3C,UAAU,YAAY,OAAO,UAAU;AAAA,EACzC;AACF;AAEA,SAAS,YAAY,KAA8B,KAAiC;AAClF,QAAM,QAAQ,IAAI,GAAG;AACrB,SAAO,OAAO,UAAU,YAAY,MAAM,SAAS,IAAI,QAAQ;AACjE;AAEA,SAAS,YAAY,KAA8B,KAAiC;AAClF,QAAM,QAAQ,IAAI,GAAG;AACrB,SAAO,OAAO,UAAU,YAAY,OAAO,SAAS,KAAK,IAAI,QAAQ;AACvE;AAEA,SAAS,gBAAgB,KAA8B,KAAiC;AACtF,QAAM,QAAQ,YAAY,KAAK,GAAG;AAClC,SAAO,UAAU,SAAY,SAAY,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,KAAK,CAAC;AACzE;AAEA,SAAS,uBAAuB,KAA8B,KAAiC;AAC7F,QAAM,QAAQ,YAAY,KAAK,GAAG;AAClC,SAAO,UAAU,SAAY,SAAY,KAAK,IAAI,GAAG,KAAK;AAC5D;AAEA,SAAS,iBAAiB,KAA8B,KAAmC;AACzF,QAAM,QAAQ,IAAI,GAAG;AACrB,MAAI,CAAC,MAAM,QAAQ,KAAK,EAAG,QAAO;AAClC,QAAM,UAAU,MAAM;AAAA,IACpB,CAAC,SAAyB,OAAO,SAAS,YAAY,KAAK,SAAS;AAAA,EACtE;AACA,SAAO,QAAQ,SAAS,IAAI,UAAU;AACxC;AAEA,SAAS,YACP,KACA,KACqC;AACrC,QAAM,QAAQ,IAAI,GAAG;AACrB,MAAI,CAAC,SAAS,OAAO,UAAU,YAAY,MAAM,QAAQ,KAAK,EAAG,QAAO;AACxE,SAAO;AACT;;;AC1JO,SAAS,8BAA8B,KAAK,wBAA+C;AAChG,SAAO;AAAA,IACL;AAAA,IACA,aAAa,OAAO;AAClB,aAAO,MAAM;AAAA,IACf;AAAA,IACA,OAAO,OAAO;AACZ,aAAO,MAAM;AAAA,IACf;AAAA,EACF;AACF;AAEO,SAAS,uCACd,QACA,cACA,UAAoD,CAAC,GACpB;AACjC,QAAM,eAAsC,CAAC;AAC7C,QAAM,cAAwB,CAAC;AAC/B,aAAW,SAAS,QAAQ;AAC1B,QAAI,CAAC,MAAM,SAAS;AAClB,kBAAY,KAAK,GAAG,MAAM,EAAE,mBAAmB;AAC/C;AAAA,IACF;AACA,QAAI,CAAC,sBAAsB,MAAM,YAAY,GAAG;AAC9C,kBAAY,KAAK,GAAG,MAAM,EAAE,0BAA0B,kBAAkB,MAAM,YAAY,CAAC,EAAE;AAC7F;AAAA,IACF;AAEA,QAAI;AACJ,QAAI;AACJ,QAAI;AACF,mBAAa,aAAa,aAAa,KAAK;AAC5C,aAAO,aAAa,SAAS,KAAK;AAAA,IACpC,SAAS,OAAO;AACd,kBAAY;AAAA,QACV,GAAG,MAAM,EAAE,mBAAmB,aAAa,EAAE,WAAW,aAAa,KAAK,CAAC;AAAA,MAC7E;AACA;AAAA,IACF;AACA,QAAI,CAAC,oBAAoB,UAAU,GAAG;AACpC,kBAAY,KAAK,GAAG,MAAM,EAAE,wBAAwB,kBAAkB,UAAU,CAAC,EAAE;AACnF;AAAA,IACF;AACA,QAAI,SAAS,QAAQ,SAAS,UAAa,CAAC,oBAAoB,IAAI,GAAG;AACrE,kBAAY,KAAK,GAAG,MAAM,EAAE,kBAAkB,kBAAkB,IAAI,CAAC,iBAAiB;AACtF,aAAO;AAAA,IACT;AAEA,iBAAa,KAAK;AAAA,MAChB,OAAO,MAAM;AAAA,MACb,QAAQ,SAAS,KAAK;AAAA,MACtB,cAAc,MAAM;AAAA,MACpB;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AACA,SAAO;AAAA,IACL,gBAAgB,aAAa;AAAA,IAC7B;AAAA,IACA,SAAS,OAAO,SAAS,aAAa;AAAA,IACtC,aAAa,mBAAmB,aAAa,QAAQ,kBAAkB,EAAE;AAAA,EAC3E;AACF;AAEO,SAAS,wBACd,QACA,cACA,UAA4B,CAAC,GACZ;AACjB,QAAM,mBAAmB,uCAAuC,QAAQ,cAAc,OAAO;AAC7F,QAAM,EAAE,aAAa,IAAI;AACzB,QAAM,YAAY,qBAAqB,cAAc,OAAO;AAC5D,QAAM,UAAU,mBAAmB,UAAU,IAAI;AAAA,IAC/C,wBAAwB,QAAQ,0BAA0B;AAAA,IAC1D,yBAAyB,QAAQ,2BAA2B;AAAA,IAC5D,SAAS,iBAAiB;AAAA,IAC1B,aAAa,iBAAiB;AAAA,EAChC,CAAC;AACD,SAAO,EAAE,gBAAgB,aAAa,IAAI,GAAG,WAAW,QAAQ;AAClE;AAEA,SAAS,mBACP,UACA,SAM6B;AAC7B,QAAM,QAAQ,SAAS,IAAI,IAAI,SAAS,sBAAsB,SAAS,IAAI;AAC3E,QAAM,UAAoB,CAAC,GAAG,QAAQ,WAAW;AACjD,MAAI,SAAS,MAAM,GAAG;AACpB,YAAQ,KAAK,2BAA2B;AAAA,EAC1C;AACA,MAAI,QAAQ,UAAU,GAAG;AACvB,YAAQ,KAAK,WAAW,QAAQ,OAAO,0BAA0B;AAAA,EACnE;AACA,MAAI,SAAS,sBAAsB,QAAQ,wBAAwB;AACjE,YAAQ;AAAA,MACN,yBAAyB,SAAS,oBAAoB,QAAQ,CAAC,CAAC,UAAU,QAAQ,sBAAsB;AAAA,IAC1G;AAAA,EACF;AACA,MAAI,QAAQ,QAAQ,yBAAyB;AAC3C,YAAQ;AAAA,MACN,0BAA0B,MAAM,QAAQ,CAAC,CAAC,UAAU,QAAQ,uBAAuB;AAAA,IACrF;AAAA,EACF;AACA,MAAI,SAAS,sBAAsB,IAAI;AACrC,YAAQ,KAAK,yBAAyB,SAAS,oBAAoB,QAAQ,CAAC,CAAC,UAAU;AAAA,EACzF;AACA,SAAO;AAAA,IACL,WAAW,QAAQ,WAAW;AAAA,IAC9B,GAAG,SAAS;AAAA,IACZ,SAAS,QAAQ;AAAA,IACjB,qBAAqB,SAAS;AAAA,IAC9B,sBAAsB;AAAA,IACtB,qBAAqB,SAAS;AAAA,IAC9B;AAAA,EACF;AACF;AAEA,SAAS,SAAS,OAAoC;AACpD,MAAI,OAAO,MAAM,SAAS,WAAW,SAAU,QAAO,MAAM,QAAQ;AACpE,MAAI,OAAO,MAAM,SAAS,UAAU,SAAU,QAAO,MAAM,QAAQ;AACnE,MAAI,MAAM,SAAS,YAAY,KAAM,QAAO;AAC5C,SAAO;AACT;AAEA,SAAS,sBAAsB,OAAiC;AAC9D,SAAO,OAAO,UAAU,YAAY,OAAO,SAAS,KAAK,KAAK,QAAQ,KAAK,SAAS;AACtF;AAEA,SAAS,oBAAoB,OAAiC;AAC5D,SAAO,OAAO,UAAU,YAAY,OAAO,SAAS,KAAK,KAAK,SAAS,KAAK,SAAS;AACvF;AAEA,SAAS,kBAAkB,OAAwB;AACjD,SAAO,OAAO,UAAU,WAAW,OAAO,KAAK,IAAI,OAAO,SAAS,SAAS;AAC9E;AAEA,SAAS,aAAa,OAAwB;AAC5C,SAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAC9D;AAEA,SAAS,mBAAmB,aAAuB,gBAAkC;AACnF,MAAI,YAAY,UAAU,eAAgB,QAAO;AACjD,SAAO;AAAA,IACL,GAAG,YAAY,MAAM,GAAG,cAAc;AAAA,IACtC,GAAG,YAAY,SAAS,cAAc;AAAA,EACxC;AACF;;;AC/JA,IAAM,kBAAkD;AAAA,EACtD,gBAAgB;AAAA,EAChB,gBAAgB;AAAA,EAChB,cAAc;AAAA,EACd,YAAY;AAAA,EACZ,SAAS;AAAA,EACT,WAAW;AAAA,EACX,aAAa;AAAA,EACb,YAAY;AACd;AAEO,SAAS,yBAAyB,SAIf;AACxB,QAAM,YAAY,QAAQ;AAC1B,MAAI,CAAC,OAAO,SAAS,SAAS,KAAK,YAAY,KAAK,YAAY,GAAG;AACjE,UAAM,IAAI;AAAA,MACR,wEAAwE,SAAS;AAAA,IACnF;AAAA,EACF;AACA,QAAM,uBAAuB,QAAQ,wBAAwB;AAC7D,SAAO;AAAA,IACL,IAAI,QAAQ,MAAM,eAAe,SAAS;AAAA,IAC1C,OAAO,OAAO;AACZ,YAAM,aAAa,MAAM,cAAc;AACvC,aAAO;AAAA,QACL,QAAQ,cAAc,YAAY,WAAW;AAAA,QAC7C;AAAA,QACA,YAAY,MAAM;AAAA,QAClB,MAAM,MAAM;AAAA,QACZ,QACE,cAAc,YAAY,gCAAgC;AAAA,MAC9D;AAAA,IACF;AAAA,EACF;AACF;AAEO,SAAS,8BACd,QACA,QACA,UAAgD,CAAC,GACnB;AAC9B,QAAM,UAAU,EAAE,GAAG,iBAAiB,GAAI,QAAQ,WAAW,CAAC,EAAG;AACjE,QAAM,SAAS,OAAO,OAAO,CAAC,UAAU,MAAM,OAAO;AACrD,QAAM,OAAO,QAAQ,QAAQ;AAC7B,QAAM,cAAc,QAAQ,eAAe;AAC3C,QAAM,kBAAkB,QAAQ,mBAAmB;AACnD,QAAM,SAAmB,CAAC;AAC1B,QAAM,kBAA4B,CAAC;AACnC,QAAM,kBAA4B,CAAC;AACnC,MAAI,kBAAkB;AACtB,MAAI,gBAAgB;AACpB,MAAI,WAAW;AACf,MAAI,iBAAiB;AAErB,aAAW,SAAS,QAAQ;AAC1B,UAAM,WAAW,cAAc,OAAO,OAAO;AAC7C,UAAM,WAAW,OAAO,OAAO,KAAK;AACpC,UAAM,YAAY,sBAAsB,OAAO,SAAS,QAAQ,OAAO;AACvE,UAAM,SAASA,UAAS,OAAO,OAAO;AACtC,uBAAmB;AACnB,qBAAiB;AACjB,WAAO,KAAK,YAAY,QAAQ;AAChC,QAAI,SAAS,WAAW,UAAU;AAChC;AACA,sBAAgB,KAAK,MAAM;AAC3B,UAAI,SAAS,EAAG;AAAA,IAClB,OAAO;AACL,sBAAgB,KAAK,MAAM;AAAA,IAC7B;AAAA,EACF;AAEA,QAAM,IAAI,OAAO;AACjB,QAAM,WAAW,KAAK,IAAI,GAAG,IAAI,QAAQ;AACzC,QAAM,KAAK,mBAAmB,QAAQ,MAAM,EAAE,MAAM,QAAQ,QAAQ,GAAG,CAAC;AACxE,QAAM,UAAoB,CAAC;AAC3B,MAAI,IAAI,KAAM,SAAQ,KAAK,iBAAiB,IAAI,0BAA0B,CAAC,EAAE;AAC7E,MAAI,WAAW;AACb,YAAQ,KAAK,iBAAiB,WAAW,4BAA4B,QAAQ,EAAE;AACjF,MAAI,GAAG,SAAS,iBAAiB;AAC/B,YAAQ,KAAK,0BAA0B,GAAG,MAAM,QAAQ,CAAC,CAAC,mBAAmB,eAAe,EAAE;AAAA,EAChG;AACA,QAAM,iBACJ,IAAI,QAAQ,WAAW,cACnB,mBACA,GAAG,QAAQ,kBACT,SACA;AAER,SAAO;AAAA,IACL,UAAU,OAAO;AAAA,IACjB;AAAA,IACA;AAAA,IACA;AAAA,IACA,UAAU,IAAI,IAAI,WAAW,IAAI;AAAA,IACjC,mBAAmB,WAAW,IAAI,iBAAiB,WAAW;AAAA,IAC9D;AAAA,IACA;AAAA,IACA,cAAc,gBAAgB;AAAA,IAC9B,aAAa;AAAA,IACb,oBAAoB,gBAAgB,SAAS,IAAI,KAAK,eAAe,IAAI;AAAA,IACzE;AAAA,IACA;AAAA,EACF;AACF;AAEA,SAAS,cACP,OACA,SACQ;AACR,SACEA,UAAS,OAAO,OAAO,IAAI,QAAQ,cAAc,MAAM,WAAW,MAAM,SAAS,WAAW;AAEhG;AAEA,SAAS,sBACP,OACA,QACA,SACQ;AACR,MAAI,WAAW,SAAU,QAAO,cAAc,OAAO,OAAO;AAC5D,MAAI,WAAW,SAAU,QAAO,QAAQ,eAAe,QAAQ;AAC/D,MAAI,WAAW,MAAO,QAAO,QAAQ,eAAe,QAAQ;AAC5D,MAAI,WAAW,QAAS,QAAO,QAAQ,eAAe,QAAQ;AAC9D,MAAI,WAAW,OAAQ,QAAO,QAAQ;AACtC,SAAO,QAAQ;AACjB;AAEA,SAASA,UAAS,OAA4B,SAAiD;AAC7F,QAAM,UAAU,MAAM;AACtB,MAAI,CAAC,QAAS,QAAO,QAAQ;AAC7B,MAAI,OAAO,QAAQ,WAAW,SAAU,QAAO,IAAI,QAAQ,SAAS;AACpE,MAAI,OAAO,QAAQ,UAAU,SAAU,QAAO,IAAI,QAAQ,QAAQ;AAClE,MAAI,QAAQ,YAAY,KAAM,QAAO,QAAQ;AAC7C,MAAI,QAAQ,YAAY,MAAO,QAAO,QAAQ;AAC9C,SAAO,QAAQ;AACjB;AAEA,SAAS,KAAK,QAA0B;AACtC,SAAO,OAAO,OAAO,CAAC,KAAK,UAAU,MAAM,OAAO,CAAC,IAAI,OAAO;AAChE;;;ACpIO,SAAS,oBACd,SAC8B;AAC9B,QAAM,YAAY,8BAA8B,QAAQ,QAAQ,QAAQ,QAAQ,QAAQ,SAAS;AACjG,QAAM,cAAc,yBAAyB,QAAQ,QAAQ,QAAQ,WAAW;AAChF,QAAM,kBAAkB,QAAQ,KAAK;AACrC,QAAM,MAAM,kBACR,wBAAwB,QAAQ,QAAQ,iBAAiB,QAAQ,GAAG,IACpE;AACJ,QAAM,cAAwB,CAAC;AAC/B,QAAM,kBAAkB,UAAU;AAClC,QAAM,oBAAoB,cAAc,cAAc;AACtD,QAAM,eAAe,QAAQ,eAAe,QAAQ,QAAQ,QAAQ;AACpE,QAAM,YAA6B,MAC/B,IAAI,QAAQ,YACV,cACA,gBACF,eACE,gBACA;AAEN,MAAI,CAAC,YAAa,aAAY,KAAK,8DAA8D;AACjG,MAAI,gBAAgB,CAAC,gBAAiB,aAAY,KAAK,wCAAwC;AAAA,WACtF,OAAO,CAAC,IAAI,QAAQ;AAC3B,gBAAY,KAAK,GAAG,IAAI,QAAQ,QAAQ,IAAI,CAAC,WAAW,oBAAoB,MAAM,EAAE,CAAC;AAEvF,QAAM,SAAS,cAAc;AAAA,IAC3B;AAAA,IACA,gBAAgB,gBAAgB;AAAA,IAChC;AAAA,IACA;AAAA,EACF,CAAC;AAED,SAAO;AAAA,IACL,UAAU,QAAQ,OAAO;AAAA,IACzB,GAAG,QAAQ,OAAO;AAAA,IAClB;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,GAAI,MAAM,EAAE,mBAAmB,IAAI,eAAe,IAAI,CAAC;AAAA,IACvD;AAAA,IACA,GAAI,cAAc,EAAE,YAAY,IAAI,CAAC;AAAA,IACrC,GAAI,MAAM,EAAE,IAAI,IAAI,CAAC;AAAA,IACrB;AAAA,EACF;AACF;AAEA,SAAS,cAAc,SAKI;AACzB,MAAI,QAAQ,oBAAoB,oBAAoB,CAAC,QAAQ,gBAAgB;AAC3E,WAAO;AAAA,EACT;AACA,MAAI,QAAQ,oBAAoB,OAAQ,QAAO;AAC/C,MAAI,QAAQ,gBAAgB,QAAQ,cAAc,YAAa,QAAO;AACtE,SAAO;AACT;","names":["rewardOf"]}

package/dist/benchmarks/index.d.ts

@@ -1,4 +1,4 @@
-export { B as BENCHMARK_SPLIT_SEED, a as BenchmarkAdapter, b as BenchmarkDatasetItem, c as BenchmarkEvaluation, d as deterministicSplit, e as routing } from '../index-DsnOpCO6.js';
-import '../run-record-BgTFzO2r.js';
+export { B as BENCHMARK_SPLIT_SEED, a as BenchmarkAdapter, b as BenchmarkDatasetItem, c as BenchmarkEvaluation, d as deterministicSplit, e as routing } from '../index-B1RKber3.js';
+import '../run-record-sItO5ftF.js';
 import '../errors-Dwqw-T_m.js';
 import '../schema-m0gsnbt3.js';

package/dist/calibration-Cpr3WaX3.d.ts

@@ -0,0 +1,101 @@
+import { T as TraceStore } from './store-CKUAgsJz.js';
+import { R as Run } from './schema-m0gsnbt3.js';
+import { O as OutcomeFilter, b as OutcomeStore } from './outcome-store-rnXLEqSn.js';
+
+/**
+ * Correlation study — "does our eval score predict real-world outcomes?"
+ *
+ * This is the load-bearing signal. Takes a TraceStore + OutcomeStore,
+ * joins on runId, computes Pearson + Spearman + bootstrap CI for every
+ * (evalMetric, outcomeMetric) pair the caller declares.
+ *
+ * Without this number the framework is ornamental. With it and r > 0.6
+ * the framework is a moat — no other agent-eval tool publishes one.
+ */
+
+interface EvalMetricSpec {
+    id: string;
+    /** Extract a scalar from a run (defaults cover score/pass/durationMs/costUsd/tokens). */
+    extract?: (run: Run, store: TraceStore) => Promise<number | null>;
+}
+interface OutcomePair {
+    evalMetric: string;
+    outcomeMetric: string;
+}
+interface CorrelationResult {
+    evalMetric: string;
+    outcomeMetric: string;
+    n: number;
+    pearson: number;
+    spearman: number;
+    /** 95% bootstrap CI for Pearson. */
+    pearsonCi95: {
+        lower: number;
+        upper: number;
+    };
+    /** Rough verdict: 'strong' ≥ 0.7, 'moderate' ≥ 0.4, else 'weak'. */
+    verdict: 'strong' | 'moderate' | 'weak';
+}
+interface CorrelationStudyResult {
+    pairs: CorrelationResult[];
+    joinedSamples: number;
+    skippedRuns: number;
+}
+interface CorrelationStudyOptions {
+    /** Only join outcomes captured within this window after run.startedAt. */
+    maxCaptureLagMs?: number;
+    /** Restrict to a subset of outcomes (cohort, region, source). */
+    outcomeFilter?: OutcomeFilter;
+    /** Which outcome per run to use when multiple exist. Default 'latest'. */
+    reduction?: 'latest' | 'mean' | 'max';
+    /** Bootstrap iterations for the CI. Default 500. */
+    bootstrapIterations?: number;
+}
+declare function correlationStudy(traceStore: TraceStore, outcomeStore: OutcomeStore, evalMetrics: EvalMetricSpec[], outcomeMetricNames: string[], options?: CorrelationStudyOptions): Promise<CorrelationStudyResult>;
+
+/**
+ * Calibration curve — binned "if eval says X, what does reality show?"
+ *
+ * Companion to correlationStudy. Raw correlation is a single number;
+ * the calibration curve shows *where* the eval is well-calibrated vs
+ * overconfident / underconfident. Buckets the eval metric, computes
+ * mean outcome per bucket, reports expected-calibration-error (ECE).
+ */
+
+interface CalibrationBin {
+    lower: number;
+    upper: number;
+    n: number;
+    evalMean: number;
+    outcomeMean: number;
+    /** |outcomeMean − evalMean|; contributes to ECE weighted by n/total. */
+    gap: number;
+}
+interface CalibrationReport {
+    evalMetric: string;
+    outcomeMetric: string;
+    n: number;
+    bins: CalibrationBin[];
+    /** Expected Calibration Error — Σ (n_i/N) × |outcomeMean_i − evalMean_i|. */
+    ece: number;
+    /** Max bin gap — upper bound on miscalibration. */
+    maxGap: number;
+}
+interface CalibrationOptions {
+    bins?: number;
+    /** Equal-width (fixed bin edges) or equal-frequency (quantile bins). */
+    binning?: 'equal-width' | 'equal-frequency';
+    /** Clip eval values to [lo, hi] before binning. */
+    range?: {
+        lo: number;
+        hi: number;
+    };
+}
+interface CalibrationPair {
+    evalScore: number;
+    outcome: number;
+}
+declare function calibrationCurve(traceStore: TraceStore, outcomeStore: OutcomeStore, evalMetric: EvalMetricSpec, outcomeMetric: string, options?: CalibrationOptions): Promise<CalibrationReport | null>;
+declare function calibrationFromPairs(inputPairs: CalibrationPair[], evalMetric: string, outcomeMetric: string, options?: CalibrationOptions): CalibrationReport | null;
+
+export { type CalibrationBin as C, type EvalMetricSpec as E, type OutcomePair as O, type CalibrationOptions as a, type CalibrationPair as b, type CalibrationReport as c, type CorrelationResult as d, type CorrelationStudyOptions as e, type CorrelationStudyResult as f, calibrationCurve as g, calibrationFromPairs as h, correlationStudy as i };