@tangle-network/agent-eval 0.83.0 → 0.84.0

package/dist/belief-state/index.d.ts

@@ -1,8 +1,9 @@
 import { c as CalibrationReport } from '../calibration-Cpr3WaX3.js';
 import { O as OffPolicyEstimate, a as OffPolicyOptions, b as OffPolicyTrajectory } from '../off-policy-DiwuKKg7.js';
 import { d as CodeAgentSessionSource, a as CodeAgentSessionIntakeOptions, c as CodeAgentSessionMetrics, C as CodeAgentSessionDiagnostic } from '../code-agent-session-BRXmavYv.js';
-import { R as RunRecord } from '../run-record-De9VarXR.js';
+import { R as RunRecord, a as RunSplitTag } from '../run-record-De9VarXR.js';
 import { T as TraceStore } from '../store-CKUAgsJz.js';
+import { R as RuntimeTrajectoryRecord, P as ProjectRuntimeTrajectoryEvidenceOptions, a as RuntimeTrajectoryEvidenceProjection } from '../runtime-trajectory-BLRiaifm.js';
 import '../schema-m0gsnbt3.js';
 import '../outcome-store-rnXLEqSn.js';
 import '../errors-Dwqw-T_m.js';
@@ -521,4 +522,82 @@ declare function runtimeDecisionPointToBeliefShadowProbeInput(point: RuntimeBeli
 declare function runtimeDecisionPointToBeliefDecisionPoint(point: RuntimeBeliefDecisionPoint, options: RuntimeBeliefDecisionPointOptions): RuntimeBeliefDecisionPointReport;
 declare function createBeliefRuntimeHookCollector(defaults: RuntimeBeliefShadowProbeInputOptions): BeliefRuntimeHookCollector;
 
-export { type AnalyzeBeliefDecisionCorpusOptions, type AnalyzeBeliefPolicyOpeOptions, type AnalyzeBeliefPolicyOptions, BELIEF_DECISION_KINDS, BELIEF_EVALUATION_CRITERIA, BELIEF_EVIDENCE_QUALITIES, BELIEF_EVIDENCE_SOURCES, type BeliefCalibrationOptions, type BeliefCalibrationRegion, type BeliefCalibrationStatus, type BeliefDecisionCorpusEvaluation, type BeliefDecisionExtractionDiagnostic, type BeliefDecisionExtractionReport, type BeliefDecisionInventoryBucket, type BeliefDecisionInventoryReport, type BeliefDecisionKind, type BeliefDecisionOutcome, type BeliefDecisionPoint, type BeliefDecisionReason, type BeliefDecisionReasonCode, type BeliefDecisionResearchEvidencePacket, type BeliefDecisionTargetSelection, type BeliefEvaluationCriterionId, type BeliefEvaluationStatus, type BeliefEvidenceQuality, type BeliefEvidenceRef, type BeliefEvidenceSource, type BeliefOffPolicyTrajectoryReport, type BeliefOpeOptions, type BeliefOpeReport, type BeliefOpeStatus, type BeliefOpeSupportDiagnostics, type BeliefOpeTargetPolicy, type BeliefPolicyAction, type BeliefPolicyDecision, type BeliefPolicyEvaluationReport, type BeliefResearchClaimScope, type BeliefResearchEvidenceGate, type BeliefResearchEvidenceStatus, type BeliefResearchGateId, type BeliefRuntimeHookCollector, type BeliefSelectivePolicy, type BeliefSelectivePolicyMetrics, type BeliefShadowProbeDiagnostic, type BeliefShadowProbeEvidenceRef, type BeliefShadowProbeInput, type BeliefShadowProbeRecord, type BeliefShadowProbeResponse, type BeliefShadowProbeRun, type BeliefShadowProbeSummary, type BeliefUtilityOptions, type BuildBeliefDecisionResearchEvidencePacketOptions, type BuildCodeAgentBeliefEvidenceCorpusOptions, type CodeAgentBeliefDecisionTargetId, type CodeAgentBeliefEvidenceCorpus, type CodeAgentBeliefSession, type EvaluateBeliefSelectivePolicyOptions, type ExtractBeliefDecisionPointsOptions, type ExtractCodeAgentBeliefDecisionPointsOptions, type RunBeliefShadowProbeOptions, type RuntimeBeliefConversionDiagnostic, type RuntimeBeliefDecisionEvidenceRef, type RuntimeBeliefDecisionPoint, type RuntimeBeliefDecisionPointOptions, type RuntimeBeliefDecisionPointReport, type RuntimeBeliefHookContext, type RuntimeBeliefHookEvent, type RuntimeBeliefHooks, type RuntimeBeliefShadowProbeInputOptions, type RuntimeBeliefShadowProbeInputReport, type SelectBeliefDecisionTargetOptions, analyzeBeliefDecisionCorpus, analyzeBeliefPolicy, beliefDecisionsToOffPolicyTrajectories, buildBeliefDecisionResearchEvidencePacket, buildCodeAgentBeliefEvidenceCorpus, calibrateBeliefDecisions, createBeliefRuntimeHookCollector, embeddedBeliefOpeTargetPolicy, evaluateBeliefOffPolicy, evaluateBeliefSelectivePolicy, extractBeliefDecisionPoints, extractCodeAgentBeliefDecisionPoints, formatBeliefShadowProbePrompt, inventoryBeliefDecisionPoints, isBeliefDecisionKind, isBeliefEvidenceSource, runBeliefShadowProbe, runtimeDecisionPointToBeliefDecisionPoint, runtimeDecisionPointToBeliefShadowProbeInput, selectBeliefDecisionTarget, thresholdSelectivePolicy };
+interface RuntimeBeliefPhase0RunRecord {
+    runId: string;
+    scenarioId?: string;
+    splitTag: RunSplitTag;
+}
+interface RuntimeBeliefDecisionLabel {
+    decisionId: string;
+    chosenAction: string;
+    outcome: BeliefDecisionOutcome;
+    confidence?: number;
+    behaviorProb?: number;
+    targetProb?: number;
+    qHat?: number | null;
+    costUsd?: number;
+    splitTag?: RunSplitTag;
+    metadata?: Record<string, unknown>;
+}
+interface BuildRuntimeBeliefPhase0MeasurementOptions extends Omit<BuildBeliefDecisionResearchEvidencePacketOptions, 'points'> {
+    runs: RuntimeBeliefPhase0RunRecord[];
+    decisions: RuntimeBeliefDecisionPoint[];
+    events?: RuntimeBeliefHookEvent[];
+    labels: RuntimeBeliefDecisionLabel[];
+    baselinePolicyId?: string;
+}
+interface RuntimeBeliefPhase0MeasurementSummary {
+    runCount: number;
+    producerDecisionCount: number;
+    lifecycleEventCount: number;
+    labelCount: number;
+    completedPointCount: number;
+    runJoinRate: number;
+    labelJoinRate: number;
+    missingRunRecordCount: number;
+    missingLabelCount: number;
+    withEvidence: number;
+    withOutcome: number;
+    withSplit: number;
+    withBehaviorProb: number;
+    withTargetProb: number;
+    baselinePolicyId: string;
+    packetStatus: BeliefDecisionResearchEvidencePacket['status'];
+    claimScope: BeliefDecisionResearchEvidencePacket['claimScope'];
+}
+interface RuntimeBeliefPhase0Measurement {
+    points: BeliefDecisionPoint[];
+    packet: BeliefDecisionResearchEvidencePacket;
+    summary: RuntimeBeliefPhase0MeasurementSummary;
+    diagnostics: string[];
+}
+declare function buildRuntimeBeliefPhase0Measurement(options: BuildRuntimeBeliefPhase0MeasurementOptions): RuntimeBeliefPhase0Measurement;
+
+type RuntimeBenchmarkTrajectoryRecord = RuntimeTrajectoryRecord & {
+    benchmark?: unknown;
+    condition?: unknown;
+    instanceId?: unknown;
+};
+interface BuildRuntimeBenchmarkBeliefPhase0MeasurementOptions extends Omit<BuildRuntimeBeliefPhase0MeasurementOptions, 'runs' | 'events' | 'decisions' | 'labels'> {
+    records: RuntimeBenchmarkTrajectoryRecord[];
+    decisions?: RuntimeBeliefDecisionPoint[];
+    defaultSplitTag?: ProjectRuntimeTrajectoryEvidenceOptions['defaultSplitTag'];
+    labels?: RuntimeBeliefDecisionLabel[];
+}
+interface RuntimeBenchmarkBeliefPhase0Summary {
+    decisionCount: number;
+    labelCount: number;
+}
+interface RuntimeBenchmarkBeliefPhase0Measurement {
+    runs: RuntimeBeliefPhase0RunRecord[];
+    events: RuntimeBeliefHookEvent[];
+    decisions: RuntimeBeliefDecisionPoint[];
+    labels: RuntimeBeliefDecisionLabel[];
+    trajectory: RuntimeTrajectoryEvidenceProjection;
+    measurement: RuntimeBeliefPhase0Measurement;
+    summary: RuntimeBenchmarkBeliefPhase0Summary;
+    diagnostics: string[];
+}
+declare function buildRuntimeBenchmarkBeliefPhase0Measurement(options: BuildRuntimeBenchmarkBeliefPhase0MeasurementOptions): RuntimeBenchmarkBeliefPhase0Measurement;
+
+export { type AnalyzeBeliefDecisionCorpusOptions, type AnalyzeBeliefPolicyOpeOptions, type AnalyzeBeliefPolicyOptions, BELIEF_DECISION_KINDS, BELIEF_EVALUATION_CRITERIA, BELIEF_EVIDENCE_QUALITIES, BELIEF_EVIDENCE_SOURCES, type BeliefCalibrationOptions, type BeliefCalibrationRegion, type BeliefCalibrationStatus, type BeliefDecisionCorpusEvaluation, type BeliefDecisionExtractionDiagnostic, type BeliefDecisionExtractionReport, type BeliefDecisionInventoryBucket, type BeliefDecisionInventoryReport, type BeliefDecisionKind, type BeliefDecisionOutcome, type BeliefDecisionPoint, type BeliefDecisionReason, type BeliefDecisionReasonCode, type BeliefDecisionResearchEvidencePacket, type BeliefDecisionTargetSelection, type BeliefEvaluationCriterionId, type BeliefEvaluationStatus, type BeliefEvidenceQuality, type BeliefEvidenceRef, type BeliefEvidenceSource, type BeliefOffPolicyTrajectoryReport, type BeliefOpeOptions, type BeliefOpeReport, type BeliefOpeStatus, type BeliefOpeSupportDiagnostics, type BeliefOpeTargetPolicy, type BeliefPolicyAction, type BeliefPolicyDecision, type BeliefPolicyEvaluationReport, type BeliefResearchClaimScope, type BeliefResearchEvidenceGate, type BeliefResearchEvidenceStatus, type BeliefResearchGateId, type BeliefRuntimeHookCollector, type BeliefSelectivePolicy, type BeliefSelectivePolicyMetrics, type BeliefShadowProbeDiagnostic, type BeliefShadowProbeEvidenceRef, type BeliefShadowProbeInput, type BeliefShadowProbeRecord, type BeliefShadowProbeResponse, type BeliefShadowProbeRun, type BeliefShadowProbeSummary, type BeliefUtilityOptions, type BuildBeliefDecisionResearchEvidencePacketOptions, type BuildCodeAgentBeliefEvidenceCorpusOptions, type BuildRuntimeBeliefPhase0MeasurementOptions, type BuildRuntimeBenchmarkBeliefPhase0MeasurementOptions, type CodeAgentBeliefDecisionTargetId, type CodeAgentBeliefEvidenceCorpus, type CodeAgentBeliefSession, type EvaluateBeliefSelectivePolicyOptions, type ExtractBeliefDecisionPointsOptions, type ExtractCodeAgentBeliefDecisionPointsOptions, type RunBeliefShadowProbeOptions, type RuntimeBeliefConversionDiagnostic, type RuntimeBeliefDecisionEvidenceRef, type RuntimeBeliefDecisionLabel, type RuntimeBeliefDecisionPoint, type RuntimeBeliefDecisionPointOptions, type RuntimeBeliefDecisionPointReport, type RuntimeBeliefHookContext, type RuntimeBeliefHookEvent, type RuntimeBeliefHooks, type RuntimeBeliefPhase0Measurement, type RuntimeBeliefPhase0MeasurementSummary, type RuntimeBeliefPhase0RunRecord, type RuntimeBeliefShadowProbeInputOptions, type RuntimeBeliefShadowProbeInputReport, type RuntimeBenchmarkBeliefPhase0Measurement, type RuntimeBenchmarkBeliefPhase0Summary, type SelectBeliefDecisionTargetOptions, analyzeBeliefDecisionCorpus, analyzeBeliefPolicy, beliefDecisionsToOffPolicyTrajectories, buildBeliefDecisionResearchEvidencePacket, buildCodeAgentBeliefEvidenceCorpus, buildRuntimeBeliefPhase0Measurement, buildRuntimeBenchmarkBeliefPhase0Measurement, calibrateBeliefDecisions, createBeliefRuntimeHookCollector, embeddedBeliefOpeTargetPolicy, evaluateBeliefOffPolicy, evaluateBeliefSelectivePolicy, extractBeliefDecisionPoints, extractCodeAgentBeliefDecisionPoints, formatBeliefShadowProbePrompt, inventoryBeliefDecisionPoints, isBeliefDecisionKind, isBeliefEvidenceSource, runBeliefShadowProbe, runtimeDecisionPointToBeliefDecisionPoint, runtimeDecisionPointToBeliefShadowProbeInput, selectBeliefDecisionTarget, thresholdSelectivePolicy };

package/dist/belief-state/index.js

@@ -8,6 +8,9 @@ import {
 import {
   calibrationFromPairs
 } from "../chunk-NPCTHQIO.js";
+import {
+  projectRuntimeTrajectoryEvidence
+} from "../chunk-T4SQEITX.js";
 import "../chunk-VI2UW6B6.js";
 import {
   offPolicyEstimateAll
@@ -127,7 +130,7 @@ function evaluateBeliefOffPolicy(points, targetPolicy, options = {}) {
   return { targetPolicyId: targetPolicy.id, ...estimates, support };
 }
 function supportDiagnostics(estimate, options) {
-  const ratio = estimate.n > 0 ? estimate.effectiveSampleSize / estimate.n : 0;
+  const ratio2 = estimate.n > 0 ? estimate.effectiveSampleSize / estimate.n : 0;
   const reasons = [...options.diagnostics];
   if (estimate.n === 0) {
     reasons.push("no valid OPE trajectories");
@@ -140,9 +143,9 @@ function supportDiagnostics(estimate, options) {
       `effective sample size ${estimate.effectiveSampleSize.toFixed(2)} below ${options.minEffectiveSampleSize}`
     );
   }
-  if (ratio < options.minEffectiveSampleRatio) {
+  if (ratio2 < options.minEffectiveSampleRatio) {
     reasons.push(
-      `effective sample ratio ${ratio.toFixed(2)} below ${options.minEffectiveSampleRatio}`
+      `effective sample ratio ${ratio2.toFixed(2)} below ${options.minEffectiveSampleRatio}`
     );
   }
   if (estimate.maxImportanceWeight > 10) {
@@ -153,7 +156,7 @@ function supportDiagnostics(estimate, options) {
     n: estimate.n,
     dropped: options.dropped,
     effectiveSampleSize: estimate.effectiveSampleSize,
-    effectiveSampleRatio: ratio,
+    effectiveSampleRatio: ratio2,
     maxImportanceWeight: estimate.maxImportanceWeight,
     reasons
   };
@@ -1622,6 +1625,157 @@ function nonNegativeNumberOrUndefined(value) {
   return number !== void 0 && number >= 0 ? number : void 0;
 }
 
+// src/belief-state/phase0-measurement.ts
+var DEFAULT_BASELINE_POLICY_ID = "always-accept-observed-action";
+function buildRuntimeBeliefPhase0Measurement(options) {
+  const runsById = new Map(options.runs.map((run) => [run.runId, run]));
+  const labelsByDecisionId = /* @__PURE__ */ new Map();
+  const diagnostics = [];
+  for (const label of options.labels) {
+    if (labelsByDecisionId.has(label.decisionId)) {
+      diagnostics.push(`${label.decisionId}: duplicate label; using the last label`);
+    }
+    labelsByDecisionId.set(label.decisionId, label);
+  }
+  const points = [];
+  let missingRunRecordCount = 0;
+  let missingLabelCount = 0;
+  for (const decision of options.decisions) {
+    const run = runsById.get(decision.runId);
+    if (!run) {
+      missingRunRecordCount += 1;
+      diagnostics.push(`${decision.id}: missing RunRecord join for runId ${decision.runId}`);
+      continue;
+    }
+    const label = labelsByDecisionId.get(decision.id);
+    if (!label) {
+      missingLabelCount += 1;
+      diagnostics.push(`${decision.id}: missing observed action/outcome label`);
+      continue;
+    }
+    const splitTag = label.splitTag ?? run.splitTag;
+    const report = runtimeDecisionPointToBeliefDecisionPoint(
+      { ...decision, scenarioId: decision.scenarioId ?? run.scenarioId },
+      {
+        chosenAction: label.chosenAction,
+        confidence: label.confidence,
+        behaviorProb: label.behaviorProb,
+        targetProb: label.targetProb,
+        qHat: label.qHat,
+        costUsd: label.costUsd,
+        outcome: label.outcome,
+        lifecycleEvents: options.events,
+        metadata: compactMetadata2({
+          baselinePolicyId: options.baselinePolicyId ?? DEFAULT_BASELINE_POLICY_ID,
+          splitTag,
+          ...label.metadata
+        })
+      }
+    );
+    diagnostics.push(...report.diagnostics.map((item) => `${item.decisionId}: ${item.reason}`));
+    if (report.point) points.push(report.point);
+  }
+  const packet = buildBeliefDecisionResearchEvidencePacket({
+    ...options,
+    points
+  });
+  return {
+    points,
+    packet,
+    summary: summarizePhase0Measurement(options, points, packet, {
+      missingRunRecordCount,
+      missingLabelCount
+    }),
+    diagnostics
+  };
+}
+function summarizePhase0Measurement(options, points, packet, counts) {
+  const producerDecisionCount = options.decisions.length;
+  return {
+    runCount: options.runs.length,
+    producerDecisionCount,
+    lifecycleEventCount: options.events?.length ?? 0,
+    labelCount: options.labels.length,
+    completedPointCount: points.length,
+    runJoinRate: ratio(producerDecisionCount - counts.missingRunRecordCount, producerDecisionCount),
+    labelJoinRate: ratio(points.length, producerDecisionCount),
+    missingRunRecordCount: counts.missingRunRecordCount,
+    missingLabelCount: counts.missingLabelCount,
+    withEvidence: points.filter((point) => point.evidence.length > 0).length,
+    withOutcome: points.filter((point) => point.outcome).length,
+    withSplit: points.filter((point) => typeof point.metadata?.splitTag === "string").length,
+    withBehaviorProb: points.filter((point) => point.behaviorProb !== void 0).length,
+    withTargetProb: points.filter((point) => point.targetProb !== void 0).length,
+    baselinePolicyId: options.baselinePolicyId ?? DEFAULT_BASELINE_POLICY_ID,
+    packetStatus: packet.status,
+    claimScope: packet.claimScope
+  };
+}
+function ratio(numerator, denominator) {
+  return denominator > 0 ? numerator / denominator : 0;
+}
+function compactMetadata2(values) {
+  const entries = Object.entries(values).filter(([, value]) => value !== void 0);
+  return entries.length > 0 ? Object.fromEntries(entries) : void 0;
+}
+
+// src/belief-state/runtime-benchmark-corpus.ts
+function buildRuntimeBenchmarkBeliefPhase0Measurement(options) {
+  const diagnostics = [];
+  const trajectory = projectRuntimeTrajectoryEvidence({
+    records: options.records,
+    defaultSplitTag: options.defaultSplitTag,
+    recordIdOf: runtimeBenchmarkRecordId,
+    scenarioIdOf: runtimeBenchmarkScenarioId
+  });
+  const decisions = options.decisions ?? [];
+  const labels = options.labels ?? [];
+  if (decisions.length === 0) {
+    diagnostics.push(
+      "no runtime decision points supplied; benchmark lifecycle events alone cannot produce belief decision rows"
+    );
+  }
+  if (labels.length === 0 && decisions.length > 0) {
+    diagnostics.push(
+      "no decision labels supplied; observed action/outcome joins will be incomplete"
+    );
+  }
+  const measurement = buildRuntimeBeliefPhase0Measurement({
+    ...options,
+    runs: trajectory.runs,
+    events: trajectory.events,
+    decisions,
+    labels
+  });
+  return {
+    runs: trajectory.runs,
+    events: trajectory.events,
+    decisions,
+    labels,
+    trajectory,
+    measurement,
+    summary: {
+      decisionCount: decisions.length,
+      labelCount: labels.length
+    },
+    diagnostics: [...trajectory.diagnostics, ...diagnostics, ...measurement.diagnostics]
+  };
+}
+function runtimeBenchmarkRecordId(record2) {
+  const parts = [
+    nonEmptyString(record2.benchmark),
+    nonEmptyString(record2.instanceId),
+    nonEmptyString(record2.condition)
+  ].filter((part) => part !== void 0);
+  return parts.length > 0 ? parts.join(":") : void 0;
+}
+function runtimeBenchmarkScenarioId(record2) {
+  return nonEmptyString(record2.instanceId);
+}
+function nonEmptyString(value) {
+  return typeof value === "string" && value.length > 0 ? value : void 0;
+}
+
 // src/belief-state/shadow-probe.ts
 var DEFAULT_CONCURRENCY = 4;
 var DEFAULT_MAX_CONTEXT_CHARS2 = 12e3;
@@ -1842,6 +1996,8 @@ export {
   beliefDecisionsToOffPolicyTrajectories,
   buildBeliefDecisionResearchEvidencePacket,
   buildCodeAgentBeliefEvidenceCorpus,
+  buildRuntimeBeliefPhase0Measurement,
+  buildRuntimeBenchmarkBeliefPhase0Measurement,
   calibrateBeliefDecisions,
   createBeliefRuntimeHookCollector,
   embeddedBeliefOpeTargetPolicy,