@tangle-network/agent-eval 0.82.0 → 0.84.0

package/CHANGELOG.md

@@ -4,6 +4,14 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval-
 
 ---
 
+## [0.83.0] — 2026-06-05 — hostedTenantFromEnv
+
+### Added
+
+- **`hostedTenantFromEnv` (`/hosted`).** Builds a `HostedTenant` config from env (the input `selfImprove({ hostedTenant })` and `emitLoopProvenance` take), with the same env precedence + overrides as `hostedClientFromEnv` — which now composes it. Returns `undefined` (not an error) when unconfigured, so a product wires `hostedTenant: hostedTenantFromEnv({ tenantId: 'my-agent' })` unconditionally and hosted ingest stays off until the env is set. Removes the env→tenant mapping every product would otherwise hand-roll when collapsing onto `selfImprove`.
+
+---
+
 ## [0.82.0] — 2026-06-05 — selfImprove forwards the full loop surface
 
 ### Changed
@@ -21,7 +29,7 @@ A deterministic offline test that drives `selfImprove` with a mock agent must no
 
 ### Added
 
-- **`aggregateJudgeVerdicts<D>` (root).** Generic judge-ensemble reducer: fan out N uncorrelated judges, mean each rubric dimension over the SURVIVORS, report the inter-rater disagreement spread, sum cost. Replaces the same reduction hand-rolled in legal (`aggregateEnsemble`), creative (`production-loop/judges.ts`), and tax (`judge-ensemble.ts`). Fail-loud: a failed judge (`perDimension: null`) is recorded in `failedJudges`, never folded into a zero; all-failed throws; a failed judge's cost is still summed. Composite reuses `weightedComposite`.
+- **`aggregateJudgeVerdicts<D>` (root).** Generic judge-ensemble reducer: fan out N uncorrelated judges, mean each rubric dimension over the SURVIVORS, report the inter-rater disagreement spread, sum cost. Replaces the same reduction hand-rolled across multiple product agents. Fail-loud: a failed judge (`perDimension: null`) is recorded in `failedJudges`, never folded into a zero; all-failed throws; a failed judge's cost is still summed. Composite reuses `weightedComposite`.
 - **`createTokenRecallChecker` (root).** The deterministic, no-LLM `CorrectnessChecker` — sibling of `createLlmCorrectnessChecker`. A produced item fulfils a requirement when its content is substantive and recalls ≥ `minRecall` of the requirement title's significant tokens. The default completion gate for apps/tests without an LLM judge.
 - **`ErrorCluster` (root + `/analyst`).** The failure-cluster element type is now a named export, so consumers import it instead of deriving `DatasetOverview['error_clusters'][number]`.
 

package/dist/belief-state/index.d.ts

@@ -1,8 +1,9 @@
 import { c as CalibrationReport } from '../calibration-Cpr3WaX3.js';
 import { O as OffPolicyEstimate, a as OffPolicyOptions, b as OffPolicyTrajectory } from '../off-policy-DiwuKKg7.js';
 import { d as CodeAgentSessionSource, a as CodeAgentSessionIntakeOptions, c as CodeAgentSessionMetrics, C as CodeAgentSessionDiagnostic } from '../code-agent-session-BRXmavYv.js';
-import { R as RunRecord } from '../run-record-De9VarXR.js';
+import { R as RunRecord, a as RunSplitTag } from '../run-record-De9VarXR.js';
 import { T as TraceStore } from '../store-CKUAgsJz.js';
+import { R as RuntimeTrajectoryRecord, P as ProjectRuntimeTrajectoryEvidenceOptions, a as RuntimeTrajectoryEvidenceProjection } from '../runtime-trajectory-BLRiaifm.js';
 import '../schema-m0gsnbt3.js';
 import '../outcome-store-rnXLEqSn.js';
 import '../errors-Dwqw-T_m.js';
@@ -521,4 +522,82 @@ declare function runtimeDecisionPointToBeliefShadowProbeInput(point: RuntimeBeli
 declare function runtimeDecisionPointToBeliefDecisionPoint(point: RuntimeBeliefDecisionPoint, options: RuntimeBeliefDecisionPointOptions): RuntimeBeliefDecisionPointReport;
 declare function createBeliefRuntimeHookCollector(defaults: RuntimeBeliefShadowProbeInputOptions): BeliefRuntimeHookCollector;
 
-export { type AnalyzeBeliefDecisionCorpusOptions, type AnalyzeBeliefPolicyOpeOptions, type AnalyzeBeliefPolicyOptions, BELIEF_DECISION_KINDS, BELIEF_EVALUATION_CRITERIA, BELIEF_EVIDENCE_QUALITIES, BELIEF_EVIDENCE_SOURCES, type BeliefCalibrationOptions, type BeliefCalibrationRegion, type BeliefCalibrationStatus, type BeliefDecisionCorpusEvaluation, type BeliefDecisionExtractionDiagnostic, type BeliefDecisionExtractionReport, type BeliefDecisionInventoryBucket, type BeliefDecisionInventoryReport, type BeliefDecisionKind, type BeliefDecisionOutcome, type BeliefDecisionPoint, type BeliefDecisionReason, type BeliefDecisionReasonCode, type BeliefDecisionResearchEvidencePacket, type BeliefDecisionTargetSelection, type BeliefEvaluationCriterionId, type BeliefEvaluationStatus, type BeliefEvidenceQuality, type BeliefEvidenceRef, type BeliefEvidenceSource, type BeliefOffPolicyTrajectoryReport, type BeliefOpeOptions, type BeliefOpeReport, type BeliefOpeStatus, type BeliefOpeSupportDiagnostics, type BeliefOpeTargetPolicy, type BeliefPolicyAction, type BeliefPolicyDecision, type BeliefPolicyEvaluationReport, type BeliefResearchClaimScope, type BeliefResearchEvidenceGate, type BeliefResearchEvidenceStatus, type BeliefResearchGateId, type BeliefRuntimeHookCollector, type BeliefSelectivePolicy, type BeliefSelectivePolicyMetrics, type BeliefShadowProbeDiagnostic, type BeliefShadowProbeEvidenceRef, type BeliefShadowProbeInput, type BeliefShadowProbeRecord, type BeliefShadowProbeResponse, type BeliefShadowProbeRun, type BeliefShadowProbeSummary, type BeliefUtilityOptions, type BuildBeliefDecisionResearchEvidencePacketOptions, type BuildCodeAgentBeliefEvidenceCorpusOptions, type CodeAgentBeliefDecisionTargetId, type CodeAgentBeliefEvidenceCorpus, type CodeAgentBeliefSession, type EvaluateBeliefSelectivePolicyOptions, type ExtractBeliefDecisionPointsOptions, type ExtractCodeAgentBeliefDecisionPointsOptions, type RunBeliefShadowProbeOptions, type RuntimeBeliefConversionDiagnostic, type RuntimeBeliefDecisionEvidenceRef, type RuntimeBeliefDecisionPoint, type RuntimeBeliefDecisionPointOptions, type RuntimeBeliefDecisionPointReport, type RuntimeBeliefHookContext, type RuntimeBeliefHookEvent, type RuntimeBeliefHooks, type RuntimeBeliefShadowProbeInputOptions, type RuntimeBeliefShadowProbeInputReport, type SelectBeliefDecisionTargetOptions, analyzeBeliefDecisionCorpus, analyzeBeliefPolicy, beliefDecisionsToOffPolicyTrajectories, buildBeliefDecisionResearchEvidencePacket, buildCodeAgentBeliefEvidenceCorpus, calibrateBeliefDecisions, createBeliefRuntimeHookCollector, embeddedBeliefOpeTargetPolicy, evaluateBeliefOffPolicy, evaluateBeliefSelectivePolicy, extractBeliefDecisionPoints, extractCodeAgentBeliefDecisionPoints, formatBeliefShadowProbePrompt, inventoryBeliefDecisionPoints, isBeliefDecisionKind, isBeliefEvidenceSource, runBeliefShadowProbe, runtimeDecisionPointToBeliefDecisionPoint, runtimeDecisionPointToBeliefShadowProbeInput, selectBeliefDecisionTarget, thresholdSelectivePolicy };
+interface RuntimeBeliefPhase0RunRecord {
+    runId: string;
+    scenarioId?: string;
+    splitTag: RunSplitTag;
+}
+interface RuntimeBeliefDecisionLabel {
+    decisionId: string;
+    chosenAction: string;
+    outcome: BeliefDecisionOutcome;
+    confidence?: number;
+    behaviorProb?: number;
+    targetProb?: number;
+    qHat?: number | null;
+    costUsd?: number;
+    splitTag?: RunSplitTag;
+    metadata?: Record<string, unknown>;
+}
+interface BuildRuntimeBeliefPhase0MeasurementOptions extends Omit<BuildBeliefDecisionResearchEvidencePacketOptions, 'points'> {
+    runs: RuntimeBeliefPhase0RunRecord[];
+    decisions: RuntimeBeliefDecisionPoint[];
+    events?: RuntimeBeliefHookEvent[];
+    labels: RuntimeBeliefDecisionLabel[];
+    baselinePolicyId?: string;
+}
+interface RuntimeBeliefPhase0MeasurementSummary {
+    runCount: number;
+    producerDecisionCount: number;
+    lifecycleEventCount: number;
+    labelCount: number;
+    completedPointCount: number;
+    runJoinRate: number;
+    labelJoinRate: number;
+    missingRunRecordCount: number;
+    missingLabelCount: number;
+    withEvidence: number;
+    withOutcome: number;
+    withSplit: number;
+    withBehaviorProb: number;
+    withTargetProb: number;
+    baselinePolicyId: string;
+    packetStatus: BeliefDecisionResearchEvidencePacket['status'];
+    claimScope: BeliefDecisionResearchEvidencePacket['claimScope'];
+}
+interface RuntimeBeliefPhase0Measurement {
+    points: BeliefDecisionPoint[];
+    packet: BeliefDecisionResearchEvidencePacket;
+    summary: RuntimeBeliefPhase0MeasurementSummary;
+    diagnostics: string[];
+}
+declare function buildRuntimeBeliefPhase0Measurement(options: BuildRuntimeBeliefPhase0MeasurementOptions): RuntimeBeliefPhase0Measurement;
+
+type RuntimeBenchmarkTrajectoryRecord = RuntimeTrajectoryRecord & {
+    benchmark?: unknown;
+    condition?: unknown;
+    instanceId?: unknown;
+};
+interface BuildRuntimeBenchmarkBeliefPhase0MeasurementOptions extends Omit<BuildRuntimeBeliefPhase0MeasurementOptions, 'runs' | 'events' | 'decisions' | 'labels'> {
+    records: RuntimeBenchmarkTrajectoryRecord[];
+    decisions?: RuntimeBeliefDecisionPoint[];
+    defaultSplitTag?: ProjectRuntimeTrajectoryEvidenceOptions['defaultSplitTag'];
+    labels?: RuntimeBeliefDecisionLabel[];
+}
+interface RuntimeBenchmarkBeliefPhase0Summary {
+    decisionCount: number;
+    labelCount: number;
+}
+interface RuntimeBenchmarkBeliefPhase0Measurement {
+    runs: RuntimeBeliefPhase0RunRecord[];
+    events: RuntimeBeliefHookEvent[];
+    decisions: RuntimeBeliefDecisionPoint[];
+    labels: RuntimeBeliefDecisionLabel[];
+    trajectory: RuntimeTrajectoryEvidenceProjection;
+    measurement: RuntimeBeliefPhase0Measurement;
+    summary: RuntimeBenchmarkBeliefPhase0Summary;
+    diagnostics: string[];
+}
+declare function buildRuntimeBenchmarkBeliefPhase0Measurement(options: BuildRuntimeBenchmarkBeliefPhase0MeasurementOptions): RuntimeBenchmarkBeliefPhase0Measurement;
+
+export { type AnalyzeBeliefDecisionCorpusOptions, type AnalyzeBeliefPolicyOpeOptions, type AnalyzeBeliefPolicyOptions, BELIEF_DECISION_KINDS, BELIEF_EVALUATION_CRITERIA, BELIEF_EVIDENCE_QUALITIES, BELIEF_EVIDENCE_SOURCES, type BeliefCalibrationOptions, type BeliefCalibrationRegion, type BeliefCalibrationStatus, type BeliefDecisionCorpusEvaluation, type BeliefDecisionExtractionDiagnostic, type BeliefDecisionExtractionReport, type BeliefDecisionInventoryBucket, type BeliefDecisionInventoryReport, type BeliefDecisionKind, type BeliefDecisionOutcome, type BeliefDecisionPoint, type BeliefDecisionReason, type BeliefDecisionReasonCode, type BeliefDecisionResearchEvidencePacket, type BeliefDecisionTargetSelection, type BeliefEvaluationCriterionId, type BeliefEvaluationStatus, type BeliefEvidenceQuality, type BeliefEvidenceRef, type BeliefEvidenceSource, type BeliefOffPolicyTrajectoryReport, type BeliefOpeOptions, type BeliefOpeReport, type BeliefOpeStatus, type BeliefOpeSupportDiagnostics, type BeliefOpeTargetPolicy, type BeliefPolicyAction, type BeliefPolicyDecision, type BeliefPolicyEvaluationReport, type BeliefResearchClaimScope, type BeliefResearchEvidenceGate, type BeliefResearchEvidenceStatus, type BeliefResearchGateId, type BeliefRuntimeHookCollector, type BeliefSelectivePolicy, type BeliefSelectivePolicyMetrics, type BeliefShadowProbeDiagnostic, type BeliefShadowProbeEvidenceRef, type BeliefShadowProbeInput, type BeliefShadowProbeRecord, type BeliefShadowProbeResponse, type BeliefShadowProbeRun, type BeliefShadowProbeSummary, type BeliefUtilityOptions, type BuildBeliefDecisionResearchEvidencePacketOptions, type BuildCodeAgentBeliefEvidenceCorpusOptions, type BuildRuntimeBeliefPhase0MeasurementOptions, type BuildRuntimeBenchmarkBeliefPhase0MeasurementOptions, type CodeAgentBeliefDecisionTargetId, type CodeAgentBeliefEvidenceCorpus, type CodeAgentBeliefSession, type EvaluateBeliefSelectivePolicyOptions, type ExtractBeliefDecisionPointsOptions, type ExtractCodeAgentBeliefDecisionPointsOptions, type RunBeliefShadowProbeOptions, type RuntimeBeliefConversionDiagnostic, type RuntimeBeliefDecisionEvidenceRef, type RuntimeBeliefDecisionLabel, type RuntimeBeliefDecisionPoint, type RuntimeBeliefDecisionPointOptions, type RuntimeBeliefDecisionPointReport, type RuntimeBeliefHookContext, type RuntimeBeliefHookEvent, type RuntimeBeliefHooks, type RuntimeBeliefPhase0Measurement, type RuntimeBeliefPhase0MeasurementSummary, type RuntimeBeliefPhase0RunRecord, type RuntimeBeliefShadowProbeInputOptions, type RuntimeBeliefShadowProbeInputReport, type RuntimeBenchmarkBeliefPhase0Measurement, type RuntimeBenchmarkBeliefPhase0Summary, type SelectBeliefDecisionTargetOptions, analyzeBeliefDecisionCorpus, analyzeBeliefPolicy, beliefDecisionsToOffPolicyTrajectories, buildBeliefDecisionResearchEvidencePacket, buildCodeAgentBeliefEvidenceCorpus, buildRuntimeBeliefPhase0Measurement, buildRuntimeBenchmarkBeliefPhase0Measurement, calibrateBeliefDecisions, createBeliefRuntimeHookCollector, embeddedBeliefOpeTargetPolicy, evaluateBeliefOffPolicy, evaluateBeliefSelectivePolicy, extractBeliefDecisionPoints, extractCodeAgentBeliefDecisionPoints, formatBeliefShadowProbePrompt, inventoryBeliefDecisionPoints, isBeliefDecisionKind, isBeliefEvidenceSource, runBeliefShadowProbe, runtimeDecisionPointToBeliefDecisionPoint, runtimeDecisionPointToBeliefShadowProbeInput, selectBeliefDecisionTarget, thresholdSelectivePolicy };

package/dist/belief-state/index.js

@@ -8,6 +8,9 @@ import {
 import {
   calibrationFromPairs
 } from "../chunk-NPCTHQIO.js";
+import {
+  projectRuntimeTrajectoryEvidence
+} from "../chunk-T4SQEITX.js";
 import "../chunk-VI2UW6B6.js";
 import {
   offPolicyEstimateAll
@@ -127,7 +130,7 @@ function evaluateBeliefOffPolicy(points, targetPolicy, options = {}) {
   return { targetPolicyId: targetPolicy.id, ...estimates, support };
 }
 function supportDiagnostics(estimate, options) {
-  const ratio = estimate.n > 0 ? estimate.effectiveSampleSize / estimate.n : 0;
+  const ratio2 = estimate.n > 0 ? estimate.effectiveSampleSize / estimate.n : 0;
   const reasons = [...options.diagnostics];
   if (estimate.n === 0) {
     reasons.push("no valid OPE trajectories");
@@ -140,9 +143,9 @@ function supportDiagnostics(estimate, options) {
       `effective sample size ${estimate.effectiveSampleSize.toFixed(2)} below ${options.minEffectiveSampleSize}`
     );
   }
-  if (ratio < options.minEffectiveSampleRatio) {
+  if (ratio2 < options.minEffectiveSampleRatio) {
     reasons.push(
-      `effective sample ratio ${ratio.toFixed(2)} below ${options.minEffectiveSampleRatio}`
+      `effective sample ratio ${ratio2.toFixed(2)} below ${options.minEffectiveSampleRatio}`
     );
   }
   if (estimate.maxImportanceWeight > 10) {
@@ -153,7 +156,7 @@ function supportDiagnostics(estimate, options) {
     n: estimate.n,
     dropped: options.dropped,
     effectiveSampleSize: estimate.effectiveSampleSize,
-    effectiveSampleRatio: ratio,
+    effectiveSampleRatio: ratio2,
     maxImportanceWeight: estimate.maxImportanceWeight,
     reasons
   };
@@ -1622,6 +1625,157 @@ function nonNegativeNumberOrUndefined(value) {
   return number !== void 0 && number >= 0 ? number : void 0;
 }
 
+// src/belief-state/phase0-measurement.ts
+var DEFAULT_BASELINE_POLICY_ID = "always-accept-observed-action";
+function buildRuntimeBeliefPhase0Measurement(options) {
+  const runsById = new Map(options.runs.map((run) => [run.runId, run]));
+  const labelsByDecisionId = /* @__PURE__ */ new Map();
+  const diagnostics = [];
+  for (const label of options.labels) {
+    if (labelsByDecisionId.has(label.decisionId)) {
+      diagnostics.push(`${label.decisionId}: duplicate label; using the last label`);
+    }
+    labelsByDecisionId.set(label.decisionId, label);
+  }
+  const points = [];
+  let missingRunRecordCount = 0;
+  let missingLabelCount = 0;
+  for (const decision of options.decisions) {
+    const run = runsById.get(decision.runId);
+    if (!run) {
+      missingRunRecordCount += 1;
+      diagnostics.push(`${decision.id}: missing RunRecord join for runId ${decision.runId}`);
+      continue;
+    }
+    const label = labelsByDecisionId.get(decision.id);
+    if (!label) {
+      missingLabelCount += 1;
+      diagnostics.push(`${decision.id}: missing observed action/outcome label`);
+      continue;
+    }
+    const splitTag = label.splitTag ?? run.splitTag;
+    const report = runtimeDecisionPointToBeliefDecisionPoint(
+      { ...decision, scenarioId: decision.scenarioId ?? run.scenarioId },
+      {
+        chosenAction: label.chosenAction,
+        confidence: label.confidence,
+        behaviorProb: label.behaviorProb,
+        targetProb: label.targetProb,
+        qHat: label.qHat,
+        costUsd: label.costUsd,
+        outcome: label.outcome,
+        lifecycleEvents: options.events,
+        metadata: compactMetadata2({
+          baselinePolicyId: options.baselinePolicyId ?? DEFAULT_BASELINE_POLICY_ID,
+          splitTag,
+          ...label.metadata
+        })
+      }
+    );
+    diagnostics.push(...report.diagnostics.map((item) => `${item.decisionId}: ${item.reason}`));
+    if (report.point) points.push(report.point);
+  }
+  const packet = buildBeliefDecisionResearchEvidencePacket({
+    ...options,
+    points
+  });
+  return {
+    points,
+    packet,
+    summary: summarizePhase0Measurement(options, points, packet, {
+      missingRunRecordCount,
+      missingLabelCount
+    }),
+    diagnostics
+  };
+}
+function summarizePhase0Measurement(options, points, packet, counts) {
+  const producerDecisionCount = options.decisions.length;
+  return {
+    runCount: options.runs.length,
+    producerDecisionCount,
+    lifecycleEventCount: options.events?.length ?? 0,
+    labelCount: options.labels.length,
+    completedPointCount: points.length,
+    runJoinRate: ratio(producerDecisionCount - counts.missingRunRecordCount, producerDecisionCount),
+    labelJoinRate: ratio(points.length, producerDecisionCount),
+    missingRunRecordCount: counts.missingRunRecordCount,
+    missingLabelCount: counts.missingLabelCount,
+    withEvidence: points.filter((point) => point.evidence.length > 0).length,
+    withOutcome: points.filter((point) => point.outcome).length,
+    withSplit: points.filter((point) => typeof point.metadata?.splitTag === "string").length,
+    withBehaviorProb: points.filter((point) => point.behaviorProb !== void 0).length,
+    withTargetProb: points.filter((point) => point.targetProb !== void 0).length,
+    baselinePolicyId: options.baselinePolicyId ?? DEFAULT_BASELINE_POLICY_ID,
+    packetStatus: packet.status,
+    claimScope: packet.claimScope
+  };
+}
+function ratio(numerator, denominator) {
+  return denominator > 0 ? numerator / denominator : 0;
+}
+function compactMetadata2(values) {
+  const entries = Object.entries(values).filter(([, value]) => value !== void 0);
+  return entries.length > 0 ? Object.fromEntries(entries) : void 0;
+}
+
+// src/belief-state/runtime-benchmark-corpus.ts
+function buildRuntimeBenchmarkBeliefPhase0Measurement(options) {
+  const diagnostics = [];
+  const trajectory = projectRuntimeTrajectoryEvidence({
+    records: options.records,
+    defaultSplitTag: options.defaultSplitTag,
+    recordIdOf: runtimeBenchmarkRecordId,
+    scenarioIdOf: runtimeBenchmarkScenarioId
+  });
+  const decisions = options.decisions ?? [];
+  const labels = options.labels ?? [];
+  if (decisions.length === 0) {
+    diagnostics.push(
+      "no runtime decision points supplied; benchmark lifecycle events alone cannot produce belief decision rows"
+    );
+  }
+  if (labels.length === 0 && decisions.length > 0) {
+    diagnostics.push(
+      "no decision labels supplied; observed action/outcome joins will be incomplete"
+    );
+  }
+  const measurement = buildRuntimeBeliefPhase0Measurement({
+    ...options,
+    runs: trajectory.runs,
+    events: trajectory.events,
+    decisions,
+    labels
+  });
+  return {
+    runs: trajectory.runs,
+    events: trajectory.events,
+    decisions,
+    labels,
+    trajectory,
+    measurement,
+    summary: {
+      decisionCount: decisions.length,
+      labelCount: labels.length
+    },
+    diagnostics: [...trajectory.diagnostics, ...diagnostics, ...measurement.diagnostics]
+  };
+}
+function runtimeBenchmarkRecordId(record2) {
+  const parts = [
+    nonEmptyString(record2.benchmark),
+    nonEmptyString(record2.instanceId),
+    nonEmptyString(record2.condition)
+  ].filter((part) => part !== void 0);
+  return parts.length > 0 ? parts.join(":") : void 0;
+}
+function runtimeBenchmarkScenarioId(record2) {
+  return nonEmptyString(record2.instanceId);
+}
+function nonEmptyString(value) {
+  return typeof value === "string" && value.length > 0 ? value : void 0;
+}
+
 // src/belief-state/shadow-probe.ts
 var DEFAULT_CONCURRENCY = 4;
 var DEFAULT_MAX_CONTEXT_CHARS2 = 12e3;
@@ -1842,6 +1996,8 @@ export {
   beliefDecisionsToOffPolicyTrajectories,
   buildBeliefDecisionResearchEvidencePacket,
   buildCodeAgentBeliefEvidenceCorpus,
+  buildRuntimeBeliefPhase0Measurement,
+  buildRuntimeBenchmarkBeliefPhase0Measurement,
   calibrateBeliefDecisions,
   createBeliefRuntimeHookCollector,
   embeddedBeliefOpeTargetPolicy,