@tangle-network/agent-eval 0.80.0 → 0.82.0

package/dist/belief-state/index.d.ts

@@ -1,11 +1,78 @@
 import { c as CalibrationReport } from '../calibration-Cpr3WaX3.js';
 import { O as OffPolicyEstimate, a as OffPolicyOptions, b as OffPolicyTrajectory } from '../off-policy-DiwuKKg7.js';
+import { d as CodeAgentSessionSource, a as CodeAgentSessionIntakeOptions, c as CodeAgentSessionMetrics, C as CodeAgentSessionDiagnostic } from '../code-agent-session-BRXmavYv.js';
+import { R as RunRecord } from '../run-record-De9VarXR.js';
 import { T as TraceStore } from '../store-CKUAgsJz.js';
 import '../schema-m0gsnbt3.js';
 import '../outcome-store-rnXLEqSn.js';
+import '../errors-Dwqw-T_m.js';
 
-type BeliefDecisionKind = 'continue' | 'verify' | 'ask' | 'retry' | 'stop' | 'memory-write' | 'memory-read' | 'tool-select' | 'skill-select' | 'workflow-select' | 'surface-promote';
-type BeliefEvidenceSource = 'run' | 'span' | 'event' | 'finding' | 'memory' | 'knowledge' | 'policy';
+declare const BELIEF_DECISION_KINDS: readonly ["continue", "verify", "ask", "retry", "stop", "memory-write", "memory-read", "tool-select", "skill-select", "workflow-select", "surface-promote"];
+type BeliefDecisionKind = (typeof BELIEF_DECISION_KINDS)[number];
+declare const BELIEF_EVIDENCE_SOURCES: readonly ["run", "span", "event", "finding", "memory", "knowledge", "policy"];
+type BeliefEvidenceSource = (typeof BELIEF_EVIDENCE_SOURCES)[number];
+declare const BELIEF_EVIDENCE_QUALITIES: readonly ["direct", "derived", "self-reported", "unverified", "stale", "contradicted"];
+type BeliefEvidenceQuality = (typeof BELIEF_EVIDENCE_QUALITIES)[number];
+declare const BELIEF_EVALUATION_CRITERIA: readonly [{
+    readonly id: "capture-integrity";
+    readonly label: "Capture integrity";
+    readonly reasonCodes: readonly ["trace-missing", "run-record-missing", "backend-integrity-missing"];
+}, {
+    readonly id: "decision-completeness";
+    readonly label: "Decision completeness";
+    readonly reasonCodes: readonly ["candidate-actions-missing", "chosen-action-missing", "decision-evidence-missing"];
+}, {
+    readonly id: "evidence-quality";
+    readonly label: "Evidence quality";
+    readonly reasonCodes: readonly ["evidence-stale", "evidence-contradictory", "evidence-unverified", "evidence-self-reported"];
+}, {
+    readonly id: "outcome-quality";
+    readonly label: "Outcome quality";
+    readonly reasonCodes: readonly ["outcome-missing", "outcome-delayed", "cost-missing"];
+}, {
+    readonly id: "calibration";
+    readonly label: "Calibration";
+    readonly reasonCodes: readonly ["confidence-missing", "calibration-unsupported", "calibration-gap-high"];
+}, {
+    readonly id: "accepted-region-risk";
+    readonly label: "Accepted-region risk";
+    readonly reasonCodes: readonly ["accepted-error-high", "coverage-too-low"];
+}, {
+    readonly id: "policy-value";
+    readonly label: "Policy value";
+    readonly reasonCodes: readonly ["utility-lift-missing", "baseline-dominates", "cost-too-high"];
+}, {
+    readonly id: "ope-support";
+    readonly label: "OPE support";
+    readonly reasonCodes: readonly ["behavior-propensity-missing", "behavior-propensity-invalid", "target-propensity-missing", "target-propensity-invalid", "effective-sample-size-low", "importance-weight-high"];
+}, {
+    readonly id: "memory-health";
+    readonly label: "Memory health";
+    readonly reasonCodes: readonly ["memory-stale", "memory-poisoning-risk", "context-bloat", "memory-write-unverified"];
+}, {
+    readonly id: "surface-attribution";
+    readonly label: "Surface attribution";
+    readonly reasonCodes: readonly ["surface-claim-unsupported", "causal-attribution-missing"];
+}, {
+    readonly id: "generalization";
+    readonly label: "Generalization";
+    readonly reasonCodes: readonly ["split-missing", "holdout-regression", "task-family-coverage-low", "leakage-risk"];
+}, {
+    readonly id: "promotion";
+    readonly label: "Promotion";
+    readonly reasonCodes: readonly ["negative-control-failed", "promotion-gate-failed", "human-review-required"];
+}];
+type BeliefEvaluationCriterionId = (typeof BELIEF_EVALUATION_CRITERIA)[number]['id'];
+type BeliefDecisionReasonCode = (typeof BELIEF_EVALUATION_CRITERIA)[number]['reasonCodes'][number];
+interface BeliefDecisionReason {
+    code: BeliefDecisionReasonCode;
+    criterion?: BeliefEvaluationCriterionId;
+    detail?: string;
+    evidenceIds?: string[];
+    metadata?: Record<string, unknown>;
+}
+declare function isBeliefDecisionKind(value: unknown): value is BeliefDecisionKind;
+declare function isBeliefEvidenceSource(value: unknown): value is BeliefEvidenceSource;
 interface BeliefEvidenceRef {
     source: BeliefEvidenceSource;
     id: string;
@@ -13,6 +80,8 @@ interface BeliefEvidenceRef {
     spanId?: string;
     eventId?: string;
     detail?: string;
+    quality?: BeliefEvidenceQuality;
+    observedAt?: string;
     metadata?: Record<string, unknown>;
 }
 interface BeliefDecisionOutcome {
@@ -38,6 +107,7 @@ interface BeliefDecisionPoint {
     costUsd?: number;
     evidence: BeliefEvidenceRef[];
     outcome?: BeliefDecisionOutcome;
+    reasons?: BeliefDecisionReason[];
     metadata?: Record<string, unknown>;
 }
 interface BeliefDecisionExtractionDiagnostic {
@@ -57,6 +127,7 @@ interface BeliefPolicyDecision {
     targetProb?: number;
     qHat?: number | null;
     reason?: string;
+    reasons?: BeliefDecisionReason[];
 }
 interface BeliefSelectivePolicy {
     id: string;
@@ -138,11 +209,6 @@ interface BeliefCalibrationOptions {
 }
 declare function calibrateBeliefDecisions(points: BeliefDecisionPoint[], options?: BeliefCalibrationOptions): CalibrationReport | null;
 
-interface ExtractBeliefDecisionPointsOptions {
-    runIds?: string[];
-}
-declare function extractBeliefDecisionPoints(store: TraceStore, options?: ExtractBeliefDecisionPointsOptions): Promise<BeliefDecisionExtractionReport>;
-
 interface BeliefOpeOptions extends OffPolicyOptions {
     minEffectiveSampleSize?: number;
     minEffectiveSampleRatio?: number;
@@ -185,4 +251,274 @@ interface AnalyzeBeliefPolicyOptions {
 }
 declare function analyzeBeliefPolicy(options: AnalyzeBeliefPolicyOptions): BeliefPolicyEvaluationReport;
 
-export { type AnalyzeBeliefPolicyOpeOptions, type AnalyzeBeliefPolicyOptions, type BeliefCalibrationOptions, type BeliefCalibrationRegion, type BeliefCalibrationStatus, type BeliefDecisionExtractionDiagnostic, type BeliefDecisionExtractionReport, type BeliefDecisionKind, type BeliefDecisionOutcome, type BeliefDecisionPoint, type BeliefEvaluationStatus, type BeliefEvidenceRef, type BeliefEvidenceSource, type BeliefOffPolicyTrajectoryReport, type BeliefOpeOptions, type BeliefOpeReport, type BeliefOpeStatus, type BeliefOpeSupportDiagnostics, type BeliefOpeTargetPolicy, type BeliefPolicyAction, type BeliefPolicyDecision, type BeliefPolicyEvaluationReport, type BeliefSelectivePolicy, type BeliefSelectivePolicyMetrics, type BeliefUtilityOptions, type EvaluateBeliefSelectivePolicyOptions, type ExtractBeliefDecisionPointsOptions, analyzeBeliefPolicy, beliefDecisionsToOffPolicyTrajectories, calibrateBeliefDecisions, embeddedBeliefOpeTargetPolicy, evaluateBeliefOffPolicy, evaluateBeliefSelectivePolicy, extractBeliefDecisionPoints, thresholdSelectivePolicy };
+type CodeAgentBeliefDecisionTargetId = 'failure-recovery' | 'tool-selection' | 'graph-completion';
+interface ExtractCodeAgentBeliefDecisionPointsOptions {
+    source: CodeAgentSessionSource;
+    entries: unknown[];
+    run: Pick<RunRecord, 'runId' | 'scenarioId' | 'outcome' | 'costUsd'>;
+    sourcePath?: string;
+}
+interface BeliefDecisionInventoryBucket {
+    id: string;
+    kind?: BeliefDecisionKind;
+    targetId?: CodeAgentBeliefDecisionTargetId;
+    n: number;
+    withOutcome: number;
+    withConfidence: number;
+    withCandidateActions: number;
+    withBehaviorProb: number;
+    withTargetProb: number;
+    successRate: number | null;
+    meanScore: number | null;
+    meanConfidence: number | null;
+}
+interface BeliefDecisionInventoryReport {
+    n: number;
+    byKind: BeliefDecisionInventoryBucket[];
+    byTarget: BeliefDecisionInventoryBucket[];
+    diagnostics: string[];
+}
+interface BeliefDecisionTargetSelection {
+    id: CodeAgentBeliefDecisionTargetId;
+    label: string;
+    points: BeliefDecisionPoint[];
+    support: BeliefDecisionInventoryBucket;
+    reasons: string[];
+}
+interface SelectBeliefDecisionTargetOptions {
+    minN?: number;
+    minOutcomeCoverage?: number;
+    preferredTargets?: CodeAgentBeliefDecisionTargetId[];
+}
+interface AnalyzeBeliefDecisionCorpusOptions {
+    points: BeliefDecisionPoint[];
+    targetId?: CodeAgentBeliefDecisionTargetId;
+    minN?: number;
+    minOutcomeCoverage?: number;
+    minAccepted?: number;
+    confidenceThreshold?: number;
+    policy?: BeliefSelectivePolicy;
+    requireOpe?: boolean;
+    policyOptions?: Partial<AnalyzeBeliefPolicyOptions>;
+}
+interface BeliefDecisionCorpusEvaluation {
+    inventory: BeliefDecisionInventoryReport;
+    target?: BeliefDecisionTargetSelection;
+    policy?: BeliefSelectivePolicy;
+    evaluation?: BeliefPolicyEvaluationReport;
+    diagnostics: string[];
+}
+declare function extractCodeAgentBeliefDecisionPoints(options: ExtractCodeAgentBeliefDecisionPointsOptions): BeliefDecisionExtractionReport;
+declare function inventoryBeliefDecisionPoints(points: BeliefDecisionPoint[]): BeliefDecisionInventoryReport;
+declare function selectBeliefDecisionTarget(points: BeliefDecisionPoint[], options?: SelectBeliefDecisionTargetOptions): BeliefDecisionTargetSelection | null;
+declare function analyzeBeliefDecisionCorpus(options: AnalyzeBeliefDecisionCorpusOptions): BeliefDecisionCorpusEvaluation;
+
+type BeliefResearchClaimScope = 'selective' | 'counterfactual';
+type BeliefResearchEvidenceStatus = 'supported' | 'blocked';
+type BeliefResearchGateId = 'corpus' | 'selective' | 'calibration' | 'ope';
+interface BeliefResearchEvidenceGate {
+    id: BeliefResearchGateId;
+    status: BeliefResearchEvidenceStatus;
+    blockers: string[];
+    caveats: string[];
+}
+interface BeliefDecisionResearchEvidencePacket {
+    claimScope: BeliefResearchClaimScope;
+    status: BeliefResearchEvidenceStatus;
+    analysis: BeliefDecisionCorpusEvaluation;
+    gates: BeliefResearchEvidenceGate[];
+    blockers: string[];
+    caveats: string[];
+}
+interface BuildBeliefDecisionResearchEvidencePacketOptions extends AnalyzeBeliefDecisionCorpusOptions {
+    claimScope?: BeliefResearchClaimScope;
+}
+declare function buildBeliefDecisionResearchEvidencePacket(options: BuildBeliefDecisionResearchEvidencePacketOptions): BeliefDecisionResearchEvidencePacket;
+
+interface CodeAgentBeliefSession extends CodeAgentSessionIntakeOptions {
+    source: CodeAgentSessionSource;
+}
+interface BuildCodeAgentBeliefEvidenceCorpusOptions extends Omit<BuildBeliefDecisionResearchEvidencePacketOptions, 'points'> {
+    sessions: CodeAgentBeliefSession[];
+}
+interface CodeAgentBeliefEvidenceCorpus {
+    runs: RunRecord[];
+    metrics: CodeAgentSessionMetrics[];
+    intakeDiagnostics: CodeAgentSessionDiagnostic[];
+    extractionDiagnostics: BeliefDecisionExtractionDiagnostic[];
+    decisions: BeliefDecisionPoint[];
+    inventory: BeliefDecisionInventoryReport;
+    evidence: BeliefDecisionResearchEvidencePacket;
+}
+declare function buildCodeAgentBeliefEvidenceCorpus(options: BuildCodeAgentBeliefEvidenceCorpusOptions): CodeAgentBeliefEvidenceCorpus;
+
+interface ExtractBeliefDecisionPointsOptions {
+    runIds?: string[];
+}
+declare function extractBeliefDecisionPoints(store: TraceStore, options?: ExtractBeliefDecisionPointsOptions): Promise<BeliefDecisionExtractionReport>;
+
+interface BeliefShadowProbeInput {
+    probeId: string;
+    decisionId: string;
+    runId: string;
+    scenarioId?: string;
+    stepIndex: number;
+    decisionKind: BeliefDecisionKind;
+    candidateActions: string[];
+    observedAction?: string;
+    evidence: BeliefShadowProbeEvidenceRef[];
+    context?: string;
+    metadata?: Record<string, unknown>;
+}
+interface BeliefShadowProbeEvidenceRef {
+    id: string;
+    source: string;
+    detail?: string;
+    quality?: BeliefEvidenceQuality;
+}
+interface BeliefShadowProbeResponse {
+    predictedAction: string;
+    confidence: number;
+    beliefSummary?: string;
+    uncertainty?: string[];
+    evidenceRefs?: string[];
+    wouldChangeMindIf?: string[];
+    targetProb?: number;
+    qHat?: number | null;
+    metadata?: Record<string, unknown>;
+}
+interface BeliefShadowProbeRecord extends BeliefShadowProbeResponse {
+    probeId: string;
+    decisionId: string;
+    runId: string;
+    scenarioId?: string;
+    stepIndex: number;
+    decisionKind: BeliefDecisionKind;
+    candidateActions: string[];
+    observedAction: string;
+    agreesWithObservedAction: boolean;
+    outcome?: BeliefDecisionOutcome;
+}
+interface BeliefShadowProbeDiagnostic {
+    decisionId: string;
+    severity: 'warning' | 'error';
+    reason: string;
+}
+interface BeliefShadowProbeSummary {
+    attempted: number;
+    completed: number;
+    dropped: number;
+    withOutcome: number;
+    withTargetProb: number;
+    meanConfidence: number | null;
+    observedAgreementRate: number | null;
+}
+interface BeliefShadowProbeRun {
+    probeId: string;
+    records: BeliefShadowProbeRecord[];
+    diagnostics: BeliefShadowProbeDiagnostic[];
+    summary: BeliefShadowProbeSummary;
+}
+interface RunBeliefShadowProbeOptions {
+    probeId: string;
+    points: BeliefDecisionPoint[];
+    probe: (input: BeliefShadowProbeInput) => BeliefShadowProbeResponse | Promise<BeliefShadowProbeResponse>;
+    contextOf?: (point: BeliefDecisionPoint) => string | undefined | Promise<string | undefined>;
+    metadataOf?: (point: BeliefDecisionPoint) => Record<string, unknown> | undefined | Promise<Record<string, unknown> | undefined>;
+    includeObservedAction?: boolean;
+    includeEvidenceDetail?: boolean;
+    includeOutcomeInRecord?: boolean;
+    requireCandidateActions?: boolean;
+    allowOutOfSetActions?: boolean;
+    concurrency?: number;
+    maxContextChars?: number;
+}
+declare function runBeliefShadowProbe(options: RunBeliefShadowProbeOptions): Promise<BeliefShadowProbeRun>;
+declare function formatBeliefShadowProbePrompt(input: BeliefShadowProbeInput): string;
+
+interface RuntimeBeliefDecisionEvidenceRef {
+    source: string;
+    id: string;
+    detail?: string;
+    quality?: BeliefEvidenceQuality;
+    metadata?: Record<string, unknown>;
+}
+interface RuntimeBeliefDecisionPoint {
+    id: string;
+    runId: string;
+    scenarioId?: string;
+    stepIndex: number;
+    kind: string;
+    candidateActions?: string[];
+    context?: string;
+    evidence?: RuntimeBeliefDecisionEvidenceRef[];
+    metadata?: Record<string, unknown>;
+}
+interface RuntimeBeliefHookEvent {
+    id: string;
+    runId: string;
+    scenarioId?: string;
+    target: string;
+    phase: string;
+    timestamp: number;
+    stepIndex?: number;
+    parentId?: string;
+    payload?: unknown;
+    metadata?: Record<string, unknown>;
+}
+interface RuntimeBeliefHookContext {
+    signal?: AbortSignal;
+}
+interface RuntimeBeliefHooks {
+    onEvent?: (event: RuntimeBeliefHookEvent, context: RuntimeBeliefHookContext) => void | Promise<void>;
+    onDecisionPoint?: (point: RuntimeBeliefDecisionPoint, context: RuntimeBeliefHookContext) => void | Promise<void>;
+}
+interface RuntimeBeliefConversionDiagnostic {
+    decisionId: string;
+    severity: 'warning' | 'error';
+    reason: string;
+}
+interface RuntimeBeliefShadowProbeInputOptions {
+    probeId: string;
+    decisionKind?: BeliefDecisionKind;
+    includeEvidenceDetail?: boolean;
+    includeLifecycleEvidence?: boolean;
+    lifecycleEvents?: RuntimeBeliefHookEvent[];
+    maxContextChars?: number;
+}
+interface RuntimeBeliefDecisionPointOptions {
+    chosenAction?: string;
+    decisionKind?: BeliefDecisionKind;
+    confidence?: number;
+    behaviorProb?: number;
+    targetProb?: number;
+    qHat?: number | null;
+    costUsd?: number;
+    outcome?: BeliefDecisionOutcome;
+    metadata?: Record<string, unknown>;
+    includeLifecycleEvidence?: boolean;
+    lifecycleEvents?: RuntimeBeliefHookEvent[];
+}
+interface RuntimeBeliefShadowProbeInputReport {
+    input?: BeliefShadowProbeInput;
+    diagnostics: RuntimeBeliefConversionDiagnostic[];
+}
+interface RuntimeBeliefDecisionPointReport {
+    point?: BeliefDecisionPoint;
+    diagnostics: RuntimeBeliefConversionDiagnostic[];
+}
+interface BeliefRuntimeHookCollector {
+    hooks: RuntimeBeliefHooks;
+    decisions: RuntimeBeliefDecisionPoint[];
+    events: RuntimeBeliefHookEvent[];
+    toShadowProbeInputs(options?: Partial<RuntimeBeliefShadowProbeInputOptions>): {
+        inputs: BeliefShadowProbeInput[];
+        diagnostics: RuntimeBeliefConversionDiagnostic[];
+    };
+    clear(): void;
+}
+declare function runtimeDecisionPointToBeliefShadowProbeInput(point: RuntimeBeliefDecisionPoint, options: RuntimeBeliefShadowProbeInputOptions): RuntimeBeliefShadowProbeInputReport;
+declare function runtimeDecisionPointToBeliefDecisionPoint(point: RuntimeBeliefDecisionPoint, options: RuntimeBeliefDecisionPointOptions): RuntimeBeliefDecisionPointReport;
+declare function createBeliefRuntimeHookCollector(defaults: RuntimeBeliefShadowProbeInputOptions): BeliefRuntimeHookCollector;
+
+export { type AnalyzeBeliefDecisionCorpusOptions, type AnalyzeBeliefPolicyOpeOptions, type AnalyzeBeliefPolicyOptions, BELIEF_DECISION_KINDS, BELIEF_EVALUATION_CRITERIA, BELIEF_EVIDENCE_QUALITIES, BELIEF_EVIDENCE_SOURCES, type BeliefCalibrationOptions, type BeliefCalibrationRegion, type BeliefCalibrationStatus, type BeliefDecisionCorpusEvaluation, type BeliefDecisionExtractionDiagnostic, type BeliefDecisionExtractionReport, type BeliefDecisionInventoryBucket, type BeliefDecisionInventoryReport, type BeliefDecisionKind, type BeliefDecisionOutcome, type BeliefDecisionPoint, type BeliefDecisionReason, type BeliefDecisionReasonCode, type BeliefDecisionResearchEvidencePacket, type BeliefDecisionTargetSelection, type BeliefEvaluationCriterionId, type BeliefEvaluationStatus, type BeliefEvidenceQuality, type BeliefEvidenceRef, type BeliefEvidenceSource, type BeliefOffPolicyTrajectoryReport, type BeliefOpeOptions, type BeliefOpeReport, type BeliefOpeStatus, type BeliefOpeSupportDiagnostics, type BeliefOpeTargetPolicy, type BeliefPolicyAction, type BeliefPolicyDecision, type BeliefPolicyEvaluationReport, type BeliefResearchClaimScope, type BeliefResearchEvidenceGate, type BeliefResearchEvidenceStatus, type BeliefResearchGateId, type BeliefRuntimeHookCollector, type BeliefSelectivePolicy, type BeliefSelectivePolicyMetrics, type BeliefShadowProbeDiagnostic, type BeliefShadowProbeEvidenceRef, type BeliefShadowProbeInput, type BeliefShadowProbeRecord, type BeliefShadowProbeResponse, type BeliefShadowProbeRun, type BeliefShadowProbeSummary, type BeliefUtilityOptions, type BuildBeliefDecisionResearchEvidencePacketOptions, type BuildCodeAgentBeliefEvidenceCorpusOptions, type CodeAgentBeliefDecisionTargetId, type CodeAgentBeliefEvidenceCorpus, type CodeAgentBeliefSession, type EvaluateBeliefSelectivePolicyOptions, type ExtractBeliefDecisionPointsOptions, type ExtractCodeAgentBeliefDecisionPointsOptions, type RunBeliefShadowProbeOptions, type RuntimeBeliefConversionDiagnostic, type RuntimeBeliefDecisionEvidenceRef, type RuntimeBeliefDecisionPoint, type RuntimeBeliefDecisionPointOptions, type RuntimeBeliefDecisionPointReport, type RuntimeBeliefHookContext, type RuntimeBeliefHookEvent, type RuntimeBeliefHooks, type RuntimeBeliefShadowProbeInputOptions, type RuntimeBeliefShadowProbeInputReport, type SelectBeliefDecisionTargetOptions, analyzeBeliefDecisionCorpus, analyzeBeliefPolicy, beliefDecisionsToOffPolicyTrajectories, buildBeliefDecisionResearchEvidencePacket, buildCodeAgentBeliefEvidenceCorpus, calibrateBeliefDecisions, createBeliefRuntimeHookCollector, embeddedBeliefOpeTargetPolicy, evaluateBeliefOffPolicy, evaluateBeliefSelectivePolicy, extractBeliefDecisionPoints, extractCodeAgentBeliefDecisionPoints, formatBeliefShadowProbePrompt, inventoryBeliefDecisionPoints, isBeliefDecisionKind, isBeliefEvidenceSource, runBeliefShadowProbe, runtimeDecisionPointToBeliefDecisionPoint, runtimeDecisionPointToBeliefShadowProbeInput, selectBeliefDecisionTarget, thresholdSelectivePolicy };