@tangle-network/agent-eval 0.117.1 → 0.118.1

package/dist/belief-state/index.d.ts

@@ -1,13 +1,354 @@
-import { c as CalibrationReport } from '../calibration-C8MTS7cw.js';
-import { O as OffPolicyEstimate, a as OffPolicyOptions, b as OffPolicyTrajectory } from '../off-policy-DiwuKKg7.js';
-import { d as CodeAgentSessionSource, a as CodeAgentSessionIntakeOptions, c as CodeAgentSessionMetrics, C as CodeAgentSessionDiagnostic } from '../code-agent-session-CjZsVd19.js';
-import { R as RunRecord, a as RunSplitTag } from '../run-record-BDH49H2E.js';
-import { T as TraceStore } from '../store-DGqD0Pyo.js';
-import { R as RuntimeTrajectoryRecord, P as ProjectRuntimeTrajectoryEvidenceOptions, a as RuntimeTrajectoryEvidenceProjection } from '../runtime-trajectory-DGBIUt4B.js';
-import '../schema-B3Q3l9Z_.js';
-import '../outcome-store-rnXLEqSn.js';
-import '@tangle-network/agent-interface';
-import '../errors-oeQrLqXC.js';
+type RunStatus = 'running' | 'completed' | 'failed' | 'aborted';
+interface BudgetSpec {
+    tokens?: number;
+    wallMs?: number;
+    calls?: number;
+    usd?: number;
+}
+interface RunOutcome$1 {
+    score?: number;
+    pass?: boolean;
+    failureClass?: FailureClass;
+    notes?: string;
+}
+/**
+ * Layer — optional classification in a nested build workflow.
+ * `builder`: the meta-agent editing a project (e.g. agent-builder Forge chat).
+ * `app-build`: sandbox harness that compiled + tested the generated scaffold.
+ * `app-runtime`: a run of the generated agent against a domain scenario.
+ * `meta`: any meta-eval (judge replay, correlation analysis).
+ */
+type RunLayer = 'builder' | 'app-build' | 'app-runtime' | 'meta' | 'custom';
+interface Run {
+    runId: string;
+    /**
+     * Stable identifier of the scenario being executed.
+     *
+     * Always populated on the persisted Run — but `TraceEmitter.startRun` accepts
+     * input WITHOUT this field, substituting a sensible default
+     * (`run.layer ?? run.tags?.['kind'] ?? 'runtime'`) when the caller has no
+     * curated scenario to anchor to (runtime / operator / meta-eval runs). This
+     * keeps the persisted shape unambiguous for downstream filters + aggregations
+     * while removing the boilerplate of inventing placeholder ids at the call site.
+     */
+    scenarioId: string;
+    variantId?: string;
+    datasetVersion?: string;
+    /** Git SHA of agent code at run time. */
+    codeSha?: string;
+    /** Hash of the prompt template + any system prompt. */
+    promptSha?: string;
+    /** Model id + date + system-prompt hash, concatenated. */
+    modelFingerprint?: string;
+    seed?: number;
+    /** Arbitrary environment markers (shell, docker version, tz). */
+    envFingerprint?: Record<string, string>;
+    /** Version of the redaction rules applied to this run. */
+    redactionVersion?: string;
+    /** Parent run in a nested build workflow. A builder run's children are
+     *  app-build runs; those children are app-runtime runs. */
+    parentRunId?: string;
+    /** Stable project identifier — groups runs across chats + sessions. */
+    projectId?: string;
+    /** Chat/conversation identifier within a project. */
+    chatId?: string;
+    /** Layer classification — hint for aggregation; not enforced. */
+    layer?: RunLayer;
+    startedAt: number;
+    endedAt?: number;
+    status: RunStatus;
+    outcome?: RunOutcome$1;
+    budget?: BudgetSpec;
+    /** Free-form labels for downstream grouping. */
+    tags?: Record<string, string>;
+}
+type SpanKind = 'agent' | 'llm' | 'tool' | 'retrieval' | 'judge' | 'sandbox' | 'custom';
+type SpanStatus = 'ok' | 'error';
+interface SpanBase {
+    spanId: string;
+    parentSpanId?: string;
+    runId: string;
+    kind: SpanKind;
+    name: string;
+    startedAt: number;
+    endedAt?: number;
+    status?: SpanStatus;
+    error?: string;
+    /** Anything not covered by typed fields. Kept deliberately free-form. */
+    attributes?: Record<string, unknown>;
+}
+interface Message {
+    role: 'system' | 'user' | 'assistant' | 'tool';
+    content: string;
+    tokens?: number;
+    /** Multi-modal content descriptors; blobs themselves live in Artifacts. */
+    images?: Array<{
+        artifactId?: string;
+        url?: string;
+        mime?: string;
+    }>;
+}
+interface LlmSpan extends SpanBase {
+    kind: 'llm';
+    model: string;
+    messages: Message[];
+    output?: string;
+    inputTokens?: number;
+    /** All generated tokens, including the reasoning subset when present. */
+    outputTokens?: number;
+    cachedTokens?: number;
+    cacheWriteTokens?: number;
+    /** Reasoning-token subset of `outputTokens`. */
+    reasoningTokens?: number;
+    costUsd?: number;
+    finishReason?: string;
+}
+interface ToolSpan extends SpanBase {
+    kind: 'tool';
+    toolName: string;
+    args: unknown;
+    /** False when the source observed the call but did not capture its arguments. */
+    argsCaptured?: boolean;
+    result?: unknown;
+    latencyMs?: number;
+}
+interface RetrievalSpan extends SpanBase {
+    kind: 'retrieval';
+    query: string;
+    hits: Array<{
+        docId: string;
+        score: number;
+        content?: string;
+    }>;
+}
+interface JudgeSpan extends SpanBase {
+    kind: 'judge';
+    judgeId: string;
+    /** Span this judgment applies to. */
+    targetSpanId: string;
+    dimension: string;
+    /** Numeric score (free-range; interpretation up to the judge). */
+    score: number;
+    rationale?: string;
+    evidence?: string;
+}
+interface SandboxSpan extends SpanBase {
+    kind: 'sandbox';
+    image?: string;
+    command?: string;
+    exitCode?: number;
+    testsTotal?: number;
+    testsPassed?: number;
+    stdoutHash?: string;
+    stderrHash?: string;
+    /** Duration in ms; the harness fills this explicitly (endedAt - startedAt may miss setup). */
+    wallMs?: number;
+}
+interface GenericSpan extends SpanBase {
+    kind: 'agent' | 'custom';
+}
+type Span = LlmSpan | ToolSpan | RetrievalSpan | JudgeSpan | SandboxSpan | GenericSpan;
+type EventKind = 'log' | 'error' | 'budget_decrement' | 'budget_breach' | 'state_mutation' | 'policy_violation' | 'redaction_applied' | 'custom';
+interface TraceEvent {
+    eventId: string;
+    runId: string;
+    spanId?: string;
+    kind: EventKind;
+    timestamp: number;
+    payload: Record<string, unknown>;
+}
+interface BudgetLedgerEntry {
+    runId: string;
+    dimension: keyof BudgetSpec;
+    limit: number;
+    consumed: number;
+    remaining: number;
+    timestamp: number;
+    breached: boolean;
+    /** Span that triggered this entry, if any. */
+    spanId?: string;
+}
+interface Artifact {
+    artifactId: string;
+    runId: string;
+    spanId?: string;
+    contentType: string;
+    sizeBytes: number;
+    /** sha256 in hex. */
+    hash: string;
+    /** External storage URL (R2, S3, filesystem path). */
+    storageUrl?: string;
+    /** Inline content for small blobs — keep under ~64KB. */
+    inlineContent?: string;
+}
+type FailureClass = 'success' | 'reasoning_error' | 'tool_selection_error' | 'tool_argument_error' | 'tool_recovery_failure' | 'hallucination' | 'instruction_following' | 'safety_refusal_miss' | 'policy_violation' | 'budget_exceeded' | 'format_drift' | 'permission_escalation' | 'pii_leak' | 'cost_overrun' | 'timeout' | 'sandbox_failure' | 'missing_user_data' | 'missing_domain_data' | 'missing_codebase_context' | 'missing_runtime_context' | 'missing_credentials' | 'missing_integration_connection' | 'missing_integration_scope' | 'integration_approval_required' | 'integration_auth_expired' | 'integration_provider_failure' | 'bad_integration_manifest' | 'unsafe_integration_write_denied' | 'stale_external_data' | 'bad_retrieval' | 'insufficient_evidence' | 'contradictory_evidence' | 'ambiguous_user_intent' | 'knowledge_readiness_blocked' | 'unknown';
+
+interface RunFilter {
+    scenarioId?: string;
+    variantId?: string;
+    status?: RunStatus;
+    since?: number;
+    until?: number;
+    tag?: {
+        key: string;
+        value: string;
+    };
+    parentRunId?: string;
+    projectId?: string;
+    chatId?: string;
+    layer?: RunLayer;
+}
+interface SpanFilter {
+    runId?: string;
+    parentSpanId?: string;
+    kind?: SpanKind;
+    name?: string;
+    toolName?: string;
+    judgeId?: string;
+    since?: number;
+    until?: number;
+}
+interface EventFilter {
+    runId?: string;
+    spanId?: string;
+    kind?: EventKind;
+    since?: number;
+    until?: number;
+}
+interface TraceStore {
+    appendRun(run: Run): Promise<void>;
+    updateRun(runId: string, patch: Partial<Run>): Promise<void>;
+    appendSpan(span: Span): Promise<void>;
+    updateSpan(spanId: string, patch: Partial<Span>): Promise<void>;
+    appendEvent(event: TraceEvent): Promise<void>;
+    appendArtifact(artifact: Artifact): Promise<void>;
+    appendBudgetEntry(entry: BudgetLedgerEntry): Promise<void>;
+    getRun(runId: string): Promise<Run | undefined>;
+    listRuns(filter?: RunFilter): Promise<Run[]>;
+    spans(filter?: SpanFilter): Promise<Span[]>;
+    events(filter?: EventFilter): Promise<TraceEvent[]>;
+    budget(runId: string): Promise<BudgetLedgerEntry[]>;
+    artifacts(runId: string): Promise<Artifact[]>;
+}
+
+/**
+ * Calibration curve — binned "if eval says X, what does reality show?"
+ *
+ * Companion to correlationStudy. Raw correlation is a single number;
+ * the calibration curve shows *where* the eval is well-calibrated vs
+ * overconfident / underconfident. Buckets the eval metric, computes
+ * mean outcome per bucket, reports expected-calibration-error (ECE).
+ */
+
+interface CalibrationBin {
+    lower: number;
+    upper: number;
+    n: number;
+    evalMean: number;
+    outcomeMean: number;
+    /** |outcomeMean − evalMean|; contributes to ECE weighted by n/total. */
+    gap: number;
+}
+interface CalibrationReport {
+    evalMetric: string;
+    outcomeMetric: string;
+    n: number;
+    bins: CalibrationBin[];
+    /** Expected Calibration Error — Σ (n_i/N) × |outcomeMean_i − evalMean_i|. */
+    ece: number;
+    /** Max bin gap — upper bound on miscalibration. */
+    maxGap: number;
+}
+
+/**
+ * Off-policy evaluation primitives.
+ *
+ * Standard inverse-probability-weighted (IPS), self-normalized
+ * importance-weighted (SNIPS), and doubly-robust (DR) estimators for the
+ * value of a *target* policy given trajectories collected under a
+ * *behavior* policy. This is the canonical RL eval task: "we have last
+ * week's runs, we changed the policy — how would the new one do without
+ * re-running?"
+ *
+ * The math here is textbook (Dudík, Langford, Li 2011 for DR; Swaminathan
+ * & Joachims 2015 for SNIPS) but the *application* to LLM-agent
+ * evaluation needs care:
+ *
+ *   - The "policy" is the (prompt, tool config, model snapshot) triple.
+ *     Two policies have the same probability over an action *iff* their
+ *     LLM call would emit the same token with the same probability —
+ *     which is generally unknowable without the model log-probs.
+ *   - For LLM agents, propensity scores must be supplied by the caller
+ *     (logged in the trace, recovered from token log-probs, or estimated
+ *     via a learned propensity model). We do NOT estimate propensity here.
+ *   - Doubly-robust requires a Q-function (model-based reward predictor).
+ *     We accept any callable; consumers pass either a tabular average,
+ *     a regression fit, or a learned reward model.
+ *
+ * Bias / variance tradeoffs:
+ *   - IPS: unbiased; high variance for small overlap, infinite variance
+ *     when target has support outside behavior.
+ *   - SNIPS: lower variance, slight bias; usually preferred in practice.
+ *   - DR: doubly-robust — unbiased if either propensity OR Q-function is
+ *     correct. Lowest practical variance when Q is decent. Use this.
+ *
+ * Caveat the panel will land: on the LLM-agent setting, propensity scores
+ * recovered from token log-probs are noisy, the action space is enormous,
+ * and overlap is often poor. These estimators are useful but not magic;
+ * complement with `replayCampaign` (exact replay where the request hashes
+ * match) for high-confidence answers and OPE for the gap.
+ */
+interface OffPolicyTrajectory {
+    /** Stable id, for traceability through the dataset. */
+    runId: string;
+    /** Reward observed under the behavior policy (the realized outcome). */
+    reward: number;
+    /**
+     * Behavior-policy probability of the action that was taken. For LLM
+     * agents this is typically `exp(sum(token_log_probs))` over the chosen
+     * trajectory. Must be in (0, 1].
+     */
+    behaviorProb: number;
+    /**
+     * Target-policy probability of the same action. For replay-style
+     * counterfactual evaluation this is what the *new* policy would have
+     * assigned to the *old* trajectory. Must be in [0, 1].
+     */
+    targetProb: number;
+    /**
+     * Optional model-based reward prediction at the same context. Used by
+     * `doublyRobust`. Set to `null` for IPS-only evaluation.
+     */
+    qHat?: number | null;
+}
+interface OffPolicyEstimate {
+    /** Estimated value of the target policy. */
+    value: number;
+    /** Standard error of the estimate. */
+    standardError: number;
+    /** Effective sample size (Kong 1992). Lower = more reliance on a few high-weight samples. */
+    effectiveSampleSize: number;
+    /** Number of trajectories used. */
+    n: number;
+    /**
+     * Diagnostic: maximum importance weight observed. Large values (>>10x
+     * mean) are a red flag — variance is dominated by a few outliers.
+     */
+    maxImportanceWeight: number;
+}
+interface OffPolicyOptions {
+    /**
+     * Cap importance weights at this value (Ionides 2008 truncated IS) to
+     * trade unbiasedness for variance reduction. Default `Infinity` (no cap).
+     * Set e.g. `10` for stable estimates when the policies are close.
+     */
+    weightCap?: number;
+    /** Reward clipping range. Default `[0, 1]`. */
+    rewardClip?: {
+        low: number;
+        high: number;
+    };
+}
 
 declare const BELIEF_DECISION_KINDS: readonly ["continue", "verify", "ask", "retry", "stop", "memory-write", "memory-read", "tool-select", "skill-select", "workflow-select", "surface-promote"];
 type BeliefDecisionKind = (typeof BELIEF_DECISION_KINDS)[number];
@@ -211,6 +552,317 @@ interface BeliefCalibrationOptions {
 }
 declare function calibrateBeliefDecisions(points: BeliefDecisionPoint[], options?: BeliefCalibrationOptions): CalibrationReport | null;
 
+type AgentProfileCellSchemaVersion = 'agent-profile-cell/v1';
+type AgentProfileDimensionValue = string | number | boolean | null;
+interface AgentProfileSource {
+    /** Runtime/profile contract being fingerprinted, e.g. `agent-interface-profile`. */
+    kind: string;
+    /** sha256 over the canonical source profile object. */
+    hash: string;
+}
+interface AgentProfileHarness {
+    id: string;
+    version?: string;
+    hash?: string;
+}
+interface AgentProfileCell {
+    schemaVersion: AgentProfileCellSchemaVersion;
+    cellId: string;
+    profileId: string;
+    sourceProfile: AgentProfileSource;
+    harness?: AgentProfileHarness;
+    model?: string;
+    promptHash?: string;
+    dimensions?: Record<string, AgentProfileDimensionValue>;
+}
+
+/**
+ * Paper-grade RunRecord schema + runtime validator.
+ *
+ * Every run that participates in a promotion gate, paper table, or
+ * researcher loop SHOULD be recorded as a `RunRecord`. The mandatory
+ * fields are exactly those the paper "Two Loops, Three Roles" requires
+ * for reproducibility: who/what/when/cost/seed/hash, plus the search vs
+ * holdout split tag and either a `searchScore` or a `holdoutScore`.
+ *
+ * This is intentionally NOT a replacement for the rich `Run` /
+ * `ProposeReviewReport` / `ScenarioResult` types already in the
+ * package. Those are runtime structures with full provenance. A
+ * `RunRecord` is the analysis-time projection — the JSON-friendly
+ * row you'd put in a parquet file or paste into a notebook.
+ *
+ * Validate at the boundary:
+ *
+ *   const rec = validateRunRecord(rawJson)         // throws on missing
+ *   const ok  = isRunRecord(rawJson)               // boolean check
+ *   const rec = parseRunRecordSafe(rawJson)        // { ok, value | error }
+ *
+ * The validator runs in pure TS — zod is intentionally NOT a
+ * dependency. Round-trip tested in `tests/run-record.test.ts`.
+ */
+
+/** Search/dev/holdout split tag. 'search' is the paper-grade alias for the
+ *  combined train+test pool that the optimizer is allowed to read. */
+type RunSplitTag = 'search' | 'dev' | 'holdout';
+interface RunTokenUsage {
+    input: number;
+    /** All generated tokens charged as output, including reasoning tokens. */
+    output: number;
+    /** Reasoning-token subset of `output`, when the provider reports it. */
+    reasoning?: number;
+    /** Prompt tokens served from a provider cache. */
+    cached?: number;
+    /** Prompt tokens written into a provider cache. */
+    cacheWrite?: number;
+}
+/**
+ * How a run's USD amount was obtained.
+ *
+ * `costUsd` remains mandatory for wire compatibility. New producers should
+ * always populate this discriminated union so a missing bill is never
+ * mistaken for an observed zero-dollar run. For `uncaptured`, `costUsd` uses
+ * the legacy `0` sentinel while this field carries the truthful null.
+ */
+type RunCostProvenance = {
+    kind: 'observed';
+    usd: number;
+} | {
+    kind: 'estimated';
+    usd: number;
+} | {
+    kind: 'uncaptured';
+    usd: null;
+};
+interface RunJudgeMetadata {
+    model: string;
+    promptVersion: string;
+    /** [0,1] confidence the judge declared. Constant judge confidence
+     *  across many runs is a fallback signal (see `canary.ts`). */
+    confidence: number;
+    /** True if the judge degraded to a fallback path (rules-only,
+     *  prior-call cache, etc.). The canary uses this to alert. */
+    fallback: boolean;
+}
+/**
+ * Per-judge / per-dimension breakdown for runs scored by an ensemble of
+ * judges over a multi-dimensional rubric.
+ *
+ * The collapsed `outcome.searchScore` / `holdoutScore` carries the
+ * composite the gate uses. The full breakdown belongs here so consumers
+ * can answer "which judge disagreed?", "which dimension dragged the
+ * composite down?", and "did half the panel fail?" without re-running.
+ *
+ * `perJudge[judgeId][dim]` is the canonical source; `perDimMean` and
+ * `composite` are convenience projections — derivable but precomputed so
+ * downstream IRR primitives (`interRaterReliability`,
+ * `corpusInterRaterAgreement`) and reporters don't pay the same
+ * aggregation twice.
+ *
+ * Fail-loud discipline: judges that errored out land in `failedJudges`
+ * by id. A missing key in `perJudge` is ambiguous (silent zero vs not
+ * run); the explicit list makes a partial-failure recorded as such.
+ */
+interface JudgeScoresRecord {
+    /** Per-judge per-dimension scores. `{ "kimi-k2.6": { helpfulness: 0.8, clarity: 0.7 }, ... }`. */
+    perJudge: Record<string, Record<string, number>>;
+    /** Per-dim mean across judges. Convenience — derivable from `perJudge`. */
+    perDimMean: Record<string, number>;
+    /** Composite mean across all dims and judges. Mirrors the score
+     *  the gate sees on `outcome.searchScore` / `holdoutScore`. */
+    composite: number;
+    /** Judges that errored or returned an unparseable verdict. Recorded
+     *  by id (e.g. `['glm-5.1']`) so a partial-failure case is explicit,
+     *  not inferred from missing keys in `perJudge`. */
+    failedJudges?: string[];
+    /** Free-form notes the judges emitted (joined across judges or
+     *  first-judge only — consumer's choice). */
+    notes?: string;
+}
+interface RunOutcome {
+    /** Score on the search/optimization split. Optional because a
+     *  holdout-only evaluation only fills `holdoutScore`. */
+    searchScore?: number;
+    /** Score on the held-out split. Optional because a search-only run
+     *  only fills `searchScore`. At least one must be present. */
+    holdoutScore?: number;
+    /** Bag of any other metric the run produced — judge dimensions,
+     *  pass/fail counters, latency stats, etc. Numeric only — keeps
+     *  reporters honest. */
+    raw: Record<string, number>;
+    /** Per-judge / per-dim breakdown. Consumers writing ensemble
+     *  judgements populate this; substrate primitives like
+     *  `interRaterReliability` and `corpusInterRaterAgreement` accept
+     *  these records as input. Optional — single-judge or scalar-only
+     *  runs leave it unset. */
+    judgeScores?: JudgeScoresRecord;
+    /** Authenticity / realness verdict — did the run build the REAL thing on the
+     *  intended infra, or fake it (see `./authenticity`)? Optional: only domains
+     *  with an authenticity config populate it. Carried in the corpus so the
+     *  flywheel / off-policy learning can optimize for real completion, not gamed
+     *  pass-rate. `score` is 0-1; `gated` is the anti-Goodhart flag — a gated run
+     *  must not count as a real success regardless of `score`. */
+    realness?: {
+        score: number;
+        gated: boolean;
+        reason?: string;
+    };
+}
+/**
+ * Mandatory paper-grade fields for a single evaluation run. Optional
+ * fields are extension points; mandatory fields throw if missing.
+ *
+ * Hash discipline:
+ *   - `promptHash` is the sha256 of the EFFECTIVE prompt sent to the
+ *     model (after any steering bundle merge).
+ *   - `configHash` is the sha256 of the effective run config (model,
+ *     temperature, tools, judges, splits). The pair (promptHash,
+ *     configHash) uniquely identifies an experiment cell.
+ *
+ * Model snapshot discipline:
+ *   - `model` MUST encode a snapshot version. Bare aliases like
+ *     `claude-sonnet-4` or `gpt-4o` are banned — they remap silently.
+ *     Use `claude-sonnet-4-6@2025-04-15` or `gpt-4o-2024-11-20`.
+ */
+interface RunRecord {
+    /** UUID for the run. */
+    runId: string;
+    /** Logical experiment grouping (a treatment vs a baseline within
+     *  the same sweep should share `experimentId`). */
+    experimentId: string;
+    /** Stable identifier for the candidate (variant) being run. The
+     *  promotion gate compares two `candidateId`s on matched items. */
+    candidateId: string;
+    /** RNG seed for the run. Always recorded — silent re-seeding is
+     *  the most common cause of non-reproducible numbers. */
+    seed: number;
+    /** Model identifier WITH snapshot version. */
+    model: string;
+    /** sha256 of the effective prompt (post-steering). */
+    promptHash: string;
+    /** sha256 of the effective config. */
+    configHash: string;
+    /** Git SHA the harness was run from. */
+    commitSha: string;
+    /** End-to-end wall-clock duration in milliseconds. */
+    wallMs: number;
+    /** Time spent queued before execution started, if known. */
+    queueMs?: number;
+    /** Total USD cost. Mandatory — runs without a cost number are
+     *  unbounded by definition and must not be admitted into the gate.
+     *  `0` is retained as the compatibility sentinel for an uncaptured amount;
+     *  inspect `costProvenance` before treating it as observed. */
+    costUsd: number;
+    /** Observed, model-priced estimate, or genuinely uncaptured USD amount.
+     *  Optional only so existing serialized RunRecords remain valid. */
+    costProvenance?: RunCostProvenance;
+    /** Token usage breakdown. */
+    tokenUsage: RunTokenUsage;
+    /** Judge-side metadata, if a judge was used. */
+    judgeMetadata?: RunJudgeMetadata;
+    /** Per-split scores + raw bag. */
+    outcome: RunOutcome;
+    /** Canonical, cross-agent failure class drawn from the shared
+     *  `FAILURE_CLASSES` taxonomy. This is the aggregation key that makes
+     *  "which failure dominates across the whole fleet" answerable in ONE
+     *  vocabulary — every agent classifies against the same enum. Producers
+     *  set it via the substrate classifier; leave unset only when the failure
+     *  genuinely can't be classified. */
+    failureClass?: FailureClass;
+    /** Free-form domain-specific failure detail, scoped UNDER `failureClass`
+     *  (e.g. failureClass='tool_recovery_failure', failureMode='forge_build_unsatisfied').
+     *  The within-agent drill-down; `failureClass` is the cross-agent key. */
+    failureMode?: string;
+    /** Which split this run was drawn from. */
+    splitTag: RunSplitTag;
+    /**
+     * Stable scenario identifier the run was scored against. Optional for
+     * backwards compatibility, but **strongly recommended**: every primitive
+     * that pairs runs by scenario (preferences, paired stats, BT tournament)
+     * keys on this. The campaign artifact populates it canonically; legacy
+     * runs without it fall back to inference from `outcome.raw.scenario_id`
+     * or `experimentId`.
+     */
+    scenarioId?: string;
+    /**
+     * Canonical identity for the agent profile cell that produced this row:
+     * profile artifact hash plus optional harness/model/prompt/reporting
+     * dimensions. Use `agentProfile.cellId` to group persona sweeps and
+     * longitudinal reports by the complete source profile, not by a loose
+     * candidate label or opaque config hash.
+     */
+    agentProfile?: AgentProfileCell;
+}
+
+type CodeAgentSessionSource = 'codex' | 'claude-code' | 'opencode' | 'kimi-code' | 'pi';
+interface CodeAgentSessionMetrics {
+    entries: number;
+    userMessages: number;
+    assistantMessages: number;
+    reasoningItems: number;
+    toolCalls: number;
+    toolOutputs: number;
+    toolErrors: number;
+    patchAttempts: number;
+    patchSuccesses: number;
+    patchFailures: number;
+    turnsStarted: number;
+    turnsCompleted: number;
+    turnsAborted: number;
+    contextCompactions: number;
+    prLinks: number;
+    fileSnapshots: number;
+    graphNodes: number;
+    graphEdges: number;
+    actionCandidates: number;
+    verificationReports: number;
+    completionDecisions: number;
+    reliabilityRows: number;
+    reliabilityLift: number;
+    inputTokens: number;
+    outputTokens: number;
+    reasoningTokens: number;
+    cachedTokens: number;
+    cacheWriteTokens: number;
+    observedCostUsd: number;
+    observedCostCaptured?: boolean;
+    wallMs: number;
+    processScore: number;
+}
+interface CodeAgentSessionDiagnostic {
+    source: CodeAgentSessionSource;
+    sessionId: string;
+    sourcePath?: string;
+    entries: number;
+    malformedLines: number;
+    inferredScore: boolean;
+    hasExplicitTerminalSignal: boolean;
+    hasQualityLabel: boolean;
+    hasTokenUsage: boolean;
+    hasCost: boolean;
+    costKind?: RunCostProvenance['kind'];
+    warnings: string[];
+}
+interface CodeAgentSessionIntakeOptions {
+    entries: unknown[];
+    malformedLines?: number;
+    sourcePath?: string;
+    experimentId?: string;
+    candidateId?: string;
+    seed?: number;
+    splitTag?: RunSplitTag;
+    scenarioId?: string;
+    model?: string;
+    promptHash?: string;
+    configHash?: string;
+    commitSha?: string;
+    score?: number;
+    /** Explicit cost receipt. Use `uncaptured` when the source says dollars
+     *  were not captured; the adapter will not relabel its compatibility $0
+     *  sentinel as observed. When omitted, source-reported cost wins, then a
+     *  token-priced estimate, then uncaptured. */
+    costProvenance?: RunCostProvenance;
+}
+
 interface BeliefOpeOptions extends OffPolicyOptions {
     minEffectiveSampleSize?: number;
     minEffectiveSampleRatio?: number;
@@ -574,6 +1226,50 @@ interface RuntimeBeliefPhase0Measurement {
 }
 declare function buildRuntimeBeliefPhase0Measurement(options: BuildRuntimeBeliefPhase0MeasurementOptions): RuntimeBeliefPhase0Measurement;
 
+interface RuntimeTrajectoryHookEvent {
+    id: string;
+    runId: string;
+    scenarioId?: string;
+    target: string;
+    phase: string;
+    timestamp: number;
+    stepIndex?: number;
+    parentId?: string;
+    payload?: unknown;
+    metadata?: Record<string, unknown>;
+}
+interface RuntimeTrajectoryRecord {
+    id?: string;
+    scenarioId?: string;
+    splitTag?: RunSplitTag;
+    runtimeEvents?: unknown;
+    [key: string]: unknown;
+}
+interface RuntimeTrajectoryRunRecord {
+    runId: string;
+    scenarioId?: string;
+    splitTag: RunSplitTag;
+}
+interface RuntimeTrajectoryEvidenceSummary {
+    recordCount: number;
+    recordWithRuntimeEventsCount: number;
+    runtimeRunCount: number;
+    lifecycleEventCount: number;
+    defaultedSplitCount: number;
+}
+interface RuntimeTrajectoryEvidenceProjection {
+    runs: RuntimeTrajectoryRunRecord[];
+    events: RuntimeTrajectoryHookEvent[];
+    summary: RuntimeTrajectoryEvidenceSummary;
+    diagnostics: string[];
+}
+interface ProjectRuntimeTrajectoryEvidenceOptions<TRecord extends RuntimeTrajectoryRecord = RuntimeTrajectoryRecord> {
+    records: TRecord[];
+    defaultSplitTag?: RunSplitTag;
+    recordIdOf?: (record: TRecord, index: number) => string | undefined;
+    scenarioIdOf?: (record: TRecord, index: number) => string | undefined;
+}
+
 type RuntimeBenchmarkTrajectoryRecord = RuntimeTrajectoryRecord & {
     benchmark?: unknown;
     condition?: unknown;