@tangle-network/agent-eval 0.117.1 → 0.118.0

package/dist/rl.d.ts

@@ -1,30 +1,446 @@
-import { R as RunRecord, a as RunSplitTag } from './run-record-BDH49H2E.js';
-export { A as AdversarialMutation } from './adversarial-B7loGVVX.js';
-import { S as Span } from './schema-B3Q3l9Z_.js';
-import { T as TraceStore } from './store-DGqD0Pyo.js';
-export { O as OffPolicyEstimate, a as OffPolicyOptions, b as OffPolicyTrajectory, d as doublyRobust, i as inverseProbabilityWeighting, o as offPolicyEstimateAll, s as selfNormalizedImportanceWeighting } from './off-policy-DiwuKKg7.js';
-import { b as OutcomeStore } from './outcome-store-rnXLEqSn.js';
-export { D as DeploymentOutcome, F as FileSystemOutcomeStore, a as FileSystemOutcomeStoreOptions, I as InMemoryOutcomeStore } from './outcome-store-rnXLEqSn.js';
-import { b as RubricPredictiveValidityReport } from './rubric-predictive-validity-p49lLVrE.js';
-import { R as Researcher, F as FailureMode, S as SteeringChange, E as ExperimentPlan, a as ExperimentResult, b as EvalCampaignResult, c as EvalCampaignOptions } from './researcher-C8XyxQsu.js';
-export { r as runEvalCampaign } from './researcher-C8XyxQsu.js';
-import { a as VerificationReport } from './multi-layer-verifier-BsqKuLyN.js';
-import { I as InterimReleaseConfidence } from './sequential-5iSVfzl2.js';
-import { C as CampaignResult } from './types-BSw1rOUB.js';
-import '@tangle-network/agent-interface';
-import './errors-oeQrLqXC.js';
-import './llm-client-qoDd18Qz.js';
-import './cost-ledger-DWy3XdJc.js';
-import './raw-provider-sink-C46HDghv.js';
-import './summary-report-C5bKFfm-.js';
-import './failure-cluster-DOAcSJ87.js';
-import './emitter-CjD7vUwv.js';
-import './integrity-DqlBiLyK.js';
-import './verdict-C9MlYujm.js';
-import './policy-edit-wG9uFEFm.js';
-import './store-C1YxJDEK.js';
-import './types-BkfcQnxV.js';
-import '@tangle-network/tcloud';
+type AgentProfileCellSchemaVersion = 'agent-profile-cell/v1';
+type AgentProfileJson = string | number | boolean | null | AgentProfileJson[] | {
+    [key: string]: AgentProfileJson;
+};
+type AgentProfileDimensionValue = string | number | boolean | null;
+interface AgentProfileSource {
+    /** Runtime/profile contract being fingerprinted, e.g. `agent-interface-profile`. */
+    kind: string;
+    /** sha256 over the canonical source profile object. */
+    hash: string;
+}
+interface AgentProfileSourceInput {
+    kind: string;
+    /** Precomputed sha256 for callers that already sign their profile artifact. */
+    hash?: string;
+    /** Full canonical runtime profile; hashed and then discarded from the cell. */
+    profile?: AgentProfileJson;
+}
+interface AgentProfileHarness {
+    id: string;
+    version?: string;
+    hash?: string;
+}
+interface AgentProfileCellInput {
+    profileId: string;
+    sourceProfile: AgentProfileSourceInput;
+    harness?: AgentProfileHarness;
+    model?: string;
+    promptHash?: string;
+    dimensions?: Record<string, AgentProfileDimensionValue>;
+}
+interface AgentProfileCell {
+    schemaVersion: AgentProfileCellSchemaVersion;
+    cellId: string;
+    profileId: string;
+    sourceProfile: AgentProfileSource;
+    harness?: AgentProfileHarness;
+    model?: string;
+    promptHash?: string;
+    dimensions?: Record<string, AgentProfileDimensionValue>;
+}
+
+type RunStatus = 'running' | 'completed' | 'failed' | 'aborted';
+interface BudgetSpec {
+    tokens?: number;
+    wallMs?: number;
+    calls?: number;
+    usd?: number;
+}
+interface RunOutcome$1 {
+    score?: number;
+    pass?: boolean;
+    failureClass?: FailureClass;
+    notes?: string;
+}
+/**
+ * Layer — optional classification in a nested build workflow.
+ * `builder`: the meta-agent editing a project (e.g. agent-builder Forge chat).
+ * `app-build`: sandbox harness that compiled + tested the generated scaffold.
+ * `app-runtime`: a run of the generated agent against a domain scenario.
+ * `meta`: any meta-eval (judge replay, correlation analysis).
+ */
+type RunLayer = 'builder' | 'app-build' | 'app-runtime' | 'meta' | 'custom';
+interface Run {
+    runId: string;
+    /**
+     * Stable identifier of the scenario being executed.
+     *
+     * Always populated on the persisted Run — but `TraceEmitter.startRun` accepts
+     * input WITHOUT this field, substituting a sensible default
+     * (`run.layer ?? run.tags?.['kind'] ?? 'runtime'`) when the caller has no
+     * curated scenario to anchor to (runtime / operator / meta-eval runs). This
+     * keeps the persisted shape unambiguous for downstream filters + aggregations
+     * while removing the boilerplate of inventing placeholder ids at the call site.
+     */
+    scenarioId: string;
+    variantId?: string;
+    datasetVersion?: string;
+    /** Git SHA of agent code at run time. */
+    codeSha?: string;
+    /** Hash of the prompt template + any system prompt. */
+    promptSha?: string;
+    /** Model id + date + system-prompt hash, concatenated. */
+    modelFingerprint?: string;
+    seed?: number;
+    /** Arbitrary environment markers (shell, docker version, tz). */
+    envFingerprint?: Record<string, string>;
+    /** Version of the redaction rules applied to this run. */
+    redactionVersion?: string;
+    /** Parent run in a nested build workflow. A builder run's children are
+     *  app-build runs; those children are app-runtime runs. */
+    parentRunId?: string;
+    /** Stable project identifier — groups runs across chats + sessions. */
+    projectId?: string;
+    /** Chat/conversation identifier within a project. */
+    chatId?: string;
+    /** Layer classification — hint for aggregation; not enforced. */
+    layer?: RunLayer;
+    startedAt: number;
+    endedAt?: number;
+    status: RunStatus;
+    outcome?: RunOutcome$1;
+    budget?: BudgetSpec;
+    /** Free-form labels for downstream grouping. */
+    tags?: Record<string, string>;
+}
+type SpanKind = 'agent' | 'llm' | 'tool' | 'retrieval' | 'judge' | 'sandbox' | 'custom';
+type SpanStatus = 'ok' | 'error';
+interface SpanBase {
+    spanId: string;
+    parentSpanId?: string;
+    runId: string;
+    kind: SpanKind;
+    name: string;
+    startedAt: number;
+    endedAt?: number;
+    status?: SpanStatus;
+    error?: string;
+    /** Anything not covered by typed fields. Kept deliberately free-form. */
+    attributes?: Record<string, unknown>;
+}
+interface Message {
+    role: 'system' | 'user' | 'assistant' | 'tool';
+    content: string;
+    tokens?: number;
+    /** Multi-modal content descriptors; blobs themselves live in Artifacts. */
+    images?: Array<{
+        artifactId?: string;
+        url?: string;
+        mime?: string;
+    }>;
+}
+interface LlmSpan extends SpanBase {
+    kind: 'llm';
+    model: string;
+    messages: Message[];
+    output?: string;
+    inputTokens?: number;
+    /** All generated tokens, including the reasoning subset when present. */
+    outputTokens?: number;
+    cachedTokens?: number;
+    cacheWriteTokens?: number;
+    /** Reasoning-token subset of `outputTokens`. */
+    reasoningTokens?: number;
+    costUsd?: number;
+    finishReason?: string;
+}
+interface ToolSpan extends SpanBase {
+    kind: 'tool';
+    toolName: string;
+    args: unknown;
+    /** False when the source observed the call but did not capture its arguments. */
+    argsCaptured?: boolean;
+    result?: unknown;
+    latencyMs?: number;
+}
+interface RetrievalSpan extends SpanBase {
+    kind: 'retrieval';
+    query: string;
+    hits: Array<{
+        docId: string;
+        score: number;
+        content?: string;
+    }>;
+}
+interface JudgeSpan extends SpanBase {
+    kind: 'judge';
+    judgeId: string;
+    /** Span this judgment applies to. */
+    targetSpanId: string;
+    dimension: string;
+    /** Numeric score (free-range; interpretation up to the judge). */
+    score: number;
+    rationale?: string;
+    evidence?: string;
+}
+interface SandboxSpan extends SpanBase {
+    kind: 'sandbox';
+    image?: string;
+    command?: string;
+    exitCode?: number;
+    testsTotal?: number;
+    testsPassed?: number;
+    stdoutHash?: string;
+    stderrHash?: string;
+    /** Duration in ms; the harness fills this explicitly (endedAt - startedAt may miss setup). */
+    wallMs?: number;
+}
+interface GenericSpan extends SpanBase {
+    kind: 'agent' | 'custom';
+}
+type Span = LlmSpan | ToolSpan | RetrievalSpan | JudgeSpan | SandboxSpan | GenericSpan;
+type EventKind = 'log' | 'error' | 'budget_decrement' | 'budget_breach' | 'state_mutation' | 'policy_violation' | 'redaction_applied' | 'custom';
+interface TraceEvent {
+    eventId: string;
+    runId: string;
+    spanId?: string;
+    kind: EventKind;
+    timestamp: number;
+    payload: Record<string, unknown>;
+}
+interface BudgetLedgerEntry {
+    runId: string;
+    dimension: keyof BudgetSpec;
+    limit: number;
+    consumed: number;
+    remaining: number;
+    timestamp: number;
+    breached: boolean;
+    /** Span that triggered this entry, if any. */
+    spanId?: string;
+}
+interface Artifact {
+    artifactId: string;
+    runId: string;
+    spanId?: string;
+    contentType: string;
+    sizeBytes: number;
+    /** sha256 in hex. */
+    hash: string;
+    /** External storage URL (R2, S3, filesystem path). */
+    storageUrl?: string;
+    /** Inline content for small blobs — keep under ~64KB. */
+    inlineContent?: string;
+}
+type FailureClass = 'success' | 'reasoning_error' | 'tool_selection_error' | 'tool_argument_error' | 'tool_recovery_failure' | 'hallucination' | 'instruction_following' | 'safety_refusal_miss' | 'policy_violation' | 'budget_exceeded' | 'format_drift' | 'permission_escalation' | 'pii_leak' | 'cost_overrun' | 'timeout' | 'sandbox_failure' | 'missing_user_data' | 'missing_domain_data' | 'missing_codebase_context' | 'missing_runtime_context' | 'missing_credentials' | 'missing_integration_connection' | 'missing_integration_scope' | 'integration_approval_required' | 'integration_auth_expired' | 'integration_provider_failure' | 'bad_integration_manifest' | 'unsafe_integration_write_denied' | 'stale_external_data' | 'bad_retrieval' | 'insufficient_evidence' | 'contradictory_evidence' | 'ambiguous_user_intent' | 'knowledge_readiness_blocked' | 'unknown';
+
+/**
+ * Paper-grade RunRecord schema + runtime validator.
+ *
+ * Every run that participates in a promotion gate, paper table, or
+ * researcher loop SHOULD be recorded as a `RunRecord`. The mandatory
+ * fields are exactly those the paper "Two Loops, Three Roles" requires
+ * for reproducibility: who/what/when/cost/seed/hash, plus the search vs
+ * holdout split tag and either a `searchScore` or a `holdoutScore`.
+ *
+ * This is intentionally NOT a replacement for the rich `Run` /
+ * `ProposeReviewReport` / `ScenarioResult` types already in the
+ * package. Those are runtime structures with full provenance. A
+ * `RunRecord` is the analysis-time projection — the JSON-friendly
+ * row you'd put in a parquet file or paste into a notebook.
+ *
+ * Validate at the boundary:
+ *
+ *   const rec = validateRunRecord(rawJson)         // throws on missing
+ *   const ok  = isRunRecord(rawJson)               // boolean check
+ *   const rec = parseRunRecordSafe(rawJson)        // { ok, value | error }
+ *
+ * The validator runs in pure TS — zod is intentionally NOT a
+ * dependency. Round-trip tested in `tests/run-record.test.ts`.
+ */
+
+/** Search/dev/holdout split tag. 'search' is the paper-grade alias for the
+ *  combined train+test pool that the optimizer is allowed to read. */
+type RunSplitTag = 'search' | 'dev' | 'holdout';
+interface RunTokenUsage {
+    input: number;
+    /** All generated tokens charged as output, including reasoning tokens. */
+    output: number;
+    /** Reasoning-token subset of `output`, when the provider reports it. */
+    reasoning?: number;
+    /** Prompt tokens served from a provider cache. */
+    cached?: number;
+    /** Prompt tokens written into a provider cache. */
+    cacheWrite?: number;
+}
+/**
+ * How a run's USD amount was obtained.
+ *
+ * `costUsd` remains mandatory for wire compatibility. New producers should
+ * always populate this discriminated union so a missing bill is never
+ * mistaken for an observed zero-dollar run. For `uncaptured`, `costUsd` uses
+ * the legacy `0` sentinel while this field carries the truthful null.
+ */
+type RunCostProvenance = {
+    kind: 'observed';
+    usd: number;
+} | {
+    kind: 'estimated';
+    usd: number;
+} | {
+    kind: 'uncaptured';
+    usd: null;
+};
+interface RunJudgeMetadata {
+    model: string;
+    promptVersion: string;
+    /** [0,1] confidence the judge declared. Constant judge confidence
+     *  across many runs is a fallback signal (see `canary.ts`). */
+    confidence: number;
+    /** True if the judge degraded to a fallback path (rules-only,
+     *  prior-call cache, etc.). The canary uses this to alert. */
+    fallback: boolean;
+}
+/**
+ * Per-judge / per-dimension breakdown for runs scored by an ensemble of
+ * judges over a multi-dimensional rubric.
+ *
+ * The collapsed `outcome.searchScore` / `holdoutScore` carries the
+ * composite the gate uses. The full breakdown belongs here so consumers
+ * can answer "which judge disagreed?", "which dimension dragged the
+ * composite down?", and "did half the panel fail?" without re-running.
+ *
+ * `perJudge[judgeId][dim]` is the canonical source; `perDimMean` and
+ * `composite` are convenience projections — derivable but precomputed so
+ * downstream IRR primitives (`interRaterReliability`,
+ * `corpusInterRaterAgreement`) and reporters don't pay the same
+ * aggregation twice.
+ *
+ * Fail-loud discipline: judges that errored out land in `failedJudges`
+ * by id. A missing key in `perJudge` is ambiguous (silent zero vs not
+ * run); the explicit list makes a partial-failure recorded as such.
+ */
+interface JudgeScoresRecord {
+    /** Per-judge per-dimension scores. `{ "kimi-k2.6": { helpfulness: 0.8, clarity: 0.7 }, ... }`. */
+    perJudge: Record<string, Record<string, number>>;
+    /** Per-dim mean across judges. Convenience — derivable from `perJudge`. */
+    perDimMean: Record<string, number>;
+    /** Composite mean across all dims and judges. Mirrors the score
+     *  the gate sees on `outcome.searchScore` / `holdoutScore`. */
+    composite: number;
+    /** Judges that errored or returned an unparseable verdict. Recorded
+     *  by id (e.g. `['glm-5.1']`) so a partial-failure case is explicit,
+     *  not inferred from missing keys in `perJudge`. */
+    failedJudges?: string[];
+    /** Free-form notes the judges emitted (joined across judges or
+     *  first-judge only — consumer's choice). */
+    notes?: string;
+}
+interface RunOutcome {
+    /** Score on the search/optimization split. Optional because a
+     *  holdout-only evaluation only fills `holdoutScore`. */
+    searchScore?: number;
+    /** Score on the held-out split. Optional because a search-only run
+     *  only fills `searchScore`. At least one must be present. */
+    holdoutScore?: number;
+    /** Bag of any other metric the run produced — judge dimensions,
+     *  pass/fail counters, latency stats, etc. Numeric only — keeps
+     *  reporters honest. */
+    raw: Record<string, number>;
+    /** Per-judge / per-dim breakdown. Consumers writing ensemble
+     *  judgements populate this; substrate primitives like
+     *  `interRaterReliability` and `corpusInterRaterAgreement` accept
+     *  these records as input. Optional — single-judge or scalar-only
+     *  runs leave it unset. */
+    judgeScores?: JudgeScoresRecord;
+    /** Authenticity / realness verdict — did the run build the REAL thing on the
+     *  intended infra, or fake it (see `./authenticity`)? Optional: only domains
+     *  with an authenticity config populate it. Carried in the corpus so the
+     *  flywheel / off-policy learning can optimize for real completion, not gamed
+     *  pass-rate. `score` is 0-1; `gated` is the anti-Goodhart flag — a gated run
+     *  must not count as a real success regardless of `score`. */
+    realness?: {
+        score: number;
+        gated: boolean;
+        reason?: string;
+    };
+}
+/**
+ * Mandatory paper-grade fields for a single evaluation run. Optional
+ * fields are extension points; mandatory fields throw if missing.
+ *
+ * Hash discipline:
+ *   - `promptHash` is the sha256 of the EFFECTIVE prompt sent to the
+ *     model (after any steering bundle merge).
+ *   - `configHash` is the sha256 of the effective run config (model,
+ *     temperature, tools, judges, splits). The pair (promptHash,
+ *     configHash) uniquely identifies an experiment cell.
+ *
+ * Model snapshot discipline:
+ *   - `model` MUST encode a snapshot version. Bare aliases like
+ *     `claude-sonnet-4` or `gpt-4o` are banned — they remap silently.
+ *     Use `claude-sonnet-4-6@2025-04-15` or `gpt-4o-2024-11-20`.
+ */
+interface RunRecord {
+    /** UUID for the run. */
+    runId: string;
+    /** Logical experiment grouping (a treatment vs a baseline within
+     *  the same sweep should share `experimentId`). */
+    experimentId: string;
+    /** Stable identifier for the candidate (variant) being run. The
+     *  promotion gate compares two `candidateId`s on matched items. */
+    candidateId: string;
+    /** RNG seed for the run. Always recorded — silent re-seeding is
+     *  the most common cause of non-reproducible numbers. */
+    seed: number;
+    /** Model identifier WITH snapshot version. */
+    model: string;
+    /** sha256 of the effective prompt (post-steering). */
+    promptHash: string;
+    /** sha256 of the effective config. */
+    configHash: string;
+    /** Git SHA the harness was run from. */
+    commitSha: string;
+    /** End-to-end wall-clock duration in milliseconds. */
+    wallMs: number;
+    /** Time spent queued before execution started, if known. */
+    queueMs?: number;
+    /** Total USD cost. Mandatory — runs without a cost number are
+     *  unbounded by definition and must not be admitted into the gate.
+     *  `0` is retained as the compatibility sentinel for an uncaptured amount;
+     *  inspect `costProvenance` before treating it as observed. */
+    costUsd: number;
+    /** Observed, model-priced estimate, or genuinely uncaptured USD amount.
+     *  Optional only so existing serialized RunRecords remain valid. */
+    costProvenance?: RunCostProvenance;
+    /** Token usage breakdown. */
+    tokenUsage: RunTokenUsage;
+    /** Judge-side metadata, if a judge was used. */
+    judgeMetadata?: RunJudgeMetadata;
+    /** Per-split scores + raw bag. */
+    outcome: RunOutcome;
+    /** Canonical, cross-agent failure class drawn from the shared
+     *  `FAILURE_CLASSES` taxonomy. This is the aggregation key that makes
+     *  "which failure dominates across the whole fleet" answerable in ONE
+     *  vocabulary — every agent classifies against the same enum. Producers
+     *  set it via the substrate classifier; leave unset only when the failure
+     *  genuinely can't be classified. */
+    failureClass?: FailureClass;
+    /** Free-form domain-specific failure detail, scoped UNDER `failureClass`
+     *  (e.g. failureClass='tool_recovery_failure', failureMode='forge_build_unsatisfied').
+     *  The within-agent drill-down; `failureClass` is the cross-agent key. */
+    failureMode?: string;
+    /** Which split this run was drawn from. */
+    splitTag: RunSplitTag;
+    /**
+     * Stable scenario identifier the run was scored against. Optional for
+     * backwards compatibility, but **strongly recommended**: every primitive
+     * that pairs runs by scenario (preferences, paired stats, BT tournament)
+     * keys on this. The campaign artifact populates it canonically; legacy
+     * runs without it fall back to inference from `outcome.raw.scenario_id`
+     * or `experimentId`.
+     */
+    scenarioId?: string;
+    /**
+     * Canonical identity for the agent profile cell that produced this row:
+     * profile artifact hash plus optional harness/model/prompt/reporting
+     * dimensions. Use `agentProfile.cellId` to group persona sweeps and
+     * longitudinal reports by the complete source profile, not by a loose
+     * candidate label or opaque config hash.
+     */
+    agentProfile?: AgentProfileCell;
+}
 
 /**
  * Adaptive curriculum / active scenario selection.
@@ -245,6 +661,24 @@ declare function compareAdaptationCurves(a: AdaptationCurve, b: AdaptationCurve,
 /** First k at which the curve's per-scenario pass rate reliably hits the threshold. */
 declare function firstPassK(curve: AdaptationCurve, threshold?: number): number | null;
 
+/**
+ * Adversarial mutation contract.
+ *
+ * `AdversarialMutation<S>` is the scenario-mutation strategy the fuzz harness
+ * (`fuzzAgent`, src/fuzz) drives: paraphrase, edge-case substitution, or
+ * compositional combination of a scenario the policy currently passes, looking
+ * for the tail inputs that break it. The harness supplies the loop; consumers
+ * supply the mutations and the failure detector.
+ */
+interface AdversarialMutation<S> {
+    id: string;
+    /**
+     * Mutate one scenario. Return null to skip; return one or more new
+     * scenarios. The harness deduplicates by `mutateScenarioId(scenario)`.
+     */
+    mutate(parent: S, rng: () => number): Promise<S[]> | S[];
+}
+
 /**
  * Test-time compute scaling curves.
  *
@@ -630,6 +1064,54 @@ declare function toAnthropicFormat(triples: PreferenceTriple[]): Array<{
     margin: number;
 }>;
 
+interface RunFilter {
+    scenarioId?: string;
+    variantId?: string;
+    status?: RunStatus;
+    since?: number;
+    until?: number;
+    tag?: {
+        key: string;
+        value: string;
+    };
+    parentRunId?: string;
+    projectId?: string;
+    chatId?: string;
+    layer?: RunLayer;
+}
+interface SpanFilter {
+    runId?: string;
+    parentSpanId?: string;
+    kind?: SpanKind;
+    name?: string;
+    toolName?: string;
+    judgeId?: string;
+    since?: number;
+    until?: number;
+}
+interface EventFilter {
+    runId?: string;
+    spanId?: string;
+    kind?: EventKind;
+    since?: number;
+    until?: number;
+}
+interface TraceStore {
+    appendRun(run: Run): Promise<void>;
+    updateRun(runId: string, patch: Partial<Run>): Promise<void>;
+    appendSpan(span: Span): Promise<void>;
+    updateSpan(spanId: string, patch: Partial<Span>): Promise<void>;
+    appendEvent(event: TraceEvent): Promise<void>;
+    appendArtifact(artifact: Artifact): Promise<void>;
+    appendBudgetEntry(entry: BudgetLedgerEntry): Promise<void>;
+    getRun(runId: string): Promise<Run | undefined>;
+    listRuns(filter?: RunFilter): Promise<Run[]>;
+    spans(filter?: SpanFilter): Promise<Span[]>;
+    events(filter?: EventFilter): Promise<TraceEvent[]>;
+    budget(runId: string): Promise<BudgetLedgerEntry[]>;
+    artifacts(runId: string): Promise<Artifact[]>;
+}
+
 /**
  * Process reward extraction — step-level credit assignment from trace spans.
  *
@@ -1044,6 +1526,451 @@ interface HarvestOptions {
  */
 declare function buildDatasetFromCorpus(corpusPath: string, config: RlDatasetConfig, opts?: HarvestOptions): Promise<RlDatasetBundle>;
 
+/**
+ * Off-policy evaluation primitives.
+ *
+ * Standard inverse-probability-weighted (IPS), self-normalized
+ * importance-weighted (SNIPS), and doubly-robust (DR) estimators for the
+ * value of a *target* policy given trajectories collected under a
+ * *behavior* policy. This is the canonical RL eval task: "we have last
+ * week's runs, we changed the policy — how would the new one do without
+ * re-running?"
+ *
+ * The math here is textbook (Dudík, Langford, Li 2011 for DR; Swaminathan
+ * & Joachims 2015 for SNIPS) but the *application* to LLM-agent
+ * evaluation needs care:
+ *
+ *   - The "policy" is the (prompt, tool config, model snapshot) triple.
+ *     Two policies have the same probability over an action *iff* their
+ *     LLM call would emit the same token with the same probability —
+ *     which is generally unknowable without the model log-probs.
+ *   - For LLM agents, propensity scores must be supplied by the caller
+ *     (logged in the trace, recovered from token log-probs, or estimated
+ *     via a learned propensity model). We do NOT estimate propensity here.
+ *   - Doubly-robust requires a Q-function (model-based reward predictor).
+ *     We accept any callable; consumers pass either a tabular average,
+ *     a regression fit, or a learned reward model.
+ *
+ * Bias / variance tradeoffs:
+ *   - IPS: unbiased; high variance for small overlap, infinite variance
+ *     when target has support outside behavior.
+ *   - SNIPS: lower variance, slight bias; usually preferred in practice.
+ *   - DR: doubly-robust — unbiased if either propensity OR Q-function is
+ *     correct. Lowest practical variance when Q is decent. Use this.
+ *
+ * Caveat the panel will land: on the LLM-agent setting, propensity scores
+ * recovered from token log-probs are noisy, the action space is enormous,
+ * and overlap is often poor. These estimators are useful but not magic;
+ * complement with `replayCampaign` (exact replay where the request hashes
+ * match) for high-confidence answers and OPE for the gap.
+ */
+interface OffPolicyTrajectory {
+    /** Stable id, for traceability through the dataset. */
+    runId: string;
+    /** Reward observed under the behavior policy (the realized outcome). */
+    reward: number;
+    /**
+     * Behavior-policy probability of the action that was taken. For LLM
+     * agents this is typically `exp(sum(token_log_probs))` over the chosen
+     * trajectory. Must be in (0, 1].
+     */
+    behaviorProb: number;
+    /**
+     * Target-policy probability of the same action. For replay-style
+     * counterfactual evaluation this is what the *new* policy would have
+     * assigned to the *old* trajectory. Must be in [0, 1].
+     */
+    targetProb: number;
+    /**
+     * Optional model-based reward prediction at the same context. Used by
+     * `doublyRobust`. Set to `null` for IPS-only evaluation.
+     */
+    qHat?: number | null;
+}
+interface OffPolicyEstimate {
+    /** Estimated value of the target policy. */
+    value: number;
+    /** Standard error of the estimate. */
+    standardError: number;
+    /** Effective sample size (Kong 1992). Lower = more reliance on a few high-weight samples. */
+    effectiveSampleSize: number;
+    /** Number of trajectories used. */
+    n: number;
+    /**
+     * Diagnostic: maximum importance weight observed. Large values (>>10x
+     * mean) are a red flag — variance is dominated by a few outliers.
+     */
+    maxImportanceWeight: number;
+}
+interface OffPolicyOptions {
+    /**
+     * Cap importance weights at this value (Ionides 2008 truncated IS) to
+     * trade unbiasedness for variance reduction. Default `Infinity` (no cap).
+     * Set e.g. `10` for stable estimates when the policies are close.
+     */
+    weightCap?: number;
+    /** Reward clipping range. Default `[0, 1]`. */
+    rewardClip?: {
+        low: number;
+        high: number;
+    };
+}
+/**
+ * Inverse Probability Weighting (Horvitz-Thompson). Unbiased estimator
+ * of E[reward under target policy]. Variance scales with the spread of
+ * target/behavior ratios.
+ */
+declare function inverseProbabilityWeighting(trajectories: OffPolicyTrajectory[], opts?: OffPolicyOptions): OffPolicyEstimate;
+/**
+ * Self-Normalized Importance Sampling. Lower variance than vanilla IPS at
+ * the cost of small bias (vanishing as N grows). The right default for
+ * LLM-agent evaluation where overlap is often poor.
+ */
+declare function selfNormalizedImportanceWeighting(trajectories: OffPolicyTrajectory[], opts?: OffPolicyOptions): OffPolicyEstimate;
+/**
+ * Doubly-robust off-policy estimator (Dudík, Langford, Li 2011).
+ *
+ *     V_DR = (1/N) * sum_i [ q_hat_i + (target_prob_i / behavior_prob_i) * (r_i - q_hat_i) ]
+ *
+ * Unbiased if EITHER:
+ *   - the importance ratios are correct (IPS-style validity), OR
+ *   - the Q-hat function is correct (model-based validity).
+ *
+ * In practice both are imperfect, but the residual bias is the *product*
+ * of both errors — much smaller than either alone. This is why DR is the
+ * default in production OPE pipelines.
+ *
+ * Requires `qHat` on every trajectory. If any are `null`, the estimator
+ * falls back to SNIPS for those entries (loud-fallback behavior; the
+ * report's `n` reflects the full set but `effectiveSampleSize` accounts
+ * for the lost variance reduction).
+ */
+declare function doublyRobust(trajectories: OffPolicyTrajectory[], opts?: OffPolicyOptions): OffPolicyEstimate;
+/**
+ * Convenience: run all three estimators and return them side-by-side.
+ * The recommended diagnostic — agreement across estimators is a much
+ * stronger signal than any single one.
+ */
+declare function offPolicyEstimateAll(trajectories: OffPolicyTrajectory[], opts?: OffPolicyOptions): {
+    ips: OffPolicyEstimate;
+    snips: OffPolicyEstimate;
+    dr: OffPolicyEstimate;
+};
+
+/**
+ * OutcomeStore — deployment outcomes attached to Run IDs.
+ *
+ * Outcomes arrive asynchronously from production telemetry after the
+ * eval run completed: user ratings, retention flags, conversion events,
+ * revenue, support-ticket rate, anything a product team can measure.
+ * The store is a peer to TraceStore — separate lifecycle, same runId
+ * foreign key.
+ *
+ * The whole point of this module is to make the meta-eval correlation
+ * question computable: `correlate(evalMetric, outcomeMetric) → r, ρ, n, CI`.
+ */
+interface DeploymentOutcome {
+    runId: string;
+    capturedAt: number;
+    /** Numeric outcomes keyed by name — retention_7d, csat, revenue_usd, etc. */
+    metrics: Record<string, number>;
+    /** Dimensions for stratified analysis — cohort, region, user_segment. */
+    labels?: Record<string, string>;
+    /** Free-form provenance (source system, pipeline version). */
+    source?: string;
+}
+interface OutcomeFilter {
+    runIds?: string[];
+    since?: number;
+    until?: number;
+    label?: {
+        key: string;
+        value: string;
+    };
+    source?: string;
+}
+interface OutcomeStore {
+    append(outcome: DeploymentOutcome): Promise<void>;
+    /** All outcomes attached to this run (a single run can have many — multiple
+     *  capture windows over deployment time). */
+    forRun(runId: string): Promise<DeploymentOutcome[]>;
+    list(filter?: OutcomeFilter): Promise<DeploymentOutcome[]>;
+}
+declare class InMemoryOutcomeStore implements OutcomeStore {
+    private items;
+    append(outcome: DeploymentOutcome): Promise<void>;
+    forRun(runId: string): Promise<DeploymentOutcome[]>;
+    list(filter?: OutcomeFilter): Promise<DeploymentOutcome[]>;
+}
+interface FileSystemOutcomeStoreOptions {
+    dir: string;
+    maxBytes?: number;
+}
+declare class FileSystemOutcomeStore implements OutcomeStore {
+    private dir;
+    private maxBytes;
+    private memo?;
+    private loaded;
+    constructor(options: FileSystemOutcomeStoreOptions);
+    private ensureDir;
+    append(outcome: DeploymentOutcome): Promise<void>;
+    private load;
+    forRun(runId: string): Promise<DeploymentOutcome[]>;
+    list(filter?: OutcomeFilter): Promise<DeploymentOutcome[]>;
+}
+
+/**
+ * Rubric predictive validity — does our eval rubric predict deployment
+ * outcomes?
+ *
+ * `correlationStudy` (already in this package) joins a `TraceStore` to an
+ * `OutcomeStore` and computes Pearson + Spearman + bootstrap CI for each
+ * (eval-metric, outcome-metric) pair. That answers "does X correlate with
+ * Y at all." `rubricPredictiveValidity` is the campaign-shaped wrapper
+ * around it: take a sequence of `RunRecord`s (the canonical campaign
+ * artifact) and a `DeploymentOutcomeStore`, join on `runId`, return a
+ * ranked verdict on every rubric whose dimension scores were captured in
+ * `outcome.raw`.
+ *
+ * The point — quoting the methodology doc — is that **without this loop
+ * every rubric is faith-based**. Once it's wired, you know which rubrics
+ * have earned their promotion power and which ones are decoration.
+ *
+ *   const validity = await rubricPredictiveValidity({
+ *     runs: lastQuarter,
+ *     outcomes: shipFlagOutcomeStore,
+ *     outcomeMetrics: ['revenue_lift', 'retention_30d', 'csat'],
+ *     rubrics: ['anti_slop', 'semantic_concept', 'tool_recovery'],
+ *   })
+ *   for (const r of validity.ranked) {
+ *     console.log(`${r.rubric} → ${r.bestOutcome}: ρ=${r.spearman.toFixed(2)}`)
+ *   }
+ *
+ * The function is intentionally read-only. Use the verdict to deprecate
+ * decorative rubrics, re-weight composite scores, or trigger a
+ * recalibration sweep when predictive validity drops below a threshold.
+ */
+
+interface RubricOutcomePair {
+    rubric: string;
+    outcome: string;
+    n: number;
+    pearson: number;
+    spearman: number;
+    ci95: {
+        low: number;
+        high: number;
+    };
+    /**
+     * Verdict bucket. `load_bearing` ≥ 0.7, `informative` ≥ 0.4,
+     * `decorative` < 0.4 in absolute correlation. A negative correlation
+     * with a desired outcome is also `decorative` — actively misleading
+     * is worse than uninformative.
+     */
+    verdict: 'load_bearing' | 'informative' | 'decorative';
+}
+interface RubricRanking {
+    rubric: string;
+    /** Outcome metric this rubric correlated best with. */
+    bestOutcome: string;
+    spearman: number;
+    pearson: number;
+    n: number;
+    verdict: RubricOutcomePair['verdict'];
+}
+interface RubricPredictiveValidityReport {
+    pairs: RubricOutcomePair[];
+    /** Per-rubric best pair, sorted descending by |spearman|. */
+    ranked: RubricRanking[];
+    joinedSamples: number;
+    skippedRuns: number;
+    /** Rubrics that were declared but never produced a usable score. */
+    rubricsWithoutData: string[];
+}
+
+/**
+ * HeldOutGate — first-class held-out paired-delta promotion gate.
+ *
+ * Encodes the "honesty override" pattern that lived inline in
+ * `~/webb/redteam/scripts/agent-eval-autoresearch.ts:138–171`.
+ * The optimizer's best-guess is one thing; what we should actually
+ * ship is another. The gate is the line between them.
+ *
+ * A candidate is promoted iff ALL three pass:
+ *
+ *   1. **Productive runs**: the candidate has at least
+ *      `minProductiveRuns` paired observations on items where BOTH
+ *      candidate and baseline produced a real (non-silent) score.
+ *   2. **Paired delta**: the lower bound of the bootstrap CI on the
+ *      median per-item delta (candidate − baseline) on the HOLDOUT
+ *      split is strictly greater than `pairedDeltaThreshold`.
+ *   3. **Overfit gap**: the candidate's gap between search-split
+ *      score and holdout-split score is no worse (more positive)
+ *      than the baseline's gap by more than `overfitGapThreshold`.
+ *      "Better on search, worse on holdout" is the canonical
+ *      overfit pattern; this catches it.
+ *
+ * The decision carries a machine-readable `rejectionCode` plus an
+ * `evidence` block with every number the gate looked at, so the
+ * downstream researcher / paper / dashboard can re-derive the
+ * verdict without re-running.
+ *
+ * See also:
+ *   - `src/statistics.ts` for `pairedBootstrap` + `wilcoxonSignedRank`
+ *   - `src/run-record.ts` for the input row schema
+ *   - `src/reference-replay.ts` for the older, reference-replay-
+ *     specific promotion path (still useful for replay-style evals).
+ */
+
+type HeldOutGateRejectionCode = 'few_runs' | 'negative_delta' | 'overfit_gap' | 'cost_ceiling';
+interface GateEvidence {
+    /** Number of paired (candidate, baseline) holdout observations used. */
+    productiveRuns: number;
+    /** Median of (candidate − baseline) paired holdout deltas. */
+    medianPairedDelta: number;
+    /** Bootstrap CI on the median paired holdout delta. */
+    pairedCI: {
+        low: number;
+        high: number;
+    };
+    /** Wilcoxon signed-rank p-value on the paired holdout deltas. */
+    pairedPValue: number;
+    /** Mean candidate score on the search split (NaN if none). */
+    searchScore: number;
+    /** Mean candidate score on the holdout split (NaN if none). */
+    holdoutScore: number;
+    /** Candidate (search − holdout) gap. */
+    overfitGap: number;
+    /** Baseline (search − holdout) gap. */
+    baselineOverfitGap: number;
+    /** Median per-task USD cost across the candidate's runs. Recorded
+     *  even when no `costPerTaskCeiling` is configured so downstream
+     *  dashboards (intelligence.tangle.tools) can render \$/task per
+     *  generation regardless of gating policy. */
+    medianCandidateCost: number;
+    /** Median per-task USD cost across the baseline runs, for
+     *  symmetric reporting. */
+    medianBaselineCost: number;
+}
+interface GateDecision$1 {
+    /** Final promote/no-promote verdict. */
+    promote: boolean;
+    /** The candidate that was evaluated. */
+    candidateId: string;
+    /** The baseline it was compared against. */
+    baselineId: string;
+    /** Every number the gate looked at, for audit + paper export. */
+    evidence: GateEvidence;
+    /** Human-readable reason. */
+    reason: string;
+    /** Machine-readable rejection code, or null on promote. */
+    rejectionCode: HeldOutGateRejectionCode | null;
+}
+
+/**
+ * Researcher interface — stable hook for an external autonomous-research
+ * agent to drive the meta-loop.
+ *
+ * Implementations live downstream (typically in a private repo that
+ * runs the actual LLM). This package ships only the contract + a
+ * `NoopResearcher` so consumers can wire the surface without being
+ * forced to implement every method up front.
+ *
+ * The four methods mirror the four stages of the paper "Two Loops,
+ * Three Roles":
+ *
+ *   inspectFailures   — given the observed runs, what failure modes
+ *                       are present? (data → diagnosis)
+ *   proposeChange     — given diagnosed failure modes, what
+ *                       structural changes should we try?
+ *                       (diagnosis → plan delta)
+ *   applyChange       — fold the proposed deltas into a concrete
+ *                       experiment plan against an existing baseline.
+ *                       (plan delta → executable plan)
+ *   evaluateChange    — run the plan, return runs + the gate verdict.
+ *                       (executable plan → verdict)
+ *
+ * Composition is the discipline: a Researcher implementation MUST
+ * keep these four steps separate and inspectable. Conflating
+ * "diagnose + propose + run" into a single LLM call defeats the
+ * point of the framework — you can't audit which step lied.
+ *
+ * THIS INTERFACE IS STABLE. Breaking changes require a new module
+ * (e.g. `Researcher2`) so existing implementations keep working.
+ */
+
+/** A diagnosed failure mode with the run-IDs that exhibit it. */
+interface FailureMode {
+    /** Short machine-readable code. Must be stable across runs of the
+     *  same researcher to enable longitudinal tracking. */
+    code: string;
+    /** Human-readable description for the paper / dashboard. */
+    description: string;
+    evidence: {
+        /** Run IDs (from `RunRecord.runId`) where this failure mode was
+         *  observed. */
+        runIds: string[];
+        /** Number of run samples that informed the diagnosis. */
+        samples: number;
+    };
+}
+/** A single steering change the researcher wants to try. */
+interface SteeringChange {
+    kind: 'reviewer_prompt' | 'skill_add' | 'skill_remove' | 'threshold' | 'budget';
+    /** Implementation-specific payload. Researcher implementations
+     *  define the schema — keep this `unknown` here to avoid coupling
+     *  the public interface to any one researcher's internal model. */
+    payload: unknown;
+    /** Why the researcher proposed this change. Goes into the audit
+     *  trail next to the failure-mode evidence. */
+    rationale: string;
+    /** Optional self-reported expected delta on the headline metric. */
+    expectedDelta?: number;
+}
+/** A single experiment plan, mapped onto the search/holdout splits. */
+interface ExperimentPlan {
+    baselineCandidateId: string;
+    proposedCandidateId: string;
+    changes: SteeringChange[];
+    /** USD ceiling for the entire experiment. The runner must stop
+     *  before exceeding this and report a partial result. */
+    evaluationBudgetUsd: number;
+    /** Item IDs (your dataset keys) for the search vs holdout splits. */
+    splits: {
+        search: string[];
+        holdout: string[];
+    };
+}
+/** Result of running a plan: every run, plus the gate verdict. */
+interface ExperimentResult {
+    plan: ExperimentPlan;
+    runs: RunRecord[];
+    gateDecision: GateDecision$1;
+}
+/**
+ * The researcher loop. Stable, four-step, inspectable.
+ *
+ *   ┌──────────┐  inspectFailures  ┌──────────┐  proposeChange ┌──────────┐
+ *   │   runs   │ ─────────────────▶│ failures │ ──────────────▶│ changes  │
+ *   └──────────┘                   └──────────┘                └────┬─────┘
+ *                                                                   │
+ *                                                                   ▼
+ *                              ┌────────────────┐  applyChange ┌────────┐
+ *                              │ ExperimentPlan │ ◀────────────│  base  │
+ *                              └────────┬───────┘              └────────┘
+ *                                       │
+ *                       evaluateChange  ▼
+ *                              ┌────────────────┐
+ *                              │ ExperimentResult│
+ *                              └────────────────┘
+ */
+interface Researcher {
+    inspectFailures(runs: RunRecord[]): Promise<FailureMode[]>;
+    proposeChange(failures: FailureMode[]): Promise<SteeringChange[]>;
+    applyChange(changes: SteeringChange[], baseline: ExperimentPlan): Promise<ExperimentPlan>;
+    evaluateChange(plan: ExperimentPlan): Promise<ExperimentResult>;
+}
+
 /**
  * `PredictiveValidityResearcher` — concrete `Researcher` implementation
  * that drives selection from outcome-anchored predictive validity.
@@ -1110,6 +2037,121 @@ declare class PredictiveValidityResearcher implements Researcher {
     getLastReport(): RubricPredictiveValidityReport | null;
 }
 
+/**
+ * Validator-output verdict — substrate primitive for "did this output pass,
+ * and how well?"
+ *
+ * Used by:
+ *   - `@tangle-network/agent-eval/matrix` — verdict per cell in the cartesian.
+ *   - `@tangle-network/agent-runtime` — Validator<Output, Verdict = DefaultVerdict>.
+ *     Runtime keeps `Validator` because it's coupled to runtime-shaped
+ *     `ValidationCtx` (iteration, signal, traceEmitter); the verdict TYPE
+ *     itself is a substrate concept and lives here.
+ *
+ * Repo layering: agent-eval is the substrate (no upward deps). Both
+ * agent-runtime and agent-knowledge consume this type FROM agent-eval —
+ * never the other way around. See CLAUDE.md "Repo layering" for the rule.
+ */
+/**
+ * Minimal verdict shape — `valid` + `score` are required; `scores` +
+ * `notes` are optional surface. Validators that need richer shapes
+ * parameterise `Validator<Output, MyVerdict>` with their own type.
+ *
+ * Need structured extras? Extend DefaultVerdict with typed fields — never
+ * serialize extras into `notes`.
+ */
+interface DefaultVerdict {
+    /** Whether the output meets the validator's pass criteria. */
+    valid: boolean;
+    /** Aggregate score in [0, 1]. Drivers use this for winner selection. */
+    score: number;
+    /** Per-dimension scores. Free-form; weighted into `score` by the validator. */
+    scores?: Record<string, number>;
+    /** Human-readable rationale; surfaces in trace + final-result `winner.verdict`. */
+    notes?: string;
+}
+
+/**
+ * Multi-layer verifier — ordered pipeline of verification layers.
+ *
+ * Different contract from {@link JudgeRunner} (which runs parallel
+ * specs against a sandbox). MultiLayerVerifier is a DAG of layers
+ * (install → typecheck → build → lint → serve → semantic → …) with
+ * dependency-based skip, per-layer findings, soft-fail semantics, and
+ * an aggregated `blendedScore` across all passed layers.
+ *
+ * Use when you want:
+ *   - ordered stages where a failing upstream stage skips downstream ones
+ *   - each stage produces rich `findings` (severity + message + evidence)
+ *   - a single composite score across stages with per-stage weights
+ *   - soft-fail stages whose failure doesn't abort the pipeline
+ *
+ * Use {@link JudgeRunner} when you want:
+ *   - N independent judges running in parallel against the same artifact
+ *   - no inter-judge dependencies
+ *   - boolean `passed` per judge + overall
+ *
+ * Both primitives compose — JudgeRunner can be invoked as a single
+ * layer inside a MultiLayerVerifier if that suits the caller.
+ */
+
+type LayerStatus = 'pass' | 'fail' | 'skipped' | 'error' | 'timeout';
+type Severity = 'critical' | 'major' | 'minor' | 'info';
+interface Finding {
+    severity: Severity;
+    message: string;
+    evidence?: string;
+    /** Optional layer name the finding belongs to (set by the verifier if omitted). */
+    layer?: string;
+    /**
+     * Free-form structured payload — used by `multiToolchainLayer` to attach
+     * `{ adapter: 'pnpm' }`, by judges to attach evidence pointers, etc.
+     * Renderers MAY interrogate; agent-eval primitives never assume shape.
+     */
+    detail?: Record<string, unknown>;
+}
+interface LayerResult {
+    layer: string;
+    status: LayerStatus;
+    /** 0..1 score, optional — layers that don't produce a numeric score omit. */
+    score?: number;
+    durationMs: number;
+    findings: Finding[];
+    /** Short human-readable summary (one line). */
+    reason?: string;
+    /**
+     * Numeric layer-level diagnostics: error counts, warning counts,
+     * cyclomatic complexity, total adapter wall-time, etc. Keyed by
+     * diagnostic name; null = "diagnostic not applicable / not measured."
+     * Renderers that know the keys can display them; ones that don't,
+     * ignore. Free-form on purpose — consumers type the value shape in
+     * their own namespace.
+     */
+    diagnostics?: Record<string, number | null>;
+    /** Any rich per-layer detail — rendered as-is by consumers that know the layer. */
+    detail?: Record<string, unknown>;
+}
+/** Extends the substrate verdict spine: `valid` = `allPass` and `score` =
+ *  `blendedScore` — derived where the report is aggregated, so spine
+ *  consumers (drivers, gates) read this report without an adapter. */
+interface VerificationReport extends DefaultVerdict {
+    layers: LayerResult[];
+    passCount: number;
+    failCount: number;
+    skippedCount: number;
+    errorCount: number;
+    /** True iff at least one scored layer ran AND every scored layer passed. */
+    allPass: boolean;
+    /**
+     * Weighted mean of `score` across contributing layers. 0 when no layers
+     * contributed. See {@link Layer.failContributesToScore} for fail semantics.
+     */
+    blendedScore: number;
+    durationMs: number;
+    startedAt: string;
+    finishedAt: string;
+}
+
 /**
  * Verifiable reward channel.
  *
@@ -1336,6 +2378,994 @@ interface DetectRewardHackingInput {
 }
 declare function detectRewardHacking(input: DetectRewardHackingInput): RewardHackingReport;
 
+type CostChannel = 'agent' | 'judge' | 'verifier' | 'analyst' | 'driver' | (string & {});
+interface ChannelRollup {
+    channel: CostChannel;
+    calls: number;
+    inputTokens: number;
+    outputTokens: number;
+    cachedTokens: number;
+    costUsd: number;
+    unpricedCalls: number;
+    unknownUsageCalls: number;
+}
+interface CostLedgerSummary {
+    totalCalls: number;
+    pendingCalls: number;
+    unresolvedCalls: number;
+    reservedCostUsd: number;
+    inputTokens: number;
+    outputTokens: number;
+    cachedTokens: number;
+    totalCostUsd: number;
+    byChannel: ChannelRollup[];
+    unpricedModels: string[];
+    fullyPriced: boolean;
+    usageComplete: boolean;
+    accountingComplete: boolean;
+    incompleteReasons: string[];
+}
+
+/**
+ * RawProviderSink — first-class persistence for the actual HTTP-level
+ * request/response bodies of every LLM provider call.
+ *
+ * Why this is a separate sink from the structured `LlmSpan`:
+ *
+ *   - `LlmSpan` records the *intent* — model name, messages, output text,
+ *     usage. It's what dashboards read; it's NOT enough for forensics.
+ *   - When a downstream consumer reports "the verifier used the wrong route"
+ *     or "tokens look right but reasoning was missing," the only way to
+ *     answer is the raw HTTP body. Span fields can lie (a proxy can echo
+ *     a different `model` value than what actually answered); the raw
+ *     response is ground truth.
+ *
+ * Default behaviour: opt-in. Pass `rawSink` to `LlmClientOptions` (or the
+ * matrix runner / BuilderSession sets it up automatically) and every
+ * request, response, and error is recorded — including retries, with the
+ * attempt index attached so a flaky call's full event chain is recoverable.
+ *
+ * Redaction is enforced at sink time. The default redactor strips
+ * `Authorization`, `X-Api-Key`, `X-Auth-Token`, `Cookie` headers and any
+ * payload field whose key matches `apiKey | api_key | bearer | password |
+ * secret | token` (case-insensitive). Override via the sink constructor or
+ * the per-call `redactor`. The `redactedFields` array on the persisted
+ * event lets a reviewer see what was stripped without exposing the values.
+ */
+type RawProviderDirection = 'request' | 'response' | 'error';
+interface RawProviderEvent {
+    /** Stable id. Generated by the sink if omitted. */
+    eventId: string;
+    /** Trace context populated by `LlmClient` when the call is wrapped in a span. */
+    runId?: string;
+    spanId?: string;
+    /**
+     * Logical provider name. Free-form so callers can use whatever id matches
+     * their topology (`'openai'`, `'anthropic'`, `'tangle-router'`, …). When
+     * omitted, derived from `baseUrl` in `LlmClientOptions`.
+     */
+    provider: string;
+    model: string;
+    /** Endpoint path, e.g. `'/v1/chat/completions'`. */
+    endpoint: string;
+    /** Base URL used for the call (already-normalised — no trailing slash). */
+    baseUrl: string;
+    /** 0-indexed retry attempt. The first attempt is 0; a retried call gets 1, 2, … */
+    attemptIndex: number;
+    direction: RawProviderDirection;
+    /** Unix ms. */
+    timestamp: number;
+    /** Wall-clock duration of the call leg. Set on `response` and `error` events; null on `request`. */
+    durationMs?: number;
+    statusCode?: number;
+    requestHeaders?: Record<string, string>;
+    requestBody?: unknown;
+    responseHeaders?: Record<string, string>;
+    responseBody?: unknown;
+    /** Set on `direction: 'error'` events. */
+    errorMessage?: string;
+    /** Field paths the redactor stripped from this event ('header:Authorization', 'body.apiKey', …). */
+    redactedFields: string[];
+}
+interface RawProviderSinkFilter {
+    runId?: string;
+    spanId?: string;
+    direction?: RawProviderDirection;
+    attemptIndex?: number;
+}
+interface RawProviderSink {
+    record(event: RawProviderEvent): Promise<void>;
+    /** Optional listing — implementations that durably persist (file, db) should support this. */
+    list?(filter?: RawProviderSinkFilter): Promise<RawProviderEvent[]>;
+    /** Optional teardown for backed implementations. */
+    close?(): Promise<void>;
+}
+type ProviderRedactor = (event: RawProviderEvent) => RawProviderEvent;
+
+/**
+ * LLM client with graceful degrade.
+ *
+ * OpenAI-compatible `/v1/chat/completions` client with:
+ *   - Exponential-backoff retry on 429 + 5xx gateway errors (502/503/504).
+ *   - Retry on transient network errors (fetch failed, AbortError, ECONNRESET).
+ *   - Graceful json_schema → json_object degrade on 400 with schema-reject body.
+ *   - Fenced-JSON stripping (```json ... ```) for models that wrap structured output.
+ *   - Configurable base URL + api key / bearer, works with LiteLLM proxies, OpenAI
+ *     directly, cli-bridge subscriptions, and any router that speaks the spec.
+ *
+ * Usage:
+ *   const { value, result } = await callLlmJson<MyType>(
+ *     { model: 'gpt-4o', messages: [...], jsonSchema: { name: 'x', schema: {...} } },
+ *     { baseUrl: 'https://router.tangle.tools/v1', apiKey: process.env.KEY },
+ *   )
+ *
+ * This is THE llm-calling seam for agent-eval primitives that need structured
+ * output (semantic concept judge, reviewer directives, critic scores). Primitives
+ * that need free-form text use `callLlm` and parse output themselves.
+ */
+
+interface LlmUsage {
+    promptTokens: number;
+    completionTokens: number;
+    totalTokens: number;
+    /** False when the provider omitted or malformed prompt/completion usage. */
+    captured?: boolean;
+    /** Proxies populate this when prompt caching is on. */
+    cachedPromptTokens?: number;
+}
+interface LlmCallResult {
+    /** The text content of the first choice. Empty string if none. */
+    content: string;
+    usage: LlmUsage;
+    /**
+     * Cost in USD. Pulled from proxy's `_response_cost` field when present;
+     * `null` when neither the proxy nor the caller can derive it.
+     */
+    costUsd: number | null;
+    /** Model name actually used (echoed from response). */
+    model: string;
+    /** Wall-clock duration of the HTTP call (last attempt, if retried). */
+    durationMs: number;
+    /**
+     * `finish_reason` echoed from the first choice (`stop`, `length`,
+     * `content_filter`, `tool_calls`, ...). `null` when the provider omits it.
+     * Exposed so a free-form `callLlm` caller CAN detect a truncated answer
+     * (`length`) instead of treating a cut-off completion as complete. Note:
+     * `callLlm` does not itself reject on it — acting on this signal is the
+     * caller's responsibility (in-repo free-form drivers do not yet enforce it).
+     */
+    finishReason?: string | null;
+    /**
+     * True when `content.trim()` is empty. An empty completion is a silent zero
+     * for free-form `callLlm` callers; this flag is the signal a caller can
+     * inspect to fail loud rather than proceed on an empty string. `callLlm`
+     * surfaces it but does not throw on it.
+     */
+    contentEmpty?: boolean;
+    /** Raw response body. */
+    raw: Record<string, unknown>;
+}
+type LlmCallMetadata = Pick<LlmCallResult, 'usage' | 'costUsd' | 'model' | 'durationMs'>;
+interface LlmClientOptions {
+    /** Base URL (without trailing slash). Must end at the `/v1` prefix. */
+    baseUrl?: string;
+    /** Bearer token — either `apiKey` or `bearer` populates `Authorization: Bearer ...`. */
+    apiKey?: string;
+    bearer?: string;
+    /** Override for the `Authorization` header (e.g. `X-Auth: ...`). Takes precedence over apiKey/bearer. */
+    authHeader?: {
+        name: string;
+        value: string;
+    };
+    /** Stable provider idempotency key, reused across retries of this logical call. */
+    idempotencyKey?: string;
+    /** Default timeout in ms. Per-call can override. */
+    defaultTimeoutMs?: number;
+    /**
+     * Caller-supplied abort signal — e.g. a campaign-wide cancel. Linked to
+     * each attempt's per-attempt timeout controller, so aborting it cancels
+     * the in-flight fetch. A caller abort is FATAL: it is not retried even
+     * though an AbortError otherwise matches the transient patterns.
+     */
+    signal?: AbortSignal;
+    /**
+     * Cross-attempt wall-clock budget in ms, measured from the first attempt.
+     * Before launching each attempt the loop checks the remaining budget and
+     * stops retrying once it is exhausted, rather than waiting the full
+     * per-attempt timeout on every retry. Bounds total time independent of
+     * total attempts × `timeoutMs`.
+     */
+    deadlineMs?: number;
+    /** Total provider attempts. Legacy option name; default 3 (1 initial + 2 retries). */
+    maxRetries?: number;
+    /** Fetch implementation — defaults to global `fetch`. Override for custom transport (e.g. tests). */
+    fetch?: typeof fetch;
+    /**
+     * Optional raw HTTP capture sink. When provided, every request, response,
+     * and error (across all retry attempts) is recorded to the sink, with auth
+     * headers and credential-shaped body fields redacted by default. This is
+     * the layer-1 forensics primitive: structured `LlmSpan`s record intent,
+     * raw events record what actually crossed the wire.
+     */
+    rawSink?: RawProviderSink;
+    /**
+     * Logical provider id attached to raw events. When omitted, derived from
+     * `baseUrl` via `providerFromBaseUrl`.
+     */
+    provider?: string;
+    /** Trace context attached to raw events; populated by emitter-aware callers. */
+    traceContext?: {
+        runId?: string;
+        spanId?: string;
+    };
+    /** Override the redaction strategy for this call. Defaults to `defaultProviderRedactor`. */
+    redactor?: ProviderRedactor;
+}
+interface LlmRouteRequirements {
+    /**
+     * Throw if `opts.baseUrl` is undefined, i.e. the call would fall back to
+     * `DEFAULT_BASE_URL`. Set this for evaluation runs where silently using
+     * the public/free-tier router is a defect — the launch reviewer needs to
+     * know exactly which provider answered.
+     */
+    requireExplicitBaseUrl?: boolean;
+    /**
+     * Allowlist of acceptable base URLs. Strings match by prefix
+     * (case-insensitive); RegExps test against the full base URL.
+     */
+    allowedBaseUrls?: Array<string | RegExp>;
+    /** Blocklist that takes precedence over `allowedBaseUrls`. */
+    blockedBaseUrls?: Array<string | RegExp>;
+    /** Throw if no auth header / api key is configured. */
+    requireAuth?: boolean;
+    /**
+     * Logical provider id the configured `baseUrl` is expected to match (via
+     * `providerFromBaseUrl`). Mainly useful when paired with `requireExplicitBaseUrl`.
+     */
+    expectedProvider?: string;
+}
+
+/**
+ * FailureClusterView — groups failed runs by (failureClass, triggerTool,
+ * argHash-prefix) so weekly reviews can prioritize the top-N clusters.
+ *
+ * Each cluster includes: N runs, scenarios affected, representative
+ * error message, a proposed mitigation hint (rule → action table).
+ */
+
+interface FailureCluster {
+    failureClass: FailureClass;
+    /** Tool name when the trigger was a tool span, else undefined. */
+    toolName?: string;
+    /** First 16 chars of argHash — clusters similar args. */
+    argPrefix?: string;
+    /**
+     * Source dimension when the trigger was a judge span (e.g. `'format'`,
+     * `'safety'`, `'correctness'`). Lets cross-template aggregators
+     * group failures by the dimension that fired without overloading
+     * `argPrefix`. Optional — clusters without this field deserialize cleanly.
+     */
+    dimension?: string;
+    runCount: number;
+    scenarioIds: string[];
+    exampleError?: string;
+    exampleRunId: string;
+}
+interface FailureClusterReport {
+    clusters: FailureCluster[];
+    totalFailures: number;
+    totalRuns: number;
+}
+
+/**
+ * Reporting helpers — production summaries and paper-quality figures — sit alongside `reporter.ts` rather
+ * than replacing it.
+ *
+ * Three artefacts:
+ *
+ *   - `summaryTable`           Markdown table of per-candidate means,
+ *                            95% bootstrap CIs, BH-adjusted Wilcoxon
+ *                            p-values, and Cohen's d versus a
+ *                            comparator candidate.
+ *   - `paretoChart`         Abstract spec for a cost vs quality
+ *                            scatter, with gate decisions overlaid.
+ *                            Returns numbers + labels — caller
+ *                            chooses the plotting library.
+ *   - `gainHistogram`
+ *                            Per-item paired holdout deltas as a
+ *                            histogram spec (bins + counts + median +
+ *                            CI). Same "data, not images" contract.
+ *
+ * The figure types are PlotSpecs — JSON-friendly, library-agnostic.
+ * They aren't React components and they aren't PNGs; they are
+ * what you'd hand to vega-lite, plotly, matplotlib, or your own
+ * Canvas renderer to draw the actual figure.
+ */
+
+interface SummaryTableRow {
+    candidateId: string;
+    n: number;
+    mean: number;
+    ciLow: number;
+    ciHigh: number;
+    /** BH-adjusted q-value vs comparator. NaN if no comparator. */
+    qValue: number;
+    /** Cohen's d vs comparator. NaN if no comparator. */
+    cohensD: number;
+}
+interface SummaryTable {
+    rows: SummaryTableRow[];
+    comparator: string | null;
+    split: 'search' | 'holdout';
+    /** Pre-rendered markdown — drop into a paper or PR. */
+    markdown: string;
+}
+interface ParetoPoint {
+    candidateId: string;
+    /** Mean USD cost per run on the chosen split. */
+    cost: number;
+    /** Mean score on the chosen split. */
+    quality: number;
+    /** Number of runs that informed this point. */
+    n: number;
+    /** Whether this candidate is on the Pareto frontier — high
+     *  quality, low cost, no dominator. */
+    onFrontier: boolean;
+    /** Optional gate verdict for this candidate, if a `GateDecision`
+     *  for it was passed in. */
+    gate?: 'promote' | 'reject_few_runs' | 'reject_negative_delta' | 'reject_overfit_gap' | null;
+}
+interface ParetoFigureSpec {
+    kind: 'pareto-cost-quality';
+    split: 'search' | 'holdout';
+    points: ParetoPoint[];
+    axes: {
+        x: 'costUsd';
+        y: 'score';
+    };
+}
+interface GainDistributionBin {
+    /** Inclusive lower edge. */
+    lo: number;
+    /** Exclusive upper edge (or inclusive if it's the last bin). */
+    hi: number;
+    /** Number of pairs whose delta lands in this bin. */
+    count: number;
+}
+interface GainDistributionFigureSpec {
+    kind: 'gain-distribution';
+    candidateId: string;
+    comparator: string;
+    split: 'search' | 'holdout';
+    /** Number of pairs used. */
+    n: number;
+    bins: GainDistributionBin[];
+    median: number;
+    ci: {
+        low: number;
+        high: number;
+    };
+}
+type ResearchReportDecision = 'promote' | 'hold' | 'reject' | 'equivalent' | 'needs_more_data';
+interface ResearchReportOptions {
+    /** Human-readable report title. */
+    title?: string;
+    /** Comparator candidate id. Required for statistical decision guidance. */
+    comparator?: string;
+    /** Which split to use for the primary decision. Default 'holdout'. */
+    split?: 'search' | 'holdout';
+    /** Confidence level used by lower-level report helpers. Default 0.95. */
+    confidence?: number;
+    /** FDR threshold for q-values. Default 0.05. */
+    fdr?: number;
+    /**
+     * Soft floor on paired observations before issuing a directional
+     * promote / reject. Below this we report `needs_more_data` and surface the
+     * minimum detectable effect at the current N. Default 20 — chosen so the
+     * Wilcoxon signed-rank approximation is reasonable and so the paired
+     * bootstrap CI has non-degenerate coverage. Hard floor is enforced at
+     * `RESEARCH_REPORT_HARD_PAIR_FLOOR` (6) regardless of this value.
+     */
+    minPairs?: number;
+    /**
+     * Region of Practical Equivalence on the paired delta. When a candidate's
+     * paired-delta CI is fully contained in `[low, high]`, the decision is
+     * `equivalent` rather than `hold`. Sourced from the domain owner — there is
+     * no statistically-defensible default.
+     */
+    rope?: {
+        low: number;
+        high: number;
+    };
+    /**
+     * Power for the minimum detectable effect (MDE) reported on each candidate.
+     * Default 0.8.
+     */
+    mdePower?: number;
+    /**
+     * Two-sided alpha for the MDE. Default matches `fdr` so the reported MDE
+     * lines up with the test the report actually runs.
+     */
+    mdeAlpha?: number;
+    /** Optional held-out gate decisions keyed by candidate id. */
+    gateDecisions?: Record<string, GateDecision$1>;
+    /** Optional failure clusters from failureClusterView. */
+    failureClusters?: FailureClusterReport;
+    /** Build gain histograms for these candidates. Defaults to all non-comparator candidates. */
+    candidateIds?: string[];
+    /** Deterministic bootstrap seed passed to gainHistogram and the posterior helper. */
+    seed?: number;
+    /** Report timestamp. Defaults to current time. */
+    generatedAt?: string;
+    /**
+     * Hash of a preregistered protocol (e.g. `signManifest({...}).contentHash`).
+     * Embedded verbatim in the report so the analysis can be cited as the
+     * preregistered one rather than a post-hoc fishing expedition.
+     */
+    preregistrationHash?: string;
+}
+interface ResearchReportRecommendation {
+    decision: ResearchReportDecision;
+    candidateId: string | null;
+    rationale: string[];
+    risks: string[];
+    nextActions: string[];
+}
+interface ResearchReportCandidate {
+    candidateId: string;
+    n: number;
+    mean: number;
+    ciLow: number;
+    ciHigh: number;
+    qValue: number;
+    cohensD: number;
+    meanDeltaVsComparator: number | null;
+    pairedN: number;
+    medianGain: number | null;
+    meanGain: number | null;
+    gainCi: {
+        low: number;
+        high: number;
+    } | null;
+    /**
+     * Bayesian-bootstrap-style posterior summaries on the paired delta. Computed
+     * from the same resamples that produce the gain CI; interpretable as
+     * "fraction of resamples in which the candidate beats the comparator on
+     * matched pairs."
+     */
+    prGreaterThanZero: number | null;
+    prInRope: number | null;
+    /**
+     * Minimum detectable effect (in score units) at the candidate's paired N,
+     * the configured power, and the configured alpha. Standardised by the
+     * observed paired-delta SD and inverted via `requiredSampleSize`. Reported
+     * for every candidate so a `needs_more_data` verdict is actionable.
+     */
+    mde: number | null;
+    onParetoFrontier: boolean;
+    gate?: ParetoPoint['gate'];
+    decision: ResearchReportDecision;
+    decisionReason: string;
+}
+interface ResearchReportMethodology {
+    /**
+     * Plain-language assumptions the report depends on. Read these first when
+     * deciding whether the verdict is load-bearing for a launch decision.
+     */
+    assumptions: string[];
+    /** Tests and estimators the verdict was computed from. */
+    methods: string[];
+    /** Alternatives the author considered and why this report didn't take them. */
+    alternatives: string[];
+    /** Failure modes — when this report should NOT drive a decision. */
+    whenNotToApply: string[];
+    /** Citations for the methodological choices above. */
+    citations: string[];
+}
+interface ResearchReport {
+    kind: 'agent-eval-research-report';
+    title: string;
+    generatedAt: string;
+    split: 'search' | 'holdout';
+    comparator: string | null;
+    /**
+     * SHA-256 over the canonicalised set of `(runId, candidateId, split)` triples
+     * the report was computed from, plus the comparator and split. Stable across
+     * key insertion order; recomputable by the reader to verify provenance.
+     */
+    runFingerprint: string;
+    preregistrationHash: string | null;
+    rope: {
+        low: number;
+        high: number;
+    } | null;
+    executiveSummary: string[];
+    recommendation: ResearchReportRecommendation;
+    candidates: ResearchReportCandidate[];
+    summary: SummaryTable;
+    charts: {
+        pareto: ParetoFigureSpec;
+        gains: GainDistributionFigureSpec[];
+    };
+    methodology: ResearchReportMethodology;
+    failureClusters?: FailureClusterReport;
+    markdown: string;
+    html: string;
+}
+
+/**
+ * TraceEmitter — hierarchical span builder that auto-parents using an
+ * internal stack. One emitter per Run; emitters do NOT share state.
+ *
+ * Convenience methods (`llm`, `tool`, `retrieval`, `judge`, `sandbox`)
+ * return a `SpanHandle` with `.end()` / `.fail()` so callers don't
+ * have to thread spanIds manually. For async workflows that can't use
+ * the stack (e.g. fan-out parallel calls), pass `parentSpanId`
+ * explicitly.
+ */
+
+interface SpanHandle<S extends Span = Span> {
+    span: S;
+    end(patch?: Partial<S>): Promise<void>;
+    fail(error: string | Error, patch?: Partial<S>): Promise<void>;
+}
+interface RunCompleteHookContext {
+    runId: string;
+    emitter: TraceEmitter;
+    store: TraceStore;
+    /** Outcome the caller passed to `endRun` (undefined for `abortRun`). */
+    outcome?: RunOutcome$1;
+    /** Final run status. */
+    status: 'completed' | 'failed' | 'aborted';
+}
+type RunCompleteHook = (ctx: RunCompleteHookContext) => Promise<void> | void;
+interface TraceEmitterOptions {
+    runId?: string;
+    /** Inject a clock for deterministic tests. */
+    now?: () => number;
+    /** Inject an id generator for deterministic tests. */
+    id?: () => string;
+    /**
+     * Hooks fired after `endRun` / `abortRun` writes the final run state.
+     * Designed for trace-analyst auto-execution, integrity assertions, and
+     * outbound notifications. Hooks run sequentially in the order supplied.
+     *
+     * By default a hook that throws is swallowed and logged as a `note` event
+     * on the run — auto-orchestration must not crash the underlying flow.
+     * Set `hookErrors: 'throw'` to propagate.
+     */
+    onRunComplete?: RunCompleteHook[];
+    /** `'swallow'` (default) | `'throw'`. */
+    hookErrors?: 'swallow' | 'throw';
+}
+declare class TraceEmitter {
+    private store;
+    private stack;
+    private _runId;
+    private now;
+    private id;
+    private hooks;
+    private hookErrors;
+    constructor(store: TraceStore, options?: TraceEmitterOptions);
+    get runId(): string;
+    get traceStore(): TraceStore;
+    /** Append a hook after construction (e.g. attach the trace analyst). */
+    addRunCompleteHook(hook: RunCompleteHook): void;
+    /**
+     * Begin a Run.
+     *
+     * `scenarioId` is required on the persisted Run shape — every Run downstream
+     * gets a non-empty scenarioId so filters and aggregations stay simple — but
+     * the INPUT here accepts it as optional. When omitted, startRun substitutes
+     * a sensible default (`run.layer ?? run.tags?.['kind'] ?? 'runtime'`) so
+     * runtime / operator / meta-eval runs that have no curated-scenario corpus
+     * to anchor to don't have to invent placeholder strings at the call site.
+     */
+    startRun(run: Omit<Run, 'runId' | 'scenarioId' | 'startedAt' | 'status'> & {
+        scenarioId?: string;
+    }): Promise<Run>;
+    endRun(outcome?: RunOutcome$1): Promise<void>;
+    abortRun(reason: string): Promise<void>;
+    private runHooks;
+    span<S extends Span = Span>(init: {
+        kind: SpanKind;
+        name: string;
+        parentSpanId?: string;
+        attributes?: Record<string, unknown>;
+    } & Partial<Omit<S, 'spanId' | 'runId' | 'startedAt' | 'kind' | 'name'>>): Promise<SpanHandle<S>>;
+    private handle;
+    private pop;
+    llm(init: Omit<LlmSpan, 'spanId' | 'runId' | 'kind' | 'startedAt'>): Promise<SpanHandle<LlmSpan>>;
+    tool(init: Omit<ToolSpan, 'spanId' | 'runId' | 'kind' | 'startedAt'>): Promise<SpanHandle<ToolSpan>>;
+    retrieval(init: Omit<RetrievalSpan, 'spanId' | 'runId' | 'kind' | 'startedAt'>): Promise<SpanHandle<RetrievalSpan>>;
+    recordJudge(verdict: Omit<JudgeSpan, 'spanId' | 'runId' | 'kind' | 'startedAt' | 'endedAt'>): Promise<JudgeSpan>;
+    sandbox(init: Omit<SandboxSpan, 'spanId' | 'runId' | 'kind' | 'startedAt'>): Promise<SpanHandle<SandboxSpan>>;
+    emit(event: {
+        kind: EventKind;
+        spanId?: string;
+        payload?: Record<string, unknown>;
+    }): Promise<TraceEvent>;
+    recordBudget(entry: Omit<BudgetLedgerEntry, 'runId' | 'timestamp'> & {
+        timestamp?: number;
+    }): Promise<BudgetLedgerEntry>;
+    recordArtifact(artifact: Omit<Artifact, 'artifactId' | 'runId'>): Promise<Artifact>;
+    /**
+     * Runs `fn` inside a span; auto-ends on success, auto-fails on throw.
+     * Returns the fn's return value. Use this for the 95% case.
+     */
+    within<T>(init: Parameters<TraceEmitter['span']>[0], fn: (handle: SpanHandle) => Promise<T>): Promise<T>;
+}
+
+/**
+ * Run-completion integrity check — at end of run, verify the expected event
+ * types were actually captured. The point is the launch-review failure mode:
+ * a run *appears* successful but the raw provider events were never written,
+ * so a downstream reviewer can't reconstruct what happened.
+ *
+ * Pattern:
+ *
+ *   const report = await assertRunCaptured(store, runId, {
+ *     llmSpansMin: 1,
+ *     judgeSpansMin: 1,
+ *     rawSink: providerSink,                  // must have ≥ 1 event for this run
+ *     requireRawCoverageOfLlmSpans: true,     // every llm span has matching raw events
+ *   })
+ *   if (!report.ok) throwIfRunIncomplete(report)  // or mark run failed and continue
+ *
+ * The function is read-only on the store and returns a structured report;
+ * the caller chooses the failure mode (throw, mark run failed, log warning).
+ * `throwIfRunIncomplete` is the convenient strict mode.
+ */
+
+interface RunIntegrityExpectations {
+    /** Minimum LLM span count. Default 0 (no requirement). */
+    llmSpansMin?: number;
+    /** Minimum judge span count. Default 0. */
+    judgeSpansMin?: number;
+    /** Minimum tool span count. Default 0. */
+    toolSpansMin?: number;
+    /**
+     * Raw provider sink to consult for capture verification. When present,
+     * the check requires at least one raw event for the run.
+     */
+    rawSink?: RawProviderSink;
+    /** Minimum raw provider event count. Default 0; ignored when `rawSink` absent. */
+    rawProviderEventsMin?: number;
+    /**
+     * Every LLM span must have at least one matching raw `request` event
+     * (matched by spanId). Catches the common bug where the structured span
+     * was emitted but the raw HTTP capture was wired to a different sink.
+     */
+    requireRawCoverageOfLlmSpans?: boolean;
+    /** Run outcome must be set (not null/undefined). Default false. */
+    requireOutcome?: boolean;
+}
+type RunIntegrityIssueCode = 'no_run' | 'missing_llm_spans' | 'missing_judge_spans' | 'missing_tool_spans' | 'missing_raw_events' | 'no_raw_sink' | 'orphan_llm_span' | 'missing_outcome';
+interface RunIntegrityIssue {
+    code: RunIntegrityIssueCode;
+    message: string;
+    detail?: Record<string, unknown>;
+}
+interface RunIntegrityReport {
+    ok: boolean;
+    runId: string;
+    llmSpanCount: number;
+    judgeSpanCount: number;
+    toolSpanCount: number;
+    rawProviderEventCount: number;
+    /**
+     * Coverage of LLM spans by raw provider events keyed on spanId.
+     * `total` is the number of LLM spans; `covered` is the count with at
+     * least one matching `request` raw event.
+     */
+    rawSpanCoverage: {
+        covered: number;
+        total: number;
+    };
+    issues: RunIntegrityIssue[];
+}
+
+/**
+ * EvalCampaign — opinionated matrix runner that wires the four
+ * capture-integrity directives by construction.
+ *
+ * The canonical benchmark shape — matrix runner → for each
+ * (variant, scenario, seed) → start a TraceEmitter → call LLMs → end the
+ * run → analyze — has a bug class at the integration boundary: raw
+ * events not captured, route silently wrong, integrity not asserted,
+ * analyst never run. The directives in `SKILL.md § Capture integrity`
+ * are the mitigations.
+ *
+ * `EvalCampaign` is the structural fix — consumers don't wire the
+ * integrity surface themselves; the campaign owns it. Specifically:
+ *
+ *   - calls `assertLlmRoute` once at preflight before any work runs
+ *   - constructs a per-run `TraceStore` and `RawProviderSink` via factories
+ *   - constructs the `TraceEmitter` with `onRunComplete: [analyst hook]`
+ *   - hands the runner an `LlmClientOptions` pre-wired with the sink and
+ *     trace context — the runner can't accidentally call an LLM without
+ *     capturing the raw HTTP envelope
+ *   - calls `assertRunCaptured` after every `endRun` and routes failures
+ *     through a configurable policy (`throw` / `mark_failed` / `log`)
+ *   - assembles per-run `RunRecord`s and runs `researchReport` at the end
+ *     so the campaign artifact is launch-decision-grade by default
+ *   - embeds the campaign fingerprint (a SHA-256 over the canonicalised
+ *     run set) and optional `preregistrationHash` in the report
+ *
+ * The runner contract is intentionally narrow: produce a `CampaignRunOutcome`
+ * given a fully-wired `CampaignRunContext`. Everything orchestration-shaped
+ * lives in the campaign. This is the inversion-of-control point — consumers
+ * stop writing matrix runners and start writing scenario-runners.
+ *
+ * Out of scope for v1 (tracked in `docs/research-report-methodology.md`):
+ *
+ *   - Distributed/cluster execution (concurrency is local async)
+ *   - Adaptive sampling / sequential interim looks
+ *   - Resume from partial state across crashes
+ *   - LLM-call retry beyond what `LlmClient` already does
+ */
+
+interface CampaignVariant<V> {
+    id: string;
+    payload: V;
+}
+interface CampaignScenario {
+    scenarioId: string;
+    /** Free-form metadata propagated to runs and reports. */
+    tags?: Record<string, string>;
+}
+interface CampaignRunContext<V> {
+    /** Stable run id. The campaign generates this; the runner does not. */
+    runId: string;
+    /** Logical experiment id (campaignId by default; overridable per-run via opts). */
+    experimentId: string;
+    variant: V;
+    variantId: string;
+    scenarioId: string;
+    scenarioTags: Record<string, string>;
+    seed: number;
+    splitTag: RunSplitTag;
+    /**
+     * The TraceEmitter for this run, with `onRunComplete` hooks pre-wired
+     * (analyst auto-execution if configured, plus integrity check). The
+     * runner MUST call `emitter.startRun` before doing any work and either
+     * `emitter.endRun` or `emitter.abortRun` before returning.
+     */
+    emitter: TraceEmitter;
+    store: TraceStore;
+    rawSink: RawProviderSink;
+    /**
+     * Pre-wired LLM client options — `rawSink` and `traceContext` are populated
+     * so any `callLlm(req, ctx.llmOpts)` automatically captures raw HTTP. The
+     * runner can spread additional fields if needed.
+     */
+    llmOpts: LlmClientOptions;
+}
+interface CampaignRunOutcome {
+    /** Did the run pass? Mirrors `RunOutcome.pass` semantics. */
+    pass: boolean;
+    /** Score for the run on its split. Maps to `searchScore` or `holdoutScore`. */
+    score: number;
+    /** Mandatory cost in USD. Use 0 + raw.cost_unknown=1 only if truly unknown. */
+    costUsd: number;
+    tokenUsage: RunTokenUsage;
+    /** Snapshot model id (e.g. `claude-sonnet-4-6@2025-04-15`). */
+    model: string;
+    /** sha256 of the effective prompt sent to the model. */
+    promptHash: string;
+    /** sha256 of the effective config (model, temperature, tools, judges, splits). */
+    configHash: string;
+    /** Optional extra numeric metrics to land in `outcome.raw`. */
+    raw?: Record<string, number>;
+    /** Canonical cross-agent failure class from the shared `FAILURE_CLASSES`
+     *  taxonomy. Propagated to `RunRecord.failureClass` so campaign runs
+     *  aggregate failures in the same vocabulary as every other producer. */
+    failureClass?: FailureClass;
+    /** Optional free-form failure detail, scoped under `failureClass`. */
+    failureMode?: string;
+    /** Optional judge metadata when a judge was used. */
+    judgeMetadata?: RunJudgeMetadata;
+    /**
+     * Optional per-judge / per-dim breakdown for ensemble-judged runs.
+     * Propagated to `outcome.judgeScores` on the resulting `RunRecord`.
+     * Single-judge or scalar-only runs leave this unset.
+     */
+    judgeScores?: JudgeScoresRecord;
+    /**
+     * Agent profile cell observed by the runner. When supplied, it overrides
+     * `EvalCampaignOptions.agentProfile` for this run and must match the
+     * outcome's `model` and `promptHash`.
+     */
+    agentProfile?: AgentProfileCell | AgentProfileCellInput;
+}
+type CampaignRunner<V> = (ctx: CampaignRunContext<V>) => Promise<CampaignRunOutcome>;
+type CampaignIntegrityPolicy = 'throw' | 'mark_failed' | 'log';
+interface EvalCampaignOptions<V> {
+    /**
+     * Stable id for the campaign. Used as the default `experimentId` on
+     * every run, and folded into the campaign fingerprint.
+     */
+    campaignId: string;
+    variants: CampaignVariant<V>[];
+    scenarios: CampaignScenario[];
+    /** Default `[0, 1, 2]`. */
+    seeds?: number[];
+    /** Default `'holdout'` — the split that anchors a launch decision. */
+    splitTag?: RunSplitTag;
+    /** Git SHA the campaign is run against. Mandatory; `RunRecord` rejects unset. */
+    commitSha: string;
+    /**
+     * LLM client config. Augmented per-run with `rawSink` and `traceContext`
+     * before being passed to the runner. The campaign asserts this config
+     * matches `routeRequirements` once at preflight.
+     */
+    llmOpts: LlmClientOptions;
+    /**
+     * Default `{ requireExplicitBaseUrl: true, requireAuth: true }` — fail
+     * loud if the campaign would silently fall back to the public router or
+     * run unauthenticated. Override with an empty object to disable.
+     */
+    routeRequirements?: LlmRouteRequirements;
+    /**
+     * Per-run TraceStore factory. Common shape: a fresh store per run keyed
+     * on `runId`. Implementations that share a store across the campaign
+     * are valid — the campaign only writes through `emitter`.
+     */
+    storeFactory: (params: CampaignFactoryParams) => TraceStore;
+    /**
+     * Per-run RawProviderSink factory. Defaults to `FileSystemRawProviderSink`
+     * rooted at `${workDir}/raw-events/${runId}` if `workDir` is supplied;
+     * otherwise required. Forensic capture is non-negotiable in a campaign
+     * run — pass `NoopRawProviderSink` explicitly if you want to opt out.
+     */
+    rawSinkFactory?: (params: CampaignFactoryParams) => RawProviderSink;
+    /**
+     * Filesystem root for default `rawSinkFactory`. Ignored if
+     * `rawSinkFactory` is supplied.
+     */
+    workDir?: string;
+    /**
+     * Extra `onRunComplete` hooks the campaign appends (after its own
+     * integrity-check hook). Pass `traceAnalystOnRunComplete(...)` here.
+     */
+    onRunComplete?: RunCompleteHook[];
+    /**
+     * Per-run integrity expectations. Defaults to:
+     *   `{ llmSpansMin: 1, requireRawCoverageOfLlmSpans: true, requireOutcome: true }`.
+     * Override (e.g. `{ llmSpansMin: 0 }`) for runs that don't call LLMs.
+     */
+    integrity?: RunIntegrityExpectations;
+    /** Behaviour when integrity fails. Default `'mark_failed'`. */
+    onIntegrityFailure?: CampaignIntegrityPolicy;
+    /**
+     * Per-run runner. Receives a fully-wired context; produces an outcome
+     * the campaign converts into a `RunRecord`.
+     */
+    runner: CampaignRunner<V>;
+    /**
+     * If set, the campaign computes `researchReport` at the end. `comparator`
+     * is a `variantId`. Other fields are forwarded verbatim.
+     */
+    report?: {
+        comparator?: string;
+    } & Omit<ResearchReportOptions, 'comparator' | 'preregistrationHash' | 'generatedAt'>;
+    /**
+     * Hash of a signed `HypothesisManifest` (see `pre-registration.ts`).
+     * Embedded in the campaign fingerprint and the research report.
+     */
+    preregistrationHash?: string;
+    /** Local concurrency. Default `1` (sequential). */
+    concurrency?: number;
+    /**
+     * Override the time source. Tests pass a mock to make wallMs deterministic.
+     */
+    now?: () => number;
+    /** Override the runId generator. Tests pin this. */
+    runId?: (params: CampaignFactoryParams) => string;
+    /**
+     * Agent profile cell for campaign runs. Static profiles can pass an object;
+     * routers or variant-specific harnesses can pass a factory. The campaign
+     * stamps the built cell onto every `RunRecord` and rejects profile/model or
+     * profile/prompt contradictions.
+     */
+    agentProfile?: AgentProfileCell | AgentProfileCellInput | ((params: CampaignFactoryParams & {
+        variant: V;
+        scenarioTags: Record<string, string>;
+    }) => AgentProfileCell | AgentProfileCellInput | Promise<AgentProfileCell | AgentProfileCellInput>);
+}
+interface CampaignFactoryParams {
+    campaignId: string;
+    runId: string;
+    variantId: string;
+    scenarioId: string;
+    seed: number;
+}
+interface FailedRun {
+    runId: string;
+    variantId: string;
+    scenarioId: string;
+    seed: number;
+    reason: string;
+    error?: string;
+}
+interface EvalCampaignResult {
+    campaignId: string;
+    /** SHA-256 over canonicalised `(variantIds, scenarioIds, seeds, comparator, splitTag, baseUrl, provider, preregistrationHash)`. */
+    campaignFingerprint: string;
+    preregistrationHash: string | null;
+    /** Successful runs only. Failed runs land in `failedRuns`. */
+    runs: RunRecord[];
+    /** Integrity reports for every successful run. */
+    integrityReports: RunIntegrityReport[];
+    failedRuns: FailedRun[];
+    /** Computed when `report` is set on options. */
+    report?: ResearchReport;
+    startedAt: string;
+    endedAt: string;
+}
+declare function runEvalCampaign<V>(opts: EvalCampaignOptions<V>): Promise<EvalCampaignResult>;
+
+/**
+ * Always-valid sequential evaluation.
+ *
+ * `researchReport` assumes a single pre-specified analysis. Real
+ * consumers run campaigns weekly / nightly / per-PR; each new run silently
+ * inflates the false-discovery rate, because the BH-FDR guarantee is for
+ * the *first* look, not the 47th. Without time-uniform inference,
+ * launch-decision teams either (a) don't peek, which forfeits the cost
+ * advantage of stop-when-decisive, or (b) peek and pretend they didn't,
+ * which forfeits scientific validity.
+ *
+ * This module ships **e-value-based confidence sequences** for paired
+ * bounded outcomes. The methodology is the predictable plug-in betting
+ * martingale of Waudby-Smith & Ramdas (2024) — provably valid at *any*
+ * stopping time. Concretely:
+ *
+ *   For paired deltas D_1, D_2, … ∈ [-c, c] with the null H_0: E[D] ≤ 0,
+ *   a betting fraction λ_i is chosen using only D_{1..i-1} (predictable
+ *   plug-in), and the running e-value is
+ *
+ *     E_t = ∏_{i=1}^{t} (1 + λ_i · D_i)
+ *
+ *   E_t is a non-negative martingale under H_0 with E[E_t] ≤ 1, so by
+ *   Ville's inequality, P(∃ t : E_t ≥ 1/α) ≤ α — we can reject the null
+ *   at any time without inflating the type-I error.
+ *
+ * Combined with `runEvalCampaign`, every consumer running rolling
+ * campaigns gains the ability to ship the moment evidence is decisive,
+ * stop-early on dead-on-arrival variants, and accumulate evidence across
+ * partial runs without spending the FDR budget. No new sweep is wasted.
+ *
+ * References:
+ *   - Howard, S. R., Ramdas, A., McAuliffe, J., Sekhon, J. (2021).
+ *     Time-uniform, nonparametric, nonasymptotic confidence sequences.
+ *     Annals of Statistics, 49(2), 1055–1080.
+ *   - Waudby-Smith, I., Ramdas, A. (2024). Estimating means of bounded
+ *     random variables by betting. JRSS B, 86(1), 1–27.
+ */
+type SequentialDecision = 'promote_now' | 'continue' | 'reject_now' | 'equivalent';
+interface InterimReleaseConfidence {
+    candidates: Array<{
+        candidateId: string;
+        decision: SequentialDecision;
+        decisionFiredAt: number | null;
+        finalEvalue: number;
+        finalPValue: number;
+        pairs: number;
+        csLow: number;
+        csHigh: number;
+    }>;
+    /**
+     * Campaign-level recommendation: pick the strongest 'promote_now', else
+     * 'continue' if any candidate is still live, else 'reject_now' if every
+     * candidate is dead, else 'equivalent'.
+     */
+    recommendation: {
+        decision: SequentialDecision;
+        candidateId: string | null;
+    };
+}
+
 /**
  * `runRLCampaign` — top-level orchestrator that runs the matrix and
  * produces every RL-ready artifact in one call.
@@ -1413,6 +3443,307 @@ interface RLCampaignResult<V> {
 }
 declare function runRLCampaign<V>(opts: RunRLCampaignOptions<V>): Promise<RLCampaignResult<V>>;
 
+/**
+ * Analyst contract — the missing orchestration layer over agent-eval's
+ * existing analyzers (analyzeTraces, MultiLayerVerifier, RunCritic,
+ * SemanticConceptJudge, JudgeFn, ...).
+ *
+ * Each existing primitive returns its own output shape. The Analyst
+ * contract is the single envelope every primitive lifts into, so a
+ * registry can run N analysts against a run and a single renderer can
+ * compose findings without knowing which analyzer produced them.
+ *
+ * The contract is intentionally domain-agnostic: nothing here knows
+ * about code, voice, RAG, or any particular agent stack. Analysts
+ * declare what INPUT KIND they need (a trace store, an artifact dir,
+ * a RunRecord, a JudgeInput, or `custom`), and the registry routes
+ * the matching input from `AnalystRunInputs`.
+ */
+
+interface EvidenceRef {
+    /**
+     * Where the evidence lives. `span` and `event` refer to OTLP trace
+     * elements; `artifact` to a file inside the run's artifact tree;
+     * `finding` to another AnalystFinding (cross-analyst chaining);
+     * `metric` to a named scalar reading the renderer knows how to read.
+     */
+    kind: 'span' | 'event' | 'artifact' | 'finding' | 'metric';
+    uri: string;
+    excerpt?: string;
+}
+
+type PolicyEditSchemaVersion = 'policy-edit/v1';
+declare const POLICY_EDIT_AXES: readonly ["carrier", "representation", "budget", "sampling", "output_contract", "tool_contract", "routing", "memory", "agent_profile", "deployment_target"];
+type PolicyEditAxis = (typeof POLICY_EDIT_AXES)[number];
+declare const POLICY_EDIT_TARGET_SURFACES: readonly ["prompt", "tool-contract", "runtime-config", "memory", "agent-profile", "code", "deployment"];
+type PolicyEditTargetSurface = (typeof POLICY_EDIT_TARGET_SURFACES)[number];
+type PolicyEditRisk = 'low' | 'medium' | 'high' | 'unknown';
+type PolicyEditGainDirection = 'increase' | 'decrease';
+type PolicyEditGainUnit = 'absolute' | 'relative' | 'percent' | 'score';
+interface PolicyEditTarget {
+    surface: PolicyEditTargetSurface;
+    /** Stable path inside the target surface, for example `system-prompt:tools`
+     * or `budget.maxTurns`. */
+    path?: string;
+    /** Optional canonical deployment identity. Store the existing cell, not a
+     * local profile shape. */
+    agentProfileCell?: AgentProfileCell;
+    /** Human label when the path is not enough for a readable audit trail. */
+    label?: string;
+}
+type PolicyEditChange = {
+    kind: 'text';
+    mode: 'append' | 'prepend' | 'replace';
+    value: string;
+    /** Required when `mode === 'replace'`; exact match only. */
+    find?: string;
+} | {
+    kind: 'json';
+    mode: 'set' | 'merge' | 'remove';
+    path: string;
+    value?: AgentProfileJson;
+};
+interface PolicyEditExpectedGain {
+    /** Metric this edit is expected to move, e.g. `holdout.composite`. */
+    metric: string;
+    direction: PolicyEditGainDirection;
+    /** Positive magnitude in the metric's native units. */
+    amount: number;
+    unit?: PolicyEditGainUnit;
+    rationale?: string;
+}
+interface PolicyEditSource {
+    findingIds: string[];
+    analystIds: string[];
+    evidenceRefs: EvidenceRef[];
+    /** Mirrors `AnalystFinding.derived_from_judge`; admission rejects it. */
+    derivedFromJudge?: boolean;
+}
+interface PolicyEdit {
+    schemaVersion: PolicyEditSchemaVersion;
+    editId: string;
+    axis: PolicyEditAxis;
+    target: PolicyEditTarget;
+    change: PolicyEditChange;
+    claim: string;
+    expectedGain: PolicyEditExpectedGain;
+    confidence: number;
+    risk: PolicyEditRisk;
+    source: PolicyEditSource;
+    rationale?: string;
+    validationPlan?: string;
+    metadata?: Record<string, unknown>;
+}
+declare const POLICY_EDIT_CANDIDATE_RECORD_SCHEMA: "tangle.policy-edit-candidate.v1";
+/** JSON-safe attribution carried with a measured candidate and its scores. */
+interface PolicyEditCandidateRecord {
+    schema: typeof POLICY_EDIT_CANDIDATE_RECORD_SCHEMA;
+    policyEdit: PolicyEdit;
+}
+
+/**
+ * Pass A substrate types — `runCampaign` is the one primitive every
+ * eval flow composes from. Three contracts in this file:
+ *
+ *   - `Scenario`            input set
+ *   - `DispatchFn`          how to run one scenario → artifact
+ *   - `CampaignResult`      defined output schema (the contract downstream tools depend on)
+ *
+ * Three more lifted from earlier substrate work (re-exported):
+ *
+ *   - `JudgeConfig`         pluggable dimensional scorer (0.38)
+ *   - `Mutator`             optimization-loop surface mutator
+ *   - `Gate`                promotion gate (`HeldOutGate` and friends adapt to this)
+ *
+ * No new architecture vs 0.38 — Pass A formalizes the shapes so consumers
+ * can build dashboards / CI gates / regression diffs against a stable schema.
+ */
+
+/** Stable identifier + kind tag for any scenario. Consumers
+ *  extend with their per-domain payload (persona, task, requirement, ...). */
+interface Scenario {
+    id: string;
+    kind: string;
+    tags?: string[];
+}
+/** The canonical judge verdict shape — one declaration, shared by campaign
+ *  judges and the multishot judge runner (which re-exports this type).
+ *
+ *  Scale is PRODUCER-DEFINED: campaign convention is [0,1]; the legacy
+ *  multishot runner emits 0-10. Cross-scale comparison must go through
+ *  `detectScale` (src/campaign/gates/statistical-heldout.ts, used by
+ *  promotion-policy) — never renormalize a producer's values in place, as
+ *  downstream thresholds (`composite >= 5` in multishot/matrix.ts, live-soak
+ *  `>= 7` gates) key on the producer's native scale. */
+interface JudgeScore {
+    dimensions: Record<string, number>;
+    composite: number;
+    notes: string;
+    /** Provider metadata for display and diagnostics; accounting uses CostLedger receipts. */
+    llmCall?: LlmCallMetadata;
+    /** Set when the judge itself failed (call error, unparseable output).
+     *  `composite`/`dimensions` carry no signal — aggregators MUST exclude
+     *  failed scores from means instead of folding them into zeros. */
+    failed?: true;
+    /** Ensemble extras (populated by `ensembleJudge`): max per-dimension
+     *  spread across surviving judges — the inter-rater signal. */
+    maxDisagreement?: number;
+    /** Ensemble extras: judge identities whose verdict failed. */
+    failedJudges?: string[];
+    /** Ensemble extras: each surviving judge's per-dimension scores. */
+    perJudge?: Record<string, Record<string, number>>;
+}
+/** Five-valued verdict taxonomy (MOSS-paper alignment). */
+type GateDecision = 'ship' | 'hold' | 'need_more_work' | 'model_ceiling' | 'arch_ceiling';
+interface GateResult {
+    decision: GateDecision;
+    reasons: string[];
+    contributingGates: Array<{
+        name: string;
+        passed: boolean;
+        detail: unknown;
+    }>;
+    delta?: number;
+}
+/** Token usage accumulated for a cell. Aliased to the canonical `RunTokenUsage`
+ *  (run-record.ts, same package) so a cell maps onto a `RunRecord` for the
+ *  backend-integrity guard with ONE source of truth — a field added to
+ *  `RunTokenUsage` is a compile error here, not a silent drift. */
+type CampaignTokenUsage = RunTokenUsage;
+interface CampaignCellResult<TArtifact> {
+    /** Manifest that produced this cell. Resumability refuses to reuse a cell
+     *  whose manifest differs from the current run. */
+    manifestHash?: string;
+    cellId: string;
+    scenarioId: string;
+    rep: number;
+    generation?: number;
+    artifact: TArtifact;
+    judgeScores: Record<string, JudgeScore>;
+    costUsd: number;
+    /** True when at least one priced receipt used the model table instead of a provider bill. */
+    costEstimated?: boolean;
+    /** Exact durable receipts required to reuse this cached result. */
+    costCallIds?: string[];
+    /** Agent-call token usage committed by `ctx.cost.runPaidCall`.
+     *  `{ input: 0, output: 0 }` when no paid agent call was recorded. */
+    tokenUsage: CampaignTokenUsage;
+    /** Concrete model from the latest committed agent receipt. Consumed by
+     *  `buildRunRecord` to pin the model when the declared profile uses a
+     *  runtime-resolved sentinel. */
+    resolvedModel?: string;
+    durationMs: number;
+    seed: number;
+    cached: boolean;
+    error?: string;
+}
+interface JudgeAggregate {
+    mean: number;
+    stdev: number;
+    ci95: [number, number];
+    n: number;
+}
+interface ScenarioAggregate {
+    meanComposite: number;
+    ci95: [number, number];
+    n: number;
+}
+interface GenerationRecord {
+    generationIndex: number;
+    candidates: GenerationCandidate[];
+    promoted: string[];
+}
+/** One scored candidate surface in a generation. `dimensions` + `scenarios`
+ *  let a reflective proposer ground its next proposal on WHICH
+ *  dimensions the candidate is weakest on and WHICH scenarios it best/worst
+ *  handled — the evidence a blind `Mutator` cannot see. */
+interface GenerationCandidate {
+    surfaceHash: string;
+    composite: number;
+    ci95: [number, number];
+    /** Exact surface this candidate mutated. */
+    parentSurfaceHash?: string;
+    /** Measured search-split composite of the exact parent surface. */
+    parentComposite?: number;
+    /** Candidate composite minus its parent's composite. Present only when the
+     *  candidate completed the designed denominator. */
+    observedDeltaFromParent?: number;
+    /** Whether this candidate had a scorable result for every designed campaign
+     *  cell and was therefore eligible for ranking, promotion, and Pareto
+     *  selection. Older externally-authored records may omit this field; loop
+     *  records always populate it. */
+    eligibleForPromotion?: boolean;
+    /** Exact denominator receipt for selection eligibility. Scores stay
+     *  descriptive: an incomplete candidate is retained with its observed score
+     *  and errors instead of receiving an invented penalty. */
+    coverage?: {
+        expectedCells: number;
+        scorableCells: number;
+        unscorableCells: Array<{
+            cellId: string;
+            reason: string;
+        }>;
+    };
+    /** Mean score per judge dimension across all cells (scenarios × reps ×
+     *  judges that reported the dimension). */
+    dimensions: Record<string, number>;
+    /** Per-scenario composite (mean over reps + judges), plus the judge's
+     *  free-form `notes` for that scenario — the "why it scored low" evidence a
+     *  reflective proposer grounds its next edit on. Keep `notes` GENERALIZABLE
+     *  (which checks/lines/dimensions failed and how), NOT case-specific ground
+     *  truth: leaking expected answers into the prompt is memorization, and the
+     *  held-out gate would reject it anyway. */
+    scenarios: Array<{
+        scenarioId: string;
+        composite: number;
+        notes?: string;
+    }>;
+    /** Proposer-supplied short label for the change. Present when the proposer
+     *  returned a `ProposedCandidate`; absent for bare-surface mutators. */
+    label?: string;
+    /** Proposer-supplied rationale — WHY this candidate was proposed. The
+     *  "because rationale Z" the audit requires to survive to the result.
+     *  Present when the proposer returned a `ProposedCandidate`. */
+    rationale?: string;
+    /** Exact structured cause threaded from the proposer, when available. */
+    candidateRecord?: PolicyEditCandidateRecord;
+}
+interface CampaignAggregates {
+    byJudge: Record<string, JudgeAggregate>;
+    byScenario: Record<string, ScenarioAggregate>;
+    /** Canonical campaign accounting, including worker and judge calls. */
+    cost: CostLedgerSummary;
+    /** Compatibility alias of `cost.totalCostUsd`. */
+    totalCostUsd: number;
+    cellsExecuted: number;
+    cellsSkipped: number;
+    cellsCached: number;
+    cellsFailed: number;
+}
+interface CampaignResult<TArtifact = unknown, TScenario extends Scenario = Scenario> {
+    /** sha256(scenarios, judges, dispatch source ref, optimizer config, seed). Stable identity for reruns. */
+    manifestHash: string;
+    seed: number;
+    startedAt: string;
+    endedAt: string;
+    durationMs: number;
+    cells: Array<CampaignCellResult<TArtifact>>;
+    aggregates: CampaignAggregates;
+    optimization?: {
+        generations: GenerationRecord[];
+        winnerSurfaceHash?: string;
+    };
+    gate?: GateResult;
+    prUrl?: string;
+    runDir: string;
+    artifactsByPath: Record<string, string>;
+    /** Substrate strips the input scenarios to id+kind for the result manifest;
+     *  consumers needing full payload look it up via the original input. The
+     *  type parameter `TScenario` is propagated for downstream consumers that
+     *  want narrowed types when extending `CampaignResult`. */
+    scenarios: Array<Pick<TScenario, 'id' | 'kind'>>;
+}
+
 /**
  * Adapters: convert measurement outputs into the canonical `RunRecord[]`
  * artifact that `replayCache`, `pairedEvalueSequence`, and
@@ -1748,4 +4079,4 @@ interface BuildPairwiseFromCampaignInput {
 }
 declare function buildPairwiseFromCampaign(input: BuildPairwiseFromCampaignInput): PairwiseOutcome[];
 
-export { ABSENT_CATEGORY, type AdaptationCurve, type AdaptationPoint, type AdaptationRunner, type AdapterContext, type BehaviorFeatures, type BradleyTerryFit, type BradleyTerryRating, type BuildPairwiseFromCampaignInput, type CellObservation, type CompareCurvesResult, type ComputeBestOfNOptions, type ComputeBestOfNResult, type ComputeCurve, type ComputeCurveBudget, type ComputeCurvePoint, type ContaminationProbeInput, type ContaminationProbeOptions, type ContaminationProbeReport, type CorpusAppendResult, type CorpusRecord, type CurriculumAllocation, DEFAULT_MIN_N_PER_FEATURE, DEFAULT_QUANTILE_BUCKETS, type DatasetFormat, type DetectRewardHackingInput, type DpoExportRow, type DpoLookups, type EasyModeOptions, type EasyModeReport, type EloOptions, type ExtractPreferencesOptions, type ExtractStepRewardsOptions, type FeatureDivergence, type FeatureShift, type FidelityReport, type FidelityVerdict, type GrpoExportRow, type GrpoLookups, type HarvestOptions, OutcomeStore, type PairwiseOutcome, type ParetoPointInput, PredictiveValidityResearcher, type PredictiveValidityResearcherOptions, type PreferenceExtractionReport, type PreferenceStrategy, type PreferenceTriple, type PrmExportRow, type PrmLookups, type PrmTrainingTriple, REPRESENTATIVE_MIN_FIDELITY, type RLCampaignResult, type RewardHackingFinding, type RewardHackingReport, type RewardHackingSignal, type RewardKind, type RewardStats, type RlDatasetBundle, type RlDatasetConfig, type RlDatasetManifest, type RlDatasetStats, type RunAdaptationCurveOptions, type RunComputeCurveOptions, type RunRLCampaignOptions, type RunwiseStepSummary, type ScenarioPerturbation, type ScenarioPerturbationKind, type SelfConsistencyOptions, type SelfConsistencyResult, type SftExportRow, type SftLookups, type SimFidelityOptions, type StepReward, type StepRewardJsonlRow, type StepScorer, type ThompsonCurriculumOptions, type VarianceCurriculumOptions, type VerifiableReward, type VerifiableRewardExtractionOptions, type VerifiableRewardSource, appendToCorpus, applyEloUpdate, bestOfN, bucketLabel, buildDatasetFromCorpus, buildPairwiseFromCampaign, buildRlDataset, campaignToRunRecords, compareAdaptationCurves, datasheetToMarkdown, defaultBehaviorFeatures, detectRewardHacking, easyModeCheck, extractPreferences, extractStepRewards, extractVerifiableReward, extractVerifiableRewardsFromRecords, filterDeterministicallyRewarded, firstPassK, fitBradleyTerry, injectIrrelevantClause, jsDivergence, observationsFromRunRecords, paretoFrontier, prmTrainingPairs, quantileEdges, readCorpus, renameVariables, runAdaptationCurve, runComputeCurve, runContaminationProbe, runRLCampaign, runwiseStepRewardSummary, selfConsistency, shuffleOrder, simFidelityReport, stepRewardsToJsonl, thompsonCurriculum, toAnthropicFormat, toDpoJsonl, toDpoRows, toGrpoJsonl, toGrpoRows, toPrmJsonl, toPrmRows, toSftJsonl, toSftRows, toTRLFormat, varianceBasedCurriculum, verificationReportToRunRecord };
+export { ABSENT_CATEGORY, type AdaptationCurve, type AdaptationPoint, type AdaptationRunner, type AdapterContext, type AdversarialMutation, type BehaviorFeatures, type BradleyTerryFit, type BradleyTerryRating, type BuildPairwiseFromCampaignInput, type CellObservation, type CompareCurvesResult, type ComputeBestOfNOptions, type ComputeBestOfNResult, type ComputeCurve, type ComputeCurveBudget, type ComputeCurvePoint, type ContaminationProbeInput, type ContaminationProbeOptions, type ContaminationProbeReport, type CorpusAppendResult, type CorpusRecord, type CurriculumAllocation, DEFAULT_MIN_N_PER_FEATURE, DEFAULT_QUANTILE_BUCKETS, type DatasetFormat, type DeploymentOutcome, type DetectRewardHackingInput, type DpoExportRow, type DpoLookups, type EasyModeOptions, type EasyModeReport, type EloOptions, type ExtractPreferencesOptions, type ExtractStepRewardsOptions, type FeatureDivergence, type FeatureShift, type FidelityReport, type FidelityVerdict, FileSystemOutcomeStore, type FileSystemOutcomeStoreOptions, type GrpoExportRow, type GrpoLookups, type HarvestOptions, InMemoryOutcomeStore, type OffPolicyEstimate, type OffPolicyOptions, type OffPolicyTrajectory, type OutcomeStore, type PairwiseOutcome, type ParetoPointInput, PredictiveValidityResearcher, type PredictiveValidityResearcherOptions, type PreferenceExtractionReport, type PreferenceStrategy, type PreferenceTriple, type PrmExportRow, type PrmLookups, type PrmTrainingTriple, REPRESENTATIVE_MIN_FIDELITY, type RLCampaignResult, type RewardHackingFinding, type RewardHackingReport, type RewardHackingSignal, type RewardKind, type RewardStats, type RlDatasetBundle, type RlDatasetConfig, type RlDatasetManifest, type RlDatasetStats, type RunAdaptationCurveOptions, type RunComputeCurveOptions, type RunRLCampaignOptions, type RunwiseStepSummary, type ScenarioPerturbation, type ScenarioPerturbationKind, type SelfConsistencyOptions, type SelfConsistencyResult, type SftExportRow, type SftLookups, type SimFidelityOptions, type StepReward, type StepRewardJsonlRow, type StepScorer, type ThompsonCurriculumOptions, type VarianceCurriculumOptions, type VerifiableReward, type VerifiableRewardExtractionOptions, type VerifiableRewardSource, appendToCorpus, applyEloUpdate, bestOfN, bucketLabel, buildDatasetFromCorpus, buildPairwiseFromCampaign, buildRlDataset, campaignToRunRecords, compareAdaptationCurves, datasheetToMarkdown, defaultBehaviorFeatures, detectRewardHacking, doublyRobust, easyModeCheck, extractPreferences, extractStepRewards, extractVerifiableReward, extractVerifiableRewardsFromRecords, filterDeterministicallyRewarded, firstPassK, fitBradleyTerry, injectIrrelevantClause, inverseProbabilityWeighting, jsDivergence, observationsFromRunRecords, offPolicyEstimateAll, paretoFrontier, prmTrainingPairs, quantileEdges, readCorpus, renameVariables, runAdaptationCurve, runComputeCurve, runContaminationProbe, runEvalCampaign, runRLCampaign, runwiseStepRewardSummary, selfConsistency, selfNormalizedImportanceWeighting, shuffleOrder, simFidelityReport, stepRewardsToJsonl, thompsonCurriculum, toAnthropicFormat, toDpoJsonl, toDpoRows, toGrpoJsonl, toGrpoRows, toPrmJsonl, toPrmRows, toSftJsonl, toSftRows, toTRLFormat, varianceBasedCurriculum, verificationReportToRunRecord };