@tangle-network/agent-eval 0.5.0 → 0.7.0

package/dist/index.d.ts

@@ -1033,86 +1033,56 @@ declare class PromptOptimizer {
     run(config: OptimizationConfig): Promise<OptimizationResult>;
 }
 
-/**
- * Dual-agent convergence bench.
- *
- * Pattern lifted from tax-agent + legal-agent: two agents take turns until
- * they converge on a consensus artifact. One proposes, the other critiques;
- * the proposer revises; repeat until a score threshold is hit or max rounds.
- *
- * Generalized so any two "agents" (gateways, local functions, anything with
- * `propose` + `critique`) compose in. Returns convergence rounds per
- * scenario + whether convergence happened.
- */
-interface DualAgentScenario {
-    id: string;
-    initialPrompt: string;
-    /** Optional context the agents can read (e.g. source documents). */
-    context?: Record<string, unknown>;
+interface SteeringRolePrompt {
+    system?: string;
+    append?: string;
 }
-interface DualAgentRound {
-    roundIndex: number;
-    proposal: string;
-    critique: string;
-    convergenceScore: number;
-}
-interface DualAgentScenarioResult {
-    scenarioId: string;
-    converged: boolean;
-    roundsToConverge: number | null;
-    finalProposal: string;
-    history: DualAgentRound[];
-    finalScore: number;
-}
-interface DualAgentBenchConfig {
-    scenarios: DualAgentScenario[];
-    maxRounds?: number;
-    /** Convergence threshold in 0..1 (default 0.85). */
-    convergenceThreshold?: number;
-    /**
-     * Propose an answer given the scenario + the critic's prior critique (if any).
-     * Returns the proposal string.
-     */
-    propose: (args: {
-        scenario: DualAgentScenario;
-        roundIndex: number;
-        priorProposal?: string;
-        priorCritique?: string;
-    }) => Promise<string>;
-    /**
-     * Critique the proposer's current output. Returns a structured critique
-     * (free text) plus a convergence score: how close the proposal is to
-     * acceptable. 1.0 = accept, 0.0 = totally off.
-     */
-    critique: (args: {
-        scenario: DualAgentScenario;
-        roundIndex: number;
-        proposal: string;
-    }) => Promise<{
-        critique: string;
-        convergenceScore: number;
-    }>;
-    /** Optional per-round hook for progress + tracing. */
-    onRoundComplete?: (info: {
-        scenarioId: string;
-        round: DualAgentRound;
-    }) => void;
+interface SteeringBundle {
+    id: string;
+    coderPrompt?: string;
+    continuePrompt?: string;
+    reviewerPrompts?: Record<string, string>;
+    skills?: string[];
+    rolePrompts?: Record<string, SteeringRolePrompt>;
+    metadata?: Record<string, unknown>;
 }
-interface DualAgentReport {
-    scenarios: DualAgentScenarioResult[];
-    aggregate: {
-        convergenceRate: number;
-        avgRoundsToConverge: number | null;
-        avgFinalScore: number;
-    };
-    config: {
-        maxRounds: number;
-        convergenceThreshold: number;
-    };
+interface SteeringDelta {
+    coderPrompt?: string;
+    continuePrompt?: string;
+    reviewerPrompts?: Record<string, string>;
+    skills?: string[];
+    rolePrompts?: Record<string, SteeringRolePrompt>;
+    metadata?: Record<string, unknown>;
 }
-declare class DualAgentBench {
-    run(config: DualAgentBenchConfig): Promise<DualAgentReport>;
+declare function mergeSteeringBundle(base: SteeringBundle, delta: SteeringDelta): SteeringBundle;
+declare function renderSteeringText(bundle: SteeringBundle): string;
+
+interface RunScore {
+    success: number;
+    goalProgress: number;
+    repoGroundedness: number;
+    driftPenalty: number;
+    toolUseQuality: number;
+    patchQuality: number;
+    testReality: number;
+    costUsd: number;
+    wallSeconds: number;
+    notes?: string[];
+}
+interface RunScoreWeights {
+    success: number;
+    goalProgress: number;
+    repoGroundedness: number;
+    driftPenalty: number;
+    toolUseQuality: number;
+    patchQuality: number;
+    testReality: number;
+    costUsd: number;
+    wallSeconds: number;
 }
+declare const DEFAULT_RUN_SCORE_WEIGHTS: RunScoreWeights;
+declare function aggregateRunScore(score: RunScore, weights?: Partial<RunScoreWeights>): number;
+declare function clamp01(value: number): number;
 
 /**
  * TraceSchema v1 — the canonical data model for agent-eval.
@@ -1606,6 +1576,137 @@ interface OtlpExport {
 /** Export a single run's spans + events in OTLP/JSON. */
 declare function exportRunAsOtlp(store: TraceStore, runId: string, resourceAttrs?: Record<string, string | number | boolean>): Promise<OtlpExport>;
 
+interface RunTrace {
+    run: Run;
+    spans: Span[];
+    events: TraceEvent[];
+    artifacts: Artifact[];
+    budget: BudgetLedgerEntry[];
+}
+interface RunCriticOptions {
+    weights?: Partial<RunScoreWeights>;
+    driftPatterns?: RegExp[];
+}
+declare class RunCritic {
+    private readonly weights?;
+    private readonly driftPatterns;
+    constructor(options?: RunCriticOptions);
+    score(store: TraceStore, runId: string): Promise<RunScore>;
+    scoreTrace(trace: RunTrace): RunScore;
+    rank(score: RunScore): number;
+    private isDrift;
+}
+
+interface PlaybookEntry {
+    instruction: string;
+    rationale: string;
+    category?: string;
+    evidence?: string;
+    weight?: number;
+    sourceRunId?: string;
+}
+interface Playbook {
+    entries: PlaybookEntry[];
+}
+declare function distillPlaybook(entries: PlaybookEntry[], options?: {
+    maxEntries?: number;
+}): Playbook;
+declare function renderPlaybookMarkdown(playbook: Playbook): string;
+
+interface OptimizationExample {
+    scenarioId: string;
+    metadata?: Record<string, unknown>;
+}
+interface SteeringEvaluation {
+    variant: SteeringBundle;
+    example: OptimizationExample;
+    trialIndex: number;
+}
+interface SteeringVariantReport {
+    variantId: string;
+    bundle: SteeringBundle;
+    mean: number;
+    ci95: {
+        lower: number;
+        upper: number;
+    };
+    scenarioScores: Record<string, {
+        mean: number;
+        n: number;
+        samples: number[];
+    }>;
+}
+interface OptimizationLoopResult {
+    winner: SteeringBundle;
+    significant: boolean;
+    reports: SteeringVariantReport[];
+    pairwise: Array<{
+        variantA: string;
+        variantB: string;
+        pValue: number;
+        qValue: number;
+        significant: boolean;
+        meanDelta: number;
+    }>;
+}
+interface OptimizationLoopConfig {
+    variants: SteeringBundle[];
+    examples: OptimizationExample[];
+    evaluate: (args: SteeringEvaluation) => Promise<RunScore>;
+    scoreWeights?: Partial<RunScoreWeights>;
+    trialsPerScenario?: number;
+}
+declare class OptimizationLoop {
+    private readonly optimizer;
+    constructor(optimizer?: PromptOptimizer);
+    run(config: OptimizationLoopConfig): Promise<OptimizationLoopResult>;
+}
+
+type SteeringOptimizerBackend = 'pairwise' | 'ax-gepa';
+interface SteeringOptimizationRow {
+    variantId: string;
+    scenarioId: string;
+    bundle: SteeringBundle;
+    score: RunScore;
+    metadata?: Record<string, unknown>;
+}
+interface SteeringOptimizationSelector {
+    backend: SteeringOptimizerBackend;
+    signature?: string;
+    labels?: string[];
+    rationale?: string;
+}
+interface SteeringOptimizationResult {
+    backend: SteeringOptimizerBackend;
+    recommendedVariantId: string;
+    rationale: string;
+    rankings: Array<{
+        variantId: string;
+        mean: number;
+        runs: number;
+    }>;
+    selector?: SteeringOptimizationSelector;
+    skipped?: boolean;
+}
+interface SteeringOptimizerConfig {
+    weights?: Partial<RunScoreWeights>;
+}
+interface AxSteeringOptimizerConfig extends SteeringOptimizerConfig {
+    provider: 'openai' | 'anthropic';
+    apiKey: string;
+    model: string;
+    teacherModel?: string;
+    minRows?: number;
+}
+declare class PairwiseSteeringOptimizer {
+    optimize(rows: SteeringOptimizationRow[], config?: SteeringOptimizerConfig): SteeringOptimizationResult;
+}
+declare class AxGepaSteeringOptimizer {
+    private readonly config;
+    constructor(config: AxSteeringOptimizerConfig);
+    optimize(rows: SteeringOptimizationRow[]): Promise<SteeringOptimizationResult>;
+}
+
 /**
  * SandboxHarness — executes a scenario in an isolated environment and
  * emits a rich SandboxSpan into the trace.
@@ -1689,113 +1790,434 @@ declare class SandboxHarness {
     run(config: HarnessConfig, emitter: TraceEmitter): Promise<SandboxHarnessResult>;
 }
 
-/**
- * TestGradedScenario — a scenario whose score comes from a test suite.
- *
- * This is the SWE-bench pattern generalized. The scenario ships:
- *   - fixture data (setup instructions)
- *   - a test command the harness runs
- *   - optional assertion overrides
- *
- * The runner emits a run, delegates to SandboxHarness, records the
- * outcome, and returns a structured verdict. Consumers bind their own
- * agent execution to this contract.
- */
-
-interface TestGradedScenario {
+type SandboxJudgeKind = 'compiler' | 'test' | 'linter' | 'security';
+interface SandboxJudgeSpec {
     id: string;
-    description?: string;
-    harness: HarnessConfig;
-    /** Optional pass threshold in 0..1 (default 1.0 = all tests must pass). */
-    passThreshold?: number;
-    /** Provenance for dataset tracking. */
-    datasetVersion?: string;
-    /** Free-form tags (difficulty, category, etc.). */
-    tags?: Record<string, string>;
+    kind: SandboxJudgeKind;
+    config: HarnessConfig;
 }
-interface TestGradedRunOptions {
-    variantId?: string;
+interface SandboxJudgeResult {
+    id: string;
+    kind: SandboxJudgeKind;
+    passed: boolean;
+    score: number;
+    summary: string;
+    detail: SandboxHarnessResult;
+}
+interface JudgeFleetOptions {
     driver?: SandboxDriver;
-    /** Metadata recorded on the Run (codeSha, promptSha, modelFingerprint, seed). */
-    provenance?: Pick<Run, 'codeSha' | 'promptSha' | 'modelFingerprint' | 'seed' | 'envFingerprint'>;
+    parallel?: boolean;
 }
-interface TestGradedRunResult {
-    runId: string;
-    scenario: TestGradedScenario;
-    harness: SandboxHarnessResult;
-    pass: boolean;
-    score: number;
-    failureClass?: FailureClass;
+declare class JudgeRunner {
+    private readonly driver;
+    constructor(driver?: SandboxDriver);
+    run(spec: SandboxJudgeSpec): Promise<SandboxJudgeResult>;
 }
-declare function runTestGradedScenario(scenario: TestGradedScenario, store: TraceStore, options?: TestGradedRunOptions): Promise<TestGradedRunResult>;
-
-/**
- * BudgetGuard — enforces token / wall-clock / call / $ caps, records
- * a ledger entry on every decrement, emits `budget_breach` + throws
- * `BudgetBreachError` when a cap is hit.
- *
- * Wraps a TraceEmitter. The emitter persists ledger entries + breach
- * events so the classifier, pipelines, and reports can all read
- * budget state from the trace corpus — no separate accounting.
- */
+declare function runJudgeFleet(specs: SandboxJudgeSpec[], options?: JudgeFleetOptions): Promise<SandboxJudgeResult[]>;
+declare function compilerJudge(id: string, config: HarnessConfig): SandboxJudgeSpec;
+declare function testJudge(id: string, config: HarnessConfig): SandboxJudgeSpec;
+declare function linterJudge(id: string, config: HarnessConfig): SandboxJudgeSpec;
+declare function securityJudge(id: string, config: HarnessConfig): SandboxJudgeSpec;
 
-declare class BudgetBreachError extends Error {
-    dimension: keyof BudgetSpec;
-    limit: number;
-    attempted: number;
-    constructor(dimension: keyof BudgetSpec, limit: number, attempted: number);
+interface HostedJudgeDimension {
+    name: string;
+    weight: number;
+    rubric: string;
 }
-declare class BudgetGuard {
-    private consumed;
-    private emitter;
-    private budget;
-    private startedAt;
-    constructor(emitter: TraceEmitter, budget: BudgetSpec, now?: () => number);
-    /** Record consumption. Throws `BudgetBreachError` if any dimension exceeds its cap. */
-    charge(delta: Partial<Record<keyof BudgetSpec, number>>, spanId?: string): Promise<void>;
-    /** Convenience: advance wall-clock budget based on elapsed wall time. */
-    tickWall(nowMs: number, spanId?: string): Promise<void>;
-    get state(): Record<keyof BudgetSpec, number>;
+interface HostedJudgeConfig {
+    model: string;
+    mode?: 'llm' | 'sandbox' | 'composite';
+    systemPrompt?: string;
+    rubricTemplate?: string;
+    temperature?: number;
+    maxTurns?: number;
+    tools?: string[];
+    dimensions?: HostedJudgeDimension[];
+    setupCommand?: string;
+    scripts?: Record<string, string>;
+}
+interface HostedJudgeRequest {
+    prompt: string;
+    response: string;
+    rubric?: string;
+    reference?: string;
+    judge: HostedJudgeConfig;
 }
+interface HostedJudgeResponse {
+    score: number;
+    reasoning: string;
+    cost: number;
+    dimensions?: Array<{
+        name: string;
+        score: number;
+        reasoning: string;
+    }>;
+    evidence?: Array<{
+        type: string;
+        content: string;
+    }>;
+    turns?: number;
+    parseFailed?: boolean;
+    rawOutput?: string;
+}
+interface HostedRunScoreRequest {
+    trace: RunTrace;
+    weights?: Partial<RunScoreWeights>;
+    driftPatterns?: string[];
+}
+interface HostedRunScoreResponse {
+    score: RunScore;
+    aggregate: number;
+    weights: RunScoreWeights;
+    notes: string[];
+}
+type HostedRunCriticConfig = Pick<RunCriticOptions, 'weights'> & {
+    driftPatterns?: string[];
+};
 
 /**
- * Failure taxonomy — canonical classes + a default classifier.
+ * Dual-agent convergence bench.
  *
- * Every failed run should end up in a named class. The classifier here
- * is rule-based (fast, deterministic); an LLM fallback can be added by
- * the consumer for novel cases and trained into the rule base over time.
+ * Pattern lifted from tax-agent + legal-agent: two agents take turns until
+ * they converge on a consensus artifact. One proposes, the other critiques;
+ * the proposer revises; repeat until a score threshold is hit or max rounds.
  *
- * Consumers call `classifyFailure(run, spans, events)` and persist the
- * returned class as `Run.outcome.failureClass`.
+ * Generalized so any two "agents" (gateways, local functions, anything with
+ * `propose` + `critique`) compose in. Returns convergence rounds per
+ * scenario + whether convergence happened.
  */
-
-interface FailureContext {
-    run: Run;
-    spans: Span[];
-    events: TraceEvent[];
-}
-interface FailureClassification {
-    failureClass: FailureClass;
-    reason: string;
-    triggerSpanId?: string;
-    triggerEventId?: string;
-}
-/** Ordered rules — first match wins. */
-interface FailureRule {
+interface DualAgentScenario {
     id: string;
-    match: (ctx: FailureContext) => {
-        failureClass: FailureClass;
-        reason: string;
-        triggerSpanId?: string;
-        triggerEventId?: string;
-    } | null;
+    initialPrompt: string;
+    /** Optional context the agents can read (e.g. source documents). */
+    context?: Record<string, unknown>;
 }
-declare const DEFAULT_RULES: FailureRule[];
-/** Classify the failure mode of a run using an ordered rule list. */
-declare function classifyFailure(ctx: FailureContext, rules?: FailureRule[]): FailureClassification;
-
-/**
- * Trajectory — ordered, structured view over a run's spans.
+interface DualAgentRound {
+    roundIndex: number;
+    proposal: string;
+    critique: string;
+    convergenceScore: number;
+}
+interface DualAgentScenarioResult {
+    scenarioId: string;
+    converged: boolean;
+    roundsToConverge: number | null;
+    finalProposal: string;
+    history: DualAgentRound[];
+    finalScore: number;
+}
+interface DualAgentBenchConfig {
+    scenarios: DualAgentScenario[];
+    maxRounds?: number;
+    /** Convergence threshold in 0..1 (default 0.85). */
+    convergenceThreshold?: number;
+    /**
+     * Propose an answer given the scenario + the critic's prior critique (if any).
+     * Returns the proposal string.
+     */
+    propose: (args: {
+        scenario: DualAgentScenario;
+        roundIndex: number;
+        priorProposal?: string;
+        priorCritique?: string;
+    }) => Promise<string>;
+    /**
+     * Critique the proposer's current output. Returns a structured critique
+     * (free text) plus a convergence score: how close the proposal is to
+     * acceptable. 1.0 = accept, 0.0 = totally off.
+     */
+    critique: (args: {
+        scenario: DualAgentScenario;
+        roundIndex: number;
+        proposal: string;
+    }) => Promise<{
+        critique: string;
+        convergenceScore: number;
+    }>;
+    /** Optional per-round hook for progress + tracing. */
+    onRoundComplete?: (info: {
+        scenarioId: string;
+        round: DualAgentRound;
+    }) => void;
+}
+interface DualAgentReport {
+    scenarios: DualAgentScenarioResult[];
+    aggregate: {
+        convergenceRate: number;
+        avgRoundsToConverge: number | null;
+        avgFinalScore: number;
+    };
+    config: {
+        maxRounds: number;
+        convergenceThreshold: number;
+    };
+}
+declare class DualAgentBench {
+    run(config: DualAgentBenchConfig): Promise<DualAgentReport>;
+}
+
+/**
+ * Propose / Verify / Review — the core multi-shot primitive.
+ *
+ *   shot N:  propose(state, priorReview) → new state
+ *            verify(state)               → pass/fail, optional layers
+ *            review(state, verification, memory) → observations + next-shot
+ *                                                   instruction + shouldContinue
+ *            memory.append(entry)
+ *
+ * Roles are strictly separated:
+ *
+ *   - The WORKER is whatever the caller wraps in `propose`. It is
+ *     stateful — caller owns its resume/session mechanism.
+ *   - The VERIFIER grades the state. It produces the ground truth.
+ *     The reviewer cannot overturn or downgrade a verification layer.
+ *   - The REVIEWER is stateless per call. Its continuity is the
+ *     `ReviewMemoryStore` — durable JSONL by default, or any store
+ *     implementing the interface. It reads memory + trace summary +
+ *     verification and directs the NEXT proposer shot.
+ *
+ * This shape is load-bearing. The reviewer never grades; the verifier
+ * never directs. Two processes, two prompts, two concerns — which is
+ * what keeps the loop from confirmation-biasing itself into "all
+ * passed" when it didn't.
+ *
+ * Short-circuits and soft-fails are both first-class:
+ *   - verify.pass === true  → reviewer LLM call is skipped, memory
+ *     records a success entry, loop exits.
+ *   - review throws         → the shot still counts; the loop uses the
+ *     last-known instruction (or `fallbackInstruction`) for the next
+ *     propose call. A transient reviewer failure must NEVER abort a
+ *     valid arc.
+ *
+ * Composable: `propose` itself can be another `runProposeReview` call.
+ * That's the dogfooding path — a harness built on this primitive is in
+ * turn evaluable by it.
+ */
+
+interface Verification {
+    pass: boolean;
+    score?: number;
+    failingLayers?: string[];
+    details?: unknown;
+}
+interface Review {
+    observations: string;
+    diagnosis: string;
+    nextShotInstruction: string;
+    shouldContinue: boolean;
+    confidence: number;
+}
+interface ReviewMemoryEntry extends Review {
+    shot: number;
+    timestamp: number;
+    verification: {
+        pass: boolean;
+        score?: number;
+        failingLayers?: string[];
+    };
+}
+interface ProposeInput<State> {
+    shot: number;
+    goal: string;
+    state: State;
+    priorReview: Review | null;
+    abortSignal: AbortSignal;
+    emitter?: TraceEmitter;
+}
+interface ProposeOutput<State, Summary = unknown> {
+    state: State;
+    traceSummary?: Summary;
+}
+interface ReviewInput<State, Summary = unknown> {
+    shot: number;
+    goal: string;
+    state: State;
+    verification: Verification;
+    traceSummary: Summary | undefined;
+    memory: ReviewMemoryEntry[];
+}
+type ProposeFn<State, Summary = unknown> = (input: ProposeInput<State>) => Promise<ProposeOutput<State, Summary>>;
+type VerifyFn<State> = (state: State) => Promise<Verification>;
+type ReviewFn<State, Summary = unknown> = (input: ReviewInput<State, Summary>) => Promise<Review>;
+interface ReviewMemoryStore {
+    load(): Promise<ReviewMemoryEntry[]>;
+    append(entry: ReviewMemoryEntry): Promise<void>;
+}
+interface ProposeReviewConfig<State, Summary = unknown> {
+    goal: string;
+    initialState: State;
+    propose: ProposeFn<State, Summary>;
+    verify: VerifyFn<State>;
+    review: ReviewFn<State, Summary>;
+    /** Hard shot cap. Default 10. */
+    maxShots?: number;
+    /** Wall-clock cap in ms. Default 10 min. */
+    maxWallMs?: number;
+    /**
+     * If the reviewer returns confidence ≤ floor on `confidenceFloorWindow`
+     * consecutive shots, terminate early. Default floor 0.3, window 2.
+     * Set window to 0 or floor to <0 to disable.
+     */
+    confidenceFloor?: number;
+    confidenceFloorWindow?: number;
+    /** Defaults to an in-memory store if omitted. */
+    memory?: ReviewMemoryStore;
+    /** If provided, emit a Run + per-shot spans. */
+    store?: TraceStore;
+    scenarioId?: string;
+    projectId?: string;
+    variantId?: string;
+    /**
+     * Used when the reviewer soft-fails on shot 1 (no prior instruction to
+     * fall back to). Default is a generic "inspect failures and fix".
+     */
+    fallbackInstruction?: string;
+}
+interface ProposeReviewShot<State, Summary = unknown> {
+    shot: number;
+    state: State;
+    verification: Verification;
+    traceSummary: Summary | undefined;
+    review: Review;
+    reviewAvailable: boolean;
+    reviewError?: string;
+    durationMs: number;
+}
+interface ProposeReviewReport<State, Summary = unknown> {
+    runId: string | null;
+    completed: boolean;
+    shots: ProposeReviewShot<State, Summary>[];
+    finalState: State;
+    finalVerification: Verification;
+    failureClass?: FailureClass;
+    wallMs: number;
+    score: number;
+}
+declare function inMemoryReviewStore(initial?: ReviewMemoryEntry[]): ReviewMemoryStore;
+declare function jsonlReviewStore(path: string): ReviewMemoryStore;
+declare function runProposeReview<State, Summary = unknown>(config: ProposeReviewConfig<State, Summary>): Promise<ProposeReviewReport<State, Summary>>;
+interface LlmJsonCall {
+    (req: {
+        system: string;
+        user: string;
+    }): Promise<unknown>;
+}
+interface LlmReviewerConfig<State, Summary = unknown> {
+    callJson: LlmJsonCall;
+    renderState?: (state: State) => string;
+    renderTraceSummary?: (summary: Summary | undefined) => string;
+    /** Appended to the default system prompt. */
+    systemPromptAddendum?: string;
+}
+declare function createLlmReviewer<State, Summary = unknown>(cfg: LlmReviewerConfig<State, Summary>): ReviewFn<State, Summary>;
+
+/**
+ * TestGradedScenario — a scenario whose score comes from a test suite.
+ *
+ * This is the SWE-bench pattern generalized. The scenario ships:
+ *   - fixture data (setup instructions)
+ *   - a test command the harness runs
+ *   - optional assertion overrides
+ *
+ * The runner emits a run, delegates to SandboxHarness, records the
+ * outcome, and returns a structured verdict. Consumers bind their own
+ * agent execution to this contract.
+ */
+
+interface TestGradedScenario {
+    id: string;
+    description?: string;
+    harness: HarnessConfig;
+    /** Optional pass threshold in 0..1 (default 1.0 = all tests must pass). */
+    passThreshold?: number;
+    /** Provenance for dataset tracking. */
+    datasetVersion?: string;
+    /** Free-form tags (difficulty, category, etc.). */
+    tags?: Record<string, string>;
+}
+interface TestGradedRunOptions {
+    variantId?: string;
+    driver?: SandboxDriver;
+    /** Metadata recorded on the Run (codeSha, promptSha, modelFingerprint, seed). */
+    provenance?: Pick<Run, 'codeSha' | 'promptSha' | 'modelFingerprint' | 'seed' | 'envFingerprint'>;
+}
+interface TestGradedRunResult {
+    runId: string;
+    scenario: TestGradedScenario;
+    harness: SandboxHarnessResult;
+    pass: boolean;
+    score: number;
+    failureClass?: FailureClass;
+}
+declare function runTestGradedScenario(scenario: TestGradedScenario, store: TraceStore, options?: TestGradedRunOptions): Promise<TestGradedRunResult>;
+
+/**
+ * BudgetGuard — enforces token / wall-clock / call / $ caps, records
+ * a ledger entry on every decrement, emits `budget_breach` + throws
+ * `BudgetBreachError` when a cap is hit.
+ *
+ * Wraps a TraceEmitter. The emitter persists ledger entries + breach
+ * events so the classifier, pipelines, and reports can all read
+ * budget state from the trace corpus — no separate accounting.
+ */
+
+declare class BudgetBreachError extends Error {
+    dimension: keyof BudgetSpec;
+    limit: number;
+    attempted: number;
+    constructor(dimension: keyof BudgetSpec, limit: number, attempted: number);
+}
+declare class BudgetGuard {
+    private consumed;
+    private emitter;
+    private budget;
+    private startedAt;
+    constructor(emitter: TraceEmitter, budget: BudgetSpec, now?: () => number);
+    /** Record consumption. Throws `BudgetBreachError` if any dimension exceeds its cap. */
+    charge(delta: Partial<Record<keyof BudgetSpec, number>>, spanId?: string): Promise<void>;
+    /** Convenience: advance wall-clock budget based on elapsed wall time. */
+    tickWall(nowMs: number, spanId?: string): Promise<void>;
+    get state(): Record<keyof BudgetSpec, number>;
+}
+
+/**
+ * Failure taxonomy — canonical classes + a default classifier.
+ *
+ * Every failed run should end up in a named class. The classifier here
+ * is rule-based (fast, deterministic); an LLM fallback can be added by
+ * the consumer for novel cases and trained into the rule base over time.
+ *
+ * Consumers call `classifyFailure(run, spans, events)` and persist the
+ * returned class as `Run.outcome.failureClass`.
+ */
+
+interface FailureContext {
+    run: Run;
+    spans: Span[];
+    events: TraceEvent[];
+}
+interface FailureClassification {
+    failureClass: FailureClass;
+    reason: string;
+    triggerSpanId?: string;
+    triggerEventId?: string;
+}
+/** Ordered rules — first match wins. */
+interface FailureRule {
+    id: string;
+    match: (ctx: FailureContext) => {
+        failureClass: FailureClass;
+        reason: string;
+        triggerSpanId?: string;
+        triggerEventId?: string;
+    } | null;
+}
+declare const DEFAULT_RULES: FailureRule[];
+/** Classify the failure mode of a run using an ordered rule list. */
+declare function classifyFailure(ctx: FailureContext, rules?: FailureRule[]): FailureClassification;
+
+/**
+ * Trajectory — ordered, structured view over a run's spans.
  *
  * A pure function `buildTrajectory(store, runId) → Trajectory` returns
  * a topologically ordered list of `TrajectoryStep` with parent-child
@@ -3102,10 +3524,25 @@ declare function resumeBuilderSession(store: TraceStore, projectId: string): Pro
  * is the highest-leverage signal the framework computes — if
  * meta_score doesn't predict runtime_score, the builder's self-scoring
  * is broken.
+ *
+ * Scaffold-only mode: when a project has no `app-runtime` runs (e.g. a
+ * scaffold-builder eval that grades compose + build without driving a
+ * runtime scenario), `kind` is `'scaffold-only'` and `complete` measures
+ * meta + build only. Consumers can tell the two apart without having to
+ * interpret null-runtime as either "not yet computed" or "N/A for this
+ * project shape".
  */
 
+type ProjectKind = 'full' | 'scaffold-only';
 interface ThreeLayerProjectReport {
     projectId: string;
+    /**
+     * `'full'` when the project has at least one `app-runtime` run;
+     * `'scaffold-only'` when it only has meta + build layers. Lets
+     * downstream consumers treat a null runtime score as expected
+     * (scaffold-only) vs. missing (full, pipeline broke).
+     */
+    kind: ProjectKind;
     builderRunId?: string;
     /** Judge-verdict score on the builder run (0..1 after normalization). */
     metaScore: number | null;
@@ -3113,10 +3550,14 @@ interface ThreeLayerProjectReport {
     /** 0..1 from the sandbox harness (testsPassed / testsTotal). */
     buildScore: number | null;
     appRuntimeRunIds: string[];
-    /** Mean of outcome.score over app-runtime runs, 0..1. */
+    /** Mean of outcome.score over app-runtime runs, 0..1. Always null in scaffold-only mode. */
     runtimeScore: number | null;
     runtimePassRate: number | null;
-    /** True when all three layers produced a score. */
+    /**
+     * Layer-aware completeness:
+     *   - `kind='full'`: all three layers scored
+     *   - `kind='scaffold-only'`: meta + build scored (runtime not applicable)
+     */
     complete: boolean;
 }
 declare function scoreProject(store: TraceStore, projectId: string): Promise<ThreeLayerProjectReport>;
@@ -3205,4 +3646,931 @@ declare class ProjectRegistry {
     projectChats(projectId: string): Promise<ChatSummary[]>;
 }
 
-export { AgentDriver, type AgentDriverConfig, type AntiSlopConfig, type AntiSlopIssue, type AntiSlopReport, type Artifact, type ArtifactCheck, type Artifact$1 as ArtifactCheckArtifact, type ArtifactResult, type ArtifactValidator, type BaselineOptions, type BaselineReport, BehaviorAssertion, type BenchmarkReport, BenchmarkRunner, type BenchmarkRunnerConfig, BudgetBreachError, type BudgetBreachFinding, type BudgetBreachReport, BudgetGuard, type BudgetLedgerEntry, type BudgetSpec, BuilderSession, type BuilderSessionInit, type CalibrationResult, CallExpectation, type CanaryLeak, type CandidateScore, type ChatSummary, type CheckResult, type CollectedArtifacts, type CompletionCriterion, type ContinuityCheck, type ContinuityCheckResult, type ContinuityReport, type ContinuitySnapshotPair, type ContractMetric, type ContractReport, ConvergenceTracker, type CorrelationReport, type CostEntry, type CostSummary, CostTracker, DEFAULT_AGENT_SLOS, DEFAULT_RULES as DEFAULT_FAILURE_RULES, DEFAULT_MUTATORS, DEFAULT_REDACTION_RULES, DEFAULT_RED_TEAM_CORPUS, Dataset, type DatasetDifficulty, type DatasetManifest, type DatasetProvenance, type DatasetScenario, type DatasetSplit, type Direction, type DivergenceOptions, type DivergenceReport, DockerSandboxDriver, type DriverResult, type DriverState, DualAgentBench, type DualAgentBenchConfig, type DualAgentReport, type DualAgentRound, type DualAgentScenario, type DualAgentScenarioResult, type EvalResult, type EventFilter, type EventKind, type ExecutorConfig, type Expectation, type Experiment, type Run$1 as ExperimentRun, type ExperimentStore, ExperimentTracker, FAILURE_CLASSES, type FailureClass, type FailureClassification, type FailureCluster, type FailureClusterReport, type FailureContext, type FailureRule, type FeedbackPattern, FileSystemTraceStore, type FileSystemTraceStoreOptions, type GenericSpan, type GoldenItem, type HarnessConfig, HoldoutAuditor, HoldoutLockedError, type ImageData, InMemoryExperimentStore, InMemoryTraceStore, InMemoryWorkspaceInspector, type InspectorContext, type JudgeAgreementReport, type JudgeConfig, type JudgeFn, type JudgeInput, type JudgePair, type JudgeReplayResult, type JudgeRubric, type JudgeScore, type JudgeSpan, type LangfuseEnvelope, type LangfuseGeneration, type LangfuseScore, type LayerCorrelation, type LlmSpan, MODEL_PRICING, type MatcherResult, type Message, type MetricSamples, type MetricVerdict, MetricsCollector, type Mutator, OTEL_AGENT_EVAL_SCOPE, type Objective, type OptimizationConfig, type OptimizationResult, type Oracle, type OracleObservation, type OracleReport, type OracleResult, type OtlpExport, type OtlpResourceSpans, type OtlpSpan, type PairwiseComparison, type ParetoResult, type PersonaConfig, type PositionalBiasResult, ProductClient, type ProductClientConfig, ProjectRegistry, type ProjectSummary, type ProjectTimelineEntry, type PromptHandle, PromptOptimizer, PromptRegistry, type PromptVariant, REDACTION_VERSION, type RedTeamCase, type RedTeamCategory, type RedTeamFinding, type RedTeamPayload, type RedTeamReport, type RedactionReport, type RedactionRule, type RegressionOptions, type RegressionSpec, type RetrievalSpan, type RobustnessResult, type RouteMap, type RubricDimension, type Run, type RunAppScenarioOptions, type RunConfig, type RunDiff, type RunFilter, type RunLayer, type RunOutcome, type RunStatus, type SandboxDriver, SandboxHarness, type SandboxHarnessResult, type SandboxResult, type SandboxSpan, type Scenario, type ScenarioCost, type ScenarioFile, ScenarioRegistry, type ScenarioResult, type SelfPreferenceResult, type SeriesConvergenceOptions, type SeriesConvergenceResult, type ShipOptions, type SliceOptions, type Slo, type SloCheckResult, type SloComparator, type SloReport, type SloSeverity, type SlopCategory, type Span, type SpanBase, type SpanFilter, type SpanHandle, type SpanKind, type SpanStatus, type StuckLoopFinding, type StuckLoopOptions, type StuckLoopReport, SubprocessSandboxDriver, TRACE_SCHEMA_VERSION, type TestGradedRunOptions, type TestGradedRunResult, type TestGradedScenario, type TestOutputParser, type TestResult, type ThreeLayerProjectReport, type ThresholdContract, TokenCounter, type TokenSpec, type ToolSpan, type ToolStats, type ToolUseMetrics, type ToolUseOptions, type ToolWasteFinding, type ToolWasteOptions, type ToolWasteReport, TraceEmitter, type TraceEmitterOptions, type TraceEvent, type TraceStore, type Trajectory, type TrajectoryStep, type Turn, type TurnMetrics, type TurnResult, type ValidationContext, type ValidationIssue, type ValidationResult, type VariantScore, type VerbosityBiasResult, type VisualDiffOptions, type VisualDiffResult, type WorkspaceAssertion, type WorkspaceAssertionResult, type WorkspaceInspector, type WorkspaceSnapshot, adversarialJudge, aggregateLlm, analyzeAntiSlop, analyzeSeries, argHash, benjaminiHochberg, bonferroni, budgetBreachView, buildTrajectory, byteLengthRange, calibrateJudge, canaryLeakView, checkCanaries, checkSlos, classifyFailure, codeExecutionJudge, cohensD, coherenceJudge, collectionPreserved, compareToBaseline, composeParsers, composeValidators, computeToolUseMetrics, confidenceInterval, containsAll, correlateLayers, createAntiSlopJudge, createCustomJudge, createDomainExpertJudge, defaultJudges, dominates, estimateCost, estimateTokens, evaluateContract, evaluateOracles, executeScenario, expectAgent, exportRunAsOtlp, failureClusterView, fileContains, fileExists, firstDivergenceView, formatBenchmarkReport, formatDriverReport, groupBy, hashContent, hashScenarios, interRaterReliability, iqr, isJudgeSpan, isLlmSpan, isRetrievalSpan, isSandboxSpan, isToolSpan, jestTestParser, jsonHasKeys, jsonShape, judgeAgreementView, judgeSpans, keyPreserved, llmSpanFromProvider, llmSpans, lowercaseMutator, mannWhitneyU, normalizeScores, notBlocked, pairedTTest, paraphraseRobustness, paretoFrontier, partialCredit, pixelDeltaRatio, politenessPrefixMutator, positionalBias, printDriverSummary, pytestTestParser, redTeamDataset, redTeamReport, redactString, redactValue, regexMatch, regexMatches, regressionView, renderMarkdownReport, replayTraceThroughJudge, requiredSampleSize, resumeBuilderSession, rowCount, rowWhere, runAssertions, runE2EWorkflow, runExpectations, runFailureClass, runTestGradedScenario, runsForScenario, scoreAllProjects, scoreContinuity, scoreProject, scoreRedTeamOutput, selfPreference, sentenceReorderMutator, statusAdvanced, stuckLoopView, textInSnapshot, toLangfuseEnvelope, toPrometheusText, toolNamesForRun, toolSpans, toolWasteView, typoMutator, urlContains, verbosityBias, visualDiff, vitestTestParser, weightedMean, welchsTTest, whitespaceCollapseMutator, wilcoxonSignedRank };
+/**
+ * OutcomeStore — deployment outcomes attached to Run IDs.
+ *
+ * Outcomes arrive asynchronously from production telemetry after the
+ * eval run completed: user ratings, retention flags, conversion events,
+ * revenue, support-ticket rate, anything a product team can measure.
+ * The store is a peer to TraceStore — separate lifecycle, same runId
+ * foreign key.
+ *
+ * The whole point of this module is to make the meta-eval correlation
+ * question computable: `correlate(evalMetric, outcomeMetric) → r, ρ, n, CI`.
+ */
+interface DeploymentOutcome {
+    runId: string;
+    capturedAt: number;
+    /** Numeric outcomes keyed by name — retention_7d, csat, revenue_usd, etc. */
+    metrics: Record<string, number>;
+    /** Dimensions for stratified analysis — cohort, region, user_segment. */
+    labels?: Record<string, string>;
+    /** Free-form provenance (source system, pipeline version). */
+    source?: string;
+}
+interface OutcomeFilter {
+    runIds?: string[];
+    since?: number;
+    until?: number;
+    label?: {
+        key: string;
+        value: string;
+    };
+    source?: string;
+}
+interface OutcomeStore {
+    append(outcome: DeploymentOutcome): Promise<void>;
+    /** All outcomes attached to this run (a single run can have many — multiple
+     *  capture windows over deployment time). */
+    forRun(runId: string): Promise<DeploymentOutcome[]>;
+    list(filter?: OutcomeFilter): Promise<DeploymentOutcome[]>;
+}
+declare class InMemoryOutcomeStore implements OutcomeStore {
+    private items;
+    append(outcome: DeploymentOutcome): Promise<void>;
+    forRun(runId: string): Promise<DeploymentOutcome[]>;
+    list(filter?: OutcomeFilter): Promise<DeploymentOutcome[]>;
+}
+interface FileSystemOutcomeStoreOptions {
+    dir: string;
+    maxBytes?: number;
+}
+declare class FileSystemOutcomeStore implements OutcomeStore {
+    private dir;
+    private maxBytes;
+    private memo?;
+    private loaded;
+    constructor(options: FileSystemOutcomeStoreOptions);
+    private ensureDir;
+    append(outcome: DeploymentOutcome): Promise<void>;
+    private load;
+    forRun(runId: string): Promise<DeploymentOutcome[]>;
+    list(filter?: OutcomeFilter): Promise<DeploymentOutcome[]>;
+}
+
+/**
+ * Correlation study — "does our eval score predict real-world outcomes?"
+ *
+ * This is the load-bearing signal. Takes a TraceStore + OutcomeStore,
+ * joins on runId, computes Pearson + Spearman + bootstrap CI for every
+ * (evalMetric, outcomeMetric) pair the caller declares.
+ *
+ * Without this number the framework is ornamental. With it and r > 0.6
+ * the framework is a moat — no other agent-eval tool publishes one.
+ */
+
+interface EvalMetricSpec {
+    id: string;
+    /** Extract a scalar from a run (defaults cover score/pass/durationMs/costUsd/tokens). */
+    extract?: (run: Run, store: TraceStore) => Promise<number | null>;
+}
+interface OutcomePair {
+    evalMetric: string;
+    outcomeMetric: string;
+}
+interface CorrelationResult {
+    evalMetric: string;
+    outcomeMetric: string;
+    n: number;
+    pearson: number;
+    spearman: number;
+    /** 95% bootstrap CI for Pearson. */
+    pearsonCi95: {
+        lower: number;
+        upper: number;
+    };
+    /** Rough verdict: 'strong' ≥ 0.7, 'moderate' ≥ 0.4, else 'weak'. */
+    verdict: 'strong' | 'moderate' | 'weak';
+}
+interface CorrelationStudyResult {
+    pairs: CorrelationResult[];
+    joinedSamples: number;
+    skippedRuns: number;
+}
+interface CorrelationStudyOptions {
+    /** Only join outcomes captured within this window after run.startedAt. */
+    maxCaptureLagMs?: number;
+    /** Restrict to a subset of outcomes (cohort, region, source). */
+    outcomeFilter?: OutcomeFilter;
+    /** Which outcome per run to use when multiple exist. Default 'latest'. */
+    reduction?: 'latest' | 'mean' | 'max';
+    /** Bootstrap iterations for the CI. Default 500. */
+    bootstrapIterations?: number;
+}
+declare function correlationStudy(traceStore: TraceStore, outcomeStore: OutcomeStore, evalMetrics: EvalMetricSpec[], outcomeMetricNames: string[], options?: CorrelationStudyOptions): Promise<CorrelationStudyResult>;
+
+/**
+ * Calibration curve — binned "if eval says X, what does reality show?"
+ *
+ * Companion to correlationStudy. Raw correlation is a single number;
+ * the calibration curve shows *where* the eval is well-calibrated vs
+ * overconfident / underconfident. Buckets the eval metric, computes
+ * mean outcome per bucket, reports expected-calibration-error (ECE).
+ */
+
+interface CalibrationBin {
+    lower: number;
+    upper: number;
+    n: number;
+    evalMean: number;
+    outcomeMean: number;
+    /** |outcomeMean − evalMean|; contributes to ECE weighted by n/total. */
+    gap: number;
+}
+interface CalibrationReport {
+    evalMetric: string;
+    outcomeMetric: string;
+    n: number;
+    bins: CalibrationBin[];
+    /** Expected Calibration Error — Σ (n_i/N) × |outcomeMean_i − evalMean_i|. */
+    ece: number;
+    /** Max bin gap — upper bound on miscalibration. */
+    maxGap: number;
+}
+interface CalibrationOptions {
+    bins?: number;
+    /** Equal-width (fixed bin edges) or equal-frequency (quantile bins). */
+    binning?: 'equal-width' | 'equal-frequency';
+    /** Clip eval values to [lo, hi] before binning. */
+    range?: {
+        lo: number;
+        hi: number;
+    };
+}
+declare function calibrationCurve(traceStore: TraceStore, outcomeStore: OutcomeStore, evalMetric: EvalMetricSpec, outcomeMetric: string, options?: CalibrationOptions): Promise<CalibrationReport | null>;
+
+/**
+ * Process Reward Modeling — per-step rubric grading.
+ *
+ * A StepRubric inspects one span and returns a score + rationale.
+ * PrmGrader applies an array of rubrics to every LLM span in a
+ * trajectory (consumers can broaden to tool/retrieval spans via the
+ * `kind` filter on each rubric).
+ *
+ * Why this matters: outcome-only eval (did the final artifact work?)
+ * gives sparse reward — most agent turns are unattributable. PRMs
+ * densify the signal so optimizers and RL fine-tuning can assign
+ * credit per turn.
+ */
+
+interface StepContext {
+    trajectory: Trajectory;
+    step: TrajectoryStep;
+    /** Steps preceding `step` in trajectory order. */
+    prior: TrajectoryStep[];
+    /** Steps following `step`. */
+    next: TrajectoryStep[];
+}
+interface StepRubric {
+    id: string;
+    /** Only grade spans of these kinds (default: all). */
+    kinds?: Array<Span['kind']>;
+    /** Weight in the aggregate score. Default 1. */
+    weight?: number;
+    /** Returns score in 0..1 + optional rationale/evidence. Return `null` to
+     *  skip grading (rubric doesn't apply to this step). */
+    grade: (ctx: StepContext) => Promise<{
+        score: number;
+        rationale?: string;
+        evidence?: string;
+    } | null>;
+}
+interface GradedStep {
+    spanId: string;
+    rubricId: string;
+    score: number;
+    weight: number;
+    rationale?: string;
+    evidence?: string;
+}
+interface PrmGradedTrace {
+    runId: string;
+    steps: GradedStep[];
+    /** Weighted mean of all graded steps; 0..1. */
+    aggregateScore: number;
+    /** Number of spans graded — useful for sanity-checking coverage. */
+    gradedCount: number;
+    /** Number of spans in the trajectory that no rubric matched. */
+    ungradedCount: number;
+}
+declare class PrmGrader {
+    private rubrics;
+    constructor(rubrics: StepRubric[]);
+    /**
+     * Grade every eligible span in a run. Emits a JudgeVerdict span for each
+     * (rubric × span) verdict so the result is visible to downstream pipelines
+     * (judgeAgreementView, etc.) — PRM is just "a judge that runs per span."
+     */
+    grade(store: TraceStore, runId: string): Promise<PrmGradedTrace>;
+}
+/** Helper: reads JudgeVerdict spans that PRM emitted so downstream pipelines
+ *  can distinguish PRM verdicts from human or top-level LLM judges. */
+declare function isPrmVerdict(verdict: JudgeSpan): boolean;
+
+/**
+ * Built-in reference rubrics. Consumers combine these with domain
+ * rubrics. All are deterministic, rule-based — cheap to run + easy
+ * to unit-test. LLM-based rubrics are trivially authored by
+ * following the StepRubric contract.
+ */
+
+/** Penalize very short or very long assistant outputs. */
+declare function outputLengthRubric(args?: {
+    minChars?: number;
+    maxChars?: number;
+    weight?: number;
+}): StepRubric;
+/** Reward tool calls that succeeded (status='ok') with an informative result. */
+declare function toolSuccessRubric(args?: {
+    weight?: number;
+}): StepRubric;
+/** Penalize tool calls that duplicate a prior call with identical args. */
+declare function toolNonRedundantRubric(args?: {
+    weight?: number;
+}): StepRubric;
+/** Penalize LLM outputs that contain common refusal markers when a refusal
+ *  is NOT expected (caller inverts weight for scenarios where refusal IS expected). */
+declare function nonRefusalRubric(args?: {
+    markers?: RegExp[];
+    weight?: number;
+}): StepRubric;
+/** Reward outputs that invoke the next-step tool the trajectory actually uses
+ *  (i.e. the LLM span announced "I will call X" and the following tool span IS X). */
+declare function toolIntentAlignmentRubric(args?: {
+    weight?: number;
+}): StepRubric;
+
+/**
+ * Export PRM-graded traces as training data for downstream reward-model
+ * fine-tuning. Canonical format is NDJSON of
+ * `{ trajectory_text, step_index, rubric, score }` so a small model can
+ * learn to predict step rewards from step context.
+ *
+ * The framework doesn't train the model — we emit the data; callers
+ * plug it into their preferred trainer (TRL, Unsloth, custom).
+ */
+
+interface PrmTrainingSample {
+    runId: string;
+    spanId: string;
+    rubricId: string;
+    score: number;
+    /** Serialized step context — step + surrounding conversation. */
+    context: {
+        priorTurns: Array<{
+            role: string;
+            content: string;
+        }>;
+        step: {
+            kind: Span['kind'];
+            text: string;
+        };
+    };
+    /** Optional evidence + rationale for auditability. */
+    rationale?: string;
+    evidence?: string;
+}
+declare function exportTrainingData(store: TraceStore, graded: PrmGradedTrace[], options?: {
+    contextWindow?: number;
+}): Promise<PrmTrainingSample[]>;
+/** NDJSON serialization — write to file or stream directly to a trainer. */
+declare function toNdjson(samples: PrmTrainingSample[]): string;
+
+/**
+ * Inference-time PRM scoring — pick the best of N candidate trajectories
+ * using a trained reward model (or a rule-based PRM as a proxy).
+ *
+ * The canonical Best-of-N pattern: generate N completions, score each
+ * with a PRM, pick the winner. Here the scoring loop is framework-agnostic
+ * — supply a TraceStore + PrmGrader + N run IDs → get ranking + winner.
+ */
+
+interface BestOfNResult {
+    winner: PrmGradedTrace;
+    ranked: PrmGradedTrace[];
+    /** Standard deviation of aggregate scores — small = candidates were homogenous. */
+    stdDev: number;
+}
+declare function prmBestOfN(store: TraceStore, grader: PrmGrader, runIds: string[]): Promise<BestOfNResult>;
+/**
+ * Weighted vote across multiple graders — use when you want a PRM ensemble
+ * (e.g. rule-based + LLM-based + trained model). Each grader produces its
+ * own ranking; we aggregate via rank-sum (Borda count) so no single grader
+ * dominates via a different score scale.
+ */
+declare function prmEnsembleBestOfN(store: TraceStore, graders: PrmGrader[], runIds: string[]): Promise<BestOfNResult>;
+
+/**
+ * Bisector — auto-locate the change that introduced an eval regression.
+ *
+ * Two shapes:
+ *   - `commitBisect` — walk an ordered SHA list, binary-search for the
+ *     first commit that fails.
+ *   - `promptBisect` — given a good and bad prompt, progressively port
+ *     paragraphs from good→bad to localize the breaking change.
+ *
+ * Generic `bisect<T>` lets callers drive any ordered state space
+ * (dataset versions, config files, CLI flag combinations).
+ */
+interface BisectOptions<T> {
+    /** State known to pass. */
+    good: T;
+    /** State known to fail. */
+    bad: T;
+    /** Equality test on state values — default Object.is. */
+    equals?: (a: T, b: T) => boolean;
+    /** Pick the halfway state between good + bad. Return null when no further
+     *  split is possible (e.g. adjacent commits). */
+    halfway: (good: T, bad: T) => T | null;
+    /** Produce a verdict for a state. */
+    runEval: (state: T) => Promise<{
+        score: number;
+        pass: boolean;
+    }>;
+    /** Hard cap on iterations (default 40 — covers ~1T ordered states). */
+    maxIterations?: number;
+}
+interface BisectStep<T> {
+    state: T;
+    score: number;
+    pass: boolean;
+}
+interface BisectResult<T> {
+    /** The first bad state — typically `bad` in the final (good, bad) adjacent pair. */
+    culprit: T;
+    /** Ordered trace of all states evaluated. */
+    path: BisectStep<T>[];
+    /** True when we narrowed to an adjacent (good, bad) pair. */
+    converged: boolean;
+    /** True when `good` itself failed or `bad` itself passed — the caller's
+     *  premise was broken. */
+    inputInconsistent: boolean;
+}
+declare function bisect<T>(options: BisectOptions<T>): Promise<BisectResult<T>>;
+/**
+ * Commit bisect — `commits` is an ordered SHA list, oldest to newest.
+ * `good` and `bad` must both be present in the list.
+ */
+declare function commitBisect(options: {
+    commits: string[];
+    good: string;
+    bad: string;
+    runEval: (sha: string) => Promise<{
+        score: number;
+        pass: boolean;
+    }>;
+    maxIterations?: number;
+}): Promise<BisectResult<string>>;
+/**
+ * Prompt bisect — splits the good and bad prompts into paragraphs, then
+ * progressively replaces paragraphs in `good` with their counterparts
+ * from `bad` to localize the offending change. Only works when the two
+ * prompts have the same paragraph count (a common editorial workflow
+ * constraint — one paragraph = one change unit).
+ */
+declare function promptBisect(options: {
+    good: string;
+    bad: string;
+    runEval: (prompt: string) => Promise<{
+        score: number;
+        pass: boolean;
+    }>;
+    maxIterations?: number;
+    paragraphSplitter?: (prompt: string) => string[];
+}): Promise<BisectResult<string> & {
+    offendingParagraphIndex?: number;
+}>;
+
+/**
+ * Counterfactual replay — "what would have happened if we'd changed
+ * exactly one thing at turn N?"
+ *
+ * The framework does NOT drive the agent — it sets up the replay
+ * context (prior spans, prior state, mutation spec) and records the
+ * resulting divergence. Consumers supply an `executeFrom(ctx)` callback
+ * that runs their agent starting from turn N with the mutation applied.
+ *
+ * Counterfactual runs are recorded as a new Run with `layer='meta'` and
+ * `parentRunId = originalRunId`, so downstream diff + correlation
+ * pipelines see them natively.
+ */
+
+type CounterfactualMutation = {
+    kind: 'swap-model';
+    at: number;
+    newModel: string;
+} | {
+    kind: 'swap-tool-result';
+    at: number;
+    newResult: unknown;
+} | {
+    kind: 'truncate-after';
+    at: number;
+} | {
+    kind: 'inject-system-message';
+    at: number;
+    content: string;
+} | {
+    kind: 'custom';
+    at: number;
+    describe: string;
+    apply: (step: TrajectoryStep) => TrajectoryStep;
+};
+interface CounterfactualContext {
+    originalRunId: string;
+    originalTrajectory: Trajectory;
+    /** Steps up to (but not including) the mutation point — the prefix the
+     *  replayed agent inherits as its prior conversation/tool history. */
+    prefix: TrajectoryStep[];
+    mutation: CounterfactualMutation;
+    /** Pre-applied mutation on the step at `mutation.at`. Consumers use this
+     *  as the FIRST step the replayed agent emits (they decide whether to
+     *  re-emit it or continue from there). */
+    mutatedStep: TrajectoryStep;
+}
+interface CounterfactualResult {
+    counterfactualRunId: string;
+    originalRunId: string;
+    mutation: CounterfactualMutation;
+    /** Structured delta summary — caller can extend via scoring. */
+    delta: {
+        originalOutcomeScore: number | null;
+        counterfactualOutcomeScore: number | null;
+        deltaScore: number | null;
+    };
+}
+interface CounterfactualRunner {
+    /**
+     * Execute the agent from `ctx.prefix` with the mutation applied.
+     * MUST emit spans into the provided emitter so they become part of
+     * the counterfactual run. MUST call emitter.endRun() with a verdict.
+     */
+    executeFrom: (ctx: CounterfactualContext, emitter: TraceEmitter) => Promise<void>;
+}
+declare function runCounterfactual(store: TraceStore, originalRunId: string, mutation: CounterfactualMutation, runner: CounterfactualRunner): Promise<CounterfactualResult>;
+/**
+ * Aggregate a batch of counterfactuals into a simple attribution table:
+ * which mutation kinds move outcomes most? (Useful when you run a grid
+ * over the same trajectory — swap-model at every llm span, swap-tool
+ * at every tool span — and want a ranked summary.)
+ */
+declare function attributeCounterfactuals(results: CounterfactualResult[]): Array<{
+    mutationKind: CounterfactualMutation['kind'];
+    n: number;
+    meanAbsDelta: number;
+    meanSignedDelta: number;
+}>;
+
+/**
+ * Full cross-trace diff — align two trajectories step-by-step, report
+ * per-step score deltas, attribute a variant's total outcome lead to
+ * specific turns.
+ *
+ * 0.5 shipped `firstDivergenceView` (finds the first differing step).
+ * This does the heavier work: full alignment via LCS, per-step
+ * contribution to score delta using PRM verdicts when available,
+ * fallback to structural heuristics (latency, token count, tool
+ * outcome) otherwise.
+ */
+
+type AlignmentOp = {
+    op: 'match';
+    a: TrajectoryStep;
+    b: TrajectoryStep;
+} | {
+    op: 'insert';
+    b: TrajectoryStep;
+} | {
+    op: 'delete';
+    a: TrajectoryStep;
+} | {
+    op: 'replace';
+    a: TrajectoryStep;
+    b: TrajectoryStep;
+};
+interface StepAttribution {
+    op: AlignmentOp;
+    /** Difference in PRM score (or null when not scored by a matching judge). */
+    prmDelta: number | null;
+    /** Difference in latency (endedAt - startedAt). */
+    latencyDeltaMs: number | null;
+    /** Difference in token count (LLM spans). */
+    tokenDelta: number | null;
+    /** Reason this step is / isn't considered a contributor to the outcome delta. */
+    note: string;
+}
+interface CrossTraceDiff {
+    runA: string;
+    runB: string;
+    alignment: AlignmentOp[];
+    attributions: StepAttribution[];
+    /** Total score delta (B - A). */
+    totalScoreDelta: number | null;
+    /** Sum of PRM deltas across matched/replaced steps. Close to
+     *  `totalScoreDelta` when PRM covers the trajectory; gap indicates
+     *  unmodeled variance. */
+    prmDeltaSum: number;
+}
+interface CrossTraceDiffOptions {
+    stepEquals?: (a: TrajectoryStep, b: TrajectoryStep) => boolean;
+}
+declare function crossTraceDiff(store: TraceStore, runA: string, runB: string, options?: CrossTraceDiffOptions): Promise<CrossTraceDiff>;
+
+/**
+ * Pre-registered hypotheses — declare what you're testing BEFORE the
+ * run, check it AFTER. Prevents p-hacking, optional stopping, and the
+ * "we ran until it looked good" failure mode.
+ *
+ * Manifest is a plain JSON-friendly object. Sign it with a content hash
+ * + timestamp; the registered record becomes immutable. Post-run,
+ * evaluate the manifest against observed results — the library refuses
+ * to let you re-interpret a different metric as the declared one.
+ */
+interface HypothesisManifest {
+    id: string;
+    /** Human prose — goes into the audit trail. */
+    hypothesis: string;
+    /** Metric the hypothesis claims to move. */
+    metric: string;
+    /** 'increase' = candidate should score higher than baseline; 'decrease' = lower. */
+    direction: 'increase' | 'decrease';
+    /** Minimum effect size to count (same units as the metric). */
+    minEffect: number;
+    /** Alpha threshold. */
+    alpha: number;
+    /** Target statistical power at which sample size was pre-computed. */
+    power: number;
+    /** Declared N per arm before running. */
+    preRegisteredN: number;
+    /** ISO8601 timestamp the manifest was registered. */
+    registeredAt: string;
+    /** Optional identifiers to tie into the trace corpus. */
+    baselineLabel?: string;
+    candidateLabel?: string;
+}
+interface SignedManifest extends HypothesisManifest {
+    /** sha256 hex of canonicalized manifest (everything except contentHash). */
+    contentHash: string;
+}
+interface HypothesisResult {
+    manifest: SignedManifest;
+    observedN: number;
+    observedEffect: number;
+    observedPValue: number;
+    /** True iff the observed effect hits the pre-declared direction with
+     *  magnitude ≥ minEffect AND p < alpha. */
+    confirmed: boolean;
+    /** Enumerated reasons the hypothesis was rejected (each a machine-tag). */
+    rejectionReasons: Array<'wrong_direction' | 'effect_too_small' | 'not_significant' | 'undersampled'>;
+    notes?: string;
+}
+declare function signManifest(m: HypothesisManifest): Promise<SignedManifest>;
+/** Verify that a signed manifest has not been tampered with. */
+declare function verifyManifest(m: SignedManifest): Promise<boolean>;
+/**
+ * Evaluate a pre-registered hypothesis against observed results.
+ * Mechanical — no re-interpretation permitted.
+ */
+declare function evaluateHypothesis(manifest: SignedManifest, observed: {
+    n: number;
+    effect: number;
+    pValue: number;
+}): Promise<HypothesisResult>;
+
+/**
+ * Self-play scenario evolution — agents generate adversarial scenarios
+ * against each other; survivors become part of the eval corpus.
+ *
+ * Framework-agnostic about how scenarios are generated. Caller supplies:
+ *   - `propose`: asks a "proposer" agent for candidate scenarios
+ *   - `scoreAgainst`: runs a target agent against a scenario and returns
+ *     its score
+ *
+ * A scenario *survives* if it reveals a meaningful score difference
+ * between two target agents (or between a target agent and itself on
+ * different runs). Survivors are promoted to a Dataset; the caller
+ * decides what to do with them (hold-out, training, regression set).
+ *
+ * Guard rails: minimum absolute score delta to consider a scenario
+ * informative; floor on absolute target score so degenerate break-all
+ * scenarios (noise, gibberish) don't flood the corpus.
+ */
+
+interface CandidateScenario {
+    id: string;
+    payload: unknown;
+    /** Free-form tags (domain, generation, parent). */
+    tags?: Record<string, string>;
+}
+interface ScoredTarget {
+    targetId: string;
+    score: number;
+}
+interface EvolutionRound {
+    round: number;
+    proposed: CandidateScenario[];
+    survived: CandidateScenario[];
+    rejected: Array<{
+        candidate: CandidateScenario;
+        reason: string;
+    }>;
+    scoredBreakdown: Array<{
+        candidate: CandidateScenario;
+        scores: ScoredTarget[];
+        spread: number;
+    }>;
+}
+interface SelfPlayOptions {
+    /** Minimum score spread across targets for a scenario to survive. Default 0.1. */
+    minSpread?: number;
+    /** Minimum floor score across targets — keeps degenerate break-all scenarios
+     *  out. Default 0.1 (if every target scores below this, discard). */
+    minAbsoluteFloor?: number;
+    /** Hard cap on survivors per round. Default 50. */
+    maxSurvivors?: number;
+    /** Rounds to run. Default 1. Each round's survivors can be fed back into
+     *  `propose` to compound. */
+    rounds?: number;
+    /** Seed for scenario id generation if proposer doesn't provide one. */
+    seed?: number;
+}
+interface SelfPlayProposer {
+    propose(round: number, priorSurvivors: CandidateScenario[]): Promise<CandidateScenario[]>;
+}
+interface SelfPlayScorer {
+    /** Score one candidate against every target; returns parallel array. */
+    scoreCandidate(candidate: CandidateScenario, targets: string[]): Promise<ScoredTarget[]>;
+}
+declare function runSelfPlay(proposer: SelfPlayProposer, scorer: SelfPlayScorer, targets: string[], options?: SelfPlayOptions): Promise<{
+    rounds: EvolutionRound[];
+    dataset: Dataset;
+}>;
+
+/**
+ * Causal attribution via factorial experiments.
+ *
+ * Run every combination of {model × prompt × scenario × seed}, then
+ * decompose observed score variance into main effects + interactions.
+ * Moves from correlational "variant B is better" to causal "the model
+ * swap accounts for 42% of the lead; the prompt change accounts for 28%;
+ * interaction is 30%."
+ *
+ * Minimal implementation: 2-way factorial (two factors at a time) with
+ * main-effect + interaction decomposition via variance of cell means.
+ * Consumers run the factorial design themselves (we don't schedule
+ * runs); this module consumes the (factorLevels, observedScores)
+ * table and does the attribution math.
+ */
+interface FactorialCell {
+    /** Map factor name → level id. e.g. { model: 'claude', prompt: 'v2' } */
+    levels: Record<string, string>;
+    /** Observed score for this cell (mean over replications if n > 1). */
+    score: number;
+    /** Number of replications averaged to produce `score`. */
+    n: number;
+}
+interface FactorContribution {
+    factor: string;
+    /** Variance attributed to this factor's main effect, as a fraction of total. */
+    shareOfVariance: number;
+    /** Range of cell means across levels of this factor. */
+    range: number;
+}
+interface InteractionContribution {
+    factors: [string, string];
+    shareOfVariance: number;
+}
+interface CausalAttributionReport {
+    totalVariance: number;
+    mainEffects: FactorContribution[];
+    interactions: InteractionContribution[];
+    /** Residual = variance unexplained by main effects + modeled interactions. */
+    residualShare: number;
+    /** Sanity: shares sum to 1 (within fp). */
+    sharesSum: number;
+}
+declare function causalAttribution(cells: FactorialCell[]): CausalAttributionReport;
+
+/**
+ * Active learning — agent-as-scenario-author.
+ *
+ * Analyzes an existing Dataset + trace corpus for coverage gaps and
+ * weak spots, returns a prioritized list of *synthesis targets*:
+ * (gap description, existing-neighbor examples, suggested direction).
+ *
+ * Does NOT call an LLM itself — the proposer agent is caller-supplied.
+ * This module's job is to identify WHERE new scenarios would compound
+ * the most information, not to author them.
+ *
+ * Gaps we detect:
+ *   - dimensions with high score variance (unstable, need more data)
+ *   - dimensions with low coverage count (undersampled)
+ *   - failure classes with clusters (systematic weakness)
+ *   - difficulty bins with no coverage
+ */
+
+type SynthesisReason = 'high-variance' | 'undersampled' | 'failure-cluster' | 'difficulty-gap';
+interface SynthesisTarget {
+    reason: SynthesisReason;
+    description: string;
+    /** Existing scenarios that are closest to the gap; caller feeds these to
+     *  their LLM proposer as few-shot examples. */
+    neighbors: DatasetScenario[];
+    /** Suggested direction — e.g. "harder variants", "edge cases of X", "failure class Y". */
+    direction: string;
+    /** Priority score — higher = more information-dense gap. 0..1. */
+    priority: number;
+}
+interface ActiveLearningOptions {
+    /** Minimum scenarios per difficulty band to count as "covered". */
+    minPerBand?: number;
+    /** Variance threshold above which a scenario's dimension is "unstable". */
+    varianceThreshold?: number;
+    /** Max synthesis targets returned. */
+    topK?: number;
+}
+declare function proposeSynthesisTargets(dataset: Dataset, traceStore: TraceStore, options?: ActiveLearningOptions): Promise<SynthesisTarget[]>;
+
+/**
+ * Reward-model export — the productizable wrapper around PRM training
+ * data. Takes a TraceStore + PrmGrader, produces an embeddable
+ * inference scorer that customers plug into their own agent stack.
+ *
+ * Two export forms:
+ *   - `exportRewardModel(store, graders)` — serializes the (step-context,
+ *     score) corpus to a framework-agnostic payload. Customer fine-tunes
+ *     their own model; we ship the scaffolding.
+ *   - `loadScorerFromTraces(store, grader)` — a zero-deps "reward model"
+ *     that literally replays the trained rubric at inference time. Works
+ *     as a reference baseline + deterministic fallback.
+ */
+
+interface ExportedRewardModel {
+    /** Version of the export format. Bump when payload shape changes. */
+    version: '1.0';
+    /** Metadata about the training corpus. */
+    metadata: {
+        nTraces: number;
+        nSamples: number;
+        rubrics: string[];
+        exportedAt: string;
+        /** Mean reward across training corpus — use as sanity check at load. */
+        meanReward: number;
+    };
+    /** NDJSON training payload suitable for most fine-tuning frameworks. */
+    trainingNdjson: string;
+}
+declare function exportRewardModel(store: TraceStore, grader: PrmGrader, runIds: string[]): Promise<ExportedRewardModel>;
+/**
+ * Zero-deps inference scorer — apply a grader to a trajectory and return
+ * its aggregate score. This is the "reward model" customers embed when
+ * they don't want (or can't) fine-tune one. Deterministic + portable.
+ */
+interface InferenceScorer {
+    /** Score a completed trajectory. Higher is better. */
+    score(trajectory: Trajectory, store: TraceStore): Promise<number>;
+    metadata: {
+        rubrics: string[];
+        deterministic: true;
+    };
+}
+declare function loadScorerFromGrader(grader: PrmGrader): InferenceScorer;
+/**
+ * Replay a trace corpus through a scorer — produces the canonical
+ * "what would this reward model have said about every run?" table.
+ * Callers use this to validate a trained model against the training
+ * corpus (expect high agreement; drift indicates overfitting).
+ */
+declare function replayScorerOverCorpus(store: TraceStore, scorer: InferenceScorer, runIds: string[]): Promise<Array<{
+    runId: string;
+    score: number;
+    outcomeScore: number | null;
+}>>;
+
+/**
+ * Governance reporting — shared types.
+ *
+ * The framework collects a `GovernanceContext` (traces + outcomes +
+ * dataset manifests + red-team results + judge calibration) and each
+ * specific template (NIST AI RMF, SOC2, EU AI Act) renders a
+ * structured report from it.
+ *
+ * Reports are machine-readable JSON first; human-readable Markdown is a
+ * pure transform on top. External auditors consume the Markdown; CI
+ * consumes the JSON.
+ */
+
+interface GovernanceContext {
+    /** Legal / org identity for the report. */
+    organization: string;
+    /** System / agent identifier. */
+    systemName: string;
+    /** ISO8601 period the report covers. */
+    periodStart: string;
+    periodEnd: string;
+    /** Versioned dataset manifests used during the period. */
+    datasets: DatasetManifest[];
+    traceStore: TraceStore;
+    outcomeStore?: OutcomeStore;
+    /** Cached red-team results for the period, if available. */
+    redTeam?: RedTeamReport;
+    /** Judge-vs-human calibration results, if measured. */
+    judgeCalibration?: CalibrationResult[];
+    /** Responsible owner for the system — role + name + email. */
+    owner: {
+        role: string;
+        name: string;
+        email: string;
+    };
+}
+interface GovernanceFinding {
+    id: string;
+    severity: 'info' | 'low' | 'medium' | 'high' | 'critical';
+    /** Control reference the finding maps to (e.g. "NIST-AI-RMF:MEASURE-2.1"). */
+    control: string;
+    summary: string;
+    evidence?: string;
+    remediation?: string;
+}
+interface GovernanceReport {
+    framework: 'NIST-AI-RMF' | 'SOC2' | 'EU-AI-ACT';
+    version: string;
+    context: Pick<GovernanceContext, 'organization' | 'systemName' | 'periodStart' | 'periodEnd' | 'owner'>;
+    summary: {
+        findings: number;
+        byeverity: Record<GovernanceFinding['severity'], number>;
+        overall: 'compliant' | 'compliant-with-findings' | 'non-compliant';
+    };
+    findings: GovernanceFinding[];
+    /** Framework-specific structured payload (mapped controls, risk class, etc.). */
+    payload: Record<string, unknown>;
+    generatedAt: string;
+}
+declare function renderMarkdown(report: GovernanceReport): string;
+declare function summarize(findings: GovernanceFinding[]): GovernanceReport['summary'];
+
+/**
+ * NIST AI RMF 1.0 — Govern / Map / Measure / Manage mapping.
+ *
+ * Each subcategory derives its status from concrete framework state:
+ *   MEASURE 2.x: do we have a calibration regime? contamination controls?
+ *   MEASURE 2.7: are red-team results available?
+ *   MANAGE 1.x: are outcome metrics captured? correlation measured?
+ *   GOVERN 1.x: dataset + prompt provenance recorded?
+ *
+ * We ship the mapping and the derivation rules; consumers supply the
+ * GovernanceContext.
+ */
+
+declare function nistAiRmfReport(ctx: GovernanceContext): Promise<GovernanceReport>;
+
+/**
+ * SOC 2 — Common Criteria 7 (system operations + change management)
+ * audit trail derived from the trace corpus.
+ *
+ * This is NOT a formal SOC2 report — that requires an external
+ * auditor. What we ship is the machine-readable *evidence* package
+ * that an auditor consumes: run counts, deploy events, access log
+ * summary, anomaly tracking, response-time SLOs.
+ */
+
+declare function soc2Report(ctx: GovernanceContext): Promise<GovernanceReport>;
+
+/**
+ * EU AI Act — risk-class classification + compliance checklist.
+ *
+ * Classification is declarative: caller supplies the domain/use-case
+ * signals (biometric? critical infrastructure? education? employment?
+ * access to services?) and we map to the Act's risk tiers:
+ *   - "unacceptable" (prohibited)
+ *   - "high"        (Annex III — strict obligations)
+ *   - "limited"     (transparency obligations)
+ *   - "minimal"     (voluntary codes of conduct)
+ *
+ * Then the compliance checklist enumerates Article 9 (risk mgmt),
+ * 10 (data + data governance), 11 (technical documentation), 13
+ * (transparency), 14 (human oversight), 15 (accuracy + robustness)
+ * requirements and flags gaps.
+ */
+
+type EuRiskClass = 'unacceptable' | 'high' | 'limited' | 'minimal';
+interface UseCaseSignals {
+    /** Used for biometric identification in public spaces? (Art. 5 — unacceptable). */
+    biometricPublic?: boolean;
+    /** Social scoring by public authorities? (Art. 5). */
+    socialScoring?: boolean;
+    /** Subliminal manipulation? (Art. 5). */
+    subliminal?: boolean;
+    /** Annex III sector: critical infrastructure / education / employment /
+     *  access to essential services / law enforcement / migration /
+     *  administration of justice / democratic processes? */
+    annexIII?: boolean;
+    /** Interacts directly with natural persons (chatbot, agent)? — limited risk. */
+    chatbot?: boolean;
+    /** Generates synthetic media (image/audio/video/text deepfakes)? — limited risk. */
+    generatesSyntheticMedia?: boolean;
+}
+declare function classifyEuAiRisk(signals: UseCaseSignals): EuRiskClass;
+declare function euAiActReport(ctx: GovernanceContext, signals: UseCaseSignals): Promise<GovernanceReport>;
+
+export { type ActiveLearningOptions, AgentDriver, type AgentDriverConfig, type AlignmentOp, type AntiSlopConfig, type AntiSlopIssue, type AntiSlopReport, type Artifact, type ArtifactCheck, type Artifact$1 as ArtifactCheckArtifact, type ArtifactResult, type ArtifactValidator, AxGepaSteeringOptimizer, type AxSteeringOptimizerConfig, type BaselineOptions, type BaselineReport, BehaviorAssertion, type BenchmarkReport, BenchmarkRunner, type BenchmarkRunnerConfig, type BestOfNResult, type BisectOptions, type BisectResult, type BisectStep, BudgetBreachError, type BudgetBreachFinding, type BudgetBreachReport, BudgetGuard, type BudgetLedgerEntry, type BudgetSpec, BuilderSession, type BuilderSessionInit, type CalibrationBin, type CalibrationOptions, type CalibrationReport, type CalibrationResult, CallExpectation, type CanaryLeak, type CandidateScenario, type CandidateScore, type CausalAttributionReport, type ChatSummary, type CheckResult, type CollectedArtifacts, type CompletionCriterion, type ContinuityCheck, type ContinuityCheckResult, type ContinuityReport, type ContinuitySnapshotPair, type ContractMetric, type ContractReport, ConvergenceTracker, type CorrelationReport, type CorrelationResult, type CorrelationStudyOptions, type CorrelationStudyResult, type CostEntry, type CostSummary, CostTracker, type CounterfactualContext, type CounterfactualMutation, type CounterfactualResult, type CounterfactualRunner, type CrossTraceDiff, type CrossTraceDiffOptions, DEFAULT_AGENT_SLOS, DEFAULT_RULES as DEFAULT_FAILURE_RULES, DEFAULT_MUTATORS, DEFAULT_REDACTION_RULES, DEFAULT_RED_TEAM_CORPUS, DEFAULT_RUN_SCORE_WEIGHTS, Dataset, type DatasetDifficulty, type DatasetManifest, type DatasetProvenance, type DatasetScenario, type DatasetSplit, type DeploymentOutcome, type Direction, type DivergenceOptions, type DivergenceReport, DockerSandboxDriver, type DriverResult, type DriverState, DualAgentBench, type DualAgentBenchConfig, type DualAgentReport, type DualAgentRound, type DualAgentScenario, type DualAgentScenarioResult, type EuRiskClass, type EvalMetricSpec, type EvalResult, type EventFilter, type EventKind, type EvolutionRound, type ExecutorConfig, type Expectation, type Experiment, type Run$1 as ExperimentRun, type ExperimentStore, ExperimentTracker, type ExportedRewardModel, FAILURE_CLASSES, type FactorContribution, type FactorialCell, type FailureClass, type FailureClassification, type FailureCluster, type FailureClusterReport, type FailureContext, type FailureRule, type FeedbackPattern, FileSystemOutcomeStore, type FileSystemOutcomeStoreOptions, FileSystemTraceStore, type FileSystemTraceStoreOptions, type GenericSpan, type GoldenItem, type GovernanceContext, type GovernanceFinding, type GovernanceReport, type GradedStep, type HarnessConfig, HoldoutAuditor, HoldoutLockedError, type HostedJudgeConfig, type HostedJudgeDimension, type HostedJudgeRequest, type HostedJudgeResponse, type HostedRunCriticConfig, type HostedRunScoreRequest, type HostedRunScoreResponse, type HypothesisManifest, type HypothesisResult, type ImageData, InMemoryExperimentStore, InMemoryOutcomeStore, InMemoryTraceStore, InMemoryWorkspaceInspector, type InferenceScorer, type InspectorContext, type InteractionContribution, type JudgeAgreementReport, type JudgeConfig, type JudgeFleetOptions, type JudgeFn, type JudgeInput, type JudgePair, type JudgeReplayResult, type JudgeRubric, JudgeRunner, type JudgeScore, type JudgeSpan, type LangfuseEnvelope, type LangfuseGeneration, type LangfuseScore, type LayerCorrelation, type LlmJsonCall, type LlmReviewerConfig, type LlmSpan, MODEL_PRICING, type MatcherResult, type Message, type MetricSamples, type MetricVerdict, MetricsCollector, type Mutator, OTEL_AGENT_EVAL_SCOPE, type Objective, type OptimizationConfig, type OptimizationExample, OptimizationLoop, type OptimizationLoopConfig, type OptimizationLoopResult, type OptimizationResult, type Oracle, type OracleObservation, type OracleReport, type OracleResult, type OtlpExport, type OtlpResourceSpans, type OtlpSpan, type OutcomeFilter, type OutcomePair, type OutcomeStore, type PairwiseComparison, PairwiseSteeringOptimizer, type ParetoResult, type PersonaConfig, type Playbook, type PlaybookEntry, type PositionalBiasResult, type PrmGradedTrace, PrmGrader, type PrmTrainingSample, ProductClient, type ProductClientConfig, type ProjectKind, ProjectRegistry, type ProjectSummary, type ProjectTimelineEntry, type PromptHandle, PromptOptimizer, PromptRegistry, type PromptVariant, type ProposeFn, type ProposeInput, type ProposeOutput, type ProposeReviewConfig, type ProposeReviewReport, type ProposeReviewShot, REDACTION_VERSION, type RedTeamCase, type RedTeamCategory, type RedTeamFinding, type RedTeamPayload, type RedTeamReport, type RedactionReport, type RedactionRule, type RegressionOptions, type RegressionSpec, type RetrievalSpan, type Review, type ReviewFn, type ReviewInput, type ReviewMemoryEntry, type ReviewMemoryStore, type RobustnessResult, type RouteMap, type RubricDimension, type Run, type RunAppScenarioOptions, type RunConfig, RunCritic, type RunCriticOptions, type RunDiff, type RunFilter, type RunLayer, type RunOutcome, type RunScore, type RunScoreWeights, type RunStatus, type RunTrace, type SandboxDriver, SandboxHarness, type SandboxHarnessResult, type SandboxJudgeKind, type SandboxJudgeResult, type SandboxJudgeSpec, type SandboxResult, type SandboxSpan, type Scenario, type ScenarioCost, type ScenarioFile, ScenarioRegistry, type ScenarioResult, type ScoredTarget, type SelfPlayOptions, type SelfPlayProposer, type SelfPlayScorer, type SelfPreferenceResult, type SeriesConvergenceOptions, type SeriesConvergenceResult, type ShipOptions, type SignedManifest, type SliceOptions, type Slo, type SloCheckResult, type SloComparator, type SloReport, type SloSeverity, type SlopCategory, type Span, type SpanBase, type SpanFilter, type SpanHandle, type SpanKind, type SpanStatus, type SteeringBundle, type SteeringDelta, type SteeringEvaluation, type SteeringOptimizationResult, type SteeringOptimizationRow, type SteeringOptimizationSelector, type SteeringOptimizerBackend, type SteeringOptimizerConfig, type SteeringRolePrompt, type SteeringVariantReport, type StepAttribution, type StepContext, type StepRubric, type StuckLoopFinding, type StuckLoopOptions, type StuckLoopReport, SubprocessSandboxDriver, type SynthesisReason, type SynthesisTarget, TRACE_SCHEMA_VERSION, type TestGradedRunOptions, type TestGradedRunResult, type TestGradedScenario, type TestOutputParser, type TestResult, type ThreeLayerProjectReport, type ThresholdContract, TokenCounter, type TokenSpec, type ToolSpan, type ToolStats, type ToolUseMetrics, type ToolUseOptions, type ToolWasteFinding, type ToolWasteOptions, type ToolWasteReport, TraceEmitter, type TraceEmitterOptions, type TraceEvent, type TraceStore, type Trajectory, type TrajectoryStep, type Turn, type TurnMetrics, type TurnResult, type UseCaseSignals, type ValidationContext, type ValidationIssue, type ValidationResult, type VariantScore, type VerbosityBiasResult, type Verification, type VerifyFn, type VisualDiffOptions, type VisualDiffResult, type WorkspaceAssertion, type WorkspaceAssertionResult, type WorkspaceInspector, type WorkspaceSnapshot, adversarialJudge, aggregateLlm, aggregateRunScore, analyzeAntiSlop, analyzeSeries, argHash, attributeCounterfactuals, benjaminiHochberg, bisect, bonferroni, budgetBreachView, buildTrajectory, byteLengthRange, calibrateJudge, calibrationCurve, canaryLeakView, causalAttribution, checkCanaries, checkSlos, clamp01, classifyEuAiRisk, classifyFailure, codeExecutionJudge, cohensD, coherenceJudge, collectionPreserved, commitBisect, compareToBaseline, compilerJudge, composeParsers, composeValidators, computeToolUseMetrics, confidenceInterval, containsAll, correlateLayers, correlationStudy, createAntiSlopJudge, createCustomJudge, createDomainExpertJudge, createLlmReviewer, crossTraceDiff, defaultJudges, distillPlaybook, dominates, estimateCost, estimateTokens, euAiActReport, evaluateContract, evaluateHypothesis, evaluateOracles, executeScenario, expectAgent, exportRewardModel, exportRunAsOtlp, exportTrainingData, failureClusterView, fileContains, fileExists, firstDivergenceView, formatBenchmarkReport, formatDriverReport, groupBy, hashContent, hashScenarios, inMemoryReviewStore, interRaterReliability, iqr, isJudgeSpan, isLlmSpan, isPrmVerdict, isRetrievalSpan, isSandboxSpan, isToolSpan, jestTestParser, jsonHasKeys, jsonShape, jsonlReviewStore, judgeAgreementView, judgeSpans, keyPreserved, linterJudge, llmSpanFromProvider, llmSpans, loadScorerFromGrader, lowercaseMutator, mannWhitneyU, mergeSteeringBundle, nistAiRmfReport, nonRefusalRubric, normalizeScores, notBlocked, outputLengthRubric, pairedTTest, paraphraseRobustness, paretoFrontier, partialCredit, pixelDeltaRatio, politenessPrefixMutator, positionalBias, printDriverSummary, prmBestOfN, prmEnsembleBestOfN, promptBisect, proposeSynthesisTargets, pytestTestParser, redTeamDataset, redTeamReport, redactString, redactValue, regexMatch, regexMatches, regressionView, renderMarkdown, renderMarkdownReport, renderPlaybookMarkdown, renderSteeringText, replayScorerOverCorpus, replayTraceThroughJudge, requiredSampleSize, resumeBuilderSession, rowCount, rowWhere, runAssertions, runCounterfactual, runE2EWorkflow, runExpectations, runFailureClass, runJudgeFleet, runProposeReview, runSelfPlay, runTestGradedScenario, runsForScenario, scoreAllProjects, scoreContinuity, scoreProject, scoreRedTeamOutput, securityJudge, selfPreference, sentenceReorderMutator, signManifest, soc2Report, statusAdvanced, stuckLoopView, summarize, testJudge, textInSnapshot, toLangfuseEnvelope, toNdjson, toPrometheusText, toolIntentAlignmentRubric, toolNamesForRun, toolNonRedundantRubric, toolSpans, toolSuccessRubric, toolWasteView, typoMutator, urlContains, verbosityBias, verifyManifest, visualDiff, vitestTestParser, weightedMean, welchsTTest, whitespaceCollapseMutator, wilcoxonSignedRank };