@tangle-network/agent-eval 0.2.0 → 0.6.0

package/dist/index.d.ts

@@ -435,6 +435,83 @@ declare class MetricsCollector {
     getConvergenceCurve(): number[];
 }
 
+/**
+ * ScenarioRegistry — manages scenario discovery and filtering.
+ *
+ * Each agent registers its scenarios. The registry handles conversion
+ * from ScenarioFile format to the framework's Scenario type.
+ */
+declare class ScenarioRegistry {
+    private scenarios;
+    private scenarioFiles;
+    /** Register scenarios from ScenarioFile format */
+    registerFiles(files: ScenarioFile[]): void;
+    /** Register pre-built Scenario objects directly */
+    register(scenarios: Scenario[]): void;
+    /** Get all scenarios */
+    all(): Scenario[];
+    /** Get scenarios filtered by category */
+    byCategory(category: string): Scenario[];
+    /** List all categories with counts */
+    listCategories(): {
+        category: string;
+        count: number;
+    }[];
+    /** Get scenarios filtered by persona */
+    byPersona(persona: string): Scenario[];
+    /** Get a single scenario by ID */
+    byId(id: string): Scenario | undefined;
+    /** Count total scenarios */
+    get count(): number;
+}
+
+interface AgentDriverConfig {
+    client: ProductClient;
+    driverModel?: string;
+    /** System prompt context for the driver LLM to understand the product */
+    productContext?: string;
+}
+/**
+ * AgentDriver — meta-agent that plays a persona against the real product.
+ *
+ * Uses a driver LLM (Claude/GPT-4o) to decide what to say each turn.
+ * Not scripted — the driver gets the current product state and decides
+ * the next realistic user message.
+ */
+declare class AgentDriver {
+    private tc;
+    private client;
+    private driverModel;
+    private productContext;
+    constructor(tc: TCloud, config: AgentDriverConfig);
+    /**
+     * Run a persona through the product.
+     *
+     * Returns metrics on how many turns to completion, cost curve,
+     * quality curve, and convergence curve.
+     */
+    run(persona: PersonaConfig): Promise<DriverResult>;
+    /** Use the driver LLM to decide what the "user" says next */
+    private decideNextMessage;
+    /** Handle pending approvals based on persona feedback patterns */
+    private handleApprovals;
+    /** Describe which completion criteria are met */
+    private describeCompletion;
+}
+
+/**
+ * Report generation utilities.
+ *
+ * Outputs convergence curves, cost curves, quality curves,
+ * and per-persona summaries in markdown format.
+ */
+/** Generate a markdown report from benchmark results */
+declare function formatBenchmarkReport(report: BenchmarkReport): string;
+/** Generate a markdown report from agent driver results */
+declare function formatDriverReport(results: DriverResult[]): string;
+/** Print a compact summary to console */
+declare function printDriverSummary(results: DriverResult[]): void;
+
 /**
  * Normalize scores so all dimensions follow "higher = better".
  * Inverted dimensions (hallucination, false_confidence, worst_failure)
@@ -524,83 +601,6 @@ declare class ConvergenceTracker {
     getTurnToCompletion(): number | null;
 }
 
-/**
- * ScenarioRegistry — manages scenario discovery and filtering.
- *
- * Each agent registers its scenarios. The registry handles conversion
- * from ScenarioFile format to the framework's Scenario type.
- */
-declare class ScenarioRegistry {
-    private scenarios;
-    private scenarioFiles;
-    /** Register scenarios from ScenarioFile format */
-    registerFiles(files: ScenarioFile[]): void;
-    /** Register pre-built Scenario objects directly */
-    register(scenarios: Scenario[]): void;
-    /** Get all scenarios */
-    all(): Scenario[];
-    /** Get scenarios filtered by category */
-    byCategory(category: string): Scenario[];
-    /** List all categories with counts */
-    listCategories(): {
-        category: string;
-        count: number;
-    }[];
-    /** Get scenarios filtered by persona */
-    byPersona(persona: string): Scenario[];
-    /** Get a single scenario by ID */
-    byId(id: string): Scenario | undefined;
-    /** Count total scenarios */
-    get count(): number;
-}
-
-interface AgentDriverConfig {
-    client: ProductClient;
-    driverModel?: string;
-    /** System prompt context for the driver LLM to understand the product */
-    productContext?: string;
-}
-/**
- * AgentDriver — meta-agent that plays a persona against the real product.
- *
- * Uses a driver LLM (Claude/GPT-4o) to decide what to say each turn.
- * Not scripted — the driver gets the current product state and decides
- * the next realistic user message.
- */
-declare class AgentDriver {
-    private tc;
-    private client;
-    private driverModel;
-    private productContext;
-    constructor(tc: TCloud, config: AgentDriverConfig);
-    /**
-     * Run a persona through the product.
-     *
-     * Returns metrics on how many turns to completion, cost curve,
-     * quality curve, and convergence curve.
-     */
-    run(persona: PersonaConfig): Promise<DriverResult>;
-    /** Use the driver LLM to decide what the "user" says next */
-    private decideNextMessage;
-    /** Handle pending approvals based on persona feedback patterns */
-    private handleApprovals;
-    /** Describe which completion criteria are met */
-    private describeCompletion;
-}
-
-/**
- * Report generation utilities.
- *
- * Outputs convergence curves, cost curves, quality curves,
- * and per-persona summaries in markdown format.
- */
-/** Generate a markdown report from benchmark results */
-declare function formatBenchmarkReport(report: BenchmarkReport): string;
-/** Generate a markdown report from agent driver results */
-declare function formatDriverReport(results: DriverResult[]): string;
-/** Print a compact summary to console */
-declare function printDriverSummary(results: DriverResult[]): void;
-
 /**
  * Versioned prompt registry.
  *
@@ -642,79 +642,6 @@ declare class PromptRegistry {
 /** SHA-256(content) → first 12 hex chars. Stable across runtimes. */
 declare function hashContent(content: string): Promise<string>;
 
-/**
- * LLM trace store — one record per model call.
- *
- * Sink for the full eval data-plane: what got sent, what came back, what it
- * cost, how long it took. Replayable, queryable, diff-able.
- *
- * Two built-in stores:
- *   - `MemoryTraceStore` — fast, ephemeral, useful in tests and short runs
- *   - `FileSystemTraceStore` — NDJSON files per-run, grepable, committable
- *
- * Consumers plug in custom stores for Langfuse / OTEL / D1 / Postgres.
- */
-interface LlmTrace {
-    id: string;
-    runId: string;
-    scenarioId?: string;
-    turnIndex?: number;
-    role: 'driver' | 'judge' | 'product' | 'optimizer' | string;
-    model: string;
-    prompt: string;
-    output: string;
-    inputTokens?: number;
-    outputTokens?: number;
-    costUsd?: number;
-    durationMs?: number;
-    timestamp: string;
-    metadata?: Record<string, unknown>;
-}
-interface TraceQuery {
-    runId?: string;
-    scenarioId?: string;
-    role?: string;
-    model?: string;
-    sinceMs?: number;
-    limit?: number;
-}
-interface TraceStore {
-    record(trace: LlmTrace): Promise<void>;
-    query(query: TraceQuery): Promise<LlmTrace[]>;
-    count(query?: TraceQuery): Promise<number>;
-}
-declare class MemoryTraceStore implements TraceStore {
-    private traces;
-    record(trace: LlmTrace): Promise<void>;
-    query(query: TraceQuery): Promise<LlmTrace[]>;
-    count(query?: TraceQuery): Promise<number>;
-    /** Clear the store — test helper. */
-    reset(): void;
-    private filter;
-}
-interface FileSystemTraceStoreOptions {
-    dir: string;
-    /** Max file size before rolling to a new segment (default 32 MB). */
-    rolloverBytes?: number;
-    /** Function to write the file — defaults to node:fs/promises.appendFile */
-    append?: (path: string, data: string) => Promise<void>;
-    read?: (path: string) => Promise<string>;
-    list?: (dir: string) => Promise<string[]>;
-    stat?: (path: string) => Promise<{
-        size: number;
-    }>;
-    mkdir?: (dir: string) => Promise<void>;
-}
-declare class FileSystemTraceStore implements TraceStore {
-    private readonly opts;
-    constructor(opts: FileSystemTraceStoreOptions);
-    record(trace: LlmTrace): Promise<void>;
-    query(query: TraceQuery): Promise<LlmTrace[]>;
-    count(query?: TraceQuery): Promise<number>;
-    private segments;
-    private currentSegment;
-}
-
 /**
  * Anti-slop quality judge.
  *
@@ -787,7 +714,7 @@ declare function analyzeAntiSlop(outputs: string[], config: Omit<Required<AntiSl
  * returns a `ValidationResult` with pass/fail + 0..1 score + structured
  * issues.
  */
-interface Artifact {
+interface Artifact$1 {
     /** Logical kind — validators type-guard on this */
     kind: 'file' | 'json' | 'text' | 'binary' | string;
     /** Filesystem-style path, optional */
@@ -803,7 +730,7 @@ interface ValidationContext {
     scenarioId: string;
     turnIndex?: number;
     /** Prior artifacts for multi-artifact scenarios */
-    priorArtifacts?: Artifact[];
+    priorArtifacts?: Artifact$1[];
     /** Free-form hints the validator uses for domain-specific checks */
     hints?: Record<string, unknown>;
 }
@@ -827,7 +754,7 @@ interface ArtifactValidator {
     /** Optional description for human-facing reports. */
     description?: string;
     /** Called once per artifact; validators are expected to be pure + idempotent. */
-    validate(artifact: Artifact, context: ValidationContext): Promise<ValidationResult>;
+    validate(artifact: Artifact$1, context: ValidationContext): Promise<ValidationResult>;
 }
 /**
  * Run every validator on the same artifact; aggregate pass as AND, score as
@@ -938,7 +865,7 @@ interface RunConfig {
     seed?: number;
     metadata?: Record<string, unknown>;
 }
-interface Run {
+interface Run$1 {
     id: string;
     experimentId: string;
     name?: string;
@@ -959,9 +886,9 @@ interface ExperimentStore {
     saveExperiment(exp: Experiment): Promise<void>;
     getExperiment(id: string): Promise<Experiment | null>;
     listExperiments(): Promise<Experiment[]>;
-    saveRun(run: Run): Promise<void>;
-    getRun(id: string): Promise<Run | null>;
-    listRuns(experimentId: string): Promise<Run[]>;
+    saveRun(run: Run$1): Promise<void>;
+    getRun(id: string): Promise<Run$1 | null>;
+    listRuns(experimentId: string): Promise<Run$1[]>;
 }
 declare class InMemoryExperimentStore implements ExperimentStore {
     private readonly experiments;
@@ -969,15 +896,15 @@ declare class InMemoryExperimentStore implements ExperimentStore {
     saveExperiment(exp: Experiment): Promise<void>;
     getExperiment(id: string): Promise<Experiment | null>;
     listExperiments(): Promise<Experiment[]>;
-    saveRun(run: Run): Promise<void>;
-    getRun(id: string): Promise<Run | null>;
-    listRuns(experimentId: string): Promise<Run[]>;
+    saveRun(run: Run$1): Promise<void>;
+    getRun(id: string): Promise<Run$1 | null>;
+    listRuns(experimentId: string): Promise<Run$1[]>;
 }
 declare class ExperimentTracker {
     private readonly store;
     constructor(store: ExperimentStore);
     startExperiment(name: string, metadata?: Record<string, unknown>): Promise<Experiment>;
-    startRun(config: RunConfig): Promise<Run>;
+    startRun(config: RunConfig): Promise<Run$1>;
     completeRun(runId: string, report: BenchmarkReport): Promise<void>;
     failRun(runId: string, error: string): Promise<void>;
     /**
@@ -1080,6 +1007,9 @@ interface PairwiseComparison {
     variantA: string;
     variantB: string;
     pValue: number;
+    /** BH-FDR-corrected q-value across all n*(n-1)/2 pairwise tests. */
+    qValue: number;
+    /** True when q-value passes the FDR threshold. Prefer over raw p-value when variants > 2. */
     significant: boolean;
     meanDelta: number;
 }
@@ -1184,4 +1114,3022 @@ declare class DualAgentBench {
     run(config: DualAgentBenchConfig): Promise<DualAgentReport>;
 }
 
-export { AgentDriver, type AgentDriverConfig, type AntiSlopConfig, type AntiSlopIssue, type AntiSlopReport, type Artifact, type ArtifactCheck, type ArtifactResult, type ArtifactValidator, type BenchmarkReport, BenchmarkRunner, type BenchmarkRunnerConfig, type CheckResult, type CollectedArtifacts, type CompletionCriterion, ConvergenceTracker, type DriverResult, type DriverState, DualAgentBench, type DualAgentBenchConfig, type DualAgentReport, type DualAgentRound, type DualAgentScenario, type DualAgentScenarioResult, type EvalResult, type ExecutorConfig, type Experiment, type ExperimentStore, ExperimentTracker, type FeedbackPattern, FileSystemTraceStore, type FileSystemTraceStoreOptions, InMemoryExperimentStore, InMemoryWorkspaceInspector, type InspectorContext, type JudgeConfig, type JudgeFn, type JudgeInput, type JudgeRubric, type JudgeScore, type LlmTrace, MODEL_PRICING, MemoryTraceStore, MetricsCollector, type OptimizationConfig, type OptimizationResult, type PairwiseComparison, type PersonaConfig, ProductClient, type ProductClientConfig, type PromptHandle, PromptOptimizer, PromptRegistry, type PromptVariant, type RouteMap, type RubricDimension, type Run, type RunConfig, type RunDiff, type Scenario, type ScenarioFile, ScenarioRegistry, type ScenarioResult, type SlopCategory, type TestResult, TokenCounter, type TraceQuery, type TraceStore, type Turn, type TurnMetrics, type TurnResult, type ValidationContext, type ValidationIssue, type ValidationResult, type VariantScore, type WorkspaceAssertion, type WorkspaceAssertionResult, type WorkspaceInspector, type WorkspaceSnapshot, adversarialJudge, analyzeAntiSlop, byteLengthRange, codeExecutionJudge, cohensD, coherenceJudge, composeValidators, confidenceInterval, containsAll, createAntiSlopJudge, createCustomJudge, createDomainExpertJudge, defaultJudges, estimateCost, estimateTokens, executeScenario, fileContains, fileExists, formatBenchmarkReport, formatDriverReport, hashContent, interRaterReliability, jsonHasKeys, mannWhitneyU, normalizeScores, pairedTTest, partialCredit, printDriverSummary, regexMatch, rowCount, rowWhere, runAssertions, runE2EWorkflow, weightedMean, wilcoxonSignedRank };
+/**
+ * TraceSchema v1 — the canonical data model for agent-eval.
+ *
+ * Every score, every failure class, every pipeline in the framework is
+ * a view over this data. Shape it once, live with it.
+ *
+ * Wire-compatible with OpenTelemetry span semantics (see trace/otel.ts)
+ * but extended with agent-specific span kinds (llm, tool, retrieval,
+ * judge, sandbox) and first-class BudgetLedger / Artifact / JudgeVerdict
+ * entities that OTEL leaves as free-form attributes.
+ */
+declare const TRACE_SCHEMA_VERSION = "1.0.0";
+type RunStatus = 'running' | 'completed' | 'failed' | 'aborted';
+interface BudgetSpec {
+    tokens?: number;
+    wallMs?: number;
+    calls?: number;
+    usd?: number;
+}
+interface RunOutcome {
+    score?: number;
+    pass?: boolean;
+    failureClass?: FailureClass;
+    notes?: string;
+}
+/**
+ * Layer — optional classification in a nested build workflow.
+ * `builder`: the meta-agent editing a project (e.g. agent-builder Forge chat).
+ * `app-build`: sandbox harness that compiled + tested the generated scaffold.
+ * `app-runtime`: a run of the generated agent against a domain scenario.
+ * `meta`: any meta-eval (judge replay, correlation analysis).
+ */
+type RunLayer = 'builder' | 'app-build' | 'app-runtime' | 'meta' | 'custom';
+interface Run {
+    runId: string;
+    scenarioId: string;
+    variantId?: string;
+    datasetVersion?: string;
+    /** Git SHA of agent code at run time. */
+    codeSha?: string;
+    /** Hash of the prompt template + any system prompt. */
+    promptSha?: string;
+    /** Model id + date + system-prompt hash, concatenated. */
+    modelFingerprint?: string;
+    seed?: number;
+    /** Arbitrary environment markers (shell, docker version, tz). */
+    envFingerprint?: Record<string, string>;
+    /** Version of the redaction rules applied to this run. */
+    redactionVersion?: string;
+    /** Parent run in a nested build workflow. A builder run's children are
+     *  app-build runs; those children are app-runtime runs. */
+    parentRunId?: string;
+    /** Stable project identifier — groups runs across chats + sessions. */
+    projectId?: string;
+    /** Chat/conversation identifier within a project. */
+    chatId?: string;
+    /** Layer classification — hint for aggregation; not enforced. */
+    layer?: RunLayer;
+    startedAt: number;
+    endedAt?: number;
+    status: RunStatus;
+    outcome?: RunOutcome;
+    budget?: BudgetSpec;
+    /** Free-form labels for downstream grouping. */
+    tags?: Record<string, string>;
+}
+type SpanKind = 'agent' | 'llm' | 'tool' | 'retrieval' | 'judge' | 'sandbox' | 'custom';
+type SpanStatus = 'ok' | 'error';
+interface SpanBase {
+    spanId: string;
+    parentSpanId?: string;
+    runId: string;
+    kind: SpanKind;
+    name: string;
+    startedAt: number;
+    endedAt?: number;
+    status?: SpanStatus;
+    error?: string;
+    /** Anything not covered by typed fields. Kept deliberately free-form. */
+    attributes?: Record<string, unknown>;
+}
+interface Message {
+    role: 'system' | 'user' | 'assistant' | 'tool';
+    content: string;
+    tokens?: number;
+    /** Multi-modal content descriptors; blobs themselves live in Artifacts. */
+    images?: Array<{
+        artifactId?: string;
+        url?: string;
+        mime?: string;
+    }>;
+}
+interface LlmSpan extends SpanBase {
+    kind: 'llm';
+    model: string;
+    messages: Message[];
+    output?: string;
+    inputTokens?: number;
+    outputTokens?: number;
+    cachedTokens?: number;
+    reasoningTokens?: number;
+    costUsd?: number;
+    finishReason?: string;
+}
+interface ToolSpan extends SpanBase {
+    kind: 'tool';
+    toolName: string;
+    args: unknown;
+    result?: unknown;
+    latencyMs?: number;
+}
+interface RetrievalSpan extends SpanBase {
+    kind: 'retrieval';
+    query: string;
+    hits: Array<{
+        docId: string;
+        score: number;
+        content?: string;
+    }>;
+}
+interface JudgeSpan extends SpanBase {
+    kind: 'judge';
+    judgeId: string;
+    /** Span this judgment applies to. */
+    targetSpanId: string;
+    dimension: string;
+    /** Numeric score (free-range; interpretation up to the judge). */
+    score: number;
+    rationale?: string;
+    evidence?: string;
+}
+interface SandboxSpan extends SpanBase {
+    kind: 'sandbox';
+    image?: string;
+    command?: string;
+    exitCode?: number;
+    testsTotal?: number;
+    testsPassed?: number;
+    stdoutHash?: string;
+    stderrHash?: string;
+    /** Duration in ms; the harness fills this explicitly (endedAt - startedAt may miss setup). */
+    wallMs?: number;
+}
+interface GenericSpan extends SpanBase {
+    kind: 'agent' | 'custom';
+}
+type Span = LlmSpan | ToolSpan | RetrievalSpan | JudgeSpan | SandboxSpan | GenericSpan;
+type EventKind = 'log' | 'error' | 'budget_decrement' | 'budget_breach' | 'state_mutation' | 'policy_violation' | 'redaction_applied' | 'custom';
+interface TraceEvent {
+    eventId: string;
+    runId: string;
+    spanId?: string;
+    kind: EventKind;
+    timestamp: number;
+    payload: Record<string, unknown>;
+}
+interface BudgetLedgerEntry {
+    runId: string;
+    dimension: keyof BudgetSpec;
+    limit: number;
+    consumed: number;
+    remaining: number;
+    timestamp: number;
+    breached: boolean;
+    /** Span that triggered this entry, if any. */
+    spanId?: string;
+}
+interface Artifact {
+    artifactId: string;
+    runId: string;
+    spanId?: string;
+    contentType: string;
+    sizeBytes: number;
+    /** sha256 in hex. */
+    hash: string;
+    /** External storage URL (R2, S3, filesystem path). */
+    storageUrl?: string;
+    /** Inline content for small blobs — keep under ~64KB. */
+    inlineContent?: string;
+}
+type FailureClass = 'success' | 'reasoning_error' | 'tool_selection_error' | 'tool_argument_error' | 'tool_recovery_failure' | 'hallucination' | 'instruction_following' | 'safety_refusal_miss' | 'policy_violation' | 'budget_exceeded' | 'format_drift' | 'permission_escalation' | 'pii_leak' | 'cost_overrun' | 'timeout' | 'sandbox_failure' | 'unknown';
+declare const FAILURE_CLASSES: readonly FailureClass[];
+declare function isLlmSpan(s: Span): s is LlmSpan;
+declare function isToolSpan(s: Span): s is ToolSpan;
+declare function isRetrievalSpan(s: Span): s is RetrievalSpan;
+declare function isJudgeSpan(s: Span): s is JudgeSpan;
+declare function isSandboxSpan(s: Span): s is SandboxSpan;
+
+interface RunFilter {
+    scenarioId?: string;
+    variantId?: string;
+    status?: RunStatus;
+    since?: number;
+    until?: number;
+    tag?: {
+        key: string;
+        value: string;
+    };
+    parentRunId?: string;
+    projectId?: string;
+    chatId?: string;
+    layer?: RunLayer;
+}
+interface SpanFilter {
+    runId?: string;
+    parentSpanId?: string;
+    kind?: SpanKind;
+    name?: string;
+    toolName?: string;
+    judgeId?: string;
+    since?: number;
+    until?: number;
+}
+interface EventFilter {
+    runId?: string;
+    spanId?: string;
+    kind?: EventKind;
+    since?: number;
+    until?: number;
+}
+interface TraceStore {
+    appendRun(run: Run): Promise<void>;
+    updateRun(runId: string, patch: Partial<Run>): Promise<void>;
+    appendSpan(span: Span): Promise<void>;
+    updateSpan(spanId: string, patch: Partial<Span>): Promise<void>;
+    appendEvent(event: TraceEvent): Promise<void>;
+    appendArtifact(artifact: Artifact): Promise<void>;
+    appendBudgetEntry(entry: BudgetLedgerEntry): Promise<void>;
+    getRun(runId: string): Promise<Run | undefined>;
+    listRuns(filter?: RunFilter): Promise<Run[]>;
+    spans(filter?: SpanFilter): Promise<Span[]>;
+    events(filter?: EventFilter): Promise<TraceEvent[]>;
+    budget(runId: string): Promise<BudgetLedgerEntry[]>;
+    artifacts(runId: string): Promise<Artifact[]>;
+}
+declare class InMemoryTraceStore implements TraceStore {
+    private runs;
+    private allSpans;
+    private allEvents;
+    private allArtifacts;
+    private allBudget;
+    appendRun(run: Run): Promise<void>;
+    updateRun(runId: string, patch: Partial<Run>): Promise<void>;
+    appendSpan(span: Span): Promise<void>;
+    updateSpan(spanId: string, patch: Partial<Span>): Promise<void>;
+    appendEvent(event: TraceEvent): Promise<void>;
+    appendArtifact(artifact: Artifact): Promise<void>;
+    appendBudgetEntry(entry: BudgetLedgerEntry): Promise<void>;
+    getRun(runId: string): Promise<Run | undefined>;
+    listRuns(filter?: RunFilter): Promise<Run[]>;
+    spans(filter?: SpanFilter): Promise<Span[]>;
+    events(filter?: EventFilter): Promise<TraceEvent[]>;
+    budget(runId: string): Promise<BudgetLedgerEntry[]>;
+    artifacts(runId: string): Promise<Artifact[]>;
+}
+interface FileSystemTraceStoreOptions {
+    dir: string;
+    /** Roll over NDJSON files when they exceed this size in bytes. Default 32 MB. */
+    maxBytes?: number;
+}
+declare class FileSystemTraceStore implements TraceStore {
+    private dir;
+    private maxBytes;
+    /** Lazy in-memory index for queries — populated on first read. */
+    private index?;
+    private loaded;
+    constructor(options: FileSystemTraceStoreOptions);
+    private ensureDir;
+    private append;
+    private insertInto;
+    private load;
+    appendRun(run: Run): Promise<void>;
+    updateRun(runId: string, patch: Partial<Run>): Promise<void>;
+    appendSpan(span: Span): Promise<void>;
+    updateSpan(spanId: string, patch: Partial<Span>): Promise<void>;
+    appendEvent(event: TraceEvent): Promise<void>;
+    appendArtifact(artifact: Artifact): Promise<void>;
+    appendBudgetEntry(entry: BudgetLedgerEntry): Promise<void>;
+    getRun(runId: string): Promise<Run | undefined>;
+    listRuns(filter?: RunFilter): Promise<Run[]>;
+    spans(filter?: SpanFilter): Promise<Span[]>;
+    events(filter?: EventFilter): Promise<TraceEvent[]>;
+    budget(runId: string): Promise<BudgetLedgerEntry[]>;
+    artifacts(runId: string): Promise<Artifact[]>;
+}
+
+/**
+ * TraceEmitter — hierarchical span builder that auto-parents using an
+ * internal stack. One emitter per Run; emitters do NOT share state.
+ *
+ * Convenience methods (`llm`, `tool`, `retrieval`, `judge`, `sandbox`)
+ * return a `SpanHandle` with `.end()` / `.fail()` so callers don't
+ * have to thread spanIds manually. For async workflows that can't use
+ * the stack (e.g. fan-out parallel calls), pass `parentSpanId`
+ * explicitly.
+ */
+
+interface SpanHandle<S extends Span = Span> {
+    span: S;
+    end(patch?: Partial<S>): Promise<void>;
+    fail(error: string | Error, patch?: Partial<S>): Promise<void>;
+}
+interface TraceEmitterOptions {
+    runId?: string;
+    /** Inject a clock for deterministic tests. */
+    now?: () => number;
+    /** Inject an id generator for deterministic tests. */
+    id?: () => string;
+}
+declare class TraceEmitter {
+    private store;
+    private stack;
+    private _runId;
+    private now;
+    private id;
+    constructor(store: TraceStore, options?: TraceEmitterOptions);
+    get runId(): string;
+    startRun(run: Omit<Run, 'runId' | 'startedAt' | 'status'>): Promise<Run>;
+    endRun(outcome?: RunOutcome): Promise<void>;
+    abortRun(reason: string): Promise<void>;
+    span<S extends Span = Span>(init: {
+        kind: SpanKind;
+        name: string;
+        parentSpanId?: string;
+        attributes?: Record<string, unknown>;
+    } & Partial<Omit<S, 'spanId' | 'runId' | 'startedAt' | 'kind' | 'name'>>): Promise<SpanHandle<S>>;
+    private handle;
+    private pop;
+    llm(init: Omit<LlmSpan, 'spanId' | 'runId' | 'kind' | 'startedAt'>): Promise<SpanHandle<LlmSpan>>;
+    tool(init: Omit<ToolSpan, 'spanId' | 'runId' | 'kind' | 'startedAt'>): Promise<SpanHandle<ToolSpan>>;
+    retrieval(init: Omit<RetrievalSpan, 'spanId' | 'runId' | 'kind' | 'startedAt'>): Promise<SpanHandle<RetrievalSpan>>;
+    recordJudge(verdict: Omit<JudgeSpan, 'spanId' | 'runId' | 'kind' | 'startedAt' | 'endedAt'>): Promise<JudgeSpan>;
+    sandbox(init: Omit<SandboxSpan, 'spanId' | 'runId' | 'kind' | 'startedAt'>): Promise<SpanHandle<SandboxSpan>>;
+    emit(event: {
+        kind: EventKind;
+        spanId?: string;
+        payload?: Record<string, unknown>;
+    }): Promise<TraceEvent>;
+    recordBudget(entry: Omit<BudgetLedgerEntry, 'runId' | 'timestamp'> & {
+        timestamp?: number;
+    }): Promise<BudgetLedgerEntry>;
+    recordArtifact(artifact: Omit<Artifact, 'artifactId' | 'runId'>): Promise<Artifact>;
+    /**
+     * Runs `fn` inside a span; auto-ends on success, auto-fails on throw.
+     * Returns the fn's return value. Use this for the 95% case.
+     */
+    within<T>(init: Parameters<TraceEmitter['span']>[0], fn: (handle: SpanHandle) => Promise<T>): Promise<T>;
+}
+/** Helper to build an LLM span handle args object from a provider-shaped response. */
+declare function llmSpanFromProvider(args: {
+    name?: string;
+    model: string;
+    messages: Message[];
+    output: string;
+    usage?: {
+        inputTokens?: number;
+        outputTokens?: number;
+        cachedTokens?: number;
+        reasoningTokens?: number;
+    };
+    costUsd?: number;
+    finishReason?: string;
+}): Omit<LlmSpan, 'spanId' | 'runId' | 'kind' | 'startedAt'>;
+
+/**
+ * Typed query helpers over TraceStore.
+ *
+ * Not a full SQL engine — a minimal, composable set of operators that
+ * cover the canned-pipeline use cases. For ad-hoc analytics, persist to
+ * NDJSON and point DuckDB at it; the schema is stable so external SQL
+ * tooling works out of the box.
+ */
+
+declare function runsForScenario(store: TraceStore, scenarioId: string): Promise<Run[]>;
+declare function llmSpans(store: TraceStore, runId?: string): Promise<LlmSpan[]>;
+declare function toolSpans(store: TraceStore, runId?: string, toolName?: string): Promise<ToolSpan[]>;
+declare function judgeSpans(store: TraceStore, runId?: string): Promise<JudgeSpan[]>;
+/** Group spans by any key selector. */
+declare function groupBy<T, K extends string | number>(items: T[], key: (t: T) => K): Map<K, T[]>;
+/** Hash tool arguments to an orderless-key-stable string for de-duplication. */
+declare function argHash(args: unknown): string;
+/** Sum an LLM-span array into aggregate token + cost. */
+declare function aggregateLlm(spans: LlmSpan[]): {
+    inputTokens: number;
+    outputTokens: number;
+    cachedTokens: number;
+    costUsd: number;
+};
+/** Pick the outcome's failure class when present, else derive 'success' from run status. */
+declare function runFailureClass(run: Run): FailureClass;
+
+/**
+ * Redaction — remove PII / secrets from trace payloads before persist.
+ *
+ * Pre-persistence rules mean raw traces in storage are already scrubbed.
+ * Unredacted variants (for debugging / post-mortems) live in a separate
+ * storage layer with stricter access controls; this module only covers
+ * the default scrub-then-persist path.
+ *
+ * Rules compose: pass an array of `RedactionRule`, each is applied in
+ * order. Strings that match get replaced with a tagged sentinel so the
+ * eval framework can count how many redactions happened per run
+ * (surfaced via `redaction_applied` events).
+ */
+interface RedactionRule {
+    id: string;
+    pattern: RegExp;
+    /** Replacement — e.g. '[PII:email]'. Defaults to `[redacted:{id}]`. */
+    replacement?: string;
+}
+interface RedactionReport {
+    redactionCount: number;
+    byRule: Record<string, number>;
+}
+/** OWASP / common-sense defaults — extend per-domain. */
+declare const DEFAULT_REDACTION_RULES: RedactionRule[];
+declare const REDACTION_VERSION = "1.0.0";
+/**
+ * Redact a single string. Returns the new string and a per-rule count of
+ * how many substitutions fired.
+ */
+declare function redactString(input: string, rules?: RedactionRule[]): {
+    output: string;
+    report: RedactionReport;
+};
+/**
+ * Walk a JSON-ish value applying `redactString` to every string leaf.
+ * Arrays and plain objects are recursed; other types pass through
+ * untouched. Circular references throw — traces should be tree-shaped.
+ */
+declare function redactValue(value: unknown, rules?: RedactionRule[], report?: RedactionReport): {
+    value: unknown;
+    report: RedactionReport;
+};
+
+/**
+ * OpenTelemetry JSON export — maps TraceSchema v1 to OTLP/JSON so
+ * traces render natively in Jaeger / Honeycomb / Langfuse / Grafana.
+ *
+ * Wire format only. We do NOT depend on the @opentelemetry SDK — that
+ * would drag in polyfills incompatible with Workers/Edge. Consumers
+ * push the JSON to their collector of choice via HTTP.
+ *
+ * Reference: OTLP 1.3.2 (ResourceSpans / ScopeSpans / Span).
+ */
+
+declare const OTEL_AGENT_EVAL_SCOPE: {
+    name: string;
+    version: string;
+};
+interface OtlpSpan {
+    traceId: string;
+    spanId: string;
+    parentSpanId?: string;
+    name: string;
+    kind: number;
+    startTimeUnixNano: string;
+    endTimeUnixNano: string;
+    attributes: Array<{
+        key: string;
+        value: {
+            stringValue?: string;
+            intValue?: string;
+            doubleValue?: number;
+            boolValue?: boolean;
+        };
+    }>;
+    events?: Array<{
+        timeUnixNano: string;
+        name: string;
+        attributes?: OtlpSpan['attributes'];
+    }>;
+    status?: {
+        code: number;
+        message?: string;
+    };
+}
+interface OtlpResourceSpans {
+    resource: {
+        attributes: OtlpSpan['attributes'];
+    };
+    scopeSpans: Array<{
+        scope: typeof OTEL_AGENT_EVAL_SCOPE;
+        spans: OtlpSpan[];
+    }>;
+}
+interface OtlpExport {
+    resourceSpans: OtlpResourceSpans[];
+}
+/** Export a single run's spans + events in OTLP/JSON. */
+declare function exportRunAsOtlp(store: TraceStore, runId: string, resourceAttrs?: Record<string, string | number | boolean>): Promise<OtlpExport>;
+
+/**
+ * SandboxHarness — executes a scenario in an isolated environment and
+ * emits a rich SandboxSpan into the trace.
+ *
+ * Two built-in drivers:
+ *   - `SubprocessSandboxDriver` — spawn in a local cwd with env vars.
+ *     Fast, no dependencies, fine for unit tests and most CI gates.
+ *   - `DockerSandboxDriver` — lifted from tangle-router's sandbox path;
+ *     shells out to `docker run`. Stronger isolation, slower startup.
+ *
+ * Consumers implement `SandboxDriver` for custom backends (Firecracker,
+ * Cloudflare sandbox product, etc.). The harness doesn't care which.
+ */
+
+interface HarnessConfig {
+    /** Setup command (e.g. "pnpm install"). Non-zero exit fails the run. */
+    setupCommand?: string;
+    /** Run command (e.g. "pnpm build"). */
+    runCommand?: string;
+    /** Test command (e.g. "pnpm test --run"). Drives the test count + pass count. */
+    testCommand?: string;
+    /** Absolute cwd for the subprocess driver. Ignored by docker driver. */
+    cwd?: string;
+    /** Max wall-clock per phase in ms. Default 10 minutes. */
+    timeoutMs?: number;
+    /** Image for the docker driver. */
+    image?: string;
+    /** Extra env vars (validated; shell-escaped). */
+    env?: Record<string, string>;
+    /** Parser for the test output — maps stdout/stderr/exit code → pass count. */
+    testParser?: TestOutputParser;
+}
+interface TestOutputParser {
+    id: string;
+    parse(stdout: string, stderr: string, exitCode: number): {
+        testsTotal: number;
+        testsPassed: number;
+    } | undefined;
+}
+interface SandboxResult {
+    phase: 'setup' | 'run' | 'test';
+    exitCode: number;
+    stdout: string;
+    stderr: string;
+    wallMs: number;
+    testsTotal?: number;
+    testsPassed?: number;
+}
+interface SandboxDriver {
+    id: string;
+    exec(phase: SandboxResult['phase'], command: string, config: HarnessConfig): Promise<SandboxResult>;
+}
+/** Vitest default summary line: "Tests  X passed | Y failed". */
+declare const vitestTestParser: TestOutputParser;
+/** Pytest default: "collected N items" + " X passed, Y failed". */
+declare const pytestTestParser: TestOutputParser;
+/** Jest: "Tests: X passed, Y total" (and optional failed). */
+declare const jestTestParser: TestOutputParser;
+/** Composite parser — tries a list of parsers in order. */
+declare function composeParsers(...parsers: TestOutputParser[]): TestOutputParser;
+declare class SubprocessSandboxDriver implements SandboxDriver {
+    id: string;
+    exec(phase: SandboxResult['phase'], command: string, config: HarnessConfig): Promise<SandboxResult>;
+}
+declare class DockerSandboxDriver implements SandboxDriver {
+    id: string;
+    exec(phase: SandboxResult['phase'], command: string, config: HarnessConfig): Promise<SandboxResult>;
+}
+interface SandboxHarnessResult {
+    passed: boolean;
+    setup?: SandboxResult;
+    run?: SandboxResult;
+    test?: SandboxResult;
+    totalWallMs: number;
+    /** Final score — 0 when no tests; otherwise testsPassed/testsTotal. */
+    score: number;
+}
+declare class SandboxHarness {
+    private driver;
+    constructor(driver?: SandboxDriver);
+    run(config: HarnessConfig, emitter: TraceEmitter): Promise<SandboxHarnessResult>;
+}
+
+/**
+ * TestGradedScenario — a scenario whose score comes from a test suite.
+ *
+ * This is the SWE-bench pattern generalized. The scenario ships:
+ *   - fixture data (setup instructions)
+ *   - a test command the harness runs
+ *   - optional assertion overrides
+ *
+ * The runner emits a run, delegates to SandboxHarness, records the
+ * outcome, and returns a structured verdict. Consumers bind their own
+ * agent execution to this contract.
+ */
+
+interface TestGradedScenario {
+    id: string;
+    description?: string;
+    harness: HarnessConfig;
+    /** Optional pass threshold in 0..1 (default 1.0 = all tests must pass). */
+    passThreshold?: number;
+    /** Provenance for dataset tracking. */
+    datasetVersion?: string;
+    /** Free-form tags (difficulty, category, etc.). */
+    tags?: Record<string, string>;
+}
+interface TestGradedRunOptions {
+    variantId?: string;
+    driver?: SandboxDriver;
+    /** Metadata recorded on the Run (codeSha, promptSha, modelFingerprint, seed). */
+    provenance?: Pick<Run, 'codeSha' | 'promptSha' | 'modelFingerprint' | 'seed' | 'envFingerprint'>;
+}
+interface TestGradedRunResult {
+    runId: string;
+    scenario: TestGradedScenario;
+    harness: SandboxHarnessResult;
+    pass: boolean;
+    score: number;
+    failureClass?: FailureClass;
+}
+declare function runTestGradedScenario(scenario: TestGradedScenario, store: TraceStore, options?: TestGradedRunOptions): Promise<TestGradedRunResult>;
+
+/**
+ * BudgetGuard — enforces token / wall-clock / call / $ caps, records
+ * a ledger entry on every decrement, emits `budget_breach` + throws
+ * `BudgetBreachError` when a cap is hit.
+ *
+ * Wraps a TraceEmitter. The emitter persists ledger entries + breach
+ * events so the classifier, pipelines, and reports can all read
+ * budget state from the trace corpus — no separate accounting.
+ */
+
+declare class BudgetBreachError extends Error {
+    dimension: keyof BudgetSpec;
+    limit: number;
+    attempted: number;
+    constructor(dimension: keyof BudgetSpec, limit: number, attempted: number);
+}
+declare class BudgetGuard {
+    private consumed;
+    private emitter;
+    private budget;
+    private startedAt;
+    constructor(emitter: TraceEmitter, budget: BudgetSpec, now?: () => number);
+    /** Record consumption. Throws `BudgetBreachError` if any dimension exceeds its cap. */
+    charge(delta: Partial<Record<keyof BudgetSpec, number>>, spanId?: string): Promise<void>;
+    /** Convenience: advance wall-clock budget based on elapsed wall time. */
+    tickWall(nowMs: number, spanId?: string): Promise<void>;
+    get state(): Record<keyof BudgetSpec, number>;
+}
+
+/**
+ * Failure taxonomy — canonical classes + a default classifier.
+ *
+ * Every failed run should end up in a named class. The classifier here
+ * is rule-based (fast, deterministic); an LLM fallback can be added by
+ * the consumer for novel cases and trained into the rule base over time.
+ *
+ * Consumers call `classifyFailure(run, spans, events)` and persist the
+ * returned class as `Run.outcome.failureClass`.
+ */
+
+interface FailureContext {
+    run: Run;
+    spans: Span[];
+    events: TraceEvent[];
+}
+interface FailureClassification {
+    failureClass: FailureClass;
+    reason: string;
+    triggerSpanId?: string;
+    triggerEventId?: string;
+}
+/** Ordered rules — first match wins. */
+interface FailureRule {
+    id: string;
+    match: (ctx: FailureContext) => {
+        failureClass: FailureClass;
+        reason: string;
+        triggerSpanId?: string;
+        triggerEventId?: string;
+    } | null;
+}
+declare const DEFAULT_RULES: FailureRule[];
+/** Classify the failure mode of a run using an ordered rule list. */
+declare function classifyFailure(ctx: FailureContext, rules?: FailureRule[]): FailureClassification;
+
+/**
+ * Trajectory — ordered, structured view over a run's spans.
+ *
+ * A pure function `buildTrajectory(store, runId) → Trajectory` returns
+ * a topologically ordered list of `TrajectoryStep` with parent-child
+ * grouping collapsed into a single line-of-agent-work. Separate
+ * analyzers (stuck-loop detection, waste ratio) live in
+ * `pipelines/` and consume the trajectory.
+ */
+
+interface TrajectoryStep {
+    index: number;
+    span: Span;
+    /** Depth in the span tree from the root. 0 = top-level. */
+    depth: number;
+    /** Events attached to this span. */
+    events: TraceEvent[];
+}
+interface Trajectory {
+    runId: string;
+    steps: TrajectoryStep[];
+    llmTurns: number;
+    toolCalls: number;
+    judgeVerdicts: number;
+    retrievals: number;
+    totalDurationMs: number;
+}
+declare function buildTrajectory(store: TraceStore, runId: string): Promise<Trajectory>;
+
+/**
+ * Tool-use metrics — derived purely from trace data.
+ *
+ * No scoring assumptions: consumers supply optional ground-truth tool
+ * selections per turn + optional "information used downstream" signals.
+ * Without those, we still compute descriptive metrics (error rate,
+ * retry rate, duplicate-call rate) that are useful on their own.
+ */
+
+interface ToolUseMetrics {
+    runId: string;
+    totalCalls: number;
+    byTool: Record<string, ToolStats>;
+    errorRate: number;
+    /** Ratio of calls with identical (toolName, argHash) already seen earlier in the same run. */
+    duplicateRate: number;
+    /** Ratio of error calls followed by ≥1 retry on same tool. */
+    retryRate: number;
+    /** Optional: of the calls agent made, fraction the evaluator marked as "correct selection". */
+    selectionAccuracy?: number;
+}
+interface ToolStats {
+    calls: number;
+    errors: number;
+    avgLatencyMs: number;
+    duplicates: number;
+}
+interface ToolUseOptions {
+    /** Map of spanId → whether the evaluator judged the tool selection correct. Optional. */
+    selectionLabels?: Record<string, boolean>;
+}
+declare function computeToolUseMetrics(store: TraceStore, runId: string, options?: ToolUseOptions): Promise<ToolUseMetrics>;
+
+/**
+ * StuckLoopView — detects when an agent calls the same tool with the
+ * same (or structurally similar) arguments ≥ N times in a short window.
+ *
+ * Rationale: agents that loop are the number-one production failure
+ * mode on long-horizon flows. The view returns (runId, toolName,
+ * argHash, occurrences, windowMs) for each detected loop plus a
+ * fraction of runs affected.
+ */
+
+interface StuckLoopFinding {
+    runId: string;
+    toolName: string;
+    argHash: string;
+    occurrences: number;
+    spanIds: string[];
+    /** Milliseconds between first and last call in the loop. */
+    windowMs: number;
+}
+interface StuckLoopReport {
+    findings: StuckLoopFinding[];
+    affectedRunRatio: number;
+    totalRuns: number;
+}
+interface StuckLoopOptions {
+    /** Minimum call count to flag a loop (default 3). */
+    minOccurrences?: number;
+    /** Filter to a specific runId; omit to scan the entire corpus. */
+    runId?: string;
+}
+declare function stuckLoopView(store: TraceStore, options?: StuckLoopOptions): Promise<StuckLoopReport>;
+
+/**
+ * ToolWasteView — fraction of tool calls whose results weren't used
+ * downstream. Without a "used" signal we fall back to structural
+ * proxies: error calls, duplicate calls, and tool calls followed by
+ * zero subsequent LLM spans are all considered waste.
+ *
+ * Consumers can pass a `usageOracle` that inspects a tool span and
+ * returns true iff the tool's result appears in a later LLM message,
+ * artifact, or state mutation — that's the canonical definition; the
+ * default heuristic is a reasonable fallback.
+ */
+
+interface ToolWasteFinding {
+    runId: string;
+    wastedCalls: number;
+    totalCalls: number;
+    wasteRate: number;
+}
+interface ToolWasteReport {
+    byRun: ToolWasteFinding[];
+    overallWasteRate: number;
+}
+interface ToolWasteOptions {
+    runId?: string;
+    usageOracle?: (tool: ToolSpan, later: {
+        llm: Awaited<ReturnType<typeof llmSpans>>;
+    }) => boolean;
+}
+declare function toolWasteView(store: TraceStore, options?: ToolWasteOptions): Promise<ToolWasteReport>;
+
+/**
+ * BudgetBreachView — aggregates breach events across the corpus.
+ *
+ * Answers: which dimensions get hit most often? Which scenarios are
+ * underbudgeted? Which variants trigger the most breaches?
+ */
+
+interface BudgetBreachFinding {
+    runId: string;
+    scenarioId: string;
+    variantId?: string;
+    dimension: keyof BudgetSpec;
+    limit: number;
+    consumed: number;
+    excessRatio: number;
+    timestamp: number;
+}
+interface BudgetBreachReport {
+    findings: BudgetBreachFinding[];
+    byDimension: Record<string, number>;
+    byScenario: Record<string, number>;
+    byVariant: Record<string, number>;
+    totalRuns: number;
+    breachedRunRatio: number;
+}
+declare function budgetBreachView(store: TraceStore, options?: {
+    scenarioId?: string;
+    variantId?: string;
+}): Promise<BudgetBreachReport>;
+
+/**
+ * FailureClusterView — groups failed runs by (failureClass, triggerTool,
+ * argHash-prefix) so weekly reviews can prioritize the top-N clusters.
+ *
+ * Each cluster includes: N runs, scenarios affected, representative
+ * error message, a proposed mitigation hint (rule → action table).
+ */
+
+interface FailureCluster {
+    failureClass: FailureClass;
+    /** Tool name when the trigger was a tool span, else undefined. */
+    toolName?: string;
+    /** First 16 chars of argHash — clusters similar args. */
+    argPrefix?: string;
+    runCount: number;
+    scenarioIds: string[];
+    exampleError?: string;
+    exampleRunId: string;
+}
+interface FailureClusterReport {
+    clusters: FailureCluster[];
+    totalFailures: number;
+    totalRuns: number;
+}
+declare function failureClusterView(store: TraceStore, options?: {
+    rules?: FailureRule[];
+    minClusterSize?: number;
+}): Promise<FailureClusterReport>;
+
+/**
+ * JudgeAgreementView — pairwise agreement between judges across the
+ * corpus, grouped by dimension.
+ *
+ * Output drives two workflows:
+ *   - Judge robustness audit: "does Claude agree with GPT at κ ≥ 0.6?"
+ *   - Calibration tracking: κ vs golden human labels over time (by
+ *     providing a `humanGoldenJudgeId`).
+ */
+
+interface JudgePair {
+    judgeA: string;
+    judgeB: string;
+    dimension: string;
+    /** Number of (targetSpanId, dimension) tuples both judges scored. */
+    commonItems: number;
+    pearson: number;
+    krippendorff: number;
+}
+interface JudgeAgreementReport {
+    pairs: JudgePair[];
+    dimensions: string[];
+    judgeIds: string[];
+}
+declare function judgeAgreementView(store: TraceStore): Promise<JudgeAgreementReport>;
+
+/**
+ * FirstDivergenceView — aligns two trajectories by step index, reports
+ * the first step where they differ.
+ *
+ * "Differ" is configurable — default is (kind, toolName if tool, model
+ * if llm). Use this view to attribute "why is variant B better?" to a
+ * specific step rather than an aggregate mean delta.
+ */
+
+interface DivergenceReport {
+    runA: string;
+    runB: string;
+    firstDivergenceIndex: number | null;
+    aStep?: TrajectoryStep;
+    bStep?: TrajectoryStep;
+    reason?: string;
+    /** Common prefix length (steps that matched). */
+    commonPrefixLen: number;
+}
+interface DivergenceOptions {
+    /** Returns true if two steps are considered equal. Default: kind + tool/model match. */
+    stepEquals?: (a: TrajectoryStep, b: TrajectoryStep) => boolean;
+}
+declare function firstDivergenceView(store: TraceStore, runA: string, runB: string, options?: DivergenceOptions): Promise<DivergenceReport>;
+
+/**
+ * Baseline regression detection.
+ *
+ * Lifted from ADC baseline.ts. Every promotion-blocking signal boils down
+ * to: "is this run measurably worse than baseline?" — with enough
+ * statistical rigor to distinguish noise from drift.
+ *
+ * Uses:
+ *   - Welch's t-test (unequal variance) for per-metric mean comparison
+ *   - Cohen's d for effect size magnitude
+ *   - IQR for stability flag (unstable samples can't be trusted for comparisons)
+ *
+ * Returns a structured verdict: improved | regressed | stable | unstable.
+ */
+interface MetricSamples {
+    /** Stable metric key (e.g. "overallScore", "firstTokenMs"). */
+    metric: string;
+    /** Whether higher values are better. */
+    higherIsBetter: boolean;
+    baseline: number[];
+    candidate: number[];
+}
+interface MetricVerdict {
+    metric: string;
+    baselineMean: number;
+    candidateMean: number;
+    delta: number;
+    cohensD: number;
+    welchT: number;
+    welchDf: number;
+    welchP: number;
+    stable: boolean;
+    /** IQR of the combined samples — used as a rough stability indicator. */
+    iqr: number;
+    verdict: 'improved' | 'regressed' | 'stable' | 'unstable';
+}
+interface BaselineReport {
+    metrics: MetricVerdict[];
+    /** True if any critical metric regressed. */
+    hasRegression: boolean;
+    /** True if any metric is unstable (too noisy to judge). */
+    hasUnstable: boolean;
+}
+interface BaselineOptions {
+    /** Effect size threshold for meaningful delta (default 0.5 — medium effect). */
+    effectThreshold?: number;
+    /** p-value threshold for statistical significance (default 0.05). */
+    alpha?: number;
+    /** IQR/mean ratio above which samples are flagged unstable (default 0.30). */
+    unstableCvThreshold?: number;
+}
+/**
+ * Compare candidate samples against baseline per metric. Verdict logic:
+ *   - unstable: IQR/|mean| > threshold on either set — not enough signal
+ *   - improved: meaningful effect in the "better" direction AND p < alpha
+ *   - regressed: meaningful effect in the "worse" direction AND p < alpha
+ *   - stable: otherwise (no significant change)
+ */
+declare function compareToBaseline(samples: MetricSamples[], options?: BaselineOptions): BaselineReport;
+/** Inter-quartile range; 0 when the sample has no spread. */
+declare function iqr(xs: number[]): number;
+/**
+ * Welch's t-test — unequal-variance two-sample t. Uses the same Student-t
+ * CDF as `pairedTTest` (via incomplete beta); falls back to normal tail
+ * when df is large.
+ */
+declare function welchsTTest(a: number[], b: number[]): {
+    t: number;
+    df: number;
+    p: number;
+};
+
+/**
+ * RegressionView — compares a candidate slice to a baseline slice on a
+ * named metric. Delegates the statistics (Welch's t-test, Cohen's d,
+ * IQR stability) to `baseline.ts`.
+ *
+ * This is the entry point for CI regression gates: "given runs tagged
+ * release=A and release=B, did any metric regress?"
+ */
+
+interface RegressionSpec {
+    metric: string;
+    higherIsBetter: boolean;
+    /** Extract a scalar from a run. Default extractors handle common metrics. */
+    extract?: (run: Run, store: TraceStore) => Promise<number | null>;
+}
+interface RegressionOptions extends BaselineOptions {
+    baseline: RunFilter;
+    candidate: RunFilter;
+}
+declare function regressionView(store: TraceStore, metrics: RegressionSpec[], options: RegressionOptions): Promise<BaselineReport>;
+
+/**
+ * SLO gates — quantified pass/fail primitives beyond score thresholds.
+ *
+ * Lifted from ADC's sandbox eval suite. Each SLO defines a metric, a
+ * threshold, and a severity (critical | warning). Critical breaches fail
+ * the eval; warnings are reported but don't gate CI. Margin is the
+ * ratio of actual to threshold for histogramming "how close are we?"
+ *
+ * Consumers assemble their own SLO arrays; DEFAULT_AGENT_SLOS covers
+ * the generic agent flow (provision, first token, pass rate, cost).
+ */
+type SloSeverity = 'critical' | 'warning';
+type SloComparator = 'lte' | 'gte';
+interface Slo {
+    /** Stable identifier — must be unique within an SLO set. */
+    id: string;
+    /** Human description, shown in reports. */
+    description: string;
+    /** Metric key looked up in the candidate record. */
+    metric: string;
+    /** Whether the metric should stay below (lte) or above (gte) threshold. */
+    comparator: SloComparator;
+    /** Threshold value. */
+    threshold: number;
+    severity: SloSeverity;
+}
+interface SloCheckResult {
+    slo: Slo;
+    actual: number | undefined;
+    passed: boolean;
+    /** actual/threshold for lte, threshold/actual for gte. >1 means safe margin; <1 means breach. 0 when actual is missing. */
+    margin: number;
+    detail: string;
+}
+interface SloReport {
+    results: SloCheckResult[];
+    passedCritical: boolean;
+    criticalBreaches: SloCheckResult[];
+    warnings: SloCheckResult[];
+}
+/**
+ * Evaluate an SLO set against a candidate metrics object. Missing metrics
+ * count as breaches — if you declared it, you must measure it.
+ */
+declare function checkSlos(metrics: Record<string, number>, slos: Slo[]): SloReport;
+/** Reference SLO set for agent-style evals. Tune per-product by cloning + overriding. */
+declare const DEFAULT_AGENT_SLOS: Slo[];
+
+/**
+ * Declarative oracles — ground-truth assertions without an LLM.
+ *
+ * Lifted from browser-agent-driver's _oracle.mjs. When you know the
+ * expected outcome exactly (a URL, a text fragment, a JSON shape), you
+ * don't need an LLM judge — you need a regex. These oracles are
+ * composable pass/fail checks over an observation bundle.
+ *
+ * Each oracle returns { pass, detail, evidence? } and has a short
+ * `id` for reporting. `evaluateOracles` runs a batch and aggregates.
+ */
+interface OracleObservation {
+    /** Final observable text output from the agent (response, page snapshot, stdout). */
+    text?: string;
+    /** Final URL — for browser-style scenarios. */
+    url?: string;
+    /** Any structured JSON the agent produced. */
+    json?: unknown;
+    /** Free-form context used by custom oracles. */
+    context?: Record<string, unknown>;
+}
+interface OracleResult {
+    id: string;
+    pass: boolean;
+    detail: string;
+    evidence?: string;
+}
+interface Oracle {
+    id: string;
+    check(obs: OracleObservation): OracleResult;
+}
+declare function textInSnapshot(needle: string, opts?: {
+    caseSensitive?: boolean;
+}): Oracle;
+declare function urlContains(fragment: string): Oracle;
+declare function jsonShape(expected: Record<string, unknown>): Oracle;
+declare function regexMatches(pattern: RegExp): Oracle;
+/**
+ * Anti-bot detector — distinguishes genuine failures from blocked navigation
+ * (cloudflare, recaptcha, etc). Returns an Oracle that PASSES when no block
+ * marker is present; on block, detail names the blocker so runners can tag
+ * results as "blocked" rather than "failed". Lifted from browser-agent-driver.
+ */
+declare function notBlocked(): Oracle;
+interface OracleReport {
+    results: OracleResult[];
+    pass: boolean;
+    passCount: number;
+    failCount: number;
+    /** 0-1 ratio of oracles passed. */
+    score: number;
+}
+/** Run all oracles against one observation and aggregate. */
+declare function evaluateOracles(obs: OracleObservation, oracles: Oracle[]): OracleReport;
+
+/**
+ * Cost tracker — token + USD accounting per scenario and per run.
+ *
+ * Lifted from tax/legal metrics.ts + tangle-router UsageEvent. Every
+ * optimizer needs to know "is the quality gain worth the cost delta?",
+ * and every dashboard needs dollars-per-completed-task. MODEL_PRICING
+ * from metrics.ts stays authoritative for estimate math; this module
+ * adds the aggregation + per-scenario roll-up that was duplicated
+ * across 4 verticals.
+ */
+interface TokenSpec {
+    inputTokens: number;
+    outputTokens: number;
+    cachedTokens?: number;
+    reasoningTokens?: number;
+}
+interface CostEntry extends TokenSpec {
+    scenarioId: string;
+    model: string;
+    /** Override estimate with an observed cost (e.g. from provider response). */
+    actualCostUsd?: number;
+    timestamp: number;
+    /** Free-form tags (variant id, round #, etc.). */
+    tags?: Record<string, string>;
+}
+interface ScenarioCost {
+    scenarioId: string;
+    entries: CostEntry[];
+    totalInputTokens: number;
+    totalOutputTokens: number;
+    totalCachedTokens: number;
+    totalCostUsd: number;
+    /** Pass flag — set by consumer via markOutcome; used for cost-per-completed-task. */
+    completed?: boolean;
+}
+declare class CostTracker {
+    private byScenario;
+    record(entry: Omit<CostEntry, 'timestamp'> & {
+        timestamp?: number;
+    }): CostEntry;
+    markOutcome(scenarioId: string, completed: boolean): void;
+    get(scenarioId: string): ScenarioCost | undefined;
+    list(): ScenarioCost[];
+    summary(): CostSummary;
+}
+interface CostSummary {
+    scenarioCount: number;
+    completedCount: number;
+    totalInputTokens: number;
+    totalOutputTokens: number;
+    totalCostUsd: number;
+    avgCostPerScenarioUsd: number;
+    /** Total USD / completed scenarios — null when nothing completed. */
+    costPerCompletedTaskUsd: number | null;
+}
+
+/**
+ * Pareto frontier — multi-objective optimization over candidate runs.
+ *
+ * Lifted from ADC pareto.ts and blueprint-agent frontier.ts. When you're
+ * trading off (cost, latency, quality) or (passRate, tokenBudget,
+ * ttfb), you rarely have a single "winner" — you have a set of
+ * non-dominated candidates. This module exposes:
+ *
+ *   - `paretoFrontier`: filter a set of candidates to the non-dominated ones
+ *   - `dominates`: does A dominate B across all objectives?
+ *
+ * Each objective is declared with a direction: 'maximize' (higher=better)
+ * or 'minimize' (lower=better). Candidates are any object; pass an
+ * `objective(candidate)` accessor.
+ */
+type Direction = 'maximize' | 'minimize';
+interface Objective<T> {
+    /** Stable label used in reports. */
+    name: string;
+    direction: Direction;
+    value: (candidate: T) => number;
+}
+interface ParetoResult<T> {
+    frontier: T[];
+    dominated: T[];
+    /** Index map: frontier[i] dominates each of dominatedBy[i]. */
+    dominanceMap: Array<{
+        dominator: T;
+        dominated: T[];
+    }>;
+}
+/** Does candidate A weakly dominate B — strictly better on at least one objective and no worse on any? */
+declare function dominates<T>(a: T, b: T, objectives: Objective<T>[]): boolean;
+/**
+ * Compute the non-dominated frontier. Candidates with NaN/Infinity on any
+ * objective are excluded (can't rank them). A candidate enters the frontier
+ * iff no other candidate dominates it.
+ */
+declare function paretoFrontier<T>(candidates: T[], objectives: Objective<T>[]): ParetoResult<T>;
+
+/**
+ * Series convergence — detects whether a sequence of scalar measurements
+ * is stabilizing, drifting, or noisy.
+ *
+ * Lifted from ADC convergence.ts. The per-turn `ConvergenceTracker` is
+ * about progress *within* a single run; this module is about drift
+ * *across* runs (e.g. "are my nightly eval scores stabilizing?").
+ *
+ * Three signals:
+ *   - stabilized: last K values have low variance (< epsilon) — done
+ *   - drifting:   recent trend is monotonic and beyond noise — regressing or improving
+ *   - noisy:      neither — keep iterating, but flag as untrustworthy for gating
+ */
+interface SeriesConvergenceOptions {
+    /** Window size for "recent" analysis (default 5). */
+    window?: number;
+    /** Coefficient-of-variation threshold below which the window is stabilized (default 0.05 = 5%). */
+    stableCv?: number;
+    /** Minimum monotone run length to call drift (default 3). */
+    driftRun?: number;
+}
+interface SeriesConvergenceResult {
+    state: 'stabilized' | 'drifting-up' | 'drifting-down' | 'noisy' | 'insufficient-data';
+    windowMean: number;
+    windowCv: number;
+    /** Longest monotonic run at the tail of the series (positive for up, negative for down). */
+    tailRun: number;
+    /** True when n ≥ window AND windowCv ≤ stableCv. */
+    stable: boolean;
+}
+declare function analyzeSeries(values: number[], options?: SeriesConvergenceOptions): SeriesConvergenceResult;
+
+/**
+ * State continuity scoring — measures how well a resumed/handed-off agent
+ * preserves prior work.
+ *
+ * Lifted from tax-agent's run-resume-eval.ts. When session 2 continues
+ * session 1's work, the key question is: did it preserve key artifacts,
+ * or start over and lose context? Each `ContinuityCheck` inspects one
+ * aspect (file preserved, key count grew, status advanced) and yields
+ * 0-1 credit; the aggregate is the simple mean.
+ *
+ * Generic over any "snapshot" shape — pass your own checks.
+ */
+interface ContinuitySnapshotPair<T> {
+    before: T;
+    after: T;
+}
+interface ContinuityCheck<T> {
+    /** Stable identifier; shown in the report. */
+    id: string;
+    /** Description of what this check measures. */
+    description: string;
+    /** Returns 0..1 credit for this dimension (1 = fully preserved/improved). */
+    score: (pair: ContinuitySnapshotPair<T>) => number;
+}
+interface ContinuityCheckResult {
+    id: string;
+    description: string;
+    score: number;
+    pass: boolean;
+}
+interface ContinuityReport {
+    results: ContinuityCheckResult[];
+    /** Mean of per-check scores, in 0..1. */
+    overallScore: number;
+    /** True iff ALL checks scored ≥ passThreshold. */
+    pass: boolean;
+}
+declare function scoreContinuity<T>(pair: ContinuitySnapshotPair<T>, checks: ContinuityCheck<T>[], options?: {
+    passThreshold?: number;
+}): ContinuityReport;
+/** Common check: a required key in a record exists and equals the prior value. */
+declare function keyPreserved<T extends Record<string, unknown>>(key: keyof T & string): ContinuityCheck<T>;
+/** Common check: a collection (array) grew or stayed the same size. */
+declare function collectionPreserved<T, K extends keyof T & string>(key: K, minRatio?: number): ContinuityCheck<T>;
+/** Common check: a status field advanced in an expected order. */
+declare function statusAdvanced<T extends Record<string, unknown>>(key: keyof T & string, progression: readonly string[]): ContinuityCheck<T>;
+
+/**
+ * Dataset — versioned, sliceable, content-hashed scenario collection.
+ *
+ * Scenarios stop being ephemeral arrays and become first-class
+ * artifacts. Every Dataset carries:
+ *   - content hash (sha256 over canonicalized scenario array)
+ *   - provenance (contributor, createdAt, sourceUrl)
+ *   - split labels (train | dev | test | holdout)
+ *   - difficulty tiers (easy | medium | hard | extreme)
+ *   - tags (free-form, per-scenario)
+ *
+ * `Dataset.slice({ difficulty, split, holdout, seed })` returns a
+ * deterministic, reproducible subset. Holdout slices are locked: you
+ * can read them but `mutate` throws, which prevents "oh I'll just
+ * tweak that one scenario" contamination drift.
+ */
+type DatasetSplit = 'train' | 'dev' | 'test' | 'holdout';
+type DatasetDifficulty = 'easy' | 'medium' | 'hard' | 'extreme';
+interface DatasetScenario {
+    id: string;
+    /** Arbitrary payload; the framework doesn't interpret it. */
+    payload: unknown;
+    split?: DatasetSplit;
+    difficulty?: DatasetDifficulty;
+    /** Canary token that MUST NOT round-trip through a correct agent output. */
+    canary?: string;
+    tags?: Record<string, string>;
+}
+interface DatasetProvenance {
+    contributor?: string;
+    createdAt: string;
+    sourceUrl?: string;
+    license?: string;
+    description?: string;
+    /** Monotonic human-readable version (e.g. "2026.04.20"). */
+    version: string;
+}
+interface DatasetManifest {
+    name: string;
+    provenance: DatasetProvenance;
+    /** sha256 hex over canonicalized scenarios. */
+    contentHash: string;
+    scenarioCount: number;
+    splitCounts: Record<DatasetSplit, number>;
+}
+interface SliceOptions {
+    split?: DatasetSplit;
+    difficulty?: DatasetDifficulty;
+    /** Number of scenarios (random sample, seeded). Omit to take all that match. */
+    limit?: number;
+    seed?: number;
+    /** Predicate narrowing. Applied after split/difficulty filters. */
+    filter?: (scenario: DatasetScenario) => boolean;
+    /** If true, include scenarios marked as holdout. Default false. */
+    includeHoldout?: boolean;
+}
+/** Locked holdouts — throws on mutate. Callers that need a mutable dataset fork it. */
+declare class HoldoutLockedError extends Error {
+    constructor(datasetName: string);
+}
+declare class Dataset {
+    readonly name: string;
+    readonly provenance: DatasetProvenance;
+    private scenarios;
+    private locked;
+    constructor(init: {
+        name: string;
+        provenance: DatasetProvenance;
+        scenarios: DatasetScenario[];
+        locked?: boolean;
+    });
+    /** All scenarios. Readonly — callers must go through `slice` or `clone`. */
+    all(): readonly DatasetScenario[];
+    get size(): number;
+    /**
+     * Deterministic sliced subset. Seed is REQUIRED when `limit` is set so
+     * the same arguments always produce the same slice across machines.
+     */
+    slice(options?: SliceOptions): DatasetScenario[];
+    /**
+     * Assemble the manifest (name + provenance + content hash + counts).
+     * Content hash is deterministic over canonicalized scenarios.
+     */
+    manifest(): Promise<DatasetManifest>;
+    /** Fresh unlocked copy — for post-release forks when mutation is needed. */
+    clone(overrides?: Partial<{
+        name: string;
+        version: string;
+    }>): Dataset;
+    lock(): void;
+    add(scenario: DatasetScenario): void;
+    remove(scenarioId: string): void;
+    /**
+     * Stable JSON-Lines serialization — deterministic byte-for-byte.
+     * Write to disk for contamination-verifiable archives.
+     */
+    toJsonl(): string;
+    static fromJsonl(jsonl: string, manifest: Omit<DatasetManifest, 'contentHash' | 'scenarioCount' | 'splitCounts'>): Dataset;
+}
+declare function hashScenarios(scenarios: DatasetScenario[]): Promise<string>;
+
+/**
+ * ContaminationGuard — ensures held-out scenarios don't leak into
+ * training/prompt paths, and flags model memorization.
+ *
+ * Three probes:
+ *   1. `checkCanaries(output, scenario)` — if the scenario carries a
+ *      canary token, it MUST NOT appear in the agent's output.
+ *      Canaries are strings that are statistically impossible to
+ *      reconstruct from the scenario description alone — so if they
+ *      echo back, the model memorized them.
+ *   2. `canaryLeakView(store)` — cross-corpus view of every run whose
+ *      output contained a canary, with the offending scenario + run.
+ *   3. `HoldoutAuditor` — wraps a Dataset and emits a structured error
+ *      on any code path that reads holdout scenarios but doesn't flag
+ *      `purpose: 'evaluation'`. Keeps engineers honest.
+ */
+
+interface CanaryLeak {
+    scenarioId: string;
+    canary: string;
+    runId?: string;
+    evidence: string;
+}
+declare function checkCanaries(output: string, scenarios: DatasetScenario[]): CanaryLeak[];
+/**
+ * Scan the LLM-output history in a corpus; returns every case where a
+ * canary from a known scenario appeared in agent output. Pass the full
+ * set of scenarios whose canaries you care about (typically the whole
+ * held-out slice).
+ */
+declare function canaryLeakView(store: TraceStore, scenarios: DatasetScenario[]): Promise<CanaryLeak[]>;
+declare class HoldoutAuditor {
+    private scenarios;
+    private accessLog;
+    constructor(scenarios: DatasetScenario[]);
+    /** Retrieve a holdout scenario for a declared purpose. Non-'evaluation' throws. */
+    get(scenarioId: string, purpose: 'evaluation' | 'debugging'): DatasetScenario;
+    getAccessLog(): ReadonlyArray<{
+        scenarioId: string;
+        purpose: string;
+        at: number;
+    }>;
+}
+
+/**
+ * Red-team battery — adversarial scenario corpus with per-category
+ * scorers.
+ *
+ * Every scenario is a `DatasetScenario` with a `redTeam: {...}` payload
+ * that declares: category, expected refusal, attack surface. Scorers
+ * inspect the agent's output + trajectory and return a PASS/FAIL per
+ * category. Aggregated via `redTeamReport`.
+ *
+ * The built-in corpus is intentionally conservative — 40 cases
+ * covering the most-seen attack patterns. Extend via `extendCorpus`.
+ */
+
+type RedTeamCategory = 'prompt_injection_direct' | 'prompt_injection_indirect' | 'jailbreak_persona' | 'jailbreak_dan' | 'pii_leak' | 'permission_escalation' | 'data_exfil' | 'policy_override';
+interface RedTeamPayload {
+    category: RedTeamCategory;
+    /** The hostile input the agent sees. */
+    input: string;
+    /**
+     * What a safe agent should do: "refuse" means explicit refusal,
+     * "ignore" means process benignly without executing the attack.
+     */
+    expected: 'refuse' | 'ignore';
+    /** Substring(s) that MUST NOT appear in output. */
+    forbiddenStrings?: string[];
+    /** Tool names the agent MUST NOT call for this scenario. */
+    forbiddenTools?: string[];
+}
+interface RedTeamCase extends DatasetScenario {
+    payload: RedTeamPayload;
+}
+interface RedTeamFinding {
+    scenarioId: string;
+    category: RedTeamCategory;
+    passed: boolean;
+    reason: string;
+    evidence?: string;
+}
+interface RedTeamReport {
+    findings: RedTeamFinding[];
+    passRateByCategory: Record<RedTeamCategory, number>;
+    overallPassRate: number;
+}
+/** Ship a canonical, small corpus. Consumers extend via `extendCorpus`. */
+declare const DEFAULT_RED_TEAM_CORPUS: RedTeamCase[];
+declare function redTeamDataset(extraCases?: RedTeamCase[]): Dataset;
+/**
+ * Score a single agent response against the red-team case.
+ * `toolCalls` should be the names of tools the agent invoked during the run.
+ */
+declare function scoreRedTeamOutput(output: string, toolCalls: string[], rtCase: RedTeamCase): RedTeamFinding;
+/** Aggregate red-team findings into per-category pass rates. */
+declare function redTeamReport(findings: RedTeamFinding[]): RedTeamReport;
+/**
+ * Extract the tool-call names from a corpus run — convenience for the
+ * common pipeline (run the scenario → score the run).
+ */
+declare function toolNamesForRun(store: TraceStore, runId: string): Promise<string[]>;
+
+/**
+ * Power analysis + multiple-comparison correction.
+ *
+ * Two jobs:
+ *   1. Before running: `requiredSampleSize({ effect, alpha, power })`
+ *      returns the N per arm needed to detect a given effect size.
+ *   2. After running: `benjaminiHochberg(pValues, fdr)` and
+ *      `bonferroni(pValues, alpha)` correct for multiple pairwise tests
+ *      so PromptOptimizer's "significant" flag is statistically honest.
+ *
+ * Fixes the correctness bug in 0.2's PromptOptimizer which applied
+ * alpha directly across n*(n-1)/2 pairwise tests without correction —
+ * dramatically inflating false-positive rate when variants ≥ 3.
+ */
+/**
+ * Required N per arm for a two-sample comparison at target effect size,
+ * alpha, and power. Uses the normal-approximation formula:
+ *
+ *   n = 2 * ( (z_{1-α/2} + z_{1-β}) / d )^2
+ *
+ * where d is Cohen's d. Returns Infinity for effect ≤ 0.
+ */
+declare function requiredSampleSize(opts: {
+    effect: number;
+    alpha?: number;
+    power?: number;
+    twoSided?: boolean;
+}): number;
+/** Bonferroni adjustment: multiply every p-value by the number of tests, clamp at 1. */
+declare function bonferroni(pValues: number[], alpha?: number): {
+    adjusted: number[];
+    significant: boolean[];
+};
+/**
+ * Benjamini–Hochberg false discovery rate. Returns adjusted q-values and
+ * significance at the target FDR. Properly handles ties and preserves
+ * monotonicity of q-values.
+ */
+declare function benjaminiHochberg(pValues: number[], fdr?: number): {
+    qValues: number[];
+    significant: boolean[];
+};
+
+/**
+ * Behavior DSL — pytest-style assertions over a run's trajectory.
+ *
+ * Shape:
+ *   expect(store, runId).toCall('search').withArgs({ q: /.+/ })
+ *   expect(store, runId).toRefuse()
+ *   expect(store, runId).toOutputMatch(/confirmed/i)
+ *   expect(store, runId).toRespectBudget('tokens')
+ *   expect(store, runId).toCompleteWithin({ wallMs: 30_000 })
+ *
+ * Each matcher returns an `Expectation` with `.check() → MatcherResult`
+ * so the DSL is composable with suite runners — you can collect all
+ * expectations into a report instead of throwing on first failure.
+ */
+
+interface MatcherResult {
+    ok: boolean;
+    detail: string;
+    evidence?: string;
+}
+interface Expectation {
+    /** Human-facing label; used in reports. */
+    label: string;
+    check(): Promise<MatcherResult>;
+}
+declare class BehaviorAssertion {
+    private store;
+    private runId;
+    constructor(store: TraceStore, runId: string);
+    toCall(toolName: string): CallExpectation;
+    toRefuse(markers?: RegExp[]): Expectation;
+    toOutputMatch(pattern: RegExp): Expectation;
+    toRespectBudget(dimension: keyof BudgetLedgerEntry['dimension'] | 'tokens' | 'wallMs' | 'calls' | 'usd'): Expectation;
+    toCompleteWithin(limits: {
+        wallMs?: number;
+        toolCalls?: number;
+        llmTurns?: number;
+    }): Expectation;
+    toNeverCall(toolName: string): Expectation;
+}
+declare class CallExpectation implements Expectation {
+    private store;
+    private runId;
+    private toolName;
+    private argMatchers;
+    private minCount;
+    private maxCount;
+    constructor(store: TraceStore, runId: string, toolName: string);
+    get label(): string;
+    withArgs(shape: Record<string, unknown | RegExp>): this;
+    times(n: number): this;
+    atLeast(n: number): this;
+    atMost(n: number): this;
+    check(): Promise<MatcherResult>;
+}
+declare function expectAgent(store: TraceStore, runId: string): BehaviorAssertion;
+/** Runs every expectation, collects results. Never throws. */
+declare function runExpectations(expectations: Expectation[]): Promise<{
+    results: Array<{
+        label: string;
+        result: MatcherResult;
+    }>;
+    pass: boolean;
+    passCount: number;
+    failCount: number;
+}>;
+
+/**
+ * Judge calibration — measure judge quality against human gold + bias.
+ *
+ * Workflow:
+ *   1. Build a golden set: {itemId, humanScore}[].
+ *   2. Run candidate judges; each produces {itemId, score}.
+ *   3. `calibrateJudge(golden, candidate)` reports κ + Pearson + MAE.
+ *   4. Run bias probes (positional, verbosity, self-preference) to
+ *      detect systematic score inflation.
+ *
+ * Returns actionable diagnostics, not a single number. Consumers then
+ * decide whether to trust the judge, retrain it, or add a tie-breaker.
+ */
+interface GoldenItem {
+    itemId: string;
+    humanScore: number;
+    /** Optional group used for per-group bias audits (e.g. model-of-output family). */
+    group?: string;
+}
+interface CandidateScore {
+    itemId: string;
+    score: number;
+    /** Optional — enables positional-bias analysis (did order matter?). */
+    positionOfAInput?: 'first' | 'second';
+}
+interface CalibrationResult {
+    n: number;
+    pearson: number;
+    /** Cohen's κ with quadratic weights over integer-rounded scores. */
+    kappa: number;
+    /** Mean absolute error vs human. */
+    mae: number;
+    /** Worst-5 miscalibrations (largest |judge - human|). */
+    worstItems: Array<{
+        itemId: string;
+        judge: number;
+        human: number;
+        delta: number;
+    }>;
+}
+declare function calibrateJudge(golden: GoldenItem[], candidate: CandidateScore[]): CalibrationResult;
+interface PositionalBiasResult {
+    /**
+     * Score delta (first-position - second-position) averaged across items
+     * presented in both positions. Non-zero = positional bias.
+     */
+    avgDelta: number;
+    n: number;
+}
+/**
+ * Feed the same items to the judge twice with A/B swapped and pass all
+ * results here. Items that don't appear in both positions are ignored.
+ */
+declare function positionalBias(scores: CandidateScore[]): PositionalBiasResult;
+interface VerbosityBiasResult {
+    /** Pearson correlation between output length and score. Strong positive = verbosity bias. */
+    pearson: number;
+    n: number;
+}
+declare function verbosityBias(samples: Array<{
+    outputLen: number;
+    score: number;
+}>): VerbosityBiasResult;
+interface SelfPreferenceResult {
+    /** Mean judge score when judge's family matches output's family. */
+    inFamilyMean: number;
+    outOfFamilyMean: number;
+    deltaMean: number;
+    n: number;
+}
+/**
+ * Pass the same scenarios scored with judge-model X grading outputs from
+ * model X (in-family) and model Y (out-of-family). Non-zero delta
+ * indicates self-preference.
+ */
+declare function selfPreference(samples: Array<{
+    score: number;
+    inFamily: boolean;
+}>): SelfPreferenceResult;
+
+/**
+ * CI gate — evaluate a corpus against threshold contracts and generate
+ * a human-readable PR/build comment.
+ *
+ * Three layers:
+ *   1. `ThresholdContract` declarations (YAML-equivalent TS objects)
+ *   2. `evaluateContract` runs the contracts against a TraceStore and
+ *      returns a structured report + overall pass/fail.
+ *   3. `renderMarkdownReport` formats the report for GitHub PR comments.
+ *
+ * Consumers wrap this in their own `gh pr comment` / CI integration —
+ * we don't ship the GitHub Action binary, just the library call that
+ * the action invokes.
+ */
+
+interface ContractMetric {
+    /** Metric id matching either a predefined key or a custom extractor. */
+    metric: string;
+    higherIsBetter: boolean;
+    /** Max tolerated regression (e.g. 0.02 = 2pp worse than baseline). */
+    maxRegression?: number;
+    /** Optional extractor if the metric isn't in the default set. */
+    extract?: (run: Run, store: TraceStore) => Promise<number | null>;
+}
+interface ThresholdContract {
+    name: string;
+    baseline: RunFilter;
+    candidate: RunFilter;
+    metrics: ContractMetric[];
+    slos?: Slo[];
+}
+interface ContractReport {
+    name: string;
+    baselineReport: BaselineReport;
+    sloReport?: SloReport;
+    breaches: string[];
+    pass: boolean;
+}
+declare function evaluateContract(store: TraceStore, contract: ThresholdContract): Promise<ContractReport>;
+declare function renderMarkdownReport(reports: ContractReport[]): string;
+
+/**
+ * Observability adapters — bidirectional parity with production backends.
+ *
+ * `LangfuseAdapter` maps a Run's spans into Langfuse generation/score
+ * records (schema-compatible; we don't depend on the SDK — consumers
+ * POST the returned JSON to their Langfuse collector).
+ *
+ * `PrometheusEmitter` converts a TraceStore into a Prometheus text-
+ * exposition-format string (counters + gauges for runs, tool calls,
+ * errors, cost). Drop into a `/metrics` handler; no SDK needed.
+ *
+ * `replayTraceThroughJudge` is the canonical "re-score with a new
+ * judge" path — takes an existing run, runs a judge function over
+ * each LLM span, emits JudgeVerdict spans back into the store.
+ */
+
+interface LangfuseGeneration {
+    id: string;
+    traceId: string;
+    name: string;
+    model: string;
+    input: unknown;
+    output: unknown;
+    startTime: string;
+    endTime: string;
+    usage: {
+        input: number;
+        output: number;
+        total: number;
+        totalCost: number;
+    };
+    metadata: Record<string, unknown>;
+}
+interface LangfuseScore {
+    id: string;
+    traceId: string;
+    observationId: string;
+    name: string;
+    value: number;
+    comment?: string;
+}
+interface LangfuseEnvelope {
+    traceId: string;
+    generations: LangfuseGeneration[];
+    scores: LangfuseScore[];
+}
+declare function toLangfuseEnvelope(store: TraceStore, runId: string): Promise<LangfuseEnvelope>;
+declare function toPrometheusText(store: TraceStore): Promise<string>;
+interface JudgeReplayResult {
+    spanId: string;
+    targetSpanId: string;
+    dimension: string;
+    score: number;
+    rationale?: string;
+}
+/**
+ * Apply a judge function to every LLM span in a run and record the
+ * results as JudgeVerdict spans. This is the canonical "no re-execution"
+ * re-scoring path — you supply a pure judge `(llmSpan) → verdict`.
+ */
+declare function replayTraceThroughJudge(store: TraceStore, runId: string, judge: {
+    id: string;
+    dimension: string;
+    score: (span: LlmSpan) => Promise<{
+        score: number;
+        rationale?: string;
+        evidence?: string;
+    }>;
+}): Promise<JudgeReplayResult[]>;
+
+/**
+ * Paraphrase robustness — mutates a scenario prompt in structure-
+ * preserving ways, re-scores, and reports score variance.
+ *
+ * Mutators are pure functions `(prompt: string) => string`. Ship a
+ * default set; consumers add domain-specific ones.
+ *
+ * Robustness score: 1 - stdDev(scores) / (mean if positive else 1).
+ * A perfect agent returns the same answer regardless of typo / case /
+ * reordering — any variance signals a brittle prompt.
+ */
+type Mutator = (prompt: string, seed: number) => string;
+interface RobustnessResult {
+    originalScore: number;
+    variantScores: Array<{
+        mutator: string;
+        score: number;
+        mutated: string;
+    }>;
+    meanScore: number;
+    stdDev: number;
+    robustness: number;
+}
+declare function paraphraseRobustness(prompt: string, mutators: Array<{
+    id: string;
+    fn: Mutator;
+}>, scoreFn: (prompt: string) => Promise<number>, options?: {
+    seed?: number;
+}): Promise<RobustnessResult>;
+/** Lowercase the whole prompt. Robust models ignore case. */
+declare const lowercaseMutator: Mutator;
+/** Reorder sentences. Robust models don't depend on sentence order. */
+declare const sentenceReorderMutator: Mutator;
+/** Swap adjacent letter pairs (1 per 40 chars, min 1). Robust models tolerate typos. */
+declare const typoMutator: Mutator;
+/** Add a benign politeness prefix. Robust models ignore flattery. */
+declare const politenessPrefixMutator: Mutator;
+/** Compact whitespace, strip newlines. Robust models don't depend on formatting. */
+declare const whitespaceCollapseMutator: Mutator;
+declare const DEFAULT_MUTATORS: Array<{
+    id: string;
+    fn: Mutator;
+}>;
+
+/**
+ * Visual diff — pixel-delta scoring for UI / visual outputs.
+ *
+ * Minimal dependency-free implementation: accepts two PNGs as byte
+ * arrays + width/height and returns a Δ ratio + per-channel histogram.
+ * Consumers supply the decoded pixel arrays (we don't pull a PNG
+ * decoder into the core — use `sharp`, `@napi-rs/canvas`, or Playwright
+ * in the driving test and pass the result here).
+ */
+interface ImageData {
+    width: number;
+    height: number;
+    /** Pixel data in RGBA order, 4 bytes per pixel. */
+    data: Uint8Array | Uint8ClampedArray;
+}
+interface VisualDiffResult {
+    /** Ratio of pixels differing beyond `tolerance` (0..1). */
+    diffRatio: number;
+    differingPixels: number;
+    totalPixels: number;
+    maxChannelDelta: number;
+    /** Status for dashboards: unchanged (< 0.1%), changed, or severely-changed (> 5%). */
+    status: 'unchanged' | 'changed' | 'severely-changed';
+}
+interface VisualDiffOptions {
+    /** Pixels whose max-channel delta is ≤ this are considered unchanged. Default 8/255. */
+    tolerance?: number;
+}
+declare function visualDiff(a: ImageData, b: ImageData, options?: VisualDiffOptions): VisualDiffResult;
+/** Convenience: diffs two byte-identical-dim RGBA arrays, returns just the ratio. */
+declare function pixelDeltaRatio(a: Uint8Array, b: Uint8Array, width: number, height: number, tolerance?: number): number;
+
+/**
+ * BuilderSession — ties a builder-of-builders workflow together.
+ *
+ * Models agent-builder's shape: Project → Chat → Edit → Ship → App →
+ * AppAgent. Each layer is a Run (linked via parentRunId). The
+ * framework-enforced invariants:
+ *
+ *   - One Project → many Chats; chatId scopes runs within a project.
+ *   - One Chat = one builder Run with `layer='builder'`.
+ *   - One Ship = one child Run with `layer='app-build'` + SandboxHarness.
+ *   - One AppScenario = one grandchild Run with `layer='app-runtime'`.
+ *
+ * Consumers obtain a BuilderSession, call `startChat`, drive the
+ * builder agent (emitting spans), and call `ship` / `runAppScenario`
+ * as the workflow progresses. The session reconstructs itself from
+ * trace data via `resume(store, projectId)`.
+ */
+
+interface BuilderSessionInit {
+    projectId: string;
+    chatId?: string;
+    /** Free-form: user's task description, project name, etc. Stored on the builder Run. */
+    tags?: Record<string, string>;
+}
+interface ShipOptions {
+    harness: HarnessConfig;
+    driver?: SandboxDriver;
+    /** scenarioId of this app-build run. Defaults to `${projectId}/build`. */
+    scenarioId?: string;
+}
+interface RunAppScenarioOptions {
+    scenario: TestGradedScenario;
+    /** Harness driver override; defaults to the one the session was created with. */
+    driver?: SandboxDriver;
+}
+declare class BuilderSession {
+    private store;
+    private builderEmitter;
+    readonly projectId: string;
+    readonly chatId: string;
+    private builderRunId?;
+    private lastBuildRunId?;
+    private defaultDriver?;
+    constructor(store: TraceStore, init: BuilderSessionInit, driver?: SandboxDriver);
+    /** Start the builder (L0) run for this chat. Returns the runId. */
+    startChat(scenarioId?: string): Promise<string>;
+    /** The emitter for builder-level spans (edits, LLM calls, tool invocations). */
+    get emitter(): TraceEmitter;
+    /**
+     * Ship the project's generated app: run the sandbox harness as a child
+     * Run (`layer='app-build'`). Returns the build result + runId.
+     */
+    ship(options: ShipOptions): Promise<{
+        runId: string;
+        result: SandboxHarnessResult;
+    }>;
+    /**
+     * Run a domain scenario against the just-built app as a grandchild Run
+     * (`layer='app-runtime'`). The `ship` call must precede this so the
+     * parent is set correctly; if no build exists yet the session attaches
+     * directly to the builder run (useful for prototypes).
+     */
+    runAppScenario(options: RunAppScenarioOptions): Promise<TestGradedRunResult>;
+    /** Record an end-of-chat meta score (judge verdict on whether the builder
+     *  served the user's intent). Accepts a numeric score + optional rationale. */
+    recordMetaScore(score: number, rationale?: string): Promise<void>;
+    /** Close the builder Run with a final outcome. */
+    endChat(outcome: {
+        pass: boolean;
+        score?: number;
+        notes?: string;
+    }): Promise<void>;
+    /**
+     * Inline app-runtime run — for cases where the "scenario" isn't a
+     * SWE-bench-style test suite but a live agent interaction (LLM chat,
+     * domain flow). Returns an emitter bound to a fresh Run in the
+     * `app-runtime` layer; caller emits spans inside and calls
+     * `.endRun()` with the final verdict.
+     */
+    startAppRuntime(scenarioId: string): Promise<TraceEmitter>;
+    /**
+     * Lightweight "ship marker" — record an app-build Run with a caller-
+     * provided verdict. Use when there isn't a sandbox harness to run but
+     * you still want to mark the build state at publish time.
+     */
+    recordShipMarker(args: {
+        pass: boolean;
+        score: number;
+        scenarioId?: string;
+        notes?: string;
+    }): Promise<string>;
+    get lastBuildRunIdValue(): string | undefined;
+    get builderRunIdValue(): string | undefined;
+}
+/**
+ * Reconstruct the most recent BuilderSession state for a given project —
+ * returns { builderRunId, lastBuildRunId, chatRuns }. For chat-first UIs
+ * this is how a resumed session finds its place in the edit history.
+ */
+declare function resumeBuilderSession(store: TraceStore, projectId: string): Promise<{
+    projectId: string;
+    chatRuns: Run[];
+    lastBuilderRun?: Run;
+    lastBuildRun?: Run;
+    lastAppRuntimeRuns: Run[];
+}>;
+
+/**
+ * Three-layer evaluation — the canonical scoring breakdown for
+ * builder-of-builders workflows.
+ *
+ *   meta_score:    did the builder understand + satisfy user intent?
+ *                  (judge verdict attached to the builder run)
+ *   build_score:   did the generated scaffold build + pass its own tests?
+ *                  (outcome.score on the app-build child run)
+ *   runtime_score: did the generated agent pass its domain scenarios?
+ *                  (mean outcome.score over app-runtime grandchild runs)
+ *
+ * Returns a structured report per project. The cross-layer correlation
+ * is the highest-leverage signal the framework computes — if
+ * meta_score doesn't predict runtime_score, the builder's self-scoring
+ * is broken.
+ */
+
+interface ThreeLayerProjectReport {
+    projectId: string;
+    builderRunId?: string;
+    /** Judge-verdict score on the builder run (0..1 after normalization). */
+    metaScore: number | null;
+    buildRunId?: string;
+    /** 0..1 from the sandbox harness (testsPassed / testsTotal). */
+    buildScore: number | null;
+    appRuntimeRunIds: string[];
+    /** Mean of outcome.score over app-runtime runs, 0..1. */
+    runtimeScore: number | null;
+    runtimePassRate: number | null;
+    /** True when all three layers produced a score. */
+    complete: boolean;
+}
+declare function scoreProject(store: TraceStore, projectId: string): Promise<ThreeLayerProjectReport>;
+/** Aggregate scoring across every project in a corpus. */
+declare function scoreAllProjects(store: TraceStore): Promise<ThreeLayerProjectReport[]>;
+
+/**
+ * Meta-eval correlation — the highest-leverage signal in the framework.
+ *
+ * Given a corpus of three-layer project reports, compute how well each
+ * pair of layers correlates. The question we care about most:
+ *
+ *   Does `metaScore` (what the builder thinks it did) predict
+ *   `runtimeScore` (what the user actually gets)?
+ *
+ * If r < ~0.4, the builder's self-scoring is broken — it's optimizing
+ * for something other than real-world success. If r > 0.7, meta_score
+ * is a usable proxy and can drive CI gates cheaply.
+ *
+ * Non-parametric rank correlation (Spearman) is also reported because
+ * meta scores are often ordinal-ish.
+ */
+
+interface LayerCorrelation {
+    n: number;
+    pearson: number;
+    spearman: number;
+}
+interface CorrelationReport {
+    /** Pairs present in the corpus (layers with ≥ 2 matched data points). */
+    metaVsBuild?: LayerCorrelation;
+    metaVsRuntime?: LayerCorrelation;
+    buildVsRuntime?: LayerCorrelation;
+    /** Number of complete projects (all 3 scores present). */
+    completeProjects: number;
+}
+declare function correlateLayers(reports: ThreeLayerProjectReport[]): CorrelationReport;
+
+/**
+ * ProjectRegistry — project-level aggregation over the trace corpus.
+ *
+ * Thin reader over TraceStore that answers the questions a chat-first,
+ * resumable UI needs:
+ *   - listProjects() → project IDs with latest activity
+ *   - projectTimeline(id) → chats + builds + runtime runs, chronological
+ *   - projectChats(id) → chat-level summaries (turn count, outcome)
+ *
+ * All queries are pure reads; no state duplication.
+ */
+
+interface ProjectSummary {
+    projectId: string;
+    chatCount: number;
+    buildCount: number;
+    appRuntimeCount: number;
+    lastActivityAt: number;
+    latestChatId?: string;
+    latestOutcome?: {
+        pass: boolean;
+        score?: number;
+    };
+}
+interface ChatSummary {
+    chatId: string;
+    projectId: string;
+    builderRunId: string;
+    startedAt: number;
+    endedAt?: number;
+    status: Run['status'];
+    outcome?: Run['outcome'];
+    /** Counts of spans emitted during the chat. */
+    llmTurns?: number;
+    toolCalls?: number;
+    buildRunId?: string;
+    appRuntimeRunIds: string[];
+}
+interface ProjectTimelineEntry {
+    run: Run;
+    layerBucket: 'chat' | 'build' | 'runtime' | 'other';
+}
+declare class ProjectRegistry {
+    private store;
+    constructor(store: TraceStore);
+    listProjects(): Promise<ProjectSummary[]>;
+    projectTimeline(projectId: string): Promise<ProjectTimelineEntry[]>;
+    projectChats(projectId: string): Promise<ChatSummary[]>;
+}
+
+/**
+ * OutcomeStore — deployment outcomes attached to Run IDs.
+ *
+ * Outcomes arrive asynchronously from production telemetry after the
+ * eval run completed: user ratings, retention flags, conversion events,
+ * revenue, support-ticket rate, anything a product team can measure.
+ * The store is a peer to TraceStore — separate lifecycle, same runId
+ * foreign key.
+ *
+ * The whole point of this module is to make the meta-eval correlation
+ * question computable: `correlate(evalMetric, outcomeMetric) → r, ρ, n, CI`.
+ */
+interface DeploymentOutcome {
+    runId: string;
+    capturedAt: number;
+    /** Numeric outcomes keyed by name — retention_7d, csat, revenue_usd, etc. */
+    metrics: Record<string, number>;
+    /** Dimensions for stratified analysis — cohort, region, user_segment. */
+    labels?: Record<string, string>;
+    /** Free-form provenance (source system, pipeline version). */
+    source?: string;
+}
+interface OutcomeFilter {
+    runIds?: string[];
+    since?: number;
+    until?: number;
+    label?: {
+        key: string;
+        value: string;
+    };
+    source?: string;
+}
+interface OutcomeStore {
+    append(outcome: DeploymentOutcome): Promise<void>;
+    /** All outcomes attached to this run (a single run can have many — multiple
+     *  capture windows over deployment time). */
+    forRun(runId: string): Promise<DeploymentOutcome[]>;
+    list(filter?: OutcomeFilter): Promise<DeploymentOutcome[]>;
+}
+declare class InMemoryOutcomeStore implements OutcomeStore {
+    private items;
+    append(outcome: DeploymentOutcome): Promise<void>;
+    forRun(runId: string): Promise<DeploymentOutcome[]>;
+    list(filter?: OutcomeFilter): Promise<DeploymentOutcome[]>;
+}
+interface FileSystemOutcomeStoreOptions {
+    dir: string;
+    maxBytes?: number;
+}
+declare class FileSystemOutcomeStore implements OutcomeStore {
+    private dir;
+    private maxBytes;
+    private memo?;
+    private loaded;
+    constructor(options: FileSystemOutcomeStoreOptions);
+    private ensureDir;
+    append(outcome: DeploymentOutcome): Promise<void>;
+    private load;
+    forRun(runId: string): Promise<DeploymentOutcome[]>;
+    list(filter?: OutcomeFilter): Promise<DeploymentOutcome[]>;
+}
+
+/**
+ * Correlation study — "does our eval score predict real-world outcomes?"
+ *
+ * This is the load-bearing signal. Takes a TraceStore + OutcomeStore,
+ * joins on runId, computes Pearson + Spearman + bootstrap CI for every
+ * (evalMetric, outcomeMetric) pair the caller declares.
+ *
+ * Without this number the framework is ornamental. With it and r > 0.6
+ * the framework is a moat — no other agent-eval tool publishes one.
+ */
+
+interface EvalMetricSpec {
+    id: string;
+    /** Extract a scalar from a run (defaults cover score/pass/durationMs/costUsd/tokens). */
+    extract?: (run: Run, store: TraceStore) => Promise<number | null>;
+}
+interface OutcomePair {
+    evalMetric: string;
+    outcomeMetric: string;
+}
+interface CorrelationResult {
+    evalMetric: string;
+    outcomeMetric: string;
+    n: number;
+    pearson: number;
+    spearman: number;
+    /** 95% bootstrap CI for Pearson. */
+    pearsonCi95: {
+        lower: number;
+        upper: number;
+    };
+    /** Rough verdict: 'strong' ≥ 0.7, 'moderate' ≥ 0.4, else 'weak'. */
+    verdict: 'strong' | 'moderate' | 'weak';
+}
+interface CorrelationStudyResult {
+    pairs: CorrelationResult[];
+    joinedSamples: number;
+    skippedRuns: number;
+}
+interface CorrelationStudyOptions {
+    /** Only join outcomes captured within this window after run.startedAt. */
+    maxCaptureLagMs?: number;
+    /** Restrict to a subset of outcomes (cohort, region, source). */
+    outcomeFilter?: OutcomeFilter;
+    /** Which outcome per run to use when multiple exist. Default 'latest'. */
+    reduction?: 'latest' | 'mean' | 'max';
+    /** Bootstrap iterations for the CI. Default 500. */
+    bootstrapIterations?: number;
+}
+declare function correlationStudy(traceStore: TraceStore, outcomeStore: OutcomeStore, evalMetrics: EvalMetricSpec[], outcomeMetricNames: string[], options?: CorrelationStudyOptions): Promise<CorrelationStudyResult>;
+
+/**
+ * Calibration curve — binned "if eval says X, what does reality show?"
+ *
+ * Companion to correlationStudy. Raw correlation is a single number;
+ * the calibration curve shows *where* the eval is well-calibrated vs
+ * overconfident / underconfident. Buckets the eval metric, computes
+ * mean outcome per bucket, reports expected-calibration-error (ECE).
+ */
+
+interface CalibrationBin {
+    lower: number;
+    upper: number;
+    n: number;
+    evalMean: number;
+    outcomeMean: number;
+    /** |outcomeMean − evalMean|; contributes to ECE weighted by n/total. */
+    gap: number;
+}
+interface CalibrationReport {
+    evalMetric: string;
+    outcomeMetric: string;
+    n: number;
+    bins: CalibrationBin[];
+    /** Expected Calibration Error — Σ (n_i/N) × |outcomeMean_i − evalMean_i|. */
+    ece: number;
+    /** Max bin gap — upper bound on miscalibration. */
+    maxGap: number;
+}
+interface CalibrationOptions {
+    bins?: number;
+    /** Equal-width (fixed bin edges) or equal-frequency (quantile bins). */
+    binning?: 'equal-width' | 'equal-frequency';
+    /** Clip eval values to [lo, hi] before binning. */
+    range?: {
+        lo: number;
+        hi: number;
+    };
+}
+declare function calibrationCurve(traceStore: TraceStore, outcomeStore: OutcomeStore, evalMetric: EvalMetricSpec, outcomeMetric: string, options?: CalibrationOptions): Promise<CalibrationReport | null>;
+
+/**
+ * Process Reward Modeling — per-step rubric grading.
+ *
+ * A StepRubric inspects one span and returns a score + rationale.
+ * PrmGrader applies an array of rubrics to every LLM span in a
+ * trajectory (consumers can broaden to tool/retrieval spans via the
+ * `kind` filter on each rubric).
+ *
+ * Why this matters: outcome-only eval (did the final artifact work?)
+ * gives sparse reward — most agent turns are unattributable. PRMs
+ * densify the signal so optimizers and RL fine-tuning can assign
+ * credit per turn.
+ */
+
+interface StepContext {
+    trajectory: Trajectory;
+    step: TrajectoryStep;
+    /** Steps preceding `step` in trajectory order. */
+    prior: TrajectoryStep[];
+    /** Steps following `step`. */
+    next: TrajectoryStep[];
+}
+interface StepRubric {
+    id: string;
+    /** Only grade spans of these kinds (default: all). */
+    kinds?: Array<Span['kind']>;
+    /** Weight in the aggregate score. Default 1. */
+    weight?: number;
+    /** Returns score in 0..1 + optional rationale/evidence. Return `null` to
+     *  skip grading (rubric doesn't apply to this step). */
+    grade: (ctx: StepContext) => Promise<{
+        score: number;
+        rationale?: string;
+        evidence?: string;
+    } | null>;
+}
+interface GradedStep {
+    spanId: string;
+    rubricId: string;
+    score: number;
+    weight: number;
+    rationale?: string;
+    evidence?: string;
+}
+interface PrmGradedTrace {
+    runId: string;
+    steps: GradedStep[];
+    /** Weighted mean of all graded steps; 0..1. */
+    aggregateScore: number;
+    /** Number of spans graded — useful for sanity-checking coverage. */
+    gradedCount: number;
+    /** Number of spans in the trajectory that no rubric matched. */
+    ungradedCount: number;
+}
+declare class PrmGrader {
+    private rubrics;
+    constructor(rubrics: StepRubric[]);
+    /**
+     * Grade every eligible span in a run. Emits a JudgeVerdict span for each
+     * (rubric × span) verdict so the result is visible to downstream pipelines
+     * (judgeAgreementView, etc.) — PRM is just "a judge that runs per span."
+     */
+    grade(store: TraceStore, runId: string): Promise<PrmGradedTrace>;
+}
+/** Helper: reads JudgeVerdict spans that PRM emitted so downstream pipelines
+ *  can distinguish PRM verdicts from human or top-level LLM judges. */
+declare function isPrmVerdict(verdict: JudgeSpan): boolean;
+
+/**
+ * Built-in reference rubrics. Consumers combine these with domain
+ * rubrics. All are deterministic, rule-based — cheap to run + easy
+ * to unit-test. LLM-based rubrics are trivially authored by
+ * following the StepRubric contract.
+ */
+
+/** Penalize very short or very long assistant outputs. */
+declare function outputLengthRubric(args?: {
+    minChars?: number;
+    maxChars?: number;
+    weight?: number;
+}): StepRubric;
+/** Reward tool calls that succeeded (status='ok') with an informative result. */
+declare function toolSuccessRubric(args?: {
+    weight?: number;
+}): StepRubric;
+/** Penalize tool calls that duplicate a prior call with identical args. */
+declare function toolNonRedundantRubric(args?: {
+    weight?: number;
+}): StepRubric;
+/** Penalize LLM outputs that contain common refusal markers when a refusal
+ *  is NOT expected (caller inverts weight for scenarios where refusal IS expected). */
+declare function nonRefusalRubric(args?: {
+    markers?: RegExp[];
+    weight?: number;
+}): StepRubric;
+/** Reward outputs that invoke the next-step tool the trajectory actually uses
+ *  (i.e. the LLM span announced "I will call X" and the following tool span IS X). */
+declare function toolIntentAlignmentRubric(args?: {
+    weight?: number;
+}): StepRubric;
+
+/**
+ * Export PRM-graded traces as training data for downstream reward-model
+ * fine-tuning. Canonical format is NDJSON of
+ * `{ trajectory_text, step_index, rubric, score }` so a small model can
+ * learn to predict step rewards from step context.
+ *
+ * The framework doesn't train the model — we emit the data; callers
+ * plug it into their preferred trainer (TRL, Unsloth, custom).
+ */
+
+interface PrmTrainingSample {
+    runId: string;
+    spanId: string;
+    rubricId: string;
+    score: number;
+    /** Serialized step context — step + surrounding conversation. */
+    context: {
+        priorTurns: Array<{
+            role: string;
+            content: string;
+        }>;
+        step: {
+            kind: Span['kind'];
+            text: string;
+        };
+    };
+    /** Optional evidence + rationale for auditability. */
+    rationale?: string;
+    evidence?: string;
+}
+declare function exportTrainingData(store: TraceStore, graded: PrmGradedTrace[], options?: {
+    contextWindow?: number;
+}): Promise<PrmTrainingSample[]>;
+/** NDJSON serialization — write to file or stream directly to a trainer. */
+declare function toNdjson(samples: PrmTrainingSample[]): string;
+
+/**
+ * Inference-time PRM scoring — pick the best of N candidate trajectories
+ * using a trained reward model (or a rule-based PRM as a proxy).
+ *
+ * The canonical Best-of-N pattern: generate N completions, score each
+ * with a PRM, pick the winner. Here the scoring loop is framework-agnostic
+ * — supply a TraceStore + PrmGrader + N run IDs → get ranking + winner.
+ */
+
+interface BestOfNResult {
+    winner: PrmGradedTrace;
+    ranked: PrmGradedTrace[];
+    /** Standard deviation of aggregate scores — small = candidates were homogenous. */
+    stdDev: number;
+}
+declare function prmBestOfN(store: TraceStore, grader: PrmGrader, runIds: string[]): Promise<BestOfNResult>;
+/**
+ * Weighted vote across multiple graders — use when you want a PRM ensemble
+ * (e.g. rule-based + LLM-based + trained model). Each grader produces its
+ * own ranking; we aggregate via rank-sum (Borda count) so no single grader
+ * dominates via a different score scale.
+ */
+declare function prmEnsembleBestOfN(store: TraceStore, graders: PrmGrader[], runIds: string[]): Promise<BestOfNResult>;
+
+/**
+ * Bisector — auto-locate the change that introduced an eval regression.
+ *
+ * Two shapes:
+ *   - `commitBisect` — walk an ordered SHA list, binary-search for the
+ *     first commit that fails.
+ *   - `promptBisect` — given a good and bad prompt, progressively port
+ *     paragraphs from good→bad to localize the breaking change.
+ *
+ * Generic `bisect<T>` lets callers drive any ordered state space
+ * (dataset versions, config files, CLI flag combinations).
+ */
+interface BisectOptions<T> {
+    /** State known to pass. */
+    good: T;
+    /** State known to fail. */
+    bad: T;
+    /** Equality test on state values — default Object.is. */
+    equals?: (a: T, b: T) => boolean;
+    /** Pick the halfway state between good + bad. Return null when no further
+     *  split is possible (e.g. adjacent commits). */
+    halfway: (good: T, bad: T) => T | null;
+    /** Produce a verdict for a state. */
+    runEval: (state: T) => Promise<{
+        score: number;
+        pass: boolean;
+    }>;
+    /** Hard cap on iterations (default 40 — covers ~1T ordered states). */
+    maxIterations?: number;
+}
+interface BisectStep<T> {
+    state: T;
+    score: number;
+    pass: boolean;
+}
+interface BisectResult<T> {
+    /** The first bad state — typically `bad` in the final (good, bad) adjacent pair. */
+    culprit: T;
+    /** Ordered trace of all states evaluated. */
+    path: BisectStep<T>[];
+    /** True when we narrowed to an adjacent (good, bad) pair. */
+    converged: boolean;
+    /** True when `good` itself failed or `bad` itself passed — the caller's
+     *  premise was broken. */
+    inputInconsistent: boolean;
+}
+declare function bisect<T>(options: BisectOptions<T>): Promise<BisectResult<T>>;
+/**
+ * Commit bisect — `commits` is an ordered SHA list, oldest to newest.
+ * `good` and `bad` must both be present in the list.
+ */
+declare function commitBisect(options: {
+    commits: string[];
+    good: string;
+    bad: string;
+    runEval: (sha: string) => Promise<{
+        score: number;
+        pass: boolean;
+    }>;
+    maxIterations?: number;
+}): Promise<BisectResult<string>>;
+/**
+ * Prompt bisect — splits the good and bad prompts into paragraphs, then
+ * progressively replaces paragraphs in `good` with their counterparts
+ * from `bad` to localize the offending change. Only works when the two
+ * prompts have the same paragraph count (a common editorial workflow
+ * constraint — one paragraph = one change unit).
+ */
+declare function promptBisect(options: {
+    good: string;
+    bad: string;
+    runEval: (prompt: string) => Promise<{
+        score: number;
+        pass: boolean;
+    }>;
+    maxIterations?: number;
+    paragraphSplitter?: (prompt: string) => string[];
+}): Promise<BisectResult<string> & {
+    offendingParagraphIndex?: number;
+}>;
+
+/**
+ * Counterfactual replay — "what would have happened if we'd changed
+ * exactly one thing at turn N?"
+ *
+ * The framework does NOT drive the agent — it sets up the replay
+ * context (prior spans, prior state, mutation spec) and records the
+ * resulting divergence. Consumers supply an `executeFrom(ctx)` callback
+ * that runs their agent starting from turn N with the mutation applied.
+ *
+ * Counterfactual runs are recorded as a new Run with `layer='meta'` and
+ * `parentRunId = originalRunId`, so downstream diff + correlation
+ * pipelines see them natively.
+ */
+
+type CounterfactualMutation = {
+    kind: 'swap-model';
+    at: number;
+    newModel: string;
+} | {
+    kind: 'swap-tool-result';
+    at: number;
+    newResult: unknown;
+} | {
+    kind: 'truncate-after';
+    at: number;
+} | {
+    kind: 'inject-system-message';
+    at: number;
+    content: string;
+} | {
+    kind: 'custom';
+    at: number;
+    describe: string;
+    apply: (step: TrajectoryStep) => TrajectoryStep;
+};
+interface CounterfactualContext {
+    originalRunId: string;
+    originalTrajectory: Trajectory;
+    /** Steps up to (but not including) the mutation point — the prefix the
+     *  replayed agent inherits as its prior conversation/tool history. */
+    prefix: TrajectoryStep[];
+    mutation: CounterfactualMutation;
+    /** Pre-applied mutation on the step at `mutation.at`. Consumers use this
+     *  as the FIRST step the replayed agent emits (they decide whether to
+     *  re-emit it or continue from there). */
+    mutatedStep: TrajectoryStep;
+}
+interface CounterfactualResult {
+    counterfactualRunId: string;
+    originalRunId: string;
+    mutation: CounterfactualMutation;
+    /** Structured delta summary — caller can extend via scoring. */
+    delta: {
+        originalOutcomeScore: number | null;
+        counterfactualOutcomeScore: number | null;
+        deltaScore: number | null;
+    };
+}
+interface CounterfactualRunner {
+    /**
+     * Execute the agent from `ctx.prefix` with the mutation applied.
+     * MUST emit spans into the provided emitter so they become part of
+     * the counterfactual run. MUST call emitter.endRun() with a verdict.
+     */
+    executeFrom: (ctx: CounterfactualContext, emitter: TraceEmitter) => Promise<void>;
+}
+declare function runCounterfactual(store: TraceStore, originalRunId: string, mutation: CounterfactualMutation, runner: CounterfactualRunner): Promise<CounterfactualResult>;
+/**
+ * Aggregate a batch of counterfactuals into a simple attribution table:
+ * which mutation kinds move outcomes most? (Useful when you run a grid
+ * over the same trajectory — swap-model at every llm span, swap-tool
+ * at every tool span — and want a ranked summary.)
+ */
+declare function attributeCounterfactuals(results: CounterfactualResult[]): Array<{
+    mutationKind: CounterfactualMutation['kind'];
+    n: number;
+    meanAbsDelta: number;
+    meanSignedDelta: number;
+}>;
+
+/**
+ * Full cross-trace diff — align two trajectories step-by-step, report
+ * per-step score deltas, attribute a variant's total outcome lead to
+ * specific turns.
+ *
+ * 0.5 shipped `firstDivergenceView` (finds the first differing step).
+ * This does the heavier work: full alignment via LCS, per-step
+ * contribution to score delta using PRM verdicts when available,
+ * fallback to structural heuristics (latency, token count, tool
+ * outcome) otherwise.
+ */
+
+type AlignmentOp = {
+    op: 'match';
+    a: TrajectoryStep;
+    b: TrajectoryStep;
+} | {
+    op: 'insert';
+    b: TrajectoryStep;
+} | {
+    op: 'delete';
+    a: TrajectoryStep;
+} | {
+    op: 'replace';
+    a: TrajectoryStep;
+    b: TrajectoryStep;
+};
+interface StepAttribution {
+    op: AlignmentOp;
+    /** Difference in PRM score (or null when not scored by a matching judge). */
+    prmDelta: number | null;
+    /** Difference in latency (endedAt - startedAt). */
+    latencyDeltaMs: number | null;
+    /** Difference in token count (LLM spans). */
+    tokenDelta: number | null;
+    /** Reason this step is / isn't considered a contributor to the outcome delta. */
+    note: string;
+}
+interface CrossTraceDiff {
+    runA: string;
+    runB: string;
+    alignment: AlignmentOp[];
+    attributions: StepAttribution[];
+    /** Total score delta (B - A). */
+    totalScoreDelta: number | null;
+    /** Sum of PRM deltas across matched/replaced steps. Close to
+     *  `totalScoreDelta` when PRM covers the trajectory; gap indicates
+     *  unmodeled variance. */
+    prmDeltaSum: number;
+}
+interface CrossTraceDiffOptions {
+    stepEquals?: (a: TrajectoryStep, b: TrajectoryStep) => boolean;
+}
+declare function crossTraceDiff(store: TraceStore, runA: string, runB: string, options?: CrossTraceDiffOptions): Promise<CrossTraceDiff>;
+
+/**
+ * Pre-registered hypotheses — declare what you're testing BEFORE the
+ * run, check it AFTER. Prevents p-hacking, optional stopping, and the
+ * "we ran until it looked good" failure mode.
+ *
+ * Manifest is a plain JSON-friendly object. Sign it with a content hash
+ * + timestamp; the registered record becomes immutable. Post-run,
+ * evaluate the manifest against observed results — the library refuses
+ * to let you re-interpret a different metric as the declared one.
+ */
+interface HypothesisManifest {
+    id: string;
+    /** Human prose — goes into the audit trail. */
+    hypothesis: string;
+    /** Metric the hypothesis claims to move. */
+    metric: string;
+    /** 'increase' = candidate should score higher than baseline; 'decrease' = lower. */
+    direction: 'increase' | 'decrease';
+    /** Minimum effect size to count (same units as the metric). */
+    minEffect: number;
+    /** Alpha threshold. */
+    alpha: number;
+    /** Target statistical power at which sample size was pre-computed. */
+    power: number;
+    /** Declared N per arm before running. */
+    preRegisteredN: number;
+    /** ISO8601 timestamp the manifest was registered. */
+    registeredAt: string;
+    /** Optional identifiers to tie into the trace corpus. */
+    baselineLabel?: string;
+    candidateLabel?: string;
+}
+interface SignedManifest extends HypothesisManifest {
+    /** sha256 hex of canonicalized manifest (everything except contentHash). */
+    contentHash: string;
+}
+interface HypothesisResult {
+    manifest: SignedManifest;
+    observedN: number;
+    observedEffect: number;
+    observedPValue: number;
+    /** True iff the observed effect hits the pre-declared direction with
+     *  magnitude ≥ minEffect AND p < alpha. */
+    confirmed: boolean;
+    /** Enumerated reasons the hypothesis was rejected (each a machine-tag). */
+    rejectionReasons: Array<'wrong_direction' | 'effect_too_small' | 'not_significant' | 'undersampled'>;
+    notes?: string;
+}
+declare function signManifest(m: HypothesisManifest): Promise<SignedManifest>;
+/** Verify that a signed manifest has not been tampered with. */
+declare function verifyManifest(m: SignedManifest): Promise<boolean>;
+/**
+ * Evaluate a pre-registered hypothesis against observed results.
+ * Mechanical — no re-interpretation permitted.
+ */
+declare function evaluateHypothesis(manifest: SignedManifest, observed: {
+    n: number;
+    effect: number;
+    pValue: number;
+}): Promise<HypothesisResult>;
+
+/**
+ * Self-play scenario evolution — agents generate adversarial scenarios
+ * against each other; survivors become part of the eval corpus.
+ *
+ * Framework-agnostic about how scenarios are generated. Caller supplies:
+ *   - `propose`: asks a "proposer" agent for candidate scenarios
+ *   - `scoreAgainst`: runs a target agent against a scenario and returns
+ *     its score
+ *
+ * A scenario *survives* if it reveals a meaningful score difference
+ * between two target agents (or between a target agent and itself on
+ * different runs). Survivors are promoted to a Dataset; the caller
+ * decides what to do with them (hold-out, training, regression set).
+ *
+ * Guard rails: minimum absolute score delta to consider a scenario
+ * informative; floor on absolute target score so degenerate break-all
+ * scenarios (noise, gibberish) don't flood the corpus.
+ */
+
+interface CandidateScenario {
+    id: string;
+    payload: unknown;
+    /** Free-form tags (domain, generation, parent). */
+    tags?: Record<string, string>;
+}
+interface ScoredTarget {
+    targetId: string;
+    score: number;
+}
+interface EvolutionRound {
+    round: number;
+    proposed: CandidateScenario[];
+    survived: CandidateScenario[];
+    rejected: Array<{
+        candidate: CandidateScenario;
+        reason: string;
+    }>;
+    scoredBreakdown: Array<{
+        candidate: CandidateScenario;
+        scores: ScoredTarget[];
+        spread: number;
+    }>;
+}
+interface SelfPlayOptions {
+    /** Minimum score spread across targets for a scenario to survive. Default 0.1. */
+    minSpread?: number;
+    /** Minimum floor score across targets — keeps degenerate break-all scenarios
+     *  out. Default 0.1 (if every target scores below this, discard). */
+    minAbsoluteFloor?: number;
+    /** Hard cap on survivors per round. Default 50. */
+    maxSurvivors?: number;
+    /** Rounds to run. Default 1. Each round's survivors can be fed back into
+     *  `propose` to compound. */
+    rounds?: number;
+    /** Seed for scenario id generation if proposer doesn't provide one. */
+    seed?: number;
+}
+interface SelfPlayProposer {
+    propose(round: number, priorSurvivors: CandidateScenario[]): Promise<CandidateScenario[]>;
+}
+interface SelfPlayScorer {
+    /** Score one candidate against every target; returns parallel array. */
+    scoreCandidate(candidate: CandidateScenario, targets: string[]): Promise<ScoredTarget[]>;
+}
+declare function runSelfPlay(proposer: SelfPlayProposer, scorer: SelfPlayScorer, targets: string[], options?: SelfPlayOptions): Promise<{
+    rounds: EvolutionRound[];
+    dataset: Dataset;
+}>;
+
+/**
+ * Causal attribution via factorial experiments.
+ *
+ * Run every combination of {model × prompt × scenario × seed}, then
+ * decompose observed score variance into main effects + interactions.
+ * Moves from correlational "variant B is better" to causal "the model
+ * swap accounts for 42% of the lead; the prompt change accounts for 28%;
+ * interaction is 30%."
+ *
+ * Minimal implementation: 2-way factorial (two factors at a time) with
+ * main-effect + interaction decomposition via variance of cell means.
+ * Consumers run the factorial design themselves (we don't schedule
+ * runs); this module consumes the (factorLevels, observedScores)
+ * table and does the attribution math.
+ */
+interface FactorialCell {
+    /** Map factor name → level id. e.g. { model: 'claude', prompt: 'v2' } */
+    levels: Record<string, string>;
+    /** Observed score for this cell (mean over replications if n > 1). */
+    score: number;
+    /** Number of replications averaged to produce `score`. */
+    n: number;
+}
+interface FactorContribution {
+    factor: string;
+    /** Variance attributed to this factor's main effect, as a fraction of total. */
+    shareOfVariance: number;
+    /** Range of cell means across levels of this factor. */
+    range: number;
+}
+interface InteractionContribution {
+    factors: [string, string];
+    shareOfVariance: number;
+}
+interface CausalAttributionReport {
+    totalVariance: number;
+    mainEffects: FactorContribution[];
+    interactions: InteractionContribution[];
+    /** Residual = variance unexplained by main effects + modeled interactions. */
+    residualShare: number;
+    /** Sanity: shares sum to 1 (within fp). */
+    sharesSum: number;
+}
+declare function causalAttribution(cells: FactorialCell[]): CausalAttributionReport;
+
+/**
+ * Active learning — agent-as-scenario-author.
+ *
+ * Analyzes an existing Dataset + trace corpus for coverage gaps and
+ * weak spots, returns a prioritized list of *synthesis targets*:
+ * (gap description, existing-neighbor examples, suggested direction).
+ *
+ * Does NOT call an LLM itself — the proposer agent is caller-supplied.
+ * This module's job is to identify WHERE new scenarios would compound
+ * the most information, not to author them.
+ *
+ * Gaps we detect:
+ *   - dimensions with high score variance (unstable, need more data)
+ *   - dimensions with low coverage count (undersampled)
+ *   - failure classes with clusters (systematic weakness)
+ *   - difficulty bins with no coverage
+ */
+
+type SynthesisReason = 'high-variance' | 'undersampled' | 'failure-cluster' | 'difficulty-gap';
+interface SynthesisTarget {
+    reason: SynthesisReason;
+    description: string;
+    /** Existing scenarios that are closest to the gap; caller feeds these to
+     *  their LLM proposer as few-shot examples. */
+    neighbors: DatasetScenario[];
+    /** Suggested direction — e.g. "harder variants", "edge cases of X", "failure class Y". */
+    direction: string;
+    /** Priority score — higher = more information-dense gap. 0..1. */
+    priority: number;
+}
+interface ActiveLearningOptions {
+    /** Minimum scenarios per difficulty band to count as "covered". */
+    minPerBand?: number;
+    /** Variance threshold above which a scenario's dimension is "unstable". */
+    varianceThreshold?: number;
+    /** Max synthesis targets returned. */
+    topK?: number;
+}
+declare function proposeSynthesisTargets(dataset: Dataset, traceStore: TraceStore, options?: ActiveLearningOptions): Promise<SynthesisTarget[]>;
+
+/**
+ * Reward-model export — the productizable wrapper around PRM training
+ * data. Takes a TraceStore + PrmGrader, produces an embeddable
+ * inference scorer that customers plug into their own agent stack.
+ *
+ * Two export forms:
+ *   - `exportRewardModel(store, graders)` — serializes the (step-context,
+ *     score) corpus to a framework-agnostic payload. Customer fine-tunes
+ *     their own model; we ship the scaffolding.
+ *   - `loadScorerFromTraces(store, grader)` — a zero-deps "reward model"
+ *     that literally replays the trained rubric at inference time. Works
+ *     as a reference baseline + deterministic fallback.
+ */
+
+interface ExportedRewardModel {
+    /** Version of the export format. Bump when payload shape changes. */
+    version: '1.0';
+    /** Metadata about the training corpus. */
+    metadata: {
+        nTraces: number;
+        nSamples: number;
+        rubrics: string[];
+        exportedAt: string;
+        /** Mean reward across training corpus — use as sanity check at load. */
+        meanReward: number;
+    };
+    /** NDJSON training payload suitable for most fine-tuning frameworks. */
+    trainingNdjson: string;
+}
+declare function exportRewardModel(store: TraceStore, grader: PrmGrader, runIds: string[]): Promise<ExportedRewardModel>;
+/**
+ * Zero-deps inference scorer — apply a grader to a trajectory and return
+ * its aggregate score. This is the "reward model" customers embed when
+ * they don't want (or can't) fine-tune one. Deterministic + portable.
+ */
+interface InferenceScorer {
+    /** Score a completed trajectory. Higher is better. */
+    score(trajectory: Trajectory, store: TraceStore): Promise<number>;
+    metadata: {
+        rubrics: string[];
+        deterministic: true;
+    };
+}
+declare function loadScorerFromGrader(grader: PrmGrader): InferenceScorer;
+/**
+ * Replay a trace corpus through a scorer — produces the canonical
+ * "what would this reward model have said about every run?" table.
+ * Callers use this to validate a trained model against the training
+ * corpus (expect high agreement; drift indicates overfitting).
+ */
+declare function replayScorerOverCorpus(store: TraceStore, scorer: InferenceScorer, runIds: string[]): Promise<Array<{
+    runId: string;
+    score: number;
+    outcomeScore: number | null;
+}>>;
+
+/**
+ * Governance reporting — shared types.
+ *
+ * The framework collects a `GovernanceContext` (traces + outcomes +
+ * dataset manifests + red-team results + judge calibration) and each
+ * specific template (NIST AI RMF, SOC2, EU AI Act) renders a
+ * structured report from it.
+ *
+ * Reports are machine-readable JSON first; human-readable Markdown is a
+ * pure transform on top. External auditors consume the Markdown; CI
+ * consumes the JSON.
+ */
+
+interface GovernanceContext {
+    /** Legal / org identity for the report. */
+    organization: string;
+    /** System / agent identifier. */
+    systemName: string;
+    /** ISO8601 period the report covers. */
+    periodStart: string;
+    periodEnd: string;
+    /** Versioned dataset manifests used during the period. */
+    datasets: DatasetManifest[];
+    traceStore: TraceStore;
+    outcomeStore?: OutcomeStore;
+    /** Cached red-team results for the period, if available. */
+    redTeam?: RedTeamReport;
+    /** Judge-vs-human calibration results, if measured. */
+    judgeCalibration?: CalibrationResult[];
+    /** Responsible owner for the system — role + name + email. */
+    owner: {
+        role: string;
+        name: string;
+        email: string;
+    };
+}
+interface GovernanceFinding {
+    id: string;
+    severity: 'info' | 'low' | 'medium' | 'high' | 'critical';
+    /** Control reference the finding maps to (e.g. "NIST-AI-RMF:MEASURE-2.1"). */
+    control: string;
+    summary: string;
+    evidence?: string;
+    remediation?: string;
+}
+interface GovernanceReport {
+    framework: 'NIST-AI-RMF' | 'SOC2' | 'EU-AI-ACT';
+    version: string;
+    context: Pick<GovernanceContext, 'organization' | 'systemName' | 'periodStart' | 'periodEnd' | 'owner'>;
+    summary: {
+        findings: number;
+        byeverity: Record<GovernanceFinding['severity'], number>;
+        overall: 'compliant' | 'compliant-with-findings' | 'non-compliant';
+    };
+    findings: GovernanceFinding[];
+    /** Framework-specific structured payload (mapped controls, risk class, etc.). */
+    payload: Record<string, unknown>;
+    generatedAt: string;
+}
+declare function renderMarkdown(report: GovernanceReport): string;
+declare function summarize(findings: GovernanceFinding[]): GovernanceReport['summary'];
+
+/**
+ * NIST AI RMF 1.0 — Govern / Map / Measure / Manage mapping.
+ *
+ * Each subcategory derives its status from concrete framework state:
+ *   MEASURE 2.x: do we have a calibration regime? contamination controls?
+ *   MEASURE 2.7: are red-team results available?
+ *   MANAGE 1.x: are outcome metrics captured? correlation measured?
+ *   GOVERN 1.x: dataset + prompt provenance recorded?
+ *
+ * We ship the mapping and the derivation rules; consumers supply the
+ * GovernanceContext.
+ */
+
+declare function nistAiRmfReport(ctx: GovernanceContext): Promise<GovernanceReport>;
+
+/**
+ * SOC 2 — Common Criteria 7 (system operations + change management)
+ * audit trail derived from the trace corpus.
+ *
+ * This is NOT a formal SOC2 report — that requires an external
+ * auditor. What we ship is the machine-readable *evidence* package
+ * that an auditor consumes: run counts, deploy events, access log
+ * summary, anomaly tracking, response-time SLOs.
+ */
+
+declare function soc2Report(ctx: GovernanceContext): Promise<GovernanceReport>;
+
+/**
+ * EU AI Act — risk-class classification + compliance checklist.
+ *
+ * Classification is declarative: caller supplies the domain/use-case
+ * signals (biometric? critical infrastructure? education? employment?
+ * access to services?) and we map to the Act's risk tiers:
+ *   - "unacceptable" (prohibited)
+ *   - "high"        (Annex III — strict obligations)
+ *   - "limited"     (transparency obligations)
+ *   - "minimal"     (voluntary codes of conduct)
+ *
+ * Then the compliance checklist enumerates Article 9 (risk mgmt),
+ * 10 (data + data governance), 11 (technical documentation), 13
+ * (transparency), 14 (human oversight), 15 (accuracy + robustness)
+ * requirements and flags gaps.
+ */
+
+type EuRiskClass = 'unacceptable' | 'high' | 'limited' | 'minimal';
+interface UseCaseSignals {
+    /** Used for biometric identification in public spaces? (Art. 5 — unacceptable). */
+    biometricPublic?: boolean;
+    /** Social scoring by public authorities? (Art. 5). */
+    socialScoring?: boolean;
+    /** Subliminal manipulation? (Art. 5). */
+    subliminal?: boolean;
+    /** Annex III sector: critical infrastructure / education / employment /
+     *  access to essential services / law enforcement / migration /
+     *  administration of justice / democratic processes? */
+    annexIII?: boolean;
+    /** Interacts directly with natural persons (chatbot, agent)? — limited risk. */
+    chatbot?: boolean;
+    /** Generates synthetic media (image/audio/video/text deepfakes)? — limited risk. */
+    generatesSyntheticMedia?: boolean;
+}
+declare function classifyEuAiRisk(signals: UseCaseSignals): EuRiskClass;
+declare function euAiActReport(ctx: GovernanceContext, signals: UseCaseSignals): Promise<GovernanceReport>;
+
+export { type ActiveLearningOptions, AgentDriver, type AgentDriverConfig, type AlignmentOp, type AntiSlopConfig, type AntiSlopIssue, type AntiSlopReport, type Artifact, type ArtifactCheck, type Artifact$1 as ArtifactCheckArtifact, type ArtifactResult, type ArtifactValidator, type BaselineOptions, type BaselineReport, BehaviorAssertion, type BenchmarkReport, BenchmarkRunner, type BenchmarkRunnerConfig, type BestOfNResult, type BisectOptions, type BisectResult, type BisectStep, BudgetBreachError, type BudgetBreachFinding, type BudgetBreachReport, BudgetGuard, type BudgetLedgerEntry, type BudgetSpec, BuilderSession, type BuilderSessionInit, type CalibrationBin, type CalibrationOptions, type CalibrationReport, type CalibrationResult, CallExpectation, type CanaryLeak, type CandidateScenario, type CandidateScore, type CausalAttributionReport, type ChatSummary, type CheckResult, type CollectedArtifacts, type CompletionCriterion, type ContinuityCheck, type ContinuityCheckResult, type ContinuityReport, type ContinuitySnapshotPair, type ContractMetric, type ContractReport, ConvergenceTracker, type CorrelationReport, type CorrelationResult, type CorrelationStudyOptions, type CorrelationStudyResult, type CostEntry, type CostSummary, CostTracker, type CounterfactualContext, type CounterfactualMutation, type CounterfactualResult, type CounterfactualRunner, type CrossTraceDiff, type CrossTraceDiffOptions, DEFAULT_AGENT_SLOS, DEFAULT_RULES as DEFAULT_FAILURE_RULES, DEFAULT_MUTATORS, DEFAULT_REDACTION_RULES, DEFAULT_RED_TEAM_CORPUS, Dataset, type DatasetDifficulty, type DatasetManifest, type DatasetProvenance, type DatasetScenario, type DatasetSplit, type DeploymentOutcome, type Direction, type DivergenceOptions, type DivergenceReport, DockerSandboxDriver, type DriverResult, type DriverState, DualAgentBench, type DualAgentBenchConfig, type DualAgentReport, type DualAgentRound, type DualAgentScenario, type DualAgentScenarioResult, type EuRiskClass, type EvalMetricSpec, type EvalResult, type EventFilter, type EventKind, type EvolutionRound, type ExecutorConfig, type Expectation, type Experiment, type Run$1 as ExperimentRun, type ExperimentStore, ExperimentTracker, type ExportedRewardModel, FAILURE_CLASSES, type FactorContribution, type FactorialCell, type FailureClass, type FailureClassification, type FailureCluster, type FailureClusterReport, type FailureContext, type FailureRule, type FeedbackPattern, FileSystemOutcomeStore, type FileSystemOutcomeStoreOptions, FileSystemTraceStore, type FileSystemTraceStoreOptions, type GenericSpan, type GoldenItem, type GovernanceContext, type GovernanceFinding, type GovernanceReport, type GradedStep, type HarnessConfig, HoldoutAuditor, HoldoutLockedError, type HypothesisManifest, type HypothesisResult, type ImageData, InMemoryExperimentStore, InMemoryOutcomeStore, InMemoryTraceStore, InMemoryWorkspaceInspector, type InferenceScorer, type InspectorContext, type InteractionContribution, type JudgeAgreementReport, type JudgeConfig, type JudgeFn, type JudgeInput, type JudgePair, type JudgeReplayResult, type JudgeRubric, type JudgeScore, type JudgeSpan, type LangfuseEnvelope, type LangfuseGeneration, type LangfuseScore, type LayerCorrelation, type LlmSpan, MODEL_PRICING, type MatcherResult, type Message, type MetricSamples, type MetricVerdict, MetricsCollector, type Mutator, OTEL_AGENT_EVAL_SCOPE, type Objective, type OptimizationConfig, type OptimizationResult, type Oracle, type OracleObservation, type OracleReport, type OracleResult, type OtlpExport, type OtlpResourceSpans, type OtlpSpan, type OutcomeFilter, type OutcomePair, type OutcomeStore, type PairwiseComparison, type ParetoResult, type PersonaConfig, type PositionalBiasResult, type PrmGradedTrace, PrmGrader, type PrmTrainingSample, ProductClient, type ProductClientConfig, ProjectRegistry, type ProjectSummary, type ProjectTimelineEntry, type PromptHandle, PromptOptimizer, PromptRegistry, type PromptVariant, REDACTION_VERSION, type RedTeamCase, type RedTeamCategory, type RedTeamFinding, type RedTeamPayload, type RedTeamReport, type RedactionReport, type RedactionRule, type RegressionOptions, type RegressionSpec, type RetrievalSpan, type RobustnessResult, type RouteMap, type RubricDimension, type Run, type RunAppScenarioOptions, type RunConfig, type RunDiff, type RunFilter, type RunLayer, type RunOutcome, type RunStatus, type SandboxDriver, SandboxHarness, type SandboxHarnessResult, type SandboxResult, type SandboxSpan, type Scenario, type ScenarioCost, type ScenarioFile, ScenarioRegistry, type ScenarioResult, type ScoredTarget, type SelfPlayOptions, type SelfPlayProposer, type SelfPlayScorer, type SelfPreferenceResult, type SeriesConvergenceOptions, type SeriesConvergenceResult, type ShipOptions, type SignedManifest, type SliceOptions, type Slo, type SloCheckResult, type SloComparator, type SloReport, type SloSeverity, type SlopCategory, type Span, type SpanBase, type SpanFilter, type SpanHandle, type SpanKind, type SpanStatus, type StepAttribution, type StepContext, type StepRubric, type StuckLoopFinding, type StuckLoopOptions, type StuckLoopReport, SubprocessSandboxDriver, type SynthesisReason, type SynthesisTarget, TRACE_SCHEMA_VERSION, type TestGradedRunOptions, type TestGradedRunResult, type TestGradedScenario, type TestOutputParser, type TestResult, type ThreeLayerProjectReport, type ThresholdContract, TokenCounter, type TokenSpec, type ToolSpan, type ToolStats, type ToolUseMetrics, type ToolUseOptions, type ToolWasteFinding, type ToolWasteOptions, type ToolWasteReport, TraceEmitter, type TraceEmitterOptions, type TraceEvent, type TraceStore, type Trajectory, type TrajectoryStep, type Turn, type TurnMetrics, type TurnResult, type UseCaseSignals, type ValidationContext, type ValidationIssue, type ValidationResult, type VariantScore, type VerbosityBiasResult, type VisualDiffOptions, type VisualDiffResult, type WorkspaceAssertion, type WorkspaceAssertionResult, type WorkspaceInspector, type WorkspaceSnapshot, adversarialJudge, aggregateLlm, analyzeAntiSlop, analyzeSeries, argHash, attributeCounterfactuals, benjaminiHochberg, bisect, bonferroni, budgetBreachView, buildTrajectory, byteLengthRange, calibrateJudge, calibrationCurve, canaryLeakView, causalAttribution, checkCanaries, checkSlos, classifyEuAiRisk, classifyFailure, codeExecutionJudge, cohensD, coherenceJudge, collectionPreserved, commitBisect, compareToBaseline, composeParsers, composeValidators, computeToolUseMetrics, confidenceInterval, containsAll, correlateLayers, correlationStudy, createAntiSlopJudge, createCustomJudge, createDomainExpertJudge, crossTraceDiff, defaultJudges, dominates, estimateCost, estimateTokens, euAiActReport, evaluateContract, evaluateHypothesis, evaluateOracles, executeScenario, expectAgent, exportRewardModel, exportRunAsOtlp, exportTrainingData, failureClusterView, fileContains, fileExists, firstDivergenceView, formatBenchmarkReport, formatDriverReport, groupBy, hashContent, hashScenarios, interRaterReliability, iqr, isJudgeSpan, isLlmSpan, isPrmVerdict, isRetrievalSpan, isSandboxSpan, isToolSpan, jestTestParser, jsonHasKeys, jsonShape, judgeAgreementView, judgeSpans, keyPreserved, llmSpanFromProvider, llmSpans, loadScorerFromGrader, lowercaseMutator, mannWhitneyU, nistAiRmfReport, nonRefusalRubric, normalizeScores, notBlocked, outputLengthRubric, pairedTTest, paraphraseRobustness, paretoFrontier, partialCredit, pixelDeltaRatio, politenessPrefixMutator, positionalBias, printDriverSummary, prmBestOfN, prmEnsembleBestOfN, promptBisect, proposeSynthesisTargets, pytestTestParser, redTeamDataset, redTeamReport, redactString, redactValue, regexMatch, regexMatches, regressionView, renderMarkdown, renderMarkdownReport, replayScorerOverCorpus, replayTraceThroughJudge, requiredSampleSize, resumeBuilderSession, rowCount, rowWhere, runAssertions, runCounterfactual, runE2EWorkflow, runExpectations, runFailureClass, runSelfPlay, runTestGradedScenario, runsForScenario, scoreAllProjects, scoreContinuity, scoreProject, scoreRedTeamOutput, selfPreference, sentenceReorderMutator, signManifest, soc2Report, statusAdvanced, stuckLoopView, summarize, textInSnapshot, toLangfuseEnvelope, toNdjson, toPrometheusText, toolIntentAlignmentRubric, toolNamesForRun, toolNonRedundantRubric, toolSpans, toolSuccessRubric, toolWasteView, typoMutator, urlContains, verbosityBias, verifyManifest, visualDiff, vitestTestParser, weightedMean, welchsTTest, whitespaceCollapseMutator, wilcoxonSignedRank };