@tangle-network/agent-eval 0.6.0 → 0.7.0

package/dist/index.d.ts

@@ -1033,86 +1033,56 @@ declare class PromptOptimizer {
     run(config: OptimizationConfig): Promise<OptimizationResult>;
 }
 
-/**
- * Dual-agent convergence bench.
- *
- * Pattern lifted from tax-agent + legal-agent: two agents take turns until
- * they converge on a consensus artifact. One proposes, the other critiques;
- * the proposer revises; repeat until a score threshold is hit or max rounds.
- *
- * Generalized so any two "agents" (gateways, local functions, anything with
- * `propose` + `critique`) compose in. Returns convergence rounds per
- * scenario + whether convergence happened.
- */
-interface DualAgentScenario {
-    id: string;
-    initialPrompt: string;
-    /** Optional context the agents can read (e.g. source documents). */
-    context?: Record<string, unknown>;
-}
-interface DualAgentRound {
-    roundIndex: number;
-    proposal: string;
-    critique: string;
-    convergenceScore: number;
-}
-interface DualAgentScenarioResult {
-    scenarioId: string;
-    converged: boolean;
-    roundsToConverge: number | null;
-    finalProposal: string;
-    history: DualAgentRound[];
-    finalScore: number;
+interface SteeringRolePrompt {
+    system?: string;
+    append?: string;
 }
-interface DualAgentBenchConfig {
-    scenarios: DualAgentScenario[];
-    maxRounds?: number;
-    /** Convergence threshold in 0..1 (default 0.85). */
-    convergenceThreshold?: number;
-    /**
-     * Propose an answer given the scenario + the critic's prior critique (if any).
-     * Returns the proposal string.
-     */
-    propose: (args: {
-        scenario: DualAgentScenario;
-        roundIndex: number;
-        priorProposal?: string;
-        priorCritique?: string;
-    }) => Promise<string>;
-    /**
-     * Critique the proposer's current output. Returns a structured critique
-     * (free text) plus a convergence score: how close the proposal is to
-     * acceptable. 1.0 = accept, 0.0 = totally off.
-     */
-    critique: (args: {
-        scenario: DualAgentScenario;
-        roundIndex: number;
-        proposal: string;
-    }) => Promise<{
-        critique: string;
-        convergenceScore: number;
-    }>;
-    /** Optional per-round hook for progress + tracing. */
-    onRoundComplete?: (info: {
-        scenarioId: string;
-        round: DualAgentRound;
-    }) => void;
+interface SteeringBundle {
+    id: string;
+    coderPrompt?: string;
+    continuePrompt?: string;
+    reviewerPrompts?: Record<string, string>;
+    skills?: string[];
+    rolePrompts?: Record<string, SteeringRolePrompt>;
+    metadata?: Record<string, unknown>;
 }
-interface DualAgentReport {
-    scenarios: DualAgentScenarioResult[];
-    aggregate: {
-        convergenceRate: number;
-        avgRoundsToConverge: number | null;
-        avgFinalScore: number;
-    };
-    config: {
-        maxRounds: number;
-        convergenceThreshold: number;
-    };
+interface SteeringDelta {
+    coderPrompt?: string;
+    continuePrompt?: string;
+    reviewerPrompts?: Record<string, string>;
+    skills?: string[];
+    rolePrompts?: Record<string, SteeringRolePrompt>;
+    metadata?: Record<string, unknown>;
 }
-declare class DualAgentBench {
-    run(config: DualAgentBenchConfig): Promise<DualAgentReport>;
+declare function mergeSteeringBundle(base: SteeringBundle, delta: SteeringDelta): SteeringBundle;
+declare function renderSteeringText(bundle: SteeringBundle): string;
+
+interface RunScore {
+    success: number;
+    goalProgress: number;
+    repoGroundedness: number;
+    driftPenalty: number;
+    toolUseQuality: number;
+    patchQuality: number;
+    testReality: number;
+    costUsd: number;
+    wallSeconds: number;
+    notes?: string[];
+}
+interface RunScoreWeights {
+    success: number;
+    goalProgress: number;
+    repoGroundedness: number;
+    driftPenalty: number;
+    toolUseQuality: number;
+    patchQuality: number;
+    testReality: number;
+    costUsd: number;
+    wallSeconds: number;
 }
+declare const DEFAULT_RUN_SCORE_WEIGHTS: RunScoreWeights;
+declare function aggregateRunScore(score: RunScore, weights?: Partial<RunScoreWeights>): number;
+declare function clamp01(value: number): number;
 
 /**
  * TraceSchema v1 — the canonical data model for agent-eval.
@@ -1606,6 +1576,137 @@ interface OtlpExport {
 /** Export a single run's spans + events in OTLP/JSON. */
 declare function exportRunAsOtlp(store: TraceStore, runId: string, resourceAttrs?: Record<string, string | number | boolean>): Promise<OtlpExport>;
 
+interface RunTrace {
+    run: Run;
+    spans: Span[];
+    events: TraceEvent[];
+    artifacts: Artifact[];
+    budget: BudgetLedgerEntry[];
+}
+interface RunCriticOptions {
+    weights?: Partial<RunScoreWeights>;
+    driftPatterns?: RegExp[];
+}
+declare class RunCritic {
+    private readonly weights?;
+    private readonly driftPatterns;
+    constructor(options?: RunCriticOptions);
+    score(store: TraceStore, runId: string): Promise<RunScore>;
+    scoreTrace(trace: RunTrace): RunScore;
+    rank(score: RunScore): number;
+    private isDrift;
+}
+
+interface PlaybookEntry {
+    instruction: string;
+    rationale: string;
+    category?: string;
+    evidence?: string;
+    weight?: number;
+    sourceRunId?: string;
+}
+interface Playbook {
+    entries: PlaybookEntry[];
+}
+declare function distillPlaybook(entries: PlaybookEntry[], options?: {
+    maxEntries?: number;
+}): Playbook;
+declare function renderPlaybookMarkdown(playbook: Playbook): string;
+
+interface OptimizationExample {
+    scenarioId: string;
+    metadata?: Record<string, unknown>;
+}
+interface SteeringEvaluation {
+    variant: SteeringBundle;
+    example: OptimizationExample;
+    trialIndex: number;
+}
+interface SteeringVariantReport {
+    variantId: string;
+    bundle: SteeringBundle;
+    mean: number;
+    ci95: {
+        lower: number;
+        upper: number;
+    };
+    scenarioScores: Record<string, {
+        mean: number;
+        n: number;
+        samples: number[];
+    }>;
+}
+interface OptimizationLoopResult {
+    winner: SteeringBundle;
+    significant: boolean;
+    reports: SteeringVariantReport[];
+    pairwise: Array<{
+        variantA: string;
+        variantB: string;
+        pValue: number;
+        qValue: number;
+        significant: boolean;
+        meanDelta: number;
+    }>;
+}
+interface OptimizationLoopConfig {
+    variants: SteeringBundle[];
+    examples: OptimizationExample[];
+    evaluate: (args: SteeringEvaluation) => Promise<RunScore>;
+    scoreWeights?: Partial<RunScoreWeights>;
+    trialsPerScenario?: number;
+}
+declare class OptimizationLoop {
+    private readonly optimizer;
+    constructor(optimizer?: PromptOptimizer);
+    run(config: OptimizationLoopConfig): Promise<OptimizationLoopResult>;
+}
+
+type SteeringOptimizerBackend = 'pairwise' | 'ax-gepa';
+interface SteeringOptimizationRow {
+    variantId: string;
+    scenarioId: string;
+    bundle: SteeringBundle;
+    score: RunScore;
+    metadata?: Record<string, unknown>;
+}
+interface SteeringOptimizationSelector {
+    backend: SteeringOptimizerBackend;
+    signature?: string;
+    labels?: string[];
+    rationale?: string;
+}
+interface SteeringOptimizationResult {
+    backend: SteeringOptimizerBackend;
+    recommendedVariantId: string;
+    rationale: string;
+    rankings: Array<{
+        variantId: string;
+        mean: number;
+        runs: number;
+    }>;
+    selector?: SteeringOptimizationSelector;
+    skipped?: boolean;
+}
+interface SteeringOptimizerConfig {
+    weights?: Partial<RunScoreWeights>;
+}
+interface AxSteeringOptimizerConfig extends SteeringOptimizerConfig {
+    provider: 'openai' | 'anthropic';
+    apiKey: string;
+    model: string;
+    teacherModel?: string;
+    minRows?: number;
+}
+declare class PairwiseSteeringOptimizer {
+    optimize(rows: SteeringOptimizationRow[], config?: SteeringOptimizerConfig): SteeringOptimizationResult;
+}
+declare class AxGepaSteeringOptimizer {
+    private readonly config;
+    constructor(config: AxSteeringOptimizerConfig);
+    optimize(rows: SteeringOptimizationRow[]): Promise<SteeringOptimizationResult>;
+}
+
 /**
  * SandboxHarness — executes a scenario in an isolated environment and
  * emits a rich SandboxSpan into the trace.
@@ -1689,6 +1790,327 @@ declare class SandboxHarness {
     run(config: HarnessConfig, emitter: TraceEmitter): Promise<SandboxHarnessResult>;
 }
 
+type SandboxJudgeKind = 'compiler' | 'test' | 'linter' | 'security';
+interface SandboxJudgeSpec {
+    id: string;
+    kind: SandboxJudgeKind;
+    config: HarnessConfig;
+}
+interface SandboxJudgeResult {
+    id: string;
+    kind: SandboxJudgeKind;
+    passed: boolean;
+    score: number;
+    summary: string;
+    detail: SandboxHarnessResult;
+}
+interface JudgeFleetOptions {
+    driver?: SandboxDriver;
+    parallel?: boolean;
+}
+declare class JudgeRunner {
+    private readonly driver;
+    constructor(driver?: SandboxDriver);
+    run(spec: SandboxJudgeSpec): Promise<SandboxJudgeResult>;
+}
+declare function runJudgeFleet(specs: SandboxJudgeSpec[], options?: JudgeFleetOptions): Promise<SandboxJudgeResult[]>;
+declare function compilerJudge(id: string, config: HarnessConfig): SandboxJudgeSpec;
+declare function testJudge(id: string, config: HarnessConfig): SandboxJudgeSpec;
+declare function linterJudge(id: string, config: HarnessConfig): SandboxJudgeSpec;
+declare function securityJudge(id: string, config: HarnessConfig): SandboxJudgeSpec;
+
+interface HostedJudgeDimension {
+    name: string;
+    weight: number;
+    rubric: string;
+}
+interface HostedJudgeConfig {
+    model: string;
+    mode?: 'llm' | 'sandbox' | 'composite';
+    systemPrompt?: string;
+    rubricTemplate?: string;
+    temperature?: number;
+    maxTurns?: number;
+    tools?: string[];
+    dimensions?: HostedJudgeDimension[];
+    setupCommand?: string;
+    scripts?: Record<string, string>;
+}
+interface HostedJudgeRequest {
+    prompt: string;
+    response: string;
+    rubric?: string;
+    reference?: string;
+    judge: HostedJudgeConfig;
+}
+interface HostedJudgeResponse {
+    score: number;
+    reasoning: string;
+    cost: number;
+    dimensions?: Array<{
+        name: string;
+        score: number;
+        reasoning: string;
+    }>;
+    evidence?: Array<{
+        type: string;
+        content: string;
+    }>;
+    turns?: number;
+    parseFailed?: boolean;
+    rawOutput?: string;
+}
+interface HostedRunScoreRequest {
+    trace: RunTrace;
+    weights?: Partial<RunScoreWeights>;
+    driftPatterns?: string[];
+}
+interface HostedRunScoreResponse {
+    score: RunScore;
+    aggregate: number;
+    weights: RunScoreWeights;
+    notes: string[];
+}
+type HostedRunCriticConfig = Pick<RunCriticOptions, 'weights'> & {
+    driftPatterns?: string[];
+};
+
+/**
+ * Dual-agent convergence bench.
+ *
+ * Pattern lifted from tax-agent + legal-agent: two agents take turns until
+ * they converge on a consensus artifact. One proposes, the other critiques;
+ * the proposer revises; repeat until a score threshold is hit or max rounds.
+ *
+ * Generalized so any two "agents" (gateways, local functions, anything with
+ * `propose` + `critique`) compose in. Returns convergence rounds per
+ * scenario + whether convergence happened.
+ */
+interface DualAgentScenario {
+    id: string;
+    initialPrompt: string;
+    /** Optional context the agents can read (e.g. source documents). */
+    context?: Record<string, unknown>;
+}
+interface DualAgentRound {
+    roundIndex: number;
+    proposal: string;
+    critique: string;
+    convergenceScore: number;
+}
+interface DualAgentScenarioResult {
+    scenarioId: string;
+    converged: boolean;
+    roundsToConverge: number | null;
+    finalProposal: string;
+    history: DualAgentRound[];
+    finalScore: number;
+}
+interface DualAgentBenchConfig {
+    scenarios: DualAgentScenario[];
+    maxRounds?: number;
+    /** Convergence threshold in 0..1 (default 0.85). */
+    convergenceThreshold?: number;
+    /**
+     * Propose an answer given the scenario + the critic's prior critique (if any).
+     * Returns the proposal string.
+     */
+    propose: (args: {
+        scenario: DualAgentScenario;
+        roundIndex: number;
+        priorProposal?: string;
+        priorCritique?: string;
+    }) => Promise<string>;
+    /**
+     * Critique the proposer's current output. Returns a structured critique
+     * (free text) plus a convergence score: how close the proposal is to
+     * acceptable. 1.0 = accept, 0.0 = totally off.
+     */
+    critique: (args: {
+        scenario: DualAgentScenario;
+        roundIndex: number;
+        proposal: string;
+    }) => Promise<{
+        critique: string;
+        convergenceScore: number;
+    }>;
+    /** Optional per-round hook for progress + tracing. */
+    onRoundComplete?: (info: {
+        scenarioId: string;
+        round: DualAgentRound;
+    }) => void;
+}
+interface DualAgentReport {
+    scenarios: DualAgentScenarioResult[];
+    aggregate: {
+        convergenceRate: number;
+        avgRoundsToConverge: number | null;
+        avgFinalScore: number;
+    };
+    config: {
+        maxRounds: number;
+        convergenceThreshold: number;
+    };
+}
+declare class DualAgentBench {
+    run(config: DualAgentBenchConfig): Promise<DualAgentReport>;
+}
+
+/**
+ * Propose / Verify / Review — the core multi-shot primitive.
+ *
+ *   shot N:  propose(state, priorReview) → new state
+ *            verify(state)               → pass/fail, optional layers
+ *            review(state, verification, memory) → observations + next-shot
+ *                                                   instruction + shouldContinue
+ *            memory.append(entry)
+ *
+ * Roles are strictly separated:
+ *
+ *   - The WORKER is whatever the caller wraps in `propose`. It is
+ *     stateful — caller owns its resume/session mechanism.
+ *   - The VERIFIER grades the state. It produces the ground truth.
+ *     The reviewer cannot overturn or downgrade a verification layer.
+ *   - The REVIEWER is stateless per call. Its continuity is the
+ *     `ReviewMemoryStore` — durable JSONL by default, or any store
+ *     implementing the interface. It reads memory + trace summary +
+ *     verification and directs the NEXT proposer shot.
+ *
+ * This shape is load-bearing. The reviewer never grades; the verifier
+ * never directs. Two processes, two prompts, two concerns — which is
+ * what keeps the loop from confirmation-biasing itself into "all
+ * passed" when it didn't.
+ *
+ * Short-circuits and soft-fails are both first-class:
+ *   - verify.pass === true  → reviewer LLM call is skipped, memory
+ *     records a success entry, loop exits.
+ *   - review throws         → the shot still counts; the loop uses the
+ *     last-known instruction (or `fallbackInstruction`) for the next
+ *     propose call. A transient reviewer failure must NEVER abort a
+ *     valid arc.
+ *
+ * Composable: `propose` itself can be another `runProposeReview` call.
+ * That's the dogfooding path — a harness built on this primitive is in
+ * turn evaluable by it.
+ */
+
+interface Verification {
+    pass: boolean;
+    score?: number;
+    failingLayers?: string[];
+    details?: unknown;
+}
+interface Review {
+    observations: string;
+    diagnosis: string;
+    nextShotInstruction: string;
+    shouldContinue: boolean;
+    confidence: number;
+}
+interface ReviewMemoryEntry extends Review {
+    shot: number;
+    timestamp: number;
+    verification: {
+        pass: boolean;
+        score?: number;
+        failingLayers?: string[];
+    };
+}
+interface ProposeInput<State> {
+    shot: number;
+    goal: string;
+    state: State;
+    priorReview: Review | null;
+    abortSignal: AbortSignal;
+    emitter?: TraceEmitter;
+}
+interface ProposeOutput<State, Summary = unknown> {
+    state: State;
+    traceSummary?: Summary;
+}
+interface ReviewInput<State, Summary = unknown> {
+    shot: number;
+    goal: string;
+    state: State;
+    verification: Verification;
+    traceSummary: Summary | undefined;
+    memory: ReviewMemoryEntry[];
+}
+type ProposeFn<State, Summary = unknown> = (input: ProposeInput<State>) => Promise<ProposeOutput<State, Summary>>;
+type VerifyFn<State> = (state: State) => Promise<Verification>;
+type ReviewFn<State, Summary = unknown> = (input: ReviewInput<State, Summary>) => Promise<Review>;
+interface ReviewMemoryStore {
+    load(): Promise<ReviewMemoryEntry[]>;
+    append(entry: ReviewMemoryEntry): Promise<void>;
+}
+interface ProposeReviewConfig<State, Summary = unknown> {
+    goal: string;
+    initialState: State;
+    propose: ProposeFn<State, Summary>;
+    verify: VerifyFn<State>;
+    review: ReviewFn<State, Summary>;
+    /** Hard shot cap. Default 10. */
+    maxShots?: number;
+    /** Wall-clock cap in ms. Default 10 min. */
+    maxWallMs?: number;
+    /**
+     * If the reviewer returns confidence ≤ floor on `confidenceFloorWindow`
+     * consecutive shots, terminate early. Default floor 0.3, window 2.
+     * Set window to 0 or floor to <0 to disable.
+     */
+    confidenceFloor?: number;
+    confidenceFloorWindow?: number;
+    /** Defaults to an in-memory store if omitted. */
+    memory?: ReviewMemoryStore;
+    /** If provided, emit a Run + per-shot spans. */
+    store?: TraceStore;
+    scenarioId?: string;
+    projectId?: string;
+    variantId?: string;
+    /**
+     * Used when the reviewer soft-fails on shot 1 (no prior instruction to
+     * fall back to). Default is a generic "inspect failures and fix".
+     */
+    fallbackInstruction?: string;
+}
+interface ProposeReviewShot<State, Summary = unknown> {
+    shot: number;
+    state: State;
+    verification: Verification;
+    traceSummary: Summary | undefined;
+    review: Review;
+    reviewAvailable: boolean;
+    reviewError?: string;
+    durationMs: number;
+}
+interface ProposeReviewReport<State, Summary = unknown> {
+    runId: string | null;
+    completed: boolean;
+    shots: ProposeReviewShot<State, Summary>[];
+    finalState: State;
+    finalVerification: Verification;
+    failureClass?: FailureClass;
+    wallMs: number;
+    score: number;
+}
+declare function inMemoryReviewStore(initial?: ReviewMemoryEntry[]): ReviewMemoryStore;
+declare function jsonlReviewStore(path: string): ReviewMemoryStore;
+declare function runProposeReview<State, Summary = unknown>(config: ProposeReviewConfig<State, Summary>): Promise<ProposeReviewReport<State, Summary>>;
+interface LlmJsonCall {
+    (req: {
+        system: string;
+        user: string;
+    }): Promise<unknown>;
+}
+interface LlmReviewerConfig<State, Summary = unknown> {
+    callJson: LlmJsonCall;
+    renderState?: (state: State) => string;
+    renderTraceSummary?: (summary: Summary | undefined) => string;
+    /** Appended to the default system prompt. */
+    systemPromptAddendum?: string;
+}
+declare function createLlmReviewer<State, Summary = unknown>(cfg: LlmReviewerConfig<State, Summary>): ReviewFn<State, Summary>;
+
 /**
  * TestGradedScenario — a scenario whose score comes from a test suite.
  *
@@ -3102,10 +3524,25 @@ declare function resumeBuilderSession(store: TraceStore, projectId: string): Pro
  * is the highest-leverage signal the framework computes — if
  * meta_score doesn't predict runtime_score, the builder's self-scoring
  * is broken.
+ *
+ * Scaffold-only mode: when a project has no `app-runtime` runs (e.g. a
+ * scaffold-builder eval that grades compose + build without driving a
+ * runtime scenario), `kind` is `'scaffold-only'` and `complete` measures
+ * meta + build only. Consumers can tell the two apart without having to
+ * interpret null-runtime as either "not yet computed" or "N/A for this
+ * project shape".
  */
 
+type ProjectKind = 'full' | 'scaffold-only';
 interface ThreeLayerProjectReport {
     projectId: string;
+    /**
+     * `'full'` when the project has at least one `app-runtime` run;
+     * `'scaffold-only'` when it only has meta + build layers. Lets
+     * downstream consumers treat a null runtime score as expected
+     * (scaffold-only) vs. missing (full, pipeline broke).
+     */
+    kind: ProjectKind;
     builderRunId?: string;
     /** Judge-verdict score on the builder run (0..1 after normalization). */
     metaScore: number | null;
@@ -3113,10 +3550,14 @@ interface ThreeLayerProjectReport {
     /** 0..1 from the sandbox harness (testsPassed / testsTotal). */
     buildScore: number | null;
     appRuntimeRunIds: string[];
-    /** Mean of outcome.score over app-runtime runs, 0..1. */
+    /** Mean of outcome.score over app-runtime runs, 0..1. Always null in scaffold-only mode. */
     runtimeScore: number | null;
     runtimePassRate: number | null;
-    /** True when all three layers produced a score. */
+    /**
+     * Layer-aware completeness:
+     *   - `kind='full'`: all three layers scored
+     *   - `kind='scaffold-only'`: meta + build scored (runtime not applicable)
+     */
     complete: boolean;
 }
 declare function scoreProject(store: TraceStore, projectId: string): Promise<ThreeLayerProjectReport>;
@@ -4132,4 +4573,4 @@ interface UseCaseSignals {
 declare function classifyEuAiRisk(signals: UseCaseSignals): EuRiskClass;
 declare function euAiActReport(ctx: GovernanceContext, signals: UseCaseSignals): Promise<GovernanceReport>;
 
-export { type ActiveLearningOptions, AgentDriver, type AgentDriverConfig, type AlignmentOp, type AntiSlopConfig, type AntiSlopIssue, type AntiSlopReport, type Artifact, type ArtifactCheck, type Artifact$1 as ArtifactCheckArtifact, type ArtifactResult, type ArtifactValidator, type BaselineOptions, type BaselineReport, BehaviorAssertion, type BenchmarkReport, BenchmarkRunner, type BenchmarkRunnerConfig, type BestOfNResult, type BisectOptions, type BisectResult, type BisectStep, BudgetBreachError, type BudgetBreachFinding, type BudgetBreachReport, BudgetGuard, type BudgetLedgerEntry, type BudgetSpec, BuilderSession, type BuilderSessionInit, type CalibrationBin, type CalibrationOptions, type CalibrationReport, type CalibrationResult, CallExpectation, type CanaryLeak, type CandidateScenario, type CandidateScore, type CausalAttributionReport, type ChatSummary, type CheckResult, type CollectedArtifacts, type CompletionCriterion, type ContinuityCheck, type ContinuityCheckResult, type ContinuityReport, type ContinuitySnapshotPair, type ContractMetric, type ContractReport, ConvergenceTracker, type CorrelationReport, type CorrelationResult, type CorrelationStudyOptions, type CorrelationStudyResult, type CostEntry, type CostSummary, CostTracker, type CounterfactualContext, type CounterfactualMutation, type CounterfactualResult, type CounterfactualRunner, type CrossTraceDiff, type CrossTraceDiffOptions, DEFAULT_AGENT_SLOS, DEFAULT_RULES as DEFAULT_FAILURE_RULES, DEFAULT_MUTATORS, DEFAULT_REDACTION_RULES, DEFAULT_RED_TEAM_CORPUS, Dataset, type DatasetDifficulty, type DatasetManifest, type DatasetProvenance, type DatasetScenario, type DatasetSplit, type DeploymentOutcome, type Direction, type DivergenceOptions, type DivergenceReport, DockerSandboxDriver, type DriverResult, type DriverState, DualAgentBench, type DualAgentBenchConfig, type DualAgentReport, type DualAgentRound, type DualAgentScenario, type DualAgentScenarioResult, type EuRiskClass, type EvalMetricSpec, type EvalResult, type EventFilter, type EventKind, type EvolutionRound, type ExecutorConfig, type Expectation, type Experiment, type Run$1 as ExperimentRun, type ExperimentStore, ExperimentTracker, type ExportedRewardModel, FAILURE_CLASSES, type FactorContribution, type FactorialCell, type FailureClass, type FailureClassification, type FailureCluster, type FailureClusterReport, type FailureContext, type FailureRule, type FeedbackPattern, FileSystemOutcomeStore, type FileSystemOutcomeStoreOptions, FileSystemTraceStore, type FileSystemTraceStoreOptions, type GenericSpan, type GoldenItem, type GovernanceContext, type GovernanceFinding, type GovernanceReport, type GradedStep, type HarnessConfig, HoldoutAuditor, HoldoutLockedError, type HypothesisManifest, type HypothesisResult, type ImageData, InMemoryExperimentStore, InMemoryOutcomeStore, InMemoryTraceStore, InMemoryWorkspaceInspector, type InferenceScorer, type InspectorContext, type InteractionContribution, type JudgeAgreementReport, type JudgeConfig, type JudgeFn, type JudgeInput, type JudgePair, type JudgeReplayResult, type JudgeRubric, type JudgeScore, type JudgeSpan, type LangfuseEnvelope, type LangfuseGeneration, type LangfuseScore, type LayerCorrelation, type LlmSpan, MODEL_PRICING, type MatcherResult, type Message, type MetricSamples, type MetricVerdict, MetricsCollector, type Mutator, OTEL_AGENT_EVAL_SCOPE, type Objective, type OptimizationConfig, type OptimizationResult, type Oracle, type OracleObservation, type OracleReport, type OracleResult, type OtlpExport, type OtlpResourceSpans, type OtlpSpan, type OutcomeFilter, type OutcomePair, type OutcomeStore, type PairwiseComparison, type ParetoResult, type PersonaConfig, type PositionalBiasResult, type PrmGradedTrace, PrmGrader, type PrmTrainingSample, ProductClient, type ProductClientConfig, ProjectRegistry, type ProjectSummary, type ProjectTimelineEntry, type PromptHandle, PromptOptimizer, PromptRegistry, type PromptVariant, REDACTION_VERSION, type RedTeamCase, type RedTeamCategory, type RedTeamFinding, type RedTeamPayload, type RedTeamReport, type RedactionReport, type RedactionRule, type RegressionOptions, type RegressionSpec, type RetrievalSpan, type RobustnessResult, type RouteMap, type RubricDimension, type Run, type RunAppScenarioOptions, type RunConfig, type RunDiff, type RunFilter, type RunLayer, type RunOutcome, type RunStatus, type SandboxDriver, SandboxHarness, type SandboxHarnessResult, type SandboxResult, type SandboxSpan, type Scenario, type ScenarioCost, type ScenarioFile, ScenarioRegistry, type ScenarioResult, type ScoredTarget, type SelfPlayOptions, type SelfPlayProposer, type SelfPlayScorer, type SelfPreferenceResult, type SeriesConvergenceOptions, type SeriesConvergenceResult, type ShipOptions, type SignedManifest, type SliceOptions, type Slo, type SloCheckResult, type SloComparator, type SloReport, type SloSeverity, type SlopCategory, type Span, type SpanBase, type SpanFilter, type SpanHandle, type SpanKind, type SpanStatus, type StepAttribution, type StepContext, type StepRubric, type StuckLoopFinding, type StuckLoopOptions, type StuckLoopReport, SubprocessSandboxDriver, type SynthesisReason, type SynthesisTarget, TRACE_SCHEMA_VERSION, type TestGradedRunOptions, type TestGradedRunResult, type TestGradedScenario, type TestOutputParser, type TestResult, type ThreeLayerProjectReport, type ThresholdContract, TokenCounter, type TokenSpec, type ToolSpan, type ToolStats, type ToolUseMetrics, type ToolUseOptions, type ToolWasteFinding, type ToolWasteOptions, type ToolWasteReport, TraceEmitter, type TraceEmitterOptions, type TraceEvent, type TraceStore, type Trajectory, type TrajectoryStep, type Turn, type TurnMetrics, type TurnResult, type UseCaseSignals, type ValidationContext, type ValidationIssue, type ValidationResult, type VariantScore, type VerbosityBiasResult, type VisualDiffOptions, type VisualDiffResult, type WorkspaceAssertion, type WorkspaceAssertionResult, type WorkspaceInspector, type WorkspaceSnapshot, adversarialJudge, aggregateLlm, analyzeAntiSlop, analyzeSeries, argHash, attributeCounterfactuals, benjaminiHochberg, bisect, bonferroni, budgetBreachView, buildTrajectory, byteLengthRange, calibrateJudge, calibrationCurve, canaryLeakView, causalAttribution, checkCanaries, checkSlos, classifyEuAiRisk, classifyFailure, codeExecutionJudge, cohensD, coherenceJudge, collectionPreserved, commitBisect, compareToBaseline, composeParsers, composeValidators, computeToolUseMetrics, confidenceInterval, containsAll, correlateLayers, correlationStudy, createAntiSlopJudge, createCustomJudge, createDomainExpertJudge, crossTraceDiff, defaultJudges, dominates, estimateCost, estimateTokens, euAiActReport, evaluateContract, evaluateHypothesis, evaluateOracles, executeScenario, expectAgent, exportRewardModel, exportRunAsOtlp, exportTrainingData, failureClusterView, fileContains, fileExists, firstDivergenceView, formatBenchmarkReport, formatDriverReport, groupBy, hashContent, hashScenarios, interRaterReliability, iqr, isJudgeSpan, isLlmSpan, isPrmVerdict, isRetrievalSpan, isSandboxSpan, isToolSpan, jestTestParser, jsonHasKeys, jsonShape, judgeAgreementView, judgeSpans, keyPreserved, llmSpanFromProvider, llmSpans, loadScorerFromGrader, lowercaseMutator, mannWhitneyU, nistAiRmfReport, nonRefusalRubric, normalizeScores, notBlocked, outputLengthRubric, pairedTTest, paraphraseRobustness, paretoFrontier, partialCredit, pixelDeltaRatio, politenessPrefixMutator, positionalBias, printDriverSummary, prmBestOfN, prmEnsembleBestOfN, promptBisect, proposeSynthesisTargets, pytestTestParser, redTeamDataset, redTeamReport, redactString, redactValue, regexMatch, regexMatches, regressionView, renderMarkdown, renderMarkdownReport, replayScorerOverCorpus, replayTraceThroughJudge, requiredSampleSize, resumeBuilderSession, rowCount, rowWhere, runAssertions, runCounterfactual, runE2EWorkflow, runExpectations, runFailureClass, runSelfPlay, runTestGradedScenario, runsForScenario, scoreAllProjects, scoreContinuity, scoreProject, scoreRedTeamOutput, selfPreference, sentenceReorderMutator, signManifest, soc2Report, statusAdvanced, stuckLoopView, summarize, textInSnapshot, toLangfuseEnvelope, toNdjson, toPrometheusText, toolIntentAlignmentRubric, toolNamesForRun, toolNonRedundantRubric, toolSpans, toolSuccessRubric, toolWasteView, typoMutator, urlContains, verbosityBias, verifyManifest, visualDiff, vitestTestParser, weightedMean, welchsTTest, whitespaceCollapseMutator, wilcoxonSignedRank };
+export { type ActiveLearningOptions, AgentDriver, type AgentDriverConfig, type AlignmentOp, type AntiSlopConfig, type AntiSlopIssue, type AntiSlopReport, type Artifact, type ArtifactCheck, type Artifact$1 as ArtifactCheckArtifact, type ArtifactResult, type ArtifactValidator, AxGepaSteeringOptimizer, type AxSteeringOptimizerConfig, type BaselineOptions, type BaselineReport, BehaviorAssertion, type BenchmarkReport, BenchmarkRunner, type BenchmarkRunnerConfig, type BestOfNResult, type BisectOptions, type BisectResult, type BisectStep, BudgetBreachError, type BudgetBreachFinding, type BudgetBreachReport, BudgetGuard, type BudgetLedgerEntry, type BudgetSpec, BuilderSession, type BuilderSessionInit, type CalibrationBin, type CalibrationOptions, type CalibrationReport, type CalibrationResult, CallExpectation, type CanaryLeak, type CandidateScenario, type CandidateScore, type CausalAttributionReport, type ChatSummary, type CheckResult, type CollectedArtifacts, type CompletionCriterion, type ContinuityCheck, type ContinuityCheckResult, type ContinuityReport, type ContinuitySnapshotPair, type ContractMetric, type ContractReport, ConvergenceTracker, type CorrelationReport, type CorrelationResult, type CorrelationStudyOptions, type CorrelationStudyResult, type CostEntry, type CostSummary, CostTracker, type CounterfactualContext, type CounterfactualMutation, type CounterfactualResult, type CounterfactualRunner, type CrossTraceDiff, type CrossTraceDiffOptions, DEFAULT_AGENT_SLOS, DEFAULT_RULES as DEFAULT_FAILURE_RULES, DEFAULT_MUTATORS, DEFAULT_REDACTION_RULES, DEFAULT_RED_TEAM_CORPUS, DEFAULT_RUN_SCORE_WEIGHTS, Dataset, type DatasetDifficulty, type DatasetManifest, type DatasetProvenance, type DatasetScenario, type DatasetSplit, type DeploymentOutcome, type Direction, type DivergenceOptions, type DivergenceReport, DockerSandboxDriver, type DriverResult, type DriverState, DualAgentBench, type DualAgentBenchConfig, type DualAgentReport, type DualAgentRound, type DualAgentScenario, type DualAgentScenarioResult, type EuRiskClass, type EvalMetricSpec, type EvalResult, type EventFilter, type EventKind, type EvolutionRound, type ExecutorConfig, type Expectation, type Experiment, type Run$1 as ExperimentRun, type ExperimentStore, ExperimentTracker, type ExportedRewardModel, FAILURE_CLASSES, type FactorContribution, type FactorialCell, type FailureClass, type FailureClassification, type FailureCluster, type FailureClusterReport, type FailureContext, type FailureRule, type FeedbackPattern, FileSystemOutcomeStore, type FileSystemOutcomeStoreOptions, FileSystemTraceStore, type FileSystemTraceStoreOptions, type GenericSpan, type GoldenItem, type GovernanceContext, type GovernanceFinding, type GovernanceReport, type GradedStep, type HarnessConfig, HoldoutAuditor, HoldoutLockedError, type HostedJudgeConfig, type HostedJudgeDimension, type HostedJudgeRequest, type HostedJudgeResponse, type HostedRunCriticConfig, type HostedRunScoreRequest, type HostedRunScoreResponse, type HypothesisManifest, type HypothesisResult, type ImageData, InMemoryExperimentStore, InMemoryOutcomeStore, InMemoryTraceStore, InMemoryWorkspaceInspector, type InferenceScorer, type InspectorContext, type InteractionContribution, type JudgeAgreementReport, type JudgeConfig, type JudgeFleetOptions, type JudgeFn, type JudgeInput, type JudgePair, type JudgeReplayResult, type JudgeRubric, JudgeRunner, type JudgeScore, type JudgeSpan, type LangfuseEnvelope, type LangfuseGeneration, type LangfuseScore, type LayerCorrelation, type LlmJsonCall, type LlmReviewerConfig, type LlmSpan, MODEL_PRICING, type MatcherResult, type Message, type MetricSamples, type MetricVerdict, MetricsCollector, type Mutator, OTEL_AGENT_EVAL_SCOPE, type Objective, type OptimizationConfig, type OptimizationExample, OptimizationLoop, type OptimizationLoopConfig, type OptimizationLoopResult, type OptimizationResult, type Oracle, type OracleObservation, type OracleReport, type OracleResult, type OtlpExport, type OtlpResourceSpans, type OtlpSpan, type OutcomeFilter, type OutcomePair, type OutcomeStore, type PairwiseComparison, PairwiseSteeringOptimizer, type ParetoResult, type PersonaConfig, type Playbook, type PlaybookEntry, type PositionalBiasResult, type PrmGradedTrace, PrmGrader, type PrmTrainingSample, ProductClient, type ProductClientConfig, type ProjectKind, ProjectRegistry, type ProjectSummary, type ProjectTimelineEntry, type PromptHandle, PromptOptimizer, PromptRegistry, type PromptVariant, type ProposeFn, type ProposeInput, type ProposeOutput, type ProposeReviewConfig, type ProposeReviewReport, type ProposeReviewShot, REDACTION_VERSION, type RedTeamCase, type RedTeamCategory, type RedTeamFinding, type RedTeamPayload, type RedTeamReport, type RedactionReport, type RedactionRule, type RegressionOptions, type RegressionSpec, type RetrievalSpan, type Review, type ReviewFn, type ReviewInput, type ReviewMemoryEntry, type ReviewMemoryStore, type RobustnessResult, type RouteMap, type RubricDimension, type Run, type RunAppScenarioOptions, type RunConfig, RunCritic, type RunCriticOptions, type RunDiff, type RunFilter, type RunLayer, type RunOutcome, type RunScore, type RunScoreWeights, type RunStatus, type RunTrace, type SandboxDriver, SandboxHarness, type SandboxHarnessResult, type SandboxJudgeKind, type SandboxJudgeResult, type SandboxJudgeSpec, type SandboxResult, type SandboxSpan, type Scenario, type ScenarioCost, type ScenarioFile, ScenarioRegistry, type ScenarioResult, type ScoredTarget, type SelfPlayOptions, type SelfPlayProposer, type SelfPlayScorer, type SelfPreferenceResult, type SeriesConvergenceOptions, type SeriesConvergenceResult, type ShipOptions, type SignedManifest, type SliceOptions, type Slo, type SloCheckResult, type SloComparator, type SloReport, type SloSeverity, type SlopCategory, type Span, type SpanBase, type SpanFilter, type SpanHandle, type SpanKind, type SpanStatus, type SteeringBundle, type SteeringDelta, type SteeringEvaluation, type SteeringOptimizationResult, type SteeringOptimizationRow, type SteeringOptimizationSelector, type SteeringOptimizerBackend, type SteeringOptimizerConfig, type SteeringRolePrompt, type SteeringVariantReport, type StepAttribution, type StepContext, type StepRubric, type StuckLoopFinding, type StuckLoopOptions, type StuckLoopReport, SubprocessSandboxDriver, type SynthesisReason, type SynthesisTarget, TRACE_SCHEMA_VERSION, type TestGradedRunOptions, type TestGradedRunResult, type TestGradedScenario, type TestOutputParser, type TestResult, type ThreeLayerProjectReport, type ThresholdContract, TokenCounter, type TokenSpec, type ToolSpan, type ToolStats, type ToolUseMetrics, type ToolUseOptions, type ToolWasteFinding, type ToolWasteOptions, type ToolWasteReport, TraceEmitter, type TraceEmitterOptions, type TraceEvent, type TraceStore, type Trajectory, type TrajectoryStep, type Turn, type TurnMetrics, type TurnResult, type UseCaseSignals, type ValidationContext, type ValidationIssue, type ValidationResult, type VariantScore, type VerbosityBiasResult, type Verification, type VerifyFn, type VisualDiffOptions, type VisualDiffResult, type WorkspaceAssertion, type WorkspaceAssertionResult, type WorkspaceInspector, type WorkspaceSnapshot, adversarialJudge, aggregateLlm, aggregateRunScore, analyzeAntiSlop, analyzeSeries, argHash, attributeCounterfactuals, benjaminiHochberg, bisect, bonferroni, budgetBreachView, buildTrajectory, byteLengthRange, calibrateJudge, calibrationCurve, canaryLeakView, causalAttribution, checkCanaries, checkSlos, clamp01, classifyEuAiRisk, classifyFailure, codeExecutionJudge, cohensD, coherenceJudge, collectionPreserved, commitBisect, compareToBaseline, compilerJudge, composeParsers, composeValidators, computeToolUseMetrics, confidenceInterval, containsAll, correlateLayers, correlationStudy, createAntiSlopJudge, createCustomJudge, createDomainExpertJudge, createLlmReviewer, crossTraceDiff, defaultJudges, distillPlaybook, dominates, estimateCost, estimateTokens, euAiActReport, evaluateContract, evaluateHypothesis, evaluateOracles, executeScenario, expectAgent, exportRewardModel, exportRunAsOtlp, exportTrainingData, failureClusterView, fileContains, fileExists, firstDivergenceView, formatBenchmarkReport, formatDriverReport, groupBy, hashContent, hashScenarios, inMemoryReviewStore, interRaterReliability, iqr, isJudgeSpan, isLlmSpan, isPrmVerdict, isRetrievalSpan, isSandboxSpan, isToolSpan, jestTestParser, jsonHasKeys, jsonShape, jsonlReviewStore, judgeAgreementView, judgeSpans, keyPreserved, linterJudge, llmSpanFromProvider, llmSpans, loadScorerFromGrader, lowercaseMutator, mannWhitneyU, mergeSteeringBundle, nistAiRmfReport, nonRefusalRubric, normalizeScores, notBlocked, outputLengthRubric, pairedTTest, paraphraseRobustness, paretoFrontier, partialCredit, pixelDeltaRatio, politenessPrefixMutator, positionalBias, printDriverSummary, prmBestOfN, prmEnsembleBestOfN, promptBisect, proposeSynthesisTargets, pytestTestParser, redTeamDataset, redTeamReport, redactString, redactValue, regexMatch, regexMatches, regressionView, renderMarkdown, renderMarkdownReport, renderPlaybookMarkdown, renderSteeringText, replayScorerOverCorpus, replayTraceThroughJudge, requiredSampleSize, resumeBuilderSession, rowCount, rowWhere, runAssertions, runCounterfactual, runE2EWorkflow, runExpectations, runFailureClass, runJudgeFleet, runProposeReview, runSelfPlay, runTestGradedScenario, runsForScenario, scoreAllProjects, scoreContinuity, scoreProject, scoreRedTeamOutput, securityJudge, selfPreference, sentenceReorderMutator, signManifest, soc2Report, statusAdvanced, stuckLoopView, summarize, testJudge, textInSnapshot, toLangfuseEnvelope, toNdjson, toPrometheusText, toolIntentAlignmentRubric, toolNamesForRun, toolNonRedundantRubric, toolSpans, toolSuccessRubric, toolWasteView, typoMutator, urlContains, verbosityBias, verifyManifest, visualDiff, vitestTestParser, weightedMean, welchsTTest, whitespaceCollapseMutator, wilcoxonSignedRank };