@tangle-network/agent-eval 0.17.3 → 0.19.0

package/dist/index.d.ts

@@ -1151,199 +1151,6 @@ declare class Dataset {
 }
 declare function hashScenarios(scenarios: DatasetScenario[]): Promise<string>;
 
-/**
- * Prompt optimizer — A/B test prompt variants with statistical rigor.
- *
- * Runs N prompt variants against a fixed scenario set, collects per-scenario
- * scores via the user-provided `scoreVariant` callback, and returns:
- *   - per-variant mean + bootstrap CI
- *   - pairwise significance (Mann-Whitney, non-parametric — works on any
- *     score distribution, not just normal)
- *   - a winner (highest mean, flagged if the lead is not significant)
- *
- * Deliberately generic — the `scoreVariant` callback does whatever domain
- * work the consumer needs (invoke the agent, judge the output, whatever),
- * and returns a number per scenario. This lets the optimizer stay small +
- * testable.
- */
-interface PromptVariant$1 {
-    id: string;
-    prompt: string;
-    metadata?: Record<string, unknown>;
-}
-interface OptimizationConfig {
-    variants: PromptVariant$1[];
-    /** How many trials per (variant, scenario) — controls CI tightness. Default 3. */
-    trialsPerScenario?: number;
-    /** Significance threshold for pairwise comparison (default 0.05). */
-    significanceLevel?: number;
-    /**
-     * The scoring callback. For each (variant, scenarioId, trialIndex), produce
-     * a score in 0..1 (or any numeric range — the optimizer only cares about
-     * monotonicity).
-     */
-    scoreVariant: (args: {
-        variant: PromptVariant$1;
-        scenarioId: string;
-        trialIndex: number;
-    }) => Promise<number>;
-    /** Scenario ids to run against. */
-    scenarioIds: string[];
-    /** Optional hook — fires after each (variant, scenario) fully scored. */
-    onScenarioComplete?: (info: {
-        variantId: string;
-        scenarioId: string;
-        scores: number[];
-    }) => void;
-}
-interface VariantScore {
-    variantId: string;
-    mean: number;
-    ci95: {
-        lower: number;
-        upper: number;
-    };
-    n: number;
-    perScenario: Record<string, {
-        mean: number;
-        n: number;
-        samples: number[];
-    }>;
-}
-interface PairwiseComparison {
-    variantA: string;
-    variantB: string;
-    pValue: number;
-    /** BH-FDR-corrected q-value across all n*(n-1)/2 pairwise tests. */
-    qValue: number;
-    /** True when q-value passes the FDR threshold. Prefer over raw p-value when variants > 2. */
-    significant: boolean;
-    meanDelta: number;
-}
-interface OptimizationResult {
-    winner: {
-        variantId: string;
-        /** True when the winner's lead vs every other variant is statistically significant. */
-        significant: boolean;
-        ciLowerBoundExceedsSecondMean: boolean;
-    };
-    scores: VariantScore[];
-    pairwise: PairwiseComparison[];
-    config: {
-        trialsPerScenario: number;
-        significanceLevel: number;
-        variants: string[];
-        scenarios: string[];
-    };
-}
-declare class PromptOptimizer {
-    run(config: OptimizationConfig): Promise<OptimizationResult>;
-}
-
-interface RunScore {
-    success: number;
-    goalProgress: number;
-    repoGroundedness: number;
-    driftPenalty: number;
-    toolUseQuality: number;
-    patchQuality: number;
-    testReality: number;
-    finalGate: number;
-    reviewerBlockers: number;
-    costUsd: number;
-    wallSeconds: number;
-    notes?: string[];
-}
-interface RunScoreWeights {
-    success: number;
-    goalProgress: number;
-    repoGroundedness: number;
-    driftPenalty: number;
-    toolUseQuality: number;
-    patchQuality: number;
-    testReality: number;
-    finalGate: number;
-    reviewerBlockers: number;
-    costUsd: number;
-    wallSeconds: number;
-}
-declare const DEFAULT_RUN_SCORE_WEIGHTS: RunScoreWeights;
-declare function aggregateRunScore(score: RunScore, weights?: Partial<RunScoreWeights>): number;
-declare function clamp01(value: number): number;
-
-interface SteeringRolePrompt {
-    system?: string;
-    append?: string;
-}
-interface SteeringBundle {
-    id: string;
-    coderPrompt?: string;
-    continuePrompt?: string;
-    reviewerPrompts?: Record<string, string>;
-    skills?: string[];
-    rolePrompts?: Record<string, SteeringRolePrompt>;
-    metadata?: Record<string, unknown>;
-}
-interface SteeringDelta {
-    coderPrompt?: string;
-    continuePrompt?: string;
-    reviewerPrompts?: Record<string, string>;
-    skills?: string[];
-    rolePrompts?: Record<string, SteeringRolePrompt>;
-    metadata?: Record<string, unknown>;
-}
-declare function mergeSteeringBundle(base: SteeringBundle, delta: SteeringDelta): SteeringBundle;
-declare function renderSteeringText(bundle: SteeringBundle): string;
-
-interface OptimizationExample {
-    scenarioId: string;
-    metadata?: Record<string, unknown>;
-}
-interface SteeringEvaluation {
-    variant: SteeringBundle;
-    example: OptimizationExample;
-    trialIndex: number;
-}
-interface SteeringVariantReport {
-    variantId: string;
-    bundle: SteeringBundle;
-    mean: number;
-    ci95: {
-        lower: number;
-        upper: number;
-    };
-    scenarioScores: Record<string, {
-        mean: number;
-        n: number;
-        samples: number[];
-    }>;
-}
-interface OptimizationLoopResult {
-    winner: SteeringBundle;
-    significant: boolean;
-    reports: SteeringVariantReport[];
-    pairwise: Array<{
-        variantA: string;
-        variantB: string;
-        pValue: number;
-        qValue: number;
-        significant: boolean;
-        meanDelta: number;
-    }>;
-}
-interface OptimizationLoopConfig {
-    variants: SteeringBundle[];
-    examples: OptimizationExample[];
-    evaluate: (args: SteeringEvaluation) => Promise<RunScore>;
-    scoreWeights?: Partial<RunScoreWeights>;
-    trialsPerScenario?: number;
-}
-declare class OptimizationLoop {
-    private readonly optimizer;
-    constructor(optimizer?: PromptOptimizer);
-    run(config: OptimizationLoopConfig): Promise<OptimizationLoopResult>;
-}
-
 type FeedbackArtifactType = 'text' | 'code' | 'plan' | 'research' | 'action' | 'ui' | 'decision' | 'data' | 'other';
 type FeedbackLabelSource = 'user' | 'judge' | 'environment' | 'metric' | 'policy' | 'system';
 type FeedbackLabelKind = 'approve' | 'reject' | 'select' | 'edit' | 'rank' | 'rate' | 'comment' | 'metric_outcome' | 'policy_block' | 'revision_request';
@@ -1432,10 +1239,12 @@ interface PreferenceMemoryEntry {
     sourceLabelId?: string;
     category?: string;
 }
-interface FeedbackOptimizerRow extends OptimizationExample {
+interface FeedbackOptimizerRow {
+    scenarioId: string;
     trajectoryId: string;
     labelKinds: FeedbackLabelKind[];
     score?: number;
+    metadata?: Record<string, unknown>;
 }
 interface FeedbackReplayResult {
     trajectoryId: string;
@@ -2070,6 +1879,61 @@ declare class D1ExperimentStore implements ExperimentStore {
     listRuns(experimentId: string): Promise<Run[]>;
 }
 
+interface SteeringRolePrompt {
+    system?: string;
+    append?: string;
+}
+interface SteeringBundle {
+    id: string;
+    coderPrompt?: string;
+    continuePrompt?: string;
+    reviewerPrompts?: Record<string, string>;
+    skills?: string[];
+    rolePrompts?: Record<string, SteeringRolePrompt>;
+    metadata?: Record<string, unknown>;
+}
+interface SteeringDelta {
+    coderPrompt?: string;
+    continuePrompt?: string;
+    reviewerPrompts?: Record<string, string>;
+    skills?: string[];
+    rolePrompts?: Record<string, SteeringRolePrompt>;
+    metadata?: Record<string, unknown>;
+}
+declare function mergeSteeringBundle(base: SteeringBundle, delta: SteeringDelta): SteeringBundle;
+declare function renderSteeringText(bundle: SteeringBundle): string;
+
+interface RunScore {
+    success: number;
+    goalProgress: number;
+    repoGroundedness: number;
+    driftPenalty: number;
+    toolUseQuality: number;
+    patchQuality: number;
+    testReality: number;
+    finalGate: number;
+    reviewerBlockers: number;
+    costUsd: number;
+    wallSeconds: number;
+    notes?: string[];
+}
+interface RunScoreWeights {
+    success: number;
+    goalProgress: number;
+    repoGroundedness: number;
+    driftPenalty: number;
+    toolUseQuality: number;
+    patchQuality: number;
+    testReality: number;
+    finalGate: number;
+    reviewerBlockers: number;
+    costUsd: number;
+    wallSeconds: number;
+}
+declare const DEFAULT_RUN_SCORE_WEIGHTS: RunScoreWeights;
+declare function aggregateRunScore(score: RunScore, weights?: Partial<RunScoreWeights>): number;
+declare function clamp01(value: number): number;
+
 /**
  * Typed query helpers over TraceStore.
  *
@@ -3809,9 +3673,9 @@ declare function toolNamesForRun(store: TraceStore, runId: string): Promise<stri
  *      returns the N per arm needed to detect a given effect size.
  *   2. After running: `benjaminiHochberg(pValues, fdr)` and
  *      `bonferroni(pValues, alpha)` correct for multiple pairwise tests
- *      so PromptOptimizer's "significant" flag is statistically honest.
+ *      so pairwise variant comparisons stay statistically honest.
  *
- * Fixes the correctness bug in 0.2's PromptOptimizer which applied
+ * Fixes the correctness bug in 0.2's pairwise optimizer which applied
  * alpha directly across n*(n-1)/2 pairwise tests without correction —
  * dramatically inflating false-positive rate when variants ≥ 3.
  */
@@ -7505,7 +7369,7 @@ declare function referenceReplayScenarioToRunScore(scenarioScore: ReferenceRepla
  *   4. Repeat for N generations OR until convergence.
  *
  * Domain-agnostic. Consumers supply:
- *   - A seed population of `PromptVariant`s.
+ *   - A seed population of `EvolvableVariant`s.
  *   - A `ScoreAdapter` that runs (variant, scenario, rep) → `TrialResult`.
  *   - A `MutateAdapter` that produces children given trace evidence.
  *   - Pareto `Objective<TrialAggregate>[]` defining the multi-objective vector.
@@ -7517,7 +7381,7 @@ declare function referenceReplayScenarioToRunScore(scenarioScore: ReferenceRepla
  * mutation primitives, persisting to disk. Those are the consumer's call.
  */
 
-interface PromptVariant<P = unknown> {
+interface EvolvableVariant<P = unknown> {
     /** Stable id for the variant — surfaces in reports and trial results. */
     id: string;
     /** Variant payload — interpretation is the consumer's responsibility. */
@@ -7571,26 +7435,26 @@ interface VariantAggregate {
 }
 interface ScoreAdapter<P = unknown> {
     score(args: {
-        variant: PromptVariant<P>;
+        variant: EvolvableVariant<P>;
         scenarioId: string;
         rep: number;
     }): Promise<TrialResult>;
 }
 interface MutateAdapter<P = unknown> {
     mutate(args: {
-        parent: PromptVariant<P>;
+        parent: EvolvableVariant<P>;
         parentAggregate: VariantAggregate;
         topTrials: TrialResult[];
         bottomTrials: TrialResult[];
         childCount: number;
         generation: number;
-    }): Promise<PromptVariant<P>[]>;
+    }): Promise<EvolvableVariant<P>[]>;
 }
 interface PromptEvolutionConfig<P = unknown> {
     runId: string;
     /** What component is being mutated — surfaces in reports + reflection prompts. */
     target: string;
-    seedVariants: PromptVariant<P>[];
+    seedVariants: EvolvableVariant<P>[];
     scenarioIds: string[];
     reps: number;
     generations: number;
@@ -7649,7 +7513,7 @@ interface GenerationReport<P = unknown> {
     runId: string;
     target: string;
     generation: number;
-    variants: PromptVariant<P>[];
+    variants: EvolvableVariant<P>[];
     aggregates: VariantAggregate[];
     /** Frontier candidates, sorted by descending crowding distance. */
     paretoFrontIds: string[];
@@ -7663,12 +7527,239 @@ interface PromptEvolutionResult<P = unknown> {
     target: string;
     generations: GenerationReport<P>[];
     /** Best variant by scalar score in the final generation. */
-    bestVariant: PromptVariant<P>;
+    bestVariant: EvolvableVariant<P>;
     /** Best aggregate (matches bestVariant). */
     bestAggregate: VariantAggregate;
 }
 declare function runPromptEvolution<P>(config: PromptEvolutionConfig<P>): Promise<PromptEvolutionResult<P>>;
 
+/**
+ * Reflective mutation — primitives for trace-conditioned prompt rewriting.
+ *
+ * Used by `prompt-evolution.ts` (and any consumer running iterative
+ * improvement). Given a parent prompt + concrete trace evidence (top trials,
+ * bottom trials, missed expectations), produce an LLM-ready prompt that
+ * proposes targeted mutations — not blind rephrasings.
+ *
+ * Why this lives outside `prompt-evolution.ts`: any consumer that wants to
+ * run reflective rewriting WITHOUT the population/Pareto machinery can
+ * import these primitives directly.
+ *
+ * Quality bar (vs. naive "mutate this prompt"):
+ *   - Show parent ↔ children diff, not just one variant
+ *   - Quote specific missed goldens with their match phrases
+ *   - Surface the model's actual emitted output side-by-side with what was expected
+ *   - Quote concrete mutation primitives so the model has a vocabulary
+ */
+interface TrialTrace {
+    /** Stable id for the trial — surfaces in the prompt for grounding. */
+    id: string;
+    /** Score the trial received on its primary metric. */
+    score: number;
+    /** Candidate inputs the agent was given (e.g., the fixture or scenario). */
+    inputName?: string;
+    /**
+     * Goldens / expectations this trial was tested against, with whether each
+     * was matched. The reflection prompt quotes the missed ones specifically.
+     */
+    expectations?: Array<{
+        id: string;
+        phrase: string;
+        matched: boolean;
+    }>;
+    /** Free-form text — what the agent actually emitted (e.g., findings, plan). */
+    emitted?: string;
+    /** Optional structured metrics (recall, precision, cost, latency). */
+    metrics?: Record<string, number>;
+}
+interface ReflectionContext {
+    /** What is being mutated — appears in the system prompt for orientation. */
+    target: string;
+    /** Current variant's payload — JSON-serialised for the prompt. */
+    parentPayload: unknown;
+    /** Best-performing trials this generation. */
+    topTrials: TrialTrace[];
+    /** Worst-performing trials this generation — the missed-golden source. */
+    bottomTrials: TrialTrace[];
+    /** How many children the mutator should propose. */
+    childCount: number;
+    /** Optional: domain-specific mutation primitives the model can pick from. */
+    mutationPrimitives?: string[];
+}
+declare const DEFAULT_MUTATION_PRIMITIVES: string[];
+/**
+ * Build the LLM-ready reflection prompt. Output is plain text — pass it as
+ * the user message. The system message should be small and stable (e.g.
+ * "Output ONLY a JSON object matching the schema below.").
+ */
+declare function buildReflectionPrompt(ctx: ReflectionContext): string;
+interface ReflectionProposal {
+    label: string;
+    rationale: string;
+    payload: unknown;
+}
+declare function parseReflectionResponse(raw: string, maxProposals?: number): ReflectionProposal[];
+
+/**
+ * Multi-shot optimization adapter.
+ *
+ * This is the canonical bridge between variable-length agent trajectories
+ * and `runPromptEvolution`. Apps provide four things:
+ *
+ *   - variants: prompt/config/tool-policy candidates
+ *   - runner: executes one full task trajectory for a variant
+ *   - scorer: turns that trajectory into score + actionable side information
+ *   - mutator: proposes new variants from top/bottom scored trials
+ *
+ * The adapter owns the boring but easy-to-get-wrong glue: stable seeds,
+ * score/cost objectives, error-to-trial conversion, ASI metric projection,
+ * and optional paired holdout gating via `HeldOutGate`.
+ */
+
+type MultiShotSplit = 'search' | 'dev' | 'holdout';
+type AsiSeverity = 'info' | 'warning' | 'error' | 'critical';
+type MultiShotVariant<P = unknown> = EvolvableVariant<P>;
+interface ActionableSideInfo {
+    /** Stable expectation/check id when available. */
+    expectationId?: string;
+    /** Human-readable diagnosis of what happened. */
+    message: string;
+    severity?: AsiSeverity;
+    /** Concrete trace excerpt, file path, tool call, screenshot id, etc. */
+    evidence?: string;
+    /** Prompt/tool/context surface likely responsible. */
+    responsibleSurface?: string;
+    /** Suggested fix in natural language. */
+    suggestion?: string;
+    /** Whether this expectation was satisfied. Defaults to false for ASI rows. */
+    matched?: boolean;
+    metadata?: Record<string, unknown>;
+}
+interface MultiShotTrace {
+    scenarioId: string;
+    /** Full turn/tool trace. Shape is intentionally app-owned. */
+    turns?: unknown[];
+    toolCalls?: unknown[];
+    artifacts?: unknown[];
+    /** Compact final output or summary used by reflection prompts. */
+    transcript?: string;
+    output?: unknown;
+    metadata?: Record<string, unknown>;
+}
+interface MultiShotRun {
+    trace: MultiShotTrace;
+    costUsd?: number;
+    durationMs?: number;
+    tokenUsage?: {
+        input?: number;
+        output?: number;
+        cached?: number;
+    };
+    metadata?: Record<string, unknown>;
+}
+interface MultiShotRunInput<P = unknown> {
+    variant: EvolvableVariant<P>;
+    scenarioId: string;
+    rep: number;
+    split: MultiShotSplit;
+    /** Stable paired seed for baseline/candidate comparisons. */
+    seed: number;
+}
+interface MultiShotRunner<P = unknown> {
+    run(input: MultiShotRunInput<P>): Promise<MultiShotRun> | MultiShotRun;
+}
+interface MultiShotScore {
+    /** Primary score in [0,1]. The adapter clamps for safety. */
+    score: number;
+    /** Pass/fail for top/bottom trial selection. Defaults to true. */
+    ok?: boolean;
+    costUsd?: number;
+    durationMs?: number;
+    metrics?: Record<string, number>;
+    asi?: ActionableSideInfo[];
+    /** Optional rich output shown to reflection mutators. */
+    emitted?: string;
+    metadata?: Record<string, unknown>;
+}
+interface MultiShotScorer<P = unknown> {
+    score(input: MultiShotRunInput<P> & {
+        run: MultiShotRun;
+    }): Promise<MultiShotScore> | MultiShotScore;
+}
+interface MultiShotTrialResult extends TrialResult {
+    split: MultiShotSplit;
+    seed: number;
+    trace?: MultiShotTrace;
+    asi?: ActionableSideInfo[];
+    emitted?: string;
+    metadata?: Record<string, unknown>;
+}
+interface MultiShotMutateAdapter<P = unknown> {
+    mutate(args: {
+        parent: EvolvableVariant<P>;
+        parentAggregate: VariantAggregate;
+        topTrials: MultiShotTrialResult[];
+        bottomTrials: MultiShotTrialResult[];
+        childCount: number;
+        generation: number;
+    }): Promise<EvolvableVariant<P>[]>;
+}
+interface MultiShotGateConfig<P = unknown> {
+    /** Search rows are optional, but enable HeldOutGate's overfit-gap check. */
+    searchScenarioIds?: string[];
+    holdoutScenarioIds: string[];
+    reps?: number;
+    gate: HeldOutGateConfig;
+    /** Convert scored trajectory runs into paper-grade RunRecords. */
+    toRunRecord(input: {
+        variant: EvolvableVariant<P>;
+        scenarioId: string;
+        rep: number;
+        split: RunSplitTag;
+        seed: number;
+        trial: MultiShotTrialResult;
+    }): RunRecord;
+}
+interface MultiShotOptimizationConfig<P = unknown> {
+    runId: string;
+    target: string;
+    seedVariants: EvolvableVariant<P>[];
+    searchScenarioIds: string[];
+    reps: number;
+    generations: number;
+    populationSize: number;
+    scoreConcurrency?: number;
+    runner: MultiShotRunner<P>;
+    scorer: MultiShotScorer<P>;
+    mutateAdapter: MultiShotMutateAdapter<P>;
+    objectives?: Objective<VariantAggregate>[];
+    scalarWeights?: Record<string, number>;
+    cache?: TrialCache;
+    earlyStopOnNoImprovement?: boolean;
+    seedBase?: number;
+    onProgress?: (event: PromptEvolutionEvent) => void;
+    gate?: MultiShotGateConfig<P>;
+}
+interface MultiShotGateResult {
+    decision: GateDecision;
+    candidateRuns: RunRecord[];
+    baselineRuns: RunRecord[];
+}
+interface MultiShotOptimizationResult<P = unknown> {
+    evolution: PromptEvolutionResult<P>;
+    /** Best candidate on the optimizer-visible search split. */
+    searchBestVariant: EvolvableVariant<P>;
+    searchBestAggregate: VariantAggregate;
+    /** Variant callers should actually ship after optional holdout gating. */
+    promotedVariant: EvolvableVariant<P>;
+    promotedAggregate: VariantAggregate;
+    /** Null when no gate was configured or the search-best candidate was the baseline. */
+    gate: MultiShotGateResult | null;
+}
+declare function runMultiShotOptimization<P>(config: MultiShotOptimizationConfig<P>): Promise<MultiShotOptimizationResult<P>>;
+declare function defaultMultiShotObjectives(): Objective<VariantAggregate>[];
+declare function trialTraceFromMultiShotTrial(trial: MultiShotTrialResult): TrialTrace;
+
 /**
  * concurrency — small primitives the evolution loop needs.
  *
@@ -7849,7 +7940,7 @@ interface LineageNode {
  * that field is part of the audit-bench convention but cheap enough to
  * accept any payload that mirrors it. Override by passing your own.
  */
-type LineageKindResolver<P> = (variant: PromptVariant<P>) => LineageKind;
+type LineageKindResolver<P> = (variant: EvolvableVariant<P>) => LineageKind;
 /**
  * Persistence shape:
  *
@@ -7874,7 +7965,7 @@ declare class LineageRecorder<P = unknown> {
     private readonly kindOf;
     constructor(path: string, kindOf?: LineageKindResolver<P>);
     upsert(node: LineageNode): Promise<void>;
-    upsertVariant(variant: PromptVariant<P>, opts?: {
+    upsertVariant(variant: EvolvableVariant<P>, opts?: {
         omitPayload?: boolean;
     }): Promise<void>;
     snapshot(): LineageNode[];
@@ -8073,7 +8164,7 @@ interface CodeMutationOutcome {
     childId?: string;
     /** Free-form one-liner: "tightened tool descriptions in forge-tools.ts". */
     description?: string;
-    /** What the runner was trying to fix (carried into PromptVariant.rationale). */
+    /** What the runner was trying to fix (carried into EvolvableVariant.rationale). */
     rationale?: string;
     /** Caller-defined diff payload. Mapped into the variant's payload by
      *  `toVariantPayload`; agent-eval treats it as opaque. */
@@ -8090,7 +8181,7 @@ interface CodeMutationOutcome {
 }
 type CodeMutationRunner<T, P> = (args: {
     slot: PoolSlot<T>;
-    parent: PromptVariant<P>;
+    parent: EvolvableVariant<P>;
     parentAggregate: VariantAggregate;
     topTrials: TrialResult[];
     bottomTrials: TrialResult[];
@@ -8105,15 +8196,15 @@ interface CreateSandboxCodeMutatorOpts<T, P> {
      * encode the diff however they want (file map, patch string, branch
      * ref, snapshot id) without agent-eval taking a stance.
      */
-    toVariantPayload(outcome: CodeMutationOutcome, parent: PromptVariant<P>): P;
+    toVariantPayload(outcome: CodeMutationOutcome, parent: EvolvableVariant<P>): P;
     /** Optional telemetry sinks. */
     mutationTelemetry?: MutationTelemetry;
     costLedger?: CostLedger;
     lineage?: LineageRecorder<P>;
     /** Override id generation. Default: `${parent.id}.g${generation}.code.${i}`. */
-    childIdFor?(parent: PromptVariant<P>, generation: number, index: number): string;
+    childIdFor?(parent: EvolvableVariant<P>, generation: number, index: number): string;
     /** Default label for the variant (visible in reports). */
-    labelFor?(outcome: CodeMutationOutcome, parent: PromptVariant<P>, generation: number, index: number): string;
+    labelFor?(outcome: CodeMutationOutcome, parent: EvolvableVariant<P>, generation: number, index: number): string;
 }
 declare function createSandboxCodeMutator<T, P>(opts: CreateSandboxCodeMutatorOpts<T, P>): MutateAdapter<P>;
 
@@ -8316,71 +8407,4 @@ declare function judgeReplayGate<TOutput>(args: JudgeReplayGateArgs<TOutput>): P
     candidateSamples: number;
 }>;
 
-/**
- * Reflective mutation — primitives for trace-conditioned prompt rewriting.
- *
- * Used by `prompt-evolution.ts` (and any consumer running iterative
- * improvement). Given a parent prompt + concrete trace evidence (top trials,
- * bottom trials, missed expectations), produce an LLM-ready prompt that
- * proposes targeted mutations — not blind rephrasings.
- *
- * Why this lives outside `prompt-evolution.ts`: any consumer that wants to
- * run reflective rewriting WITHOUT the population/Pareto machinery can
- * import these primitives directly.
- *
- * Quality bar (vs. naive "mutate this prompt"):
- *   - Show parent ↔ children diff, not just one variant
- *   - Quote specific missed goldens with their match phrases
- *   - Surface the model's actual emitted output side-by-side with what was expected
- *   - Quote concrete mutation primitives so the model has a vocabulary
- */
-interface TrialTrace {
-    /** Stable id for the trial — surfaces in the prompt for grounding. */
-    id: string;
-    /** Score the trial received on its primary metric. */
-    score: number;
-    /** Candidate inputs the agent was given (e.g., the fixture or scenario). */
-    inputName?: string;
-    /**
-     * Goldens / expectations this trial was tested against, with whether each
-     * was matched. The reflection prompt quotes the missed ones specifically.
-     */
-    expectations?: Array<{
-        id: string;
-        phrase: string;
-        matched: boolean;
-    }>;
-    /** Free-form text — what the agent actually emitted (e.g., findings, plan). */
-    emitted?: string;
-    /** Optional structured metrics (recall, precision, cost, latency). */
-    metrics?: Record<string, number>;
-}
-interface ReflectionContext {
-    /** What is being mutated — appears in the system prompt for orientation. */
-    target: string;
-    /** Current variant's payload — JSON-serialised for the prompt. */
-    parentPayload: unknown;
-    /** Best-performing trials this generation. */
-    topTrials: TrialTrace[];
-    /** Worst-performing trials this generation — the missed-golden source. */
-    bottomTrials: TrialTrace[];
-    /** How many children the mutator should propose. */
-    childCount: number;
-    /** Optional: domain-specific mutation primitives the model can pick from. */
-    mutationPrimitives?: string[];
-}
-declare const DEFAULT_MUTATION_PRIMITIVES: string[];
-/**
- * Build the LLM-ready reflection prompt. Output is plain text — pass it as
- * the user message. The system message should be small and stable (e.g.
- * "Output ONLY a JSON object matching the schema below.").
- */
-declare function buildReflectionPrompt(ctx: ReflectionContext): string;
-interface ReflectionProposal {
-    label: string;
-    rationale: string;
-    payload: unknown;
-}
-declare function parseReflectionResponse(raw: string, maxProposals?: number): ReflectionProposal[];
-
-export { type ActionExecutionPolicy, type ActionPolicyDecision, type ActiveLearningOptions, type AdapterRun, AgentDriver, type AgentDriverConfig, type AlignmentOp, type AntiSlopConfig, type AntiSlopIssue, type AntiSlopReport, type Artifact$1 as Artifact, type ArtifactCheck, type Artifact as ArtifactCheckArtifact, type ArtifactResult, type ArtifactValidator, AxGepaSteeringOptimizer, type AxSteeringOptimizerConfig, BENCHMARK_SPLIT_SEED, type BaselineOptions, type BaselineReport, BehaviorAssertion, type BenchmarkAdapter, type BenchmarkDatasetItem, type BenchmarkEvaluation, type BenchmarkReport, BenchmarkRunner, type BenchmarkRunnerConfig, type BestOfNResult, type BisectOptions, type BisectResult, type BisectStep, type BootstrapOptions, type BootstrapResult, BudgetBreachError, type BudgetBreachFinding, type BudgetBreachReport, BudgetGuard, type BudgetLedgerEntry, type BudgetSpec, BuilderSession, type BuilderSessionInit, type CalibrationBin, type CalibrationOptions, type CalibrationReport, type CalibrationResult, CallExpectation, type CanaryAlert, type CanaryKind, type CanaryLeak, type CanaryOptions, type CanaryReport, type CanarySeverity, type CandidateScenario, type CandidateScore, type CausalAttributionReport, type ChatSummary, type CheckResult, type CodeMutationOutcome, type CodeMutationRunner, type CollectedArtifacts, type CommandRunner, type CompletionCriterion, type CompositePolicy, type ConceptComplexity, type ConceptFinding, type ConceptSpec, type ConceptWeightStrategy, type ContinuityCheck, type ContinuityCheckResult, type ContinuityReport, type ContinuitySnapshotPair, type ContractMetric, type ContractReport, type ControlActionFailureMode, type ControlActionOutcome, type ControlBudget, type ControlContext, type ControlDecision, type ControlEvalResult, type ControlRunResult, type ControlRuntimeConfig, type ControlRuntimeError, type ControlSeverity, type ControlStep, type ControlStopPolicies, ConvergenceTracker, type CorrelationReport, type CorrelationResult, type CorrelationStudyOptions, type CorrelationStudyResult, type CostEntry, CostLedger, type CostLedgerGeneration, type CostLedgerSnapshot, type CostSummary, CostTracker, type CounterfactualContext, type CounterfactualMutation, type CounterfactualResult, type CounterfactualRunner, type CreateCompositeMutatorOpts, type CreateDefaultReviewerOptions, type CreateSandboxCodeMutatorOpts, type CreateSandboxPoolOpts, type CrossTraceDiff, type CrossTraceDiffOptions, D1ExperimentStore, type D1ExperimentStoreOptions, type D1Like, type D1PreparedStatementLike, DEFAULT_AGENT_SLOS, DEFAULT_COMPLEXITY_WEIGHTS, DEFAULT_RULES as DEFAULT_FAILURE_RULES, DEFAULT_FINDERS, DEFAULT_HARNESS_OBJECTIVES, DEFAULT_MUTATION_PRIMITIVES, DEFAULT_MUTATORS, DEFAULT_REDACTION_RULES, DEFAULT_RED_TEAM_CORPUS, DEFAULT_RUN_SCORE_WEIGHTS, DEFAULT_SEVERITY_WEIGHTS, Dataset, type DatasetDifficulty, type DatasetManifest, type DatasetProvenance, type DatasetScenario, type DatasetSplit, type DeployFamily, type DeployGateLayerInput, type DeployRunResult, type DeployRunner, type DeploymentOutcome, type DirEntry, type Direction, type DivergenceOptions, type DivergenceReport, DockerSandboxDriver, type DriverResult, type DriverState, DualAgentBench, type DualAgentBenchConfig, type DualAgentReport, type DualAgentRound, type DualAgentScenario, type DualAgentScenarioResult, ERROR_COUNT_PATTERNS, type ErrorCountPattern, type EuRiskClass, type EvalMetricSpec, type EvalResult, type EventFilter, type EventKind, type EvolutionRound, type PromptVariant as EvolvableVariant, type ExecutorConfig, type Expectation, type Experiment, type ExperimentPlan, type ExperimentResult, type Run as ExperimentRun, type ExperimentStore, ExperimentTracker, type ExportedRewardModel, type ExtractOptions, type ExtractResult, FAILURE_CLASSES, type FactorContribution, type FactorialCell, type FailureClass, type FailureClassification, type FailureCluster, type FailureClusterReport, type FailureContext, type FailureMode, type FailureRule, type FeedbackArtifactType, type FeedbackAttempt, type FeedbackLabel, type FeedbackLabelKind, type FeedbackLabelSource, type FeedbackOptimizerRow, type FeedbackOutcome, type FeedbackPattern, type FeedbackReplayAdapter, type FeedbackReplayResult, type FeedbackSeverity, type FeedbackSplitPolicy, type FeedbackTask, type FeedbackTrajectory, type FeedbackTrajectoryFilter, type FeedbackTrajectoryStore, FileSystemExperimentStore, type FileSystemExperimentStoreOptions, FileSystemFeedbackTrajectoryStore, FileSystemOutcomeStore, type FileSystemOutcomeStoreOptions, FileSystemTraceStore, type FileSystemTraceStoreOptions, type Finding, type FlowAction, type FlowLayerEnv, type FlowLayerFactoryInput, type FlowRunner, type FlowRunnerStepResult, type FlowSpec, type FlowStep, type GainDistributionBin, type GainDistributionFigureSpec, type GainDistributionOptions, type GateDecision, type GateEvidence, type GenerationReport, type GenericSpan, type GoldenItem, type GoldenSeverity, type GoldenSpec, type GovernanceContext, type GovernanceFinding, type GovernanceReport, type GradedStep, type HarnessAdapter, type HarnessConfig, type HarnessExperimentConfig, type HarnessExperimentResult, type HarnessIntervention, type HarnessRunRequest, type HarnessRunResult, type HarnessScenario, type HarnessSelection, type HarnessVariant, type HarnessVariantReport, HeldOutGate, type HeldOutGateConfig, type HeldOutGateRejectionCode, HoldoutAuditor, HoldoutLockedError, type HostedJudgeConfig, type HostedJudgeDimension, type HostedJudgeRequest, type HostedJudgeResponse, type HostedRunCriticConfig, type HostedRunScoreRequest, type HostedRunScoreResponse, type HypothesisManifest, type HypothesisResult, INTENT_MATCH_JUDGE_VERSION, type ImageData, InMemoryExperimentStore, InMemoryFeedbackTrajectoryStore, InMemoryOutcomeStore, InMemoryTraceStore, InMemoryTrialCache, InMemoryWorkspaceInspector, type InferenceScorer, type InspectorContext, type IntentMatchInput, type IntentMatchOptions, type IntentMatchResult, type InteractionContribution, JsonlTrialCache, type JudgeAgreementReport, type JudgeConfig, type JudgeFleetOptions, type JudgeFn, type JudgeInput, type JudgePair, type JudgeReplayGateArgs, type JudgeReplayResult, type JudgeRubric, JudgeRunner, type JudgeScore, type JudgeSpan, type KeywordConceptSpec, type KeywordCoverageFinding, type KeywordCoverageOptions, type KeywordCoverageResult, type LangfuseEnvelope, type LangfuseGeneration, type LangfuseScore, type Layer, type LayerCorrelation, type LayerResult, type LayerStatus, type LineageKind, type LineageKindResolver, type LineageNode, LineageRecorder, LlmCallError, type LlmCallRequest, type LlmCallResult, LlmClient, type LlmClientOptions, type LlmJsonCall, type LlmMessage, type LlmReviewerConfig, type LlmSpan, type LlmUsage, LockedJsonlAppender, MODEL_PRICING, type MatchResult, type MatcherResult, type MeasurementPolicy, type MergeOptions, type Message, type MetricSamples, type MetricVerdict, MetricsCollector, type MuffledFinder, type MuffledFinding, MultiLayerVerifier, type MultiToolchainLayerConfig, type MutateAdapter, type MutationAttempt, type MutationChannel, MutationTelemetry, type Mutator, Mutex, NoopResearcher, OTEL_AGENT_EVAL_SCOPE, type Objective, type OptimizationConfig, type OptimizationExample, OptimizationLoop, type OptimizationLoopConfig, type OptimizationLoopResult, type OptimizationResult, type Oracle, type OracleObservation, type OracleReport, type OracleResult, type OrthogonalityInput, type OrthogonalityResult, type OtlpExport, type OtlpResourceSpans, type OtlpSpan, type OutcomeFilter, type OutcomePair, type OutcomeStore, type PairedBootstrapOptions, type PairedBootstrapResult, type PairwiseComparison, PairwiseSteeringOptimizer, type ParetoFigureSpec, type ParetoPoint, type ParetoResult, type PersonaConfig, type Playbook, type PlaybookEntry, type PoolSlot, type PositionalBiasResult, type PreferenceMemoryEntry, type PrmGradedTrace, PrmGrader, type PrmTrainingSample, ProductClient, type ProductClientConfig, type ProjectKind, ProjectRegistry, type ProjectSummary, type ProjectTimelineEntry, type PromptEvolutionConfig, type PromptEvolutionEvent, type PromptEvolutionResult, type PromptHandle, PromptOptimizer, PromptRegistry, type TrialResult as PromptTrialResult, type PromptVariant$1 as PromptVariant, type ProposeFn, type ProposeInput, type ProposeOutput, type ProposeReviewConfig, type ProposeReviewControlAction, type ProposeReviewControlConfig, type ProposeReviewControlResult, type ProposeReviewControlState, type ProposeReviewReport, type ProposeReviewShot, type ProposedSideEffect, REDACTION_VERSION, type RedTeamCase, type RedTeamCategory, type RedTeamFinding, type RedTeamPayload, type RedTeamReport, type RedactionReport, type RedactionRule, type ReferenceMatchResult, type ReferenceReplayAdapter, type ReferenceReplayAdapterFn, type ReferenceReplayAdapterLike, type ReferenceReplayAggregate, type ReferenceReplayCandidate, type ReferenceReplayCase, type ReferenceReplayCaseRun, type ReferenceReplayExecutionScenario, type ReferenceReplayItem, type ReferenceReplayMatch, type ReferenceReplayMatchStrategy, type ReferenceReplayMatcher, type ReferenceReplayPromotionDecision, type ReferenceReplayPromotionPolicy, type ReferenceReplayRun, type ReferenceReplayRunContext, type ReferenceReplayRunOptions, type ReferenceReplayRunStore, type ReferenceReplayScenario, type ReferenceReplayScenarioScore, type ReferenceReplayScore, type ReferenceReplayScoreOptions, type ReferenceReplaySplit, type ReferenceReplaySplitComparison, type ReferenceReplaySteeringRowsOptions, type ReflectionContext, type ReflectionProposal, type RegressionOptions, type RegressionSpec, type Researcher, type RetrievalSpan, type Review, type ReviewFn, type ReviewInput, type ReviewMemoryEntry, type ReviewMemoryStore, type ReviewerMemoryEntry, type ReviewerOutput, type ReviewerPromptInput, type ReviewerSoftFailDefaults, type ReviewerVerificationSummary, type RobustnessResult, type RouteMap, type RubricDimension, type Run$1 as Run, type RunAppScenarioOptions, type RunCommandInput, type RunCommandResult, type RunConfig, RunCritic, type RunCriticOptions, type RunDiff, type RunFilter, type RunJudgeMetadata, type RunLayer, type RunOutcome, type RunRecord, RunRecordValidationError, type RunScore, type RunScoreWeights, type RunSplitTag, type RunStatus, type RunTokenUsage, type RunTrace, SEMANTIC_CONCEPT_JUDGE_VERSION, type SandboxDriver, SandboxHarness, type SandboxHarnessResult, type SandboxJudgeKind, type SandboxJudgeResult, type SandboxJudgeSpec, type SandboxPool, type SandboxResult, type SandboxSpan, type ScanOptions, type Scenario, type ScenarioAggregate, type ScenarioCost, type ScenarioFile, ScenarioRegistry, type ScenarioResult, type ScoreAdapter, type ScoredTarget, type SelfPlayOptions, type SelfPlayProposer, type SelfPlayScorer, type SelfPreferenceResult, type SemanticConceptJudgeInput, type SemanticConceptJudgeOptions, type SemanticConceptJudgeResult, type SeriesConvergenceOptions, type SeriesConvergenceResult, type Severity, type ShipOptions, type SignedManifest, type SliceOptions, type Slo, type SloCheckResult, type SloComparator, type SloReport, type SloSeverity, type SlopCategory, type SlotFactory, type Span, type SpanBase, type SpanFilter, type SpanHandle, type SpanKind, type SpanStatus, type SteeringBundle, type SteeringChange, type SteeringDelta, type SteeringEvaluation, type SteeringOptimizationResult, type SteeringOptimizationRow, type SteeringOptimizationSelector, type SteeringOptimizerBackend, type SteeringOptimizerConfig, type SteeringRolePrompt, type SteeringVariantReport, type StepAttribution, type StepContext, type StepRubric, type StopDecision, type StuckLoopFinding, type StuckLoopOptions, type StuckLoopReport, SubprocessSandboxDriver, type SubprocessSandboxDriverOptions, type SummaryTable, type SummaryTableOptions, type SummaryTableRow, type SynthesisReason, type SynthesisTarget, TRACE_SCHEMA_VERSION, type TestGradedRunOptions, type TestGradedRunResult, type TestGradedScenario, type TestOutputParser, type TestResult, type ThreeLayerProjectReport, type ThresholdContract, TokenCounter, type TokenSpec, type ToolSpan, type ToolStats, type ToolUseMetrics, type ToolUseOptions, type ToolWasteFinding, type ToolWasteOptions, type ToolWasteReport, TraceEmitter, type TraceEmitterOptions, type TraceEvent, type TraceStore, type Trajectory, type TrajectoryStep, type TrialAttempt, type TrialCache, TrialTelemetry, type TrialTrace, type Turn, type TurnMetrics, type TurnResult, UNIVERSAL_FINDERS, type UseCaseSignals, type ValidationContext, type ValidationIssue, type ValidationResult, type VariantAggregate, type VariantScore, type VerbosityBiasResult, type Verdict, type Verification, type VerificationReport, type VerifyContext, type VerifyFn, type VerifyOptions, type VisualDiffOptions, type VisualDiffResult, type ViteDeployRunnerInput, type WorkflowTopology, type WorkspaceAssertion, type WorkspaceAssertionResult, type WorkspaceInspector, type WorkspaceSnapshot, type WranglerDeployRunnerInput, adversarialJudge, aggregateLlm, aggregateRunScore, allCriticalPassed, analyzeAntiSlop, analyzeSeries, argHash, assignFeedbackSplit, attributeCounterfactuals, deterministicSplit as benchmarkDeterministicSplit, index as benchmarks, benjaminiHochberg, bhAdjust, bisect, bonferroni, bootstrapCi, budgetBreachView, buildReflectionPrompt, buildReviewerPrompt, buildTrajectory, byteLengthRange, calibrateJudge, calibrationCurve, callLlm, callLlmJson, canaryLeakView, causalAttribution, checkCanaries, checkSlos, clamp01, classifyEuAiRisk, classifyFailure, codeExecutionJudge, cohensD, coherenceJudge, collectionPreserved, commitBisect, compareReferenceReplay, compareToBaseline, compilerJudge, composeParsers, composeValidators, computeToolUseMetrics, confidenceInterval, containsAll, controlFailureClassFromVerification, controlRunToFeedbackTrajectory, correlateLayers, correlationStudy, createAntiSlopJudge, createCompositeMutator, createCustomJudge, createDefaultReviewer, createDomainExpertJudge, createFeedbackTrajectory, createIntentMatchJudge, createLlmReviewer, createSandboxCodeMutator, createSandboxPool, createSemanticConceptJudge, crossTraceDiff, crowdingDistance, decideReferenceReplayPromotion, decideReferenceReplayRunPromotion, defaultJudges, defaultReferenceReplayMatcher, deployGateLayer, distillPlaybook, dominates, estimateCost, estimateTokens, euAiActReport, evaluateActionPolicy, evaluateContract, evaluateHypothesis, evaluateOracles, executeScenario, expectAgent, exportRewardModel, exportRunAsOtlp, exportTrainingData, extractAssetUrls, extractErrorCount, failureClusterView, feedbackTrajectoriesToDatasetScenarios, feedbackTrajectoriesToOptimizerRows, feedbackTrajectoryToDatasetScenario, feedbackTrajectoryToOptimizerRow, fileContains, fileExists, findAutoMatchNoExpectation, findConstructorCwdDropped, findFallbackToPass, findLiteralTruePass, findSkipCountsAsPass, firstDivergenceView, flowLayer, formatBenchmarkReport, formatDriverReport, formatFindings, gainHistogram, precision as goldenPrecision, gradeSemanticStatus, groupBy, hashContent, hashScenarios, htmlContainsElement, inMemoryReferenceReplayStore, inMemoryReviewStore, interRaterReliability, iqr, isJudgeSpan, isLlmSpan, isPrmVerdict, isRetrievalSpan, isRunRecord, isSandboxSpan, isToolSpan, jestTestParser, jsonHasKeys, jsonShape, jsonlReferenceReplayStore, jsonlReviewStore, judgeAgreementView, judgeReplayGate, judgeSpans, keyPreserved, linterJudge, llmSpanFromProvider, llmSpans, loadScorerFromGrader, localCommandRunner, lowercaseMutator, mannWhitneyU, matchGoldens, mergeLayerResults, mergeSteeringBundle, multiToolchainLayer, nistAiRmfReport, nonRefusalRubric, normalizeScores, notBlocked, objectiveEval, outputLengthRubric, pairedBootstrap, pairedTTest, pairedWilcoxon, paraphraseRobustness, paretoChart, paretoFrontier, paretoFrontierWithCrowding, parseFeedbackTrajectoriesJsonl, parseReflectionResponse, parseRunRecordSafe, partialCredit, passOrthogonality, pixelDeltaRatio, politenessPrefixMutator, positionalBias, printDriverSummary, prmBestOfN, prmEnsembleBestOfN, probeLlm, promptBisect, proposeSynthesisTargets, pytestTestParser, redTeamDataset, redTeamReport, redactString, redactValue, referenceReplayRunsToSteeringRows, referenceReplayScenarioToRunScore, regexMatch, regexMatches, regressionView, renderMarkdown, renderMarkdownReport, renderPlaybookMarkdown, renderPreferenceMemoryMarkdown, renderSteeringText, replayFeedbackTrajectories, replayFeedbackTrajectory, replayScorerOverCorpus, replayTraceThroughJudge, requiredSampleSize, resetLockedAppendersForTesting, resumeBuilderSession, roundTripRunRecord, rowCount, rowWhere, runAgentControlLoop, runAssertions, runCanaries, runCounterfactual, runE2EWorkflow, runExpectations, runFailureClass, runHarnessExperiment, runIntentMatchJudge, runJudgeFleet, runKeywordCoverageJudge, runKeywordCoverageJudgeUrl, runPromptEvolution, runProposeReview, runProposeReviewAsControlLoop, runReferenceReplay, runSelfPlay, runSemanticConceptJudge, runTestGradedScenario, runsForScenario, scalarScore, scanForMuffledGates, scoreAllProjects, scoreContinuity, scoreProject, scoreRedTeamOutput, scoreReferenceReplay, securityJudge, selectHarnessVariant, selfPreference, sentenceReorderMutator, serializeFeedbackTrajectoriesJsonl, signManifest, soc2Report, statusAdvanced, stopOnNoProgress, stopOnRepeatedAction, stripFencedJson, stuckLoopView, subjectiveEval, summarize, summarizeHarnessResults, summarizePreferenceMemory, summaryTable, testJudge, textInSnapshot, toLangfuseEnvelope, toNdjson, toPrometheusText, toolIntentAlignmentRubric, toolNamesForRun, toolNonRedundantRubric, toolSpans, toolSuccessRubric, toolWasteView, typoMutator, urlContains, validateRunRecord, verbosityBias, verifyManifest, visualDiff, viteDeployRunner, vitestTestParser, weightedMean, weightedRecall, welchsTTest, whitespaceCollapseMutator, wilcoxonSignedRank, withAssignedFeedbackSplit, wranglerDeployRunner };
+export { type ActionExecutionPolicy, type ActionPolicyDecision, type ActionableSideInfo, type ActiveLearningOptions, type AdapterRun, AgentDriver, type AgentDriverConfig, type AlignmentOp, type AntiSlopConfig, type AntiSlopIssue, type AntiSlopReport, type Artifact$1 as Artifact, type ArtifactCheck, type Artifact as ArtifactCheckArtifact, type ArtifactResult, type ArtifactValidator, type AsiSeverity, AxGepaSteeringOptimizer, type AxSteeringOptimizerConfig, BENCHMARK_SPLIT_SEED, type BaselineOptions, type BaselineReport, BehaviorAssertion, type BenchmarkAdapter, type BenchmarkDatasetItem, type BenchmarkEvaluation, type BenchmarkReport, BenchmarkRunner, type BenchmarkRunnerConfig, type BestOfNResult, type BisectOptions, type BisectResult, type BisectStep, type BootstrapOptions, type BootstrapResult, BudgetBreachError, type BudgetBreachFinding, type BudgetBreachReport, BudgetGuard, type BudgetLedgerEntry, type BudgetSpec, BuilderSession, type BuilderSessionInit, type CalibrationBin, type CalibrationOptions, type CalibrationReport, type CalibrationResult, CallExpectation, type CanaryAlert, type CanaryKind, type CanaryLeak, type CanaryOptions, type CanaryReport, type CanarySeverity, type CandidateScenario, type CandidateScore, type CausalAttributionReport, type ChatSummary, type CheckResult, type CodeMutationOutcome, type CodeMutationRunner, type CollectedArtifacts, type CommandRunner, type CompletionCriterion, type CompositePolicy, type ConceptComplexity, type ConceptFinding, type ConceptSpec, type ConceptWeightStrategy, type ContinuityCheck, type ContinuityCheckResult, type ContinuityReport, type ContinuitySnapshotPair, type ContractMetric, type ContractReport, type ControlActionFailureMode, type ControlActionOutcome, type ControlBudget, type ControlContext, type ControlDecision, type ControlEvalResult, type ControlRunResult, type ControlRuntimeConfig, type ControlRuntimeError, type ControlSeverity, type ControlStep, type ControlStopPolicies, ConvergenceTracker, type CorrelationReport, type CorrelationResult, type CorrelationStudyOptions, type CorrelationStudyResult, type CostEntry, CostLedger, type CostLedgerGeneration, type CostLedgerSnapshot, type CostSummary, CostTracker, type CounterfactualContext, type CounterfactualMutation, type CounterfactualResult, type CounterfactualRunner, type CreateCompositeMutatorOpts, type CreateDefaultReviewerOptions, type CreateSandboxCodeMutatorOpts, type CreateSandboxPoolOpts, type CrossTraceDiff, type CrossTraceDiffOptions, D1ExperimentStore, type D1ExperimentStoreOptions, type D1Like, type D1PreparedStatementLike, DEFAULT_AGENT_SLOS, DEFAULT_COMPLEXITY_WEIGHTS, DEFAULT_RULES as DEFAULT_FAILURE_RULES, DEFAULT_FINDERS, DEFAULT_HARNESS_OBJECTIVES, DEFAULT_MUTATION_PRIMITIVES, DEFAULT_MUTATORS, DEFAULT_REDACTION_RULES, DEFAULT_RED_TEAM_CORPUS, DEFAULT_RUN_SCORE_WEIGHTS, DEFAULT_SEVERITY_WEIGHTS, Dataset, type DatasetDifficulty, type DatasetManifest, type DatasetProvenance, type DatasetScenario, type DatasetSplit, type DeployFamily, type DeployGateLayerInput, type DeployRunResult, type DeployRunner, type DeploymentOutcome, type DirEntry, type Direction, type DivergenceOptions, type DivergenceReport, DockerSandboxDriver, type DriverResult, type DriverState, DualAgentBench, type DualAgentBenchConfig, type DualAgentReport, type DualAgentRound, type DualAgentScenario, type DualAgentScenarioResult, ERROR_COUNT_PATTERNS, type ErrorCountPattern, type EuRiskClass, type EvalMetricSpec, type EvalResult, type EventFilter, type EventKind, type EvolutionRound, type EvolvableVariant, type ExecutorConfig, type Expectation, type Experiment, type ExperimentPlan, type ExperimentResult, type Run as ExperimentRun, type ExperimentStore, ExperimentTracker, type ExportedRewardModel, type ExtractOptions, type ExtractResult, FAILURE_CLASSES, type FactorContribution, type FactorialCell, type FailureClass, type FailureClassification, type FailureCluster, type FailureClusterReport, type FailureContext, type FailureMode, type FailureRule, type FeedbackArtifactType, type FeedbackAttempt, type FeedbackLabel, type FeedbackLabelKind, type FeedbackLabelSource, type FeedbackOptimizerRow, type FeedbackOutcome, type FeedbackPattern, type FeedbackReplayAdapter, type FeedbackReplayResult, type FeedbackSeverity, type FeedbackSplitPolicy, type FeedbackTask, type FeedbackTrajectory, type FeedbackTrajectoryFilter, type FeedbackTrajectoryStore, FileSystemExperimentStore, type FileSystemExperimentStoreOptions, FileSystemFeedbackTrajectoryStore, FileSystemOutcomeStore, type FileSystemOutcomeStoreOptions, FileSystemTraceStore, type FileSystemTraceStoreOptions, type Finding, type FlowAction, type FlowLayerEnv, type FlowLayerFactoryInput, type FlowRunner, type FlowRunnerStepResult, type FlowSpec, type FlowStep, type GainDistributionBin, type GainDistributionFigureSpec, type GainDistributionOptions, type GateDecision, type GateEvidence, type GenerationReport, type GenericSpan, type GoldenItem, type GoldenSeverity, type GoldenSpec, type GovernanceContext, type GovernanceFinding, type GovernanceReport, type GradedStep, type HarnessAdapter, type HarnessConfig, type HarnessExperimentConfig, type HarnessExperimentResult, type HarnessIntervention, type HarnessRunRequest, type HarnessRunResult, type HarnessScenario, type HarnessSelection, type HarnessVariant, type HarnessVariantReport, HeldOutGate, type HeldOutGateConfig, type HeldOutGateRejectionCode, HoldoutAuditor, HoldoutLockedError, type HostedJudgeConfig, type HostedJudgeDimension, type HostedJudgeRequest, type HostedJudgeResponse, type HostedRunCriticConfig, type HostedRunScoreRequest, type HostedRunScoreResponse, type HypothesisManifest, type HypothesisResult, INTENT_MATCH_JUDGE_VERSION, type ImageData, InMemoryExperimentStore, InMemoryFeedbackTrajectoryStore, InMemoryOutcomeStore, InMemoryTraceStore, InMemoryTrialCache, InMemoryWorkspaceInspector, type InferenceScorer, type InspectorContext, type IntentMatchInput, type IntentMatchOptions, type IntentMatchResult, type InteractionContribution, JsonlTrialCache, type JudgeAgreementReport, type JudgeConfig, type JudgeFleetOptions, type JudgeFn, type JudgeInput, type JudgePair, type JudgeReplayGateArgs, type JudgeReplayResult, type JudgeRubric, JudgeRunner, type JudgeScore, type JudgeSpan, type KeywordConceptSpec, type KeywordCoverageFinding, type KeywordCoverageOptions, type KeywordCoverageResult, type LangfuseEnvelope, type LangfuseGeneration, type LangfuseScore, type Layer, type LayerCorrelation, type LayerResult, type LayerStatus, type LineageKind, type LineageKindResolver, type LineageNode, LineageRecorder, LlmCallError, type LlmCallRequest, type LlmCallResult, LlmClient, type LlmClientOptions, type LlmJsonCall, type LlmMessage, type LlmReviewerConfig, type LlmSpan, type LlmUsage, LockedJsonlAppender, MODEL_PRICING, type MatchResult, type MatcherResult, type MeasurementPolicy, type MergeOptions, type Message, type MetricSamples, type MetricVerdict, MetricsCollector, type MuffledFinder, type MuffledFinding, MultiLayerVerifier, type MultiShotGateConfig, type MultiShotGateResult, type MultiShotMutateAdapter, type MultiShotOptimizationConfig, type MultiShotOptimizationResult, type MultiShotRun, type MultiShotRunInput, type MultiShotRunner, type MultiShotScore, type MultiShotScorer, type MultiShotSplit, type MultiShotTrace, type MultiShotTrialResult, type MultiShotVariant, type MultiToolchainLayerConfig, type MutateAdapter, type MutationAttempt, type MutationChannel, MutationTelemetry, type Mutator, Mutex, NoopResearcher, OTEL_AGENT_EVAL_SCOPE, type Objective, type Oracle, type OracleObservation, type OracleReport, type OracleResult, type OrthogonalityInput, type OrthogonalityResult, type OtlpExport, type OtlpResourceSpans, type OtlpSpan, type OutcomeFilter, type OutcomePair, type OutcomeStore, type PairedBootstrapOptions, type PairedBootstrapResult, PairwiseSteeringOptimizer, type ParetoFigureSpec, type ParetoPoint, type ParetoResult, type PersonaConfig, type Playbook, type PlaybookEntry, type PoolSlot, type PositionalBiasResult, type PreferenceMemoryEntry, type PrmGradedTrace, PrmGrader, type PrmTrainingSample, ProductClient, type ProductClientConfig, type ProjectKind, ProjectRegistry, type ProjectSummary, type ProjectTimelineEntry, type PromptEvolutionConfig, type PromptEvolutionEvent, type PromptEvolutionResult, type PromptHandle, PromptRegistry, type TrialResult as PromptTrialResult, type ProposeFn, type ProposeInput, type ProposeOutput, type ProposeReviewConfig, type ProposeReviewControlAction, type ProposeReviewControlConfig, type ProposeReviewControlResult, type ProposeReviewControlState, type ProposeReviewReport, type ProposeReviewShot, type ProposedSideEffect, REDACTION_VERSION, type RedTeamCase, type RedTeamCategory, type RedTeamFinding, type RedTeamPayload, type RedTeamReport, type RedactionReport, type RedactionRule, type ReferenceMatchResult, type ReferenceReplayAdapter, type ReferenceReplayAdapterFn, type ReferenceReplayAdapterLike, type ReferenceReplayAggregate, type ReferenceReplayCandidate, type ReferenceReplayCase, type ReferenceReplayCaseRun, type ReferenceReplayExecutionScenario, type ReferenceReplayItem, type ReferenceReplayMatch, type ReferenceReplayMatchStrategy, type ReferenceReplayMatcher, type ReferenceReplayPromotionDecision, type ReferenceReplayPromotionPolicy, type ReferenceReplayRun, type ReferenceReplayRunContext, type ReferenceReplayRunOptions, type ReferenceReplayRunStore, type ReferenceReplayScenario, type ReferenceReplayScenarioScore, type ReferenceReplayScore, type ReferenceReplayScoreOptions, type ReferenceReplaySplit, type ReferenceReplaySplitComparison, type ReferenceReplaySteeringRowsOptions, type ReflectionContext, type ReflectionProposal, type RegressionOptions, type RegressionSpec, type Researcher, type RetrievalSpan, type Review, type ReviewFn, type ReviewInput, type ReviewMemoryEntry, type ReviewMemoryStore, type ReviewerMemoryEntry, type ReviewerOutput, type ReviewerPromptInput, type ReviewerSoftFailDefaults, type ReviewerVerificationSummary, type RobustnessResult, type RouteMap, type RubricDimension, type Run$1 as Run, type RunAppScenarioOptions, type RunCommandInput, type RunCommandResult, type RunConfig, RunCritic, type RunCriticOptions, type RunDiff, type RunFilter, type RunJudgeMetadata, type RunLayer, type RunOutcome, type RunRecord, RunRecordValidationError, type RunScore, type RunScoreWeights, type RunSplitTag, type RunStatus, type RunTokenUsage, type RunTrace, SEMANTIC_CONCEPT_JUDGE_VERSION, type SandboxDriver, SandboxHarness, type SandboxHarnessResult, type SandboxJudgeKind, type SandboxJudgeResult, type SandboxJudgeSpec, type SandboxPool, type SandboxResult, type SandboxSpan, type ScanOptions, type Scenario, type ScenarioAggregate, type ScenarioCost, type ScenarioFile, ScenarioRegistry, type ScenarioResult, type ScoreAdapter, type ScoredTarget, type SelfPlayOptions, type SelfPlayProposer, type SelfPlayScorer, type SelfPreferenceResult, type SemanticConceptJudgeInput, type SemanticConceptJudgeOptions, type SemanticConceptJudgeResult, type SeriesConvergenceOptions, type SeriesConvergenceResult, type Severity, type ShipOptions, type SignedManifest, type SliceOptions, type Slo, type SloCheckResult, type SloComparator, type SloReport, type SloSeverity, type SlopCategory, type SlotFactory, type Span, type SpanBase, type SpanFilter, type SpanHandle, type SpanKind, type SpanStatus, type SteeringBundle, type SteeringChange, type SteeringDelta, type SteeringOptimizationResult, type SteeringOptimizationRow, type SteeringOptimizationSelector, type SteeringOptimizerBackend, type SteeringOptimizerConfig, type SteeringRolePrompt, type StepAttribution, type StepContext, type StepRubric, type StopDecision, type StuckLoopFinding, type StuckLoopOptions, type StuckLoopReport, SubprocessSandboxDriver, type SubprocessSandboxDriverOptions, type SummaryTable, type SummaryTableOptions, type SummaryTableRow, type SynthesisReason, type SynthesisTarget, TRACE_SCHEMA_VERSION, type TestGradedRunOptions, type TestGradedRunResult, type TestGradedScenario, type TestOutputParser, type TestResult, type ThreeLayerProjectReport, type ThresholdContract, TokenCounter, type TokenSpec, type ToolSpan, type ToolStats, type ToolUseMetrics, type ToolUseOptions, type ToolWasteFinding, type ToolWasteOptions, type ToolWasteReport, TraceEmitter, type TraceEmitterOptions, type TraceEvent, type TraceStore, type Trajectory, type TrajectoryStep, type TrialAttempt, type TrialCache, TrialTelemetry, type TrialTrace, type Turn, type TurnMetrics, type TurnResult, UNIVERSAL_FINDERS, type UseCaseSignals, type ValidationContext, type ValidationIssue, type ValidationResult, type VariantAggregate, type VerbosityBiasResult, type Verdict, type Verification, type VerificationReport, type VerifyContext, type VerifyFn, type VerifyOptions, type VisualDiffOptions, type VisualDiffResult, type ViteDeployRunnerInput, type WorkflowTopology, type WorkspaceAssertion, type WorkspaceAssertionResult, type WorkspaceInspector, type WorkspaceSnapshot, type WranglerDeployRunnerInput, adversarialJudge, aggregateLlm, aggregateRunScore, allCriticalPassed, analyzeAntiSlop, analyzeSeries, argHash, assignFeedbackSplit, attributeCounterfactuals, deterministicSplit as benchmarkDeterministicSplit, index as benchmarks, benjaminiHochberg, bhAdjust, bisect, bonferroni, bootstrapCi, budgetBreachView, buildReflectionPrompt, buildReviewerPrompt, buildTrajectory, byteLengthRange, calibrateJudge, calibrationCurve, callLlm, callLlmJson, canaryLeakView, causalAttribution, checkCanaries, checkSlos, clamp01, classifyEuAiRisk, classifyFailure, codeExecutionJudge, cohensD, coherenceJudge, collectionPreserved, commitBisect, compareReferenceReplay, compareToBaseline, compilerJudge, composeParsers, composeValidators, computeToolUseMetrics, confidenceInterval, containsAll, controlFailureClassFromVerification, controlRunToFeedbackTrajectory, correlateLayers, correlationStudy, createAntiSlopJudge, createCompositeMutator, createCustomJudge, createDefaultReviewer, createDomainExpertJudge, createFeedbackTrajectory, createIntentMatchJudge, createLlmReviewer, createSandboxCodeMutator, createSandboxPool, createSemanticConceptJudge, crossTraceDiff, crowdingDistance, decideReferenceReplayPromotion, decideReferenceReplayRunPromotion, defaultJudges, defaultMultiShotObjectives, defaultReferenceReplayMatcher, deployGateLayer, distillPlaybook, dominates, estimateCost, estimateTokens, euAiActReport, evaluateActionPolicy, evaluateContract, evaluateHypothesis, evaluateOracles, executeScenario, expectAgent, exportRewardModel, exportRunAsOtlp, exportTrainingData, extractAssetUrls, extractErrorCount, failureClusterView, feedbackTrajectoriesToDatasetScenarios, feedbackTrajectoriesToOptimizerRows, feedbackTrajectoryToDatasetScenario, feedbackTrajectoryToOptimizerRow, fileContains, fileExists, findAutoMatchNoExpectation, findConstructorCwdDropped, findFallbackToPass, findLiteralTruePass, findSkipCountsAsPass, firstDivergenceView, flowLayer, formatBenchmarkReport, formatDriverReport, formatFindings, gainHistogram, precision as goldenPrecision, gradeSemanticStatus, groupBy, hashContent, hashScenarios, htmlContainsElement, inMemoryReferenceReplayStore, inMemoryReviewStore, interRaterReliability, iqr, isJudgeSpan, isLlmSpan, isPrmVerdict, isRetrievalSpan, isRunRecord, isSandboxSpan, isToolSpan, jestTestParser, jsonHasKeys, jsonShape, jsonlReferenceReplayStore, jsonlReviewStore, judgeAgreementView, judgeReplayGate, judgeSpans, keyPreserved, linterJudge, llmSpanFromProvider, llmSpans, loadScorerFromGrader, localCommandRunner, lowercaseMutator, mannWhitneyU, matchGoldens, mergeLayerResults, mergeSteeringBundle, multiToolchainLayer, nistAiRmfReport, nonRefusalRubric, normalizeScores, notBlocked, objectiveEval, outputLengthRubric, pairedBootstrap, pairedTTest, pairedWilcoxon, paraphraseRobustness, paretoChart, paretoFrontier, paretoFrontierWithCrowding, parseFeedbackTrajectoriesJsonl, parseReflectionResponse, parseRunRecordSafe, partialCredit, passOrthogonality, pixelDeltaRatio, politenessPrefixMutator, positionalBias, printDriverSummary, prmBestOfN, prmEnsembleBestOfN, probeLlm, promptBisect, proposeSynthesisTargets, pytestTestParser, redTeamDataset, redTeamReport, redactString, redactValue, referenceReplayRunsToSteeringRows, referenceReplayScenarioToRunScore, regexMatch, regexMatches, regressionView, renderMarkdown, renderMarkdownReport, renderPlaybookMarkdown, renderPreferenceMemoryMarkdown, renderSteeringText, replayFeedbackTrajectories, replayFeedbackTrajectory, replayScorerOverCorpus, replayTraceThroughJudge, requiredSampleSize, resetLockedAppendersForTesting, resumeBuilderSession, roundTripRunRecord, rowCount, rowWhere, runAgentControlLoop, runAssertions, runCanaries, runCounterfactual, runE2EWorkflow, runExpectations, runFailureClass, runHarnessExperiment, runIntentMatchJudge, runJudgeFleet, runKeywordCoverageJudge, runKeywordCoverageJudgeUrl, runMultiShotOptimization, runPromptEvolution, runProposeReview, runProposeReviewAsControlLoop, runReferenceReplay, runSelfPlay, runSemanticConceptJudge, runTestGradedScenario, runsForScenario, scalarScore, scanForMuffledGates, scoreAllProjects, scoreContinuity, scoreProject, scoreRedTeamOutput, scoreReferenceReplay, securityJudge, selectHarnessVariant, selfPreference, sentenceReorderMutator, serializeFeedbackTrajectoriesJsonl, signManifest, soc2Report, statusAdvanced, stopOnNoProgress, stopOnRepeatedAction, stripFencedJson, stuckLoopView, subjectiveEval, summarize, summarizeHarnessResults, summarizePreferenceMemory, summaryTable, testJudge, textInSnapshot, toLangfuseEnvelope, toNdjson, toPrometheusText, toolIntentAlignmentRubric, toolNamesForRun, toolNonRedundantRubric, toolSpans, toolSuccessRubric, toolWasteView, trialTraceFromMultiShotTrial, typoMutator, urlContains, validateRunRecord, verbosityBias, verifyManifest, visualDiff, viteDeployRunner, vitestTestParser, weightedMean, weightedRecall, welchsTTest, whitespaceCollapseMutator, wilcoxonSignedRank, withAssignedFeedbackSplit, wranglerDeployRunner };