@tangle-network/agent-eval 0.40.5 → 0.42.0

package/dist/summary-report-DuZXOk7K.d.ts

@@ -1,917 +0,0 @@
-import { R as RunRecord, a as RunSplitTag } from './run-record-BGY6bHRh.js';
-import { F as FailureClusterReport } from './failure-cluster-Cw65_5FY.js';
-
-/**
- * HeldOutGate — first-class held-out paired-delta promotion gate.
- *
- * Encodes the "honesty override" pattern that lived inline in
- * `~/webb/redteam/scripts/agent-eval-autoresearch.ts:138–171`.
- * The optimizer's best-guess is one thing; what we should actually
- * ship is another. The gate is the line between them.
- *
- * A candidate is promoted iff ALL three pass:
- *
- *   1. **Productive runs**: the candidate has at least
- *      `minProductiveRuns` paired observations on items where BOTH
- *      candidate and baseline produced a real (non-silent) score.
- *   2. **Paired delta**: the lower bound of the bootstrap CI on the
- *      median per-item delta (candidate − baseline) on the HOLDOUT
- *      split is strictly greater than `pairedDeltaThreshold`.
- *   3. **Overfit gap**: the candidate's gap between search-split
- *      score and holdout-split score is no worse (more positive)
- *      than the baseline's gap by more than `overfitGapThreshold`.
- *      "Better on search, worse on holdout" is the canonical
- *      overfit pattern; this catches it.
- *
- * The decision carries a machine-readable `rejectionCode` plus an
- * `evidence` block with every number the gate looked at, so the
- * downstream researcher / paper / dashboard can re-derive the
- * verdict without re-running.
- *
- * See also:
- *   - `src/statistics.ts` for `pairedBootstrap` + `wilcoxonSignedRank`
- *   - `src/run-record.ts` for the input row schema
- *   - `src/reference-replay.ts` for the older, reference-replay-
- *     specific promotion path (still useful for replay-style evals).
- */
-
-type HeldOutGateRejectionCode = 'few_runs' | 'negative_delta' | 'overfit_gap';
-interface HeldOutGateConfig {
-    /** Minimum number of paired (candidate, baseline) holdout observations
-     *  required before the gate will even consider promoting. Default 3. */
-    minProductiveRuns?: number;
-    /** The bootstrap-CI lower bound on the median paired holdout delta
-     *  must exceed this to promote. Default 0. */
-    pairedDeltaThreshold?: number;
-    /** Maximum allowed worsening of (search − holdout) gap relative to
-     *  baseline. Default 0.15 (i.e. candidate may overfit by up to 15
-     *  absolute score points more than baseline before rejection). */
-    overfitGapThreshold?: number;
-    /** Stable label of the baseline candidate. Required — paper-grade
-     *  evaluation never compares two unlabelled candidates. */
-    baselineKey: string;
-    /** Confidence level for the bootstrap CI. Default 0.95. */
-    confidence?: number;
-    /** Bootstrap resamples. Default 2000. */
-    bootstrapResamples?: number;
-    /** Optional deterministic seed for the bootstrap. Default undefined
-     *  (Math.random). */
-    seed?: number;
-}
-interface GateEvidence {
-    /** Number of paired (candidate, baseline) holdout observations used. */
-    productiveRuns: number;
-    /** Median of (candidate − baseline) paired holdout deltas. */
-    medianPairedDelta: number;
-    /** Bootstrap CI on the median paired holdout delta. */
-    pairedCI: {
-        low: number;
-        high: number;
-    };
-    /** Wilcoxon signed-rank p-value on the paired holdout deltas. */
-    pairedPValue: number;
-    /** Mean candidate score on the search split (NaN if none). */
-    searchScore: number;
-    /** Mean candidate score on the holdout split (NaN if none). */
-    holdoutScore: number;
-    /** Candidate (search − holdout) gap. */
-    overfitGap: number;
-    /** Baseline (search − holdout) gap. */
-    baselineOverfitGap: number;
-}
-interface GateDecision {
-    /** Final promote/no-promote verdict. */
-    promote: boolean;
-    /** The candidate that was evaluated. */
-    candidateId: string;
-    /** The baseline it was compared against. */
-    baselineId: string;
-    /** Every number the gate looked at, for audit + paper export. */
-    evidence: GateEvidence;
-    /** Human-readable reason. */
-    reason: string;
-    /** Machine-readable rejection code, or null on promote. */
-    rejectionCode: HeldOutGateRejectionCode | null;
-}
-/**
- * Held-out paired-delta promotion gate. Construct once with config,
- * call `evaluate(candidateRuns, baselineRuns)` per (candidate,
- * baseline) pair. Stateless across calls.
- */
-declare class HeldOutGate {
-    private readonly minProductiveRuns;
-    private readonly pairedDeltaThreshold;
-    private readonly overfitGapThreshold;
-    private readonly baselineKey;
-    private readonly confidence;
-    private readonly resamples;
-    private readonly seed?;
-    constructor(config: HeldOutGateConfig);
-    /** Decide whether `candidate` should replace `baseline`. Pairing
-     *  is by (experimentId, seed) — identical experiment + seed pairs
-     *  the candidate run with the matching baseline run. Pairs without
-     *  a holdout score on both sides are dropped. */
-    evaluate(candidate: RunRecord[], baseline: RunRecord[]): GateDecision;
-}
-
-/**
- * Pareto frontier — multi-objective optimization over candidate runs.
- *
- * Lifted from ADC pareto.ts and blueprint-agent frontier.ts. When you're
- * trading off (cost, latency, quality) or (passRate, tokenBudget,
- * ttfb), you rarely have a single "winner" — you have a set of
- * non-dominated candidates. This module exposes:
- *
- *   - `paretoFrontier`: filter a set of candidates to the non-dominated ones
- *   - `dominates`: does A dominate B across all objectives?
- *
- * Each objective is declared with a direction: 'maximize' (higher=better)
- * or 'minimize' (lower=better). Candidates are any object; pass an
- * `objective(candidate)` accessor.
- */
-type Direction = 'maximize' | 'minimize';
-interface Objective<T> {
-    /** Stable label used in reports. */
-    name: string;
-    direction: Direction;
-    value: (candidate: T) => number;
-}
-interface ParetoResult<T> {
-    frontier: T[];
-    dominated: T[];
-    /** Index map: frontier[i] dominates each of dominatedBy[i]. */
-    dominanceMap: Array<{
-        dominator: T;
-        dominated: T[];
-    }>;
-}
-/** Does candidate A weakly dominate B — strictly better on at least one objective and no worse on any? */
-declare function dominates<T>(a: T, b: T, objectives: Objective<T>[]): boolean;
-/**
- * Compute the non-dominated frontier. Candidates with NaN/Infinity on any
- * objective are excluded (can't rank them). A candidate enters the frontier
- * iff no other candidate dominates it.
- */
-declare function paretoFrontier<T>(candidates: T[], objectives: Objective<T>[]): ParetoResult<T>;
-/**
- * Weighted-sum scalarisation. Use as a tie-break / single-winner selector
- * when callers don't want to consume a frontier. Each objective contributes
- * its normalised value (0..1 via min-max across the candidate pool) times
- * its weight; missing weights default to 1/N.
- *
- * Direction is honoured automatically — `minimize` axes have their values
- * inverted before scaling so "higher scalar = better" always holds.
- */
-declare function scalarScore<T>(candidates: T[], objectives: Objective<T>[], options?: {
-    weights?: Partial<Record<string, number>>;
-}): Array<{
-    candidate: T;
-    score: number;
-}>;
-/**
- * NSGA-II crowding distance — secondary sort for ties on the frontier.
- *
- * When the Pareto front collapses to a single point (or many candidates tie
- * on dominance), naive selection picks arbitrarily and the population
- * degenerates over generations. NSGA-II preserves diversity by preferring
- * candidates with more empty space around them on the frontier.
- *
- * Returns an array of `{ candidate, distance }` in the SAME order as the
- * input. Higher distance = more isolated = should be preferred when
- * preserving diversity.
- */
-declare function crowdingDistance<T>(candidates: T[], objectives: Objective<T>[]): Array<{
-    candidate: T;
-    distance: number;
-}>;
-/**
- * Pareto frontier with tie-break by crowding distance — the canonical
- * NSGA-II selection step. Returns the frontier sorted by descending crowding
- * distance so callers can `.slice(0, k)` to pick K diverse winners.
- */
-declare function paretoFrontierWithCrowding<T>(candidates: T[], objectives: Objective<T>[]): Array<{
-    candidate: T;
-    distance: number;
-}>;
-
-/**
- * PromptEvolutionLoop — population-based reflective-mutation evolution.
- *
- * Above the existing `AxGepaSteeringOptimizer` (which RANKS variants),
- * this loop GENERATES variants. Each generation:
- *   1. Score the population across (variant × scenario × rep).
- *   2. Pick survivors from the Pareto frontier (with crowding-distance tie-break).
- *   3. Ask the mutator for replacements until population size is restored.
- *   4. Repeat for N generations OR until convergence.
- *
- * Domain-agnostic. Consumers supply:
- *   - A seed population of `EvolvableVariant`s.
- *   - A `ScoreAdapter` that runs (variant, scenario, rep) → `TrialResult`.
- *   - A `MutateAdapter` that produces children given trace evidence.
- *   - Pareto `Objective<TrialAggregate>[]` defining the multi-objective vector.
- *
- * The loop owns: population management, parallel scheduling (concurrency-
- * limited), Pareto selection with crowding distance, generation reporting.
- *
- * It does NOT own: rendering trials to a model, executing prompts, choosing
- * mutation primitives, persisting to disk. Those are the consumer's call.
- */
-
-interface EvolvableVariant<P = unknown> {
-    /** Stable id for the variant — surfaces in reports and trial results. */
-    id: string;
-    /** Variant payload — interpretation is the consumer's responsibility. */
-    payload: P;
-    /** Generation index (0 = seed, then 1, 2, ...). */
-    generation: number;
-    /** Parent variant id when produced via mutation; absent for seeds. */
-    parentId?: string;
-    /** Human label for reports. */
-    label: string;
-    /** What the mutator was trying to fix. */
-    rationale?: string;
-}
-interface TrialResult {
-    variantId: string;
-    scenarioId: string;
-    rep: number;
-    ok: boolean;
-    /** Primary scalar score the consumer cares about (e.g., recall, accuracy). */
-    score: number;
-    /** Token cost (or any cost-like dimension). */
-    cost?: number;
-    /** Wall time in ms. */
-    durationMs?: number;
-    /** Free-form metric bag for objective accessors. */
-    metrics?: Record<string, number>;
-    error?: string;
-    /**
-     * Whether the judge LLM call(s) that produced this trial's score actually
-     * completed. `undefined` means "consumer didn't report"; `false` means
-     * the judge aborted/failed and the score is synthetic (typically 0 or
-     * partial). `aggregateTrials({mode: 'exclude-failed'})` skips these
-     * trials so a silent-zero judge can't pollute the composite.
-     */
-    judgeSucceeded?: boolean;
-    /** Number of judge attempts (informational, populated by `withJudgeRetry`). */
-    judgeAttempts?: number;
-    /** Last judge error message when `judgeSucceeded === false`. */
-    judgeError?: string;
-}
-/** Aggregated trial summary for one (variant, scenario) pair across reps. */
-interface ScenarioAggregate {
-    variantId: string;
-    scenarioId: string;
-    meanScore: number;
-    meanCost: number;
-    meanDurationMs: number;
-    okRate: number;
-    trials: number;
-    /** Mean of every numeric metric across reps. */
-    metrics: Record<string, number>;
-}
-/** Aggregated trial summary for one variant across all scenarios. */
-interface VariantAggregate {
-    variantId: string;
-    meanScore: number;
-    meanCost: number;
-    meanDurationMs: number;
-    okRate: number;
-    scenarios: ScenarioAggregate[];
-    /** Mean of every numeric metric, averaged across scenarios. */
-    metrics: Record<string, number>;
-}
-interface ScoreAdapter<P = unknown> {
-    score(args: {
-        variant: EvolvableVariant<P>;
-        scenarioId: string;
-        rep: number;
-    }): Promise<TrialResult>;
-}
-interface MutateAdapter<P = unknown> {
-    mutate(args: {
-        parent: EvolvableVariant<P>;
-        parentAggregate: VariantAggregate;
-        topTrials: TrialResult[];
-        bottomTrials: TrialResult[];
-        childCount: number;
-        generation: number;
-    }): Promise<EvolvableVariant<P>[]>;
-}
-interface PromptEvolutionConfig<P = unknown> {
-    runId: string;
-    /** What component is being mutated — surfaces in reports + reflection prompts. */
-    target: string;
-    seedVariants: EvolvableVariant<P>[];
-    scenarioIds: string[];
-    reps: number;
-    generations: number;
-    populationSize: number;
-    /** Maximum concurrent score() calls. */
-    scoreConcurrency: number;
-    scoreAdapter: ScoreAdapter<P>;
-    mutateAdapter: MutateAdapter<P>;
-    /** Pareto objectives over `VariantAggregate`. Ordered by importance. */
-    objectives: Objective<VariantAggregate>[];
-    /** Optional weights for the scalar tie-break selector (by objective name). */
-    scalarWeights?: Record<string, number>;
-    /** Stop early if a generation produces no Pareto improvement. Default true. */
-    earlyStopOnNoImprovement?: boolean;
-    onProgress?: (event: PromptEvolutionEvent) => void;
-    /**
-     * Optional cache key for memoising scored (variantId, scenarioId, rep)
-     * tuples. When provided AND a cache instance is passed, repeated trials
-     * skip re-scoring. Cache keys are stable across runs.
-     */
-    cache?: TrialCache;
-}
-interface TrialCache {
-    get(key: string): TrialResult | undefined;
-    set(key: string, value: TrialResult): void;
-}
-declare class InMemoryTrialCache implements TrialCache {
-    private store;
-    get(key: string): TrialResult | undefined;
-    set(key: string, value: TrialResult): void;
-    size(): number;
-    clear(): void;
-}
-type PromptEvolutionEvent = {
-    type: 'generation-start';
-    generation: number;
-    populationSize: number;
-} | {
-    type: 'trial-complete';
-    generation: number;
-    variantId: string;
-    scenarioId: string;
-    rep: number;
-    ok: boolean;
-    score: number;
-    cached: boolean;
-} | {
-    type: 'generation-complete';
-    report: GenerationReport<unknown>;
-} | {
-    type: 'converged';
-    generation: number;
-    reason: string;
-};
-interface GenerationReport<P = unknown> {
-    runId: string;
-    target: string;
-    generation: number;
-    variants: EvolvableVariant<P>[];
-    aggregates: VariantAggregate[];
-    /** Frontier candidates, sorted by descending crowding distance. */
-    paretoFrontIds: string[];
-    /** Scalar-best variant id — used for the single "winner" if callers want one. */
-    winnerId: string;
-    /** Trials that fed this generation (kept for downstream reporting). */
-    trials: TrialResult[];
-}
-interface PromptEvolutionResult<P = unknown> {
-    runId: string;
-    target: string;
-    generations: GenerationReport<P>[];
-    /** Best variant by scalar score in the final generation. */
-    bestVariant: EvolvableVariant<P>;
-    /** Best aggregate (matches bestVariant). */
-    bestAggregate: VariantAggregate;
-}
-declare function runPromptEvolution<P>(config: PromptEvolutionConfig<P>): Promise<PromptEvolutionResult<P>>;
-
-/**
- * Reflective mutation — primitives for trace-conditioned prompt rewriting.
- *
- * Used by `prompt-evolution.ts` (and any consumer running iterative
- * improvement). Given a parent prompt + concrete trace evidence (top trials,
- * bottom trials, missed expectations), produce an LLM-ready prompt that
- * proposes targeted mutations — not blind rephrasings.
- *
- * Why this lives outside `prompt-evolution.ts`: any consumer that wants to
- * run reflective rewriting WITHOUT the population/Pareto machinery can
- * import these primitives directly.
- *
- * Quality bar (vs. naive "mutate this prompt"):
- *   - Show parent ↔ children diff, not just one variant
- *   - Quote specific missed goldens with their match phrases
- *   - Surface the model's actual emitted output side-by-side with what was expected
- *   - Quote concrete mutation primitives so the model has a vocabulary
- */
-interface TrialTrace {
-    /** Stable id for the trial — surfaces in the prompt for grounding. */
-    id: string;
-    /** Score the trial received on its primary metric. */
-    score: number;
-    /** Candidate inputs the agent was given (e.g., the fixture or scenario). */
-    inputName?: string;
-    /**
-     * Goldens / expectations this trial was tested against, with whether each
-     * was matched. The reflection prompt quotes the missed ones specifically.
-     */
-    expectations?: Array<{
-        id: string;
-        phrase: string;
-        matched: boolean;
-    }>;
-    /** Free-form text — what the agent actually emitted (e.g., findings, plan). */
-    emitted?: string;
-    /** Optional structured metrics (recall, precision, cost, latency). */
-    metrics?: Record<string, number>;
-}
-interface ReflectionContext {
-    /** What is being mutated — appears in the system prompt for orientation. */
-    target: string;
-    /** Current variant's payload — JSON-serialised for the prompt. */
-    parentPayload: unknown;
-    /** Best-performing trials this generation. */
-    topTrials: TrialTrace[];
-    /** Worst-performing trials this generation — the missed-golden source. */
-    bottomTrials: TrialTrace[];
-    /** How many children the mutator should propose. */
-    childCount: number;
-    /** Optional: domain-specific mutation primitives the model can pick from. */
-    mutationPrimitives?: string[];
-}
-declare const DEFAULT_MUTATION_PRIMITIVES: string[];
-/**
- * Build the LLM-ready reflection prompt. Output is plain text — pass it as
- * the user message. The system message should be small and stable (e.g.
- * "Output ONLY a JSON object matching the schema below.").
- */
-declare function buildReflectionPrompt(ctx: ReflectionContext): string;
-interface ReflectionProposal {
-    label: string;
-    rationale: string;
-    payload: unknown;
-}
-declare function parseReflectionResponse(raw: string, maxProposals?: number): ReflectionProposal[];
-
-/**
- * Multi-shot optimization adapter.
- *
- * This is the canonical bridge between variable-length agent trajectories
- * and `runPromptEvolution`. Apps provide four things:
- *
- *   - variants: prompt/config/tool-policy candidates
- *   - runner: executes one full task trajectory for a variant
- *   - scorer: turns that trajectory into score + actionable side information
- *   - mutator: proposes new variants from top/bottom scored trials
- *
- * The adapter owns the boring but easy-to-get-wrong glue: stable seeds,
- * score/cost objectives, error-to-trial conversion, ASI metric projection,
- * and optional paired holdout gating via `HeldOutGate`.
- */
-
-type MultiShotSplit = 'search' | 'dev' | 'holdout';
-type AsiSeverity = 'info' | 'warning' | 'error' | 'critical';
-type MultiShotVariant<P = unknown> = EvolvableVariant<P>;
-interface ActionableSideInfo {
-    /** Stable expectation/check id when available. */
-    expectationId?: string;
-    /** Human-readable diagnosis of what happened. */
-    message: string;
-    severity?: AsiSeverity;
-    /** Concrete trace excerpt, file path, tool call, screenshot id, etc. */
-    evidence?: string;
-    /** Prompt/tool/context surface likely responsible. */
-    responsibleSurface?: string;
-    /** Suggested fix in natural language. */
-    suggestion?: string;
-    /** Whether this expectation was satisfied. Defaults to false for ASI rows. */
-    matched?: boolean;
-    metadata?: Record<string, unknown>;
-}
-interface MultiShotTrace {
-    scenarioId: string;
-    /** Full turn/tool trace. Shape is intentionally app-owned. */
-    turns?: unknown[];
-    toolCalls?: unknown[];
-    artifacts?: unknown[];
-    /** Compact final output or summary used by reflection prompts. */
-    transcript?: string;
-    output?: unknown;
-    metadata?: Record<string, unknown>;
-}
-interface MultiShotRun {
-    trace: MultiShotTrace;
-    costUsd?: number;
-    durationMs?: number;
-    tokenUsage?: {
-        input?: number;
-        output?: number;
-        cached?: number;
-    };
-    metadata?: Record<string, unknown>;
-}
-interface MultiShotRunInput<P = unknown> {
-    variant: EvolvableVariant<P>;
-    scenarioId: string;
-    rep: number;
-    split: MultiShotSplit;
-    /** Stable paired seed for baseline/candidate comparisons. */
-    seed: number;
-}
-interface MultiShotRunner<P = unknown> {
-    run(input: MultiShotRunInput<P>): Promise<MultiShotRun> | MultiShotRun;
-}
-interface MultiShotScore {
-    /** Primary score in [0,1]. The adapter clamps for safety. */
-    score: number;
-    /** Pass/fail for top/bottom trial selection. Defaults to true. */
-    ok?: boolean;
-    costUsd?: number;
-    durationMs?: number;
-    metrics?: Record<string, number>;
-    asi?: ActionableSideInfo[];
-    /** Optional rich output shown to reflection mutators. */
-    emitted?: string;
-    metadata?: Record<string, unknown>;
-}
-interface MultiShotScorer<P = unknown> {
-    score(input: MultiShotRunInput<P> & {
-        run: MultiShotRun;
-    }): Promise<MultiShotScore> | MultiShotScore;
-}
-interface MultiShotTrialResult extends TrialResult {
-    split: MultiShotSplit;
-    seed: number;
-    trace?: MultiShotTrace;
-    asi?: ActionableSideInfo[];
-    emitted?: string;
-    metadata?: Record<string, unknown>;
-}
-interface MultiShotMutateAdapter<P = unknown> {
-    mutate(args: {
-        parent: EvolvableVariant<P>;
-        parentAggregate: VariantAggregate;
-        topTrials: MultiShotTrialResult[];
-        bottomTrials: MultiShotTrialResult[];
-        childCount: number;
-        generation: number;
-    }): Promise<EvolvableVariant<P>[]>;
-}
-interface MultiShotGateConfig<P = unknown> {
-    /** Search rows are optional, but enable HeldOutGate's overfit-gap check. */
-    searchScenarioIds?: string[];
-    holdoutScenarioIds: string[];
-    reps?: number;
-    gate: HeldOutGateConfig;
-    /** Convert scored trajectory runs into paper-grade RunRecords. */
-    toRunRecord(input: {
-        variant: EvolvableVariant<P>;
-        scenarioId: string;
-        rep: number;
-        split: RunSplitTag;
-        seed: number;
-        trial: MultiShotTrialResult;
-    }): RunRecord;
-}
-interface MultiShotOptimizationConfig<P = unknown> {
-    runId: string;
-    target: string;
-    seedVariants: EvolvableVariant<P>[];
-    searchScenarioIds: string[];
-    reps: number;
-    generations: number;
-    populationSize: number;
-    scoreConcurrency?: number;
-    runner: MultiShotRunner<P>;
-    scorer: MultiShotScorer<P>;
-    mutateAdapter: MultiShotMutateAdapter<P>;
-    objectives?: Objective<VariantAggregate>[];
-    scalarWeights?: Record<string, number>;
-    cache?: TrialCache;
-    earlyStopOnNoImprovement?: boolean;
-    seedBase?: number;
-    onProgress?: (event: PromptEvolutionEvent) => void;
-    gate?: MultiShotGateConfig<P>;
-}
-interface MultiShotGateResult {
-    decision: GateDecision;
-    candidateRuns: RunRecord[];
-    baselineRuns: RunRecord[];
-}
-interface MultiShotOptimizationResult<P = unknown> {
-    evolution: PromptEvolutionResult<P>;
-    /** Best candidate on the optimizer-visible search split. */
-    searchBestVariant: EvolvableVariant<P>;
-    searchBestAggregate: VariantAggregate;
-    /** Variant callers should actually ship after optional holdout gating. */
-    promotedVariant: EvolvableVariant<P>;
-    promotedAggregate: VariantAggregate;
-    /** Null when no gate was configured or the search-best candidate was the baseline. */
-    gate: MultiShotGateResult | null;
-}
-declare function runMultiShotOptimization<P>(config: MultiShotOptimizationConfig<P>): Promise<MultiShotOptimizationResult<P>>;
-declare function defaultMultiShotObjectives(): Objective<VariantAggregate>[];
-declare function trialTraceFromMultiShotTrial(trial: MultiShotTrialResult): TrialTrace;
-
-/**
- * Reporting helpers — production summaries and paper-quality figures — sit alongside `reporter.ts` rather
- * than replacing it.
- *
- * Three artefacts:
- *
- *   - `summaryTable`           Markdown table of per-candidate means,
- *                            95% bootstrap CIs, BH-adjusted Wilcoxon
- *                            p-values, and Cohen's d versus a
- *                            comparator candidate.
- *   - `paretoChart`         Abstract spec for a cost vs quality
- *                            scatter, with gate decisions overlaid.
- *                            Returns numbers + labels — caller
- *                            chooses the plotting library.
- *   - `gainHistogram`
- *                            Per-item paired holdout deltas as a
- *                            histogram spec (bins + counts + median +
- *                            CI). Same "data, not images" contract.
- *
- * The figure types are PlotSpecs — JSON-friendly, library-agnostic.
- * They aren't React components and they aren't PNGs; they are
- * what you'd hand to vega-lite, plotly, matplotlib, or your own
- * Canvas renderer to draw the actual figure.
- */
-
-interface SummaryTableOptions {
-    /** Comparator candidate id. Wilcoxon + Cohen's d are computed
-     *  versus this candidate. Required for paired stats columns. */
-    comparator?: string;
-    /** Which split to read scores from. Default 'holdout'. */
-    split?: 'search' | 'holdout';
-    /** Confidence level for the bootstrap CI on the mean. Default 0.95. */
-    confidence?: number;
-    /** FDR for BH adjustment of the comparison p-values. Default 0.05. */
-    fdr?: number;
-}
-interface SummaryTableRow {
-    candidateId: string;
-    n: number;
-    mean: number;
-    ciLow: number;
-    ciHigh: number;
-    /** BH-adjusted q-value vs comparator. NaN if no comparator. */
-    qValue: number;
-    /** Cohen's d vs comparator. NaN if no comparator. */
-    cohensD: number;
-}
-interface SummaryTable {
-    rows: SummaryTableRow[];
-    comparator: string | null;
-    split: 'search' | 'holdout';
-    /** Pre-rendered markdown — drop into a paper or PR. */
-    markdown: string;
-}
-/**
- * Table 1 helper. Buckets runs by `candidateId`, computes mean +
- * bootstrap CI on the chosen split, and (when a comparator is given)
- * BH-adjusted Wilcoxon p + Cohen's d versus that comparator.
- */
-declare function summaryTable(runs: RunRecord[], opts?: SummaryTableOptions): SummaryTable;
-interface ParetoPoint {
-    candidateId: string;
-    /** Mean USD cost per run on the chosen split. */
-    cost: number;
-    /** Mean score on the chosen split. */
-    quality: number;
-    /** Number of runs that informed this point. */
-    n: number;
-    /** Whether this candidate is on the Pareto frontier — high
-     *  quality, low cost, no dominator. */
-    onFrontier: boolean;
-    /** Optional gate verdict for this candidate, if a `GateDecision`
-     *  for it was passed in. */
-    gate?: 'promote' | 'reject_few_runs' | 'reject_negative_delta' | 'reject_overfit_gap' | null;
-}
-interface ParetoFigureSpec {
-    kind: 'pareto-cost-quality';
-    split: 'search' | 'holdout';
-    points: ParetoPoint[];
-    axes: {
-        x: 'costUsd';
-        y: 'score';
-    };
-}
-/**
- * Cost vs quality scatter spec. `gateDecisions` is keyed by
- * candidate id; if present, every point picks up the gate verdict
- * for overlay.
- */
-declare function paretoChart(runs: RunRecord[], opts?: {
-    split?: 'search' | 'holdout';
-    gateDecisions?: Record<string, GateDecision>;
-}): ParetoFigureSpec;
-interface GainDistributionBin {
-    /** Inclusive lower edge. */
-    lo: number;
-    /** Exclusive upper edge (or inclusive if it's the last bin). */
-    hi: number;
-    /** Number of pairs whose delta lands in this bin. */
-    count: number;
-}
-interface GainDistributionFigureSpec {
-    kind: 'gain-distribution';
-    candidateId: string;
-    comparator: string;
-    split: 'search' | 'holdout';
-    /** Number of pairs used. */
-    n: number;
-    bins: GainDistributionBin[];
-    median: number;
-    ci: {
-        low: number;
-        high: number;
-    };
-}
-interface GainDistributionOptions {
-    /** Number of histogram bins. Default 11 (so the centre is exact at 0). */
-    bins?: number;
-    /** Which split to use. Default 'holdout'. */
-    split?: 'search' | 'holdout';
-    /** Confidence level for the CI. Default 0.95. */
-    confidence?: number;
-    /** Bootstrap resamples. Default 2000. */
-    resamples?: number;
-    /** Deterministic seed. */
-    seed?: number;
-}
-/**
- * Held-out improvement distribution: per-pair delta (candidate −
- * comparator), histogrammed. Includes the bootstrap CI on the median
- * delta — same primitive the promotion gate uses.
- */
-declare function gainHistogram(runs: RunRecord[], candidateId: string, comparator: string, opts?: GainDistributionOptions): GainDistributionFigureSpec;
-type ResearchReportDecision = 'promote' | 'hold' | 'reject' | 'equivalent' | 'needs_more_data';
-/**
- * Hard floor below which a paired comparison is treated as uninformative
- * regardless of `minPairs`. Mirrors the lower limit on Wilcoxon signed-rank
- * exact tables; below this the test has no power to separate effect sizes.
- */
-declare const RESEARCH_REPORT_HARD_PAIR_FLOOR = 6;
-interface ResearchReportOptions {
-    /** Human-readable report title. */
-    title?: string;
-    /** Comparator candidate id. Required for statistical decision guidance. */
-    comparator?: string;
-    /** Which split to use for the primary decision. Default 'holdout'. */
-    split?: 'search' | 'holdout';
-    /** Confidence level used by lower-level report helpers. Default 0.95. */
-    confidence?: number;
-    /** FDR threshold for q-values. Default 0.05. */
-    fdr?: number;
-    /**
-     * Soft floor on paired observations before issuing a directional
-     * promote / reject. Below this we report `needs_more_data` and surface the
-     * minimum detectable effect at the current N. Default 20 — chosen so the
-     * Wilcoxon signed-rank approximation is reasonable and so the paired
-     * bootstrap CI has non-degenerate coverage. Hard floor is enforced at
-     * `RESEARCH_REPORT_HARD_PAIR_FLOOR` (6) regardless of this value.
-     */
-    minPairs?: number;
-    /**
-     * Region of Practical Equivalence on the paired delta. When a candidate's
-     * paired-delta CI is fully contained in `[low, high]`, the decision is
-     * `equivalent` rather than `hold`. Sourced from the domain owner — there is
-     * no statistically-defensible default.
-     */
-    rope?: {
-        low: number;
-        high: number;
-    };
-    /**
-     * Power for the minimum detectable effect (MDE) reported on each candidate.
-     * Default 0.8.
-     */
-    mdePower?: number;
-    /**
-     * Two-sided alpha for the MDE. Default matches `fdr` so the reported MDE
-     * lines up with the test the report actually runs.
-     */
-    mdeAlpha?: number;
-    /** Optional held-out gate decisions keyed by candidate id. */
-    gateDecisions?: Record<string, GateDecision>;
-    /** Optional failure clusters from failureClusterView. */
-    failureClusters?: FailureClusterReport;
-    /** Build gain histograms for these candidates. Defaults to all non-comparator candidates. */
-    candidateIds?: string[];
-    /** Deterministic bootstrap seed passed to gainHistogram and the posterior helper. */
-    seed?: number;
-    /** Report timestamp. Defaults to current time. */
-    generatedAt?: string;
-    /**
-     * Hash of a preregistered protocol (e.g. `signManifest({...}).contentHash`).
-     * Embedded verbatim in the report so the analysis can be cited as the
-     * preregistered one rather than a post-hoc fishing expedition.
-     */
-    preregistrationHash?: string;
-}
-interface ResearchReportRecommendation {
-    decision: ResearchReportDecision;
-    candidateId: string | null;
-    rationale: string[];
-    risks: string[];
-    nextActions: string[];
-}
-interface ResearchReportCandidate {
-    candidateId: string;
-    n: number;
-    mean: number;
-    ciLow: number;
-    ciHigh: number;
-    qValue: number;
-    cohensD: number;
-    meanDeltaVsComparator: number | null;
-    pairedN: number;
-    medianGain: number | null;
-    meanGain: number | null;
-    gainCi: {
-        low: number;
-        high: number;
-    } | null;
-    /**
-     * Bayesian-bootstrap-style posterior summaries on the paired delta. Computed
-     * from the same resamples that produce the gain CI; interpretable as
-     * "fraction of resamples in which the candidate beats the comparator on
-     * matched pairs."
-     */
-    prGreaterThanZero: number | null;
-    prInRope: number | null;
-    /**
-     * Minimum detectable effect (in score units) at the candidate's paired N,
-     * the configured power, and the configured alpha. Standardised by the
-     * observed paired-delta SD and inverted via `requiredSampleSize`. Reported
-     * for every candidate so a `needs_more_data` verdict is actionable.
-     */
-    mde: number | null;
-    onParetoFrontier: boolean;
-    gate?: ParetoPoint['gate'];
-    decision: ResearchReportDecision;
-    decisionReason: string;
-}
-interface ResearchReportMethodology {
-    /**
-     * Plain-language assumptions the report depends on. Read these first when
-     * deciding whether the verdict is load-bearing for a launch decision.
-     */
-    assumptions: string[];
-    /** Tests and estimators the verdict was computed from. */
-    methods: string[];
-    /** Alternatives the author considered and why this report didn't take them. */
-    alternatives: string[];
-    /** Failure modes — when this report should NOT drive a decision. */
-    whenNotToApply: string[];
-    /** Citations for the methodological choices above. */
-    citations: string[];
-}
-interface ResearchReport {
-    kind: 'agent-eval-research-report';
-    title: string;
-    generatedAt: string;
-    split: 'search' | 'holdout';
-    comparator: string | null;
-    /**
-     * SHA-256 over the canonicalised set of `(runId, candidateId, split)` triples
-     * the report was computed from, plus the comparator and split. Stable across
-     * key insertion order; recomputable by the reader to verify provenance.
-     */
-    runFingerprint: string;
-    preregistrationHash: string | null;
-    rope: {
-        low: number;
-        high: number;
-    } | null;
-    executiveSummary: string[];
-    recommendation: ResearchReportRecommendation;
-    candidates: ResearchReportCandidate[];
-    summary: SummaryTable;
-    charts: {
-        pareto: ParetoFigureSpec;
-        gains: GainDistributionFigureSpec[];
-    };
-    methodology: ResearchReportMethodology;
-    failureClusters?: FailureClusterReport;
-    markdown: string;
-    html: string;
-}
-/**
- * Executive research report for CPO / AI-lead / launch-review consumption.
- *
- * Composes:
- *   - `summaryTable`         marginal stats with BH-FDR-adjusted q-values
- *   - `paretoChart`           cost-vs-quality frontier with gate overlay
- *   - `gainHistogram`         per-candidate paired-delta distribution
- *   - paired posterior (this file): bootstrap CI on median, Pr(Δ>0),
- *                              Pr(Δ∈ROPE), MDE at the configured power
- *
- * Decisions are made on paired evidence — never on marginal means alone —
- * and respect any held-out gate decision the caller passes through. The
- * report embeds a SHA-256 fingerprint of the input run set and, optionally,
- * the hash of a preregistered protocol so a downstream reader can verify
- * provenance and that the analysis was the preregistered one.
- *
- * Async because the fingerprint uses Web Crypto via `hashJson`; deterministic
- * for any fixed `runs`, `seed`, and ROPE.
- */
-declare function researchReport(runs: RunRecord[], opts?: ResearchReportOptions): Promise<ResearchReport>;
-
-export { gainHistogram as $, type ActionableSideInfo as A, trialTraceFromMultiShotTrial as B, type GainDistributionBin as C, DEFAULT_MUTATION_PRIMITIVES as D, type EvolvableVariant as E, type GainDistributionFigureSpec as F, type GenerationReport as G, type GainDistributionOptions as H, InMemoryTrialCache as I, type ParetoFigureSpec as J, type ParetoPoint as K, RESEARCH_REPORT_HARD_PAIR_FLOOR as L, type MultiShotGateConfig as M, type ResearchReport as N, type ResearchReportCandidate as O, type PromptEvolutionConfig as P, type ResearchReportDecision as Q, type ReflectionContext as R, type ScenarioAggregate as S, type TrialCache as T, type ResearchReportMethodology as U, type VariantAggregate as V, type ResearchReportOptions as W, type ResearchReportRecommendation as X, type SummaryTable as Y, type SummaryTableOptions as Z, type SummaryTableRow as _, type AsiSeverity as a, paretoChart as a0, researchReport as a1, summaryTable as a2, type GateDecision as a3, type HeldOutGateConfig as a4, type Objective as a5, type ParetoResult as a6, type Direction as a7, type GateEvidence as a8, HeldOutGate as a9, type HeldOutGateRejectionCode as aa, crowdingDistance as ab, dominates as ac, paretoFrontier as ad, paretoFrontierWithCrowding as ae, scalarScore as af, type MultiShotGateResult as b, type MultiShotMutateAdapter as c, type MultiShotOptimizationConfig as d, type MultiShotOptimizationResult as e, type MultiShotRun as f, type MultiShotRunInput as g, type MultiShotRunner as h, type MultiShotScore as i, type MultiShotScorer as j, type MultiShotSplit as k, type MultiShotTrace as l, type MultiShotTrialResult as m, type MultiShotVariant as n, type MutateAdapter as o, type PromptEvolutionEvent as p, type PromptEvolutionResult as q, type ReflectionProposal as r, type ScoreAdapter as s, type TrialResult as t, type TrialTrace as u, buildReflectionPrompt as v, defaultMultiShotObjectives as w, parseReflectionResponse as x, runMultiShotOptimization as y, runPromptEvolution as z };