@tangle-network/agent-eval 0.20.11 → 0.21.0

package/dist/multi-shot-optimization-Bvtz294B.d.ts

@@ -0,0 +1,598 @@
+import { a as RunRecord, R as RunSplitTag } from './run-record-CX_jcAyr.js';
+
+/**
+ * HeldOutGate — first-class held-out paired-delta promotion gate.
+ *
+ * Encodes the "honesty override" pattern that lived inline in
+ * `~/webb/redteam/scripts/agent-eval-autoresearch.ts:138–171`.
+ * The optimizer's best-guess is one thing; what we should actually
+ * ship is another. The gate is the line between them.
+ *
+ * A candidate is promoted iff ALL three pass:
+ *
+ *   1. **Productive runs**: the candidate has at least
+ *      `minProductiveRuns` paired observations on items where BOTH
+ *      candidate and baseline produced a real (non-silent) score.
+ *   2. **Paired delta**: the lower bound of the bootstrap CI on the
+ *      median per-item delta (candidate − baseline) on the HOLDOUT
+ *      split is strictly greater than `pairedDeltaThreshold`.
+ *   3. **Overfit gap**: the candidate's gap between search-split
+ *      score and holdout-split score is no worse (more positive)
+ *      than the baseline's gap by more than `overfitGapThreshold`.
+ *      "Better on search, worse on holdout" is the canonical
+ *      overfit pattern; this catches it.
+ *
+ * The decision carries a machine-readable `rejectionCode` plus an
+ * `evidence` block with every number the gate looked at, so the
+ * downstream researcher / paper / dashboard can re-derive the
+ * verdict without re-running.
+ *
+ * See also:
+ *   - `src/paired-stats.ts` for `pairedBootstrap` + `pairedWilcoxon`
+ *   - `src/run-record.ts` for the input row schema
+ *   - `src/reference-replay.ts` for the older, reference-replay-
+ *     specific promotion path (still useful for replay-style evals).
+ */
+
+type HeldOutGateRejectionCode = 'few_runs' | 'negative_delta' | 'overfit_gap';
+interface HeldOutGateConfig {
+    /** Minimum number of paired (candidate, baseline) holdout observations
+     *  required before the gate will even consider promoting. Default 3. */
+    minProductiveRuns?: number;
+    /** The bootstrap-CI lower bound on the median paired holdout delta
+     *  must exceed this to promote. Default 0. */
+    pairedDeltaThreshold?: number;
+    /** Maximum allowed worsening of (search − holdout) gap relative to
+     *  baseline. Default 0.15 (i.e. candidate may overfit by up to 15
+     *  absolute score points more than baseline before rejection). */
+    overfitGapThreshold?: number;
+    /** Stable label of the baseline candidate. Required — paper-grade
+     *  evaluation never compares two unlabelled candidates. */
+    baselineKey: string;
+    /** Confidence level for the bootstrap CI. Default 0.95. */
+    confidence?: number;
+    /** Bootstrap resamples. Default 2000. */
+    bootstrapResamples?: number;
+    /** Optional deterministic seed for the bootstrap. Default undefined
+     *  (Math.random). */
+    seed?: number;
+}
+interface GateEvidence {
+    /** Number of paired (candidate, baseline) holdout observations used. */
+    productiveRuns: number;
+    /** Median of (candidate − baseline) paired holdout deltas. */
+    medianPairedDelta: number;
+    /** Bootstrap CI on the median paired holdout delta. */
+    pairedCI: {
+        low: number;
+        high: number;
+    };
+    /** Wilcoxon signed-rank p-value on the paired holdout deltas. */
+    pairedPValue: number;
+    /** Mean candidate score on the search split (NaN if none). */
+    searchScore: number;
+    /** Mean candidate score on the holdout split (NaN if none). */
+    holdoutScore: number;
+    /** Candidate (search − holdout) gap. */
+    overfitGap: number;
+    /** Baseline (search − holdout) gap. */
+    baselineOverfitGap: number;
+}
+interface GateDecision {
+    /** Final promote/no-promote verdict. */
+    promote: boolean;
+    /** The candidate that was evaluated. */
+    candidateId: string;
+    /** The baseline it was compared against. */
+    baselineId: string;
+    /** Every number the gate looked at, for audit + paper export. */
+    evidence: GateEvidence;
+    /** Human-readable reason. */
+    reason: string;
+    /** Machine-readable rejection code, or null on promote. */
+    rejectionCode: HeldOutGateRejectionCode | null;
+}
+/**
+ * Held-out paired-delta promotion gate. Construct once with config,
+ * call `evaluate(candidateRuns, baselineRuns)` per (candidate,
+ * baseline) pair. Stateless across calls.
+ */
+declare class HeldOutGate {
+    private readonly minProductiveRuns;
+    private readonly pairedDeltaThreshold;
+    private readonly overfitGapThreshold;
+    private readonly baselineKey;
+    private readonly confidence;
+    private readonly resamples;
+    private readonly seed?;
+    constructor(config: HeldOutGateConfig);
+    /** Decide whether `candidate` should replace `baseline`. Pairing
+     *  is by (experimentId, seed) — identical experiment + seed pairs
+     *  the candidate run with the matching baseline run. Pairs without
+     *  a holdout score on both sides are dropped. */
+    evaluate(candidate: RunRecord[], baseline: RunRecord[]): GateDecision;
+}
+
+/**
+ * Pareto frontier — multi-objective optimization over candidate runs.
+ *
+ * Lifted from ADC pareto.ts and blueprint-agent frontier.ts. When you're
+ * trading off (cost, latency, quality) or (passRate, tokenBudget,
+ * ttfb), you rarely have a single "winner" — you have a set of
+ * non-dominated candidates. This module exposes:
+ *
+ *   - `paretoFrontier`: filter a set of candidates to the non-dominated ones
+ *   - `dominates`: does A dominate B across all objectives?
+ *
+ * Each objective is declared with a direction: 'maximize' (higher=better)
+ * or 'minimize' (lower=better). Candidates are any object; pass an
+ * `objective(candidate)` accessor.
+ */
+type Direction = 'maximize' | 'minimize';
+interface Objective<T> {
+    /** Stable label used in reports. */
+    name: string;
+    direction: Direction;
+    value: (candidate: T) => number;
+}
+interface ParetoResult<T> {
+    frontier: T[];
+    dominated: T[];
+    /** Index map: frontier[i] dominates each of dominatedBy[i]. */
+    dominanceMap: Array<{
+        dominator: T;
+        dominated: T[];
+    }>;
+}
+/** Does candidate A weakly dominate B — strictly better on at least one objective and no worse on any? */
+declare function dominates<T>(a: T, b: T, objectives: Objective<T>[]): boolean;
+/**
+ * Compute the non-dominated frontier. Candidates with NaN/Infinity on any
+ * objective are excluded (can't rank them). A candidate enters the frontier
+ * iff no other candidate dominates it.
+ */
+declare function paretoFrontier<T>(candidates: T[], objectives: Objective<T>[]): ParetoResult<T>;
+/**
+ * Weighted-sum scalarisation. Use as a tie-break / single-winner selector
+ * when callers don't want to consume a frontier. Each objective contributes
+ * its normalised value (0..1 via min-max across the candidate pool) times
+ * its weight; missing weights default to 1/N.
+ *
+ * Direction is honoured automatically — `minimize` axes have their values
+ * inverted before scaling so "higher scalar = better" always holds.
+ */
+declare function scalarScore<T>(candidates: T[], objectives: Objective<T>[], options?: {
+    weights?: Partial<Record<string, number>>;
+}): Array<{
+    candidate: T;
+    score: number;
+}>;
+/**
+ * NSGA-II crowding distance — secondary sort for ties on the frontier.
+ *
+ * When the Pareto front collapses to a single point (or many candidates tie
+ * on dominance), naive selection picks arbitrarily and the population
+ * degenerates over generations. NSGA-II preserves diversity by preferring
+ * candidates with more empty space around them on the frontier.
+ *
+ * Returns an array of `{ candidate, distance }` in the SAME order as the
+ * input. Higher distance = more isolated = should be preferred when
+ * preserving diversity.
+ */
+declare function crowdingDistance<T>(candidates: T[], objectives: Objective<T>[]): Array<{
+    candidate: T;
+    distance: number;
+}>;
+/**
+ * Pareto frontier with tie-break by crowding distance — the canonical
+ * NSGA-II selection step. Returns the frontier sorted by descending crowding
+ * distance so callers can `.slice(0, k)` to pick K diverse winners.
+ */
+declare function paretoFrontierWithCrowding<T>(candidates: T[], objectives: Objective<T>[]): Array<{
+    candidate: T;
+    distance: number;
+}>;
+
+/**
+ * PromptEvolutionLoop — population-based reflective-mutation evolution.
+ *
+ * Above the existing `AxGepaSteeringOptimizer` (which RANKS variants),
+ * this loop GENERATES variants. Each generation:
+ *   1. Score the population across (variant × scenario × rep).
+ *   2. Pick survivors from the Pareto frontier (with crowding-distance tie-break).
+ *   3. Ask the mutator for replacements until population size is restored.
+ *   4. Repeat for N generations OR until convergence.
+ *
+ * Domain-agnostic. Consumers supply:
+ *   - A seed population of `EvolvableVariant`s.
+ *   - A `ScoreAdapter` that runs (variant, scenario, rep) → `TrialResult`.
+ *   - A `MutateAdapter` that produces children given trace evidence.
+ *   - Pareto `Objective<TrialAggregate>[]` defining the multi-objective vector.
+ *
+ * The loop owns: population management, parallel scheduling (concurrency-
+ * limited), Pareto selection with crowding distance, generation reporting.
+ *
+ * It does NOT own: rendering trials to a model, executing prompts, choosing
+ * mutation primitives, persisting to disk. Those are the consumer's call.
+ */
+
+interface EvolvableVariant<P = unknown> {
+    /** Stable id for the variant — surfaces in reports and trial results. */
+    id: string;
+    /** Variant payload — interpretation is the consumer's responsibility. */
+    payload: P;
+    /** Generation index (0 = seed, then 1, 2, ...). */
+    generation: number;
+    /** Parent variant id when produced via mutation; absent for seeds. */
+    parentId?: string;
+    /** Human label for reports. */
+    label: string;
+    /** What the mutator was trying to fix. */
+    rationale?: string;
+}
+interface TrialResult {
+    variantId: string;
+    scenarioId: string;
+    rep: number;
+    ok: boolean;
+    /** Primary scalar score the consumer cares about (e.g., recall, accuracy). */
+    score: number;
+    /** Token cost (or any cost-like dimension). */
+    cost?: number;
+    /** Wall time in ms. */
+    durationMs?: number;
+    /** Free-form metric bag for objective accessors. */
+    metrics?: Record<string, number>;
+    error?: string;
+}
+/** Aggregated trial summary for one (variant, scenario) pair across reps. */
+interface ScenarioAggregate {
+    variantId: string;
+    scenarioId: string;
+    meanScore: number;
+    meanCost: number;
+    meanDurationMs: number;
+    okRate: number;
+    trials: number;
+    /** Mean of every numeric metric across reps. */
+    metrics: Record<string, number>;
+}
+/** Aggregated trial summary for one variant across all scenarios. */
+interface VariantAggregate {
+    variantId: string;
+    meanScore: number;
+    meanCost: number;
+    meanDurationMs: number;
+    okRate: number;
+    scenarios: ScenarioAggregate[];
+    /** Mean of every numeric metric, averaged across scenarios. */
+    metrics: Record<string, number>;
+}
+interface ScoreAdapter<P = unknown> {
+    score(args: {
+        variant: EvolvableVariant<P>;
+        scenarioId: string;
+        rep: number;
+    }): Promise<TrialResult>;
+}
+interface MutateAdapter<P = unknown> {
+    mutate(args: {
+        parent: EvolvableVariant<P>;
+        parentAggregate: VariantAggregate;
+        topTrials: TrialResult[];
+        bottomTrials: TrialResult[];
+        childCount: number;
+        generation: number;
+    }): Promise<EvolvableVariant<P>[]>;
+}
+interface PromptEvolutionConfig<P = unknown> {
+    runId: string;
+    /** What component is being mutated — surfaces in reports + reflection prompts. */
+    target: string;
+    seedVariants: EvolvableVariant<P>[];
+    scenarioIds: string[];
+    reps: number;
+    generations: number;
+    populationSize: number;
+    /** Maximum concurrent score() calls. */
+    scoreConcurrency: number;
+    scoreAdapter: ScoreAdapter<P>;
+    mutateAdapter: MutateAdapter<P>;
+    /** Pareto objectives over `VariantAggregate`. Ordered by importance. */
+    objectives: Objective<VariantAggregate>[];
+    /** Optional weights for the scalar tie-break selector (by objective name). */
+    scalarWeights?: Record<string, number>;
+    /** Stop early if a generation produces no Pareto improvement. Default true. */
+    earlyStopOnNoImprovement?: boolean;
+    onProgress?: (event: PromptEvolutionEvent) => void;
+    /**
+     * Optional cache key for memoising scored (variantId, scenarioId, rep)
+     * tuples. When provided AND a cache instance is passed, repeated trials
+     * skip re-scoring. Cache keys are stable across runs.
+     */
+    cache?: TrialCache;
+}
+interface TrialCache {
+    get(key: string): TrialResult | undefined;
+    set(key: string, value: TrialResult): void;
+}
+declare class InMemoryTrialCache implements TrialCache {
+    private store;
+    get(key: string): TrialResult | undefined;
+    set(key: string, value: TrialResult): void;
+    size(): number;
+    clear(): void;
+}
+type PromptEvolutionEvent = {
+    type: 'generation-start';
+    generation: number;
+    populationSize: number;
+} | {
+    type: 'trial-complete';
+    generation: number;
+    variantId: string;
+    scenarioId: string;
+    rep: number;
+    ok: boolean;
+    score: number;
+    cached: boolean;
+} | {
+    type: 'generation-complete';
+    report: GenerationReport<unknown>;
+} | {
+    type: 'converged';
+    generation: number;
+    reason: string;
+};
+interface GenerationReport<P = unknown> {
+    runId: string;
+    target: string;
+    generation: number;
+    variants: EvolvableVariant<P>[];
+    aggregates: VariantAggregate[];
+    /** Frontier candidates, sorted by descending crowding distance. */
+    paretoFrontIds: string[];
+    /** Scalar-best variant id — used for the single "winner" if callers want one. */
+    winnerId: string;
+    /** Trials that fed this generation (kept for downstream reporting). */
+    trials: TrialResult[];
+}
+interface PromptEvolutionResult<P = unknown> {
+    runId: string;
+    target: string;
+    generations: GenerationReport<P>[];
+    /** Best variant by scalar score in the final generation. */
+    bestVariant: EvolvableVariant<P>;
+    /** Best aggregate (matches bestVariant). */
+    bestAggregate: VariantAggregate;
+}
+declare function runPromptEvolution<P>(config: PromptEvolutionConfig<P>): Promise<PromptEvolutionResult<P>>;
+
+/**
+ * Reflective mutation — primitives for trace-conditioned prompt rewriting.
+ *
+ * Used by `prompt-evolution.ts` (and any consumer running iterative
+ * improvement). Given a parent prompt + concrete trace evidence (top trials,
+ * bottom trials, missed expectations), produce an LLM-ready prompt that
+ * proposes targeted mutations — not blind rephrasings.
+ *
+ * Why this lives outside `prompt-evolution.ts`: any consumer that wants to
+ * run reflective rewriting WITHOUT the population/Pareto machinery can
+ * import these primitives directly.
+ *
+ * Quality bar (vs. naive "mutate this prompt"):
+ *   - Show parent ↔ children diff, not just one variant
+ *   - Quote specific missed goldens with their match phrases
+ *   - Surface the model's actual emitted output side-by-side with what was expected
+ *   - Quote concrete mutation primitives so the model has a vocabulary
+ */
+interface TrialTrace {
+    /** Stable id for the trial — surfaces in the prompt for grounding. */
+    id: string;
+    /** Score the trial received on its primary metric. */
+    score: number;
+    /** Candidate inputs the agent was given (e.g., the fixture or scenario). */
+    inputName?: string;
+    /**
+     * Goldens / expectations this trial was tested against, with whether each
+     * was matched. The reflection prompt quotes the missed ones specifically.
+     */
+    expectations?: Array<{
+        id: string;
+        phrase: string;
+        matched: boolean;
+    }>;
+    /** Free-form text — what the agent actually emitted (e.g., findings, plan). */
+    emitted?: string;
+    /** Optional structured metrics (recall, precision, cost, latency). */
+    metrics?: Record<string, number>;
+}
+interface ReflectionContext {
+    /** What is being mutated — appears in the system prompt for orientation. */
+    target: string;
+    /** Current variant's payload — JSON-serialised for the prompt. */
+    parentPayload: unknown;
+    /** Best-performing trials this generation. */
+    topTrials: TrialTrace[];
+    /** Worst-performing trials this generation — the missed-golden source. */
+    bottomTrials: TrialTrace[];
+    /** How many children the mutator should propose. */
+    childCount: number;
+    /** Optional: domain-specific mutation primitives the model can pick from. */
+    mutationPrimitives?: string[];
+}
+declare const DEFAULT_MUTATION_PRIMITIVES: string[];
+/**
+ * Build the LLM-ready reflection prompt. Output is plain text — pass it as
+ * the user message. The system message should be small and stable (e.g.
+ * "Output ONLY a JSON object matching the schema below.").
+ */
+declare function buildReflectionPrompt(ctx: ReflectionContext): string;
+interface ReflectionProposal {
+    label: string;
+    rationale: string;
+    payload: unknown;
+}
+declare function parseReflectionResponse(raw: string, maxProposals?: number): ReflectionProposal[];
+
+/**
+ * Multi-shot optimization adapter.
+ *
+ * This is the canonical bridge between variable-length agent trajectories
+ * and `runPromptEvolution`. Apps provide four things:
+ *
+ *   - variants: prompt/config/tool-policy candidates
+ *   - runner: executes one full task trajectory for a variant
+ *   - scorer: turns that trajectory into score + actionable side information
+ *   - mutator: proposes new variants from top/bottom scored trials
+ *
+ * The adapter owns the boring but easy-to-get-wrong glue: stable seeds,
+ * score/cost objectives, error-to-trial conversion, ASI metric projection,
+ * and optional paired holdout gating via `HeldOutGate`.
+ */
+
+type MultiShotSplit = 'search' | 'dev' | 'holdout';
+type AsiSeverity = 'info' | 'warning' | 'error' | 'critical';
+type MultiShotVariant<P = unknown> = EvolvableVariant<P>;
+interface ActionableSideInfo {
+    /** Stable expectation/check id when available. */
+    expectationId?: string;
+    /** Human-readable diagnosis of what happened. */
+    message: string;
+    severity?: AsiSeverity;
+    /** Concrete trace excerpt, file path, tool call, screenshot id, etc. */
+    evidence?: string;
+    /** Prompt/tool/context surface likely responsible. */
+    responsibleSurface?: string;
+    /** Suggested fix in natural language. */
+    suggestion?: string;
+    /** Whether this expectation was satisfied. Defaults to false for ASI rows. */
+    matched?: boolean;
+    metadata?: Record<string, unknown>;
+}
+interface MultiShotTrace {
+    scenarioId: string;
+    /** Full turn/tool trace. Shape is intentionally app-owned. */
+    turns?: unknown[];
+    toolCalls?: unknown[];
+    artifacts?: unknown[];
+    /** Compact final output or summary used by reflection prompts. */
+    transcript?: string;
+    output?: unknown;
+    metadata?: Record<string, unknown>;
+}
+interface MultiShotRun {
+    trace: MultiShotTrace;
+    costUsd?: number;
+    durationMs?: number;
+    tokenUsage?: {
+        input?: number;
+        output?: number;
+        cached?: number;
+    };
+    metadata?: Record<string, unknown>;
+}
+interface MultiShotRunInput<P = unknown> {
+    variant: EvolvableVariant<P>;
+    scenarioId: string;
+    rep: number;
+    split: MultiShotSplit;
+    /** Stable paired seed for baseline/candidate comparisons. */
+    seed: number;
+}
+interface MultiShotRunner<P = unknown> {
+    run(input: MultiShotRunInput<P>): Promise<MultiShotRun> | MultiShotRun;
+}
+interface MultiShotScore {
+    /** Primary score in [0,1]. The adapter clamps for safety. */
+    score: number;
+    /** Pass/fail for top/bottom trial selection. Defaults to true. */
+    ok?: boolean;
+    costUsd?: number;
+    durationMs?: number;
+    metrics?: Record<string, number>;
+    asi?: ActionableSideInfo[];
+    /** Optional rich output shown to reflection mutators. */
+    emitted?: string;
+    metadata?: Record<string, unknown>;
+}
+interface MultiShotScorer<P = unknown> {
+    score(input: MultiShotRunInput<P> & {
+        run: MultiShotRun;
+    }): Promise<MultiShotScore> | MultiShotScore;
+}
+interface MultiShotTrialResult extends TrialResult {
+    split: MultiShotSplit;
+    seed: number;
+    trace?: MultiShotTrace;
+    asi?: ActionableSideInfo[];
+    emitted?: string;
+    metadata?: Record<string, unknown>;
+}
+interface MultiShotMutateAdapter<P = unknown> {
+    mutate(args: {
+        parent: EvolvableVariant<P>;
+        parentAggregate: VariantAggregate;
+        topTrials: MultiShotTrialResult[];
+        bottomTrials: MultiShotTrialResult[];
+        childCount: number;
+        generation: number;
+    }): Promise<EvolvableVariant<P>[]>;
+}
+interface MultiShotGateConfig<P = unknown> {
+    /** Search rows are optional, but enable HeldOutGate's overfit-gap check. */
+    searchScenarioIds?: string[];
+    holdoutScenarioIds: string[];
+    reps?: number;
+    gate: HeldOutGateConfig;
+    /** Convert scored trajectory runs into paper-grade RunRecords. */
+    toRunRecord(input: {
+        variant: EvolvableVariant<P>;
+        scenarioId: string;
+        rep: number;
+        split: RunSplitTag;
+        seed: number;
+        trial: MultiShotTrialResult;
+    }): RunRecord;
+}
+interface MultiShotOptimizationConfig<P = unknown> {
+    runId: string;
+    target: string;
+    seedVariants: EvolvableVariant<P>[];
+    searchScenarioIds: string[];
+    reps: number;
+    generations: number;
+    populationSize: number;
+    scoreConcurrency?: number;
+    runner: MultiShotRunner<P>;
+    scorer: MultiShotScorer<P>;
+    mutateAdapter: MultiShotMutateAdapter<P>;
+    objectives?: Objective<VariantAggregate>[];
+    scalarWeights?: Record<string, number>;
+    cache?: TrialCache;
+    earlyStopOnNoImprovement?: boolean;
+    seedBase?: number;
+    onProgress?: (event: PromptEvolutionEvent) => void;
+    gate?: MultiShotGateConfig<P>;
+}
+interface MultiShotGateResult {
+    decision: GateDecision;
+    candidateRuns: RunRecord[];
+    baselineRuns: RunRecord[];
+}
+interface MultiShotOptimizationResult<P = unknown> {
+    evolution: PromptEvolutionResult<P>;
+    /** Best candidate on the optimizer-visible search split. */
+    searchBestVariant: EvolvableVariant<P>;
+    searchBestAggregate: VariantAggregate;
+    /** Variant callers should actually ship after optional holdout gating. */
+    promotedVariant: EvolvableVariant<P>;
+    promotedAggregate: VariantAggregate;
+    /** Null when no gate was configured or the search-best candidate was the baseline. */
+    gate: MultiShotGateResult | null;
+}
+declare function runMultiShotOptimization<P>(config: MultiShotOptimizationConfig<P>): Promise<MultiShotOptimizationResult<P>>;
+declare function defaultMultiShotObjectives(): Objective<VariantAggregate>[];
+declare function trialTraceFromMultiShotTrial(trial: MultiShotTrialResult): TrialTrace;
+
+export { type ActionableSideInfo as A, type TrialTrace as B, buildReflectionPrompt as C, DEFAULT_MUTATION_PRIMITIVES as D, type EvolvableVariant as E, crowdingDistance as F, type GateDecision as G, HeldOutGate as H, InMemoryTrialCache as I, defaultMultiShotObjectives as J, dominates as K, paretoFrontier as L, type MutateAdapter as M, paretoFrontierWithCrowding as N, type Objective as O, type ParetoResult as P, parseReflectionResponse as Q, type ReflectionContext as R, type ScenarioAggregate as S, type TrialCache as T, runMultiShotOptimization as U, type VariantAggregate as V, runPromptEvolution as W, scalarScore as X, trialTraceFromMultiShotTrial as Y, type TrialResult as a, type AsiSeverity as b, type Direction as c, type GateEvidence as d, type GenerationReport as e, type HeldOutGateConfig as f, type HeldOutGateRejectionCode as g, type MultiShotGateConfig as h, type MultiShotGateResult as i, type MultiShotMutateAdapter as j, type MultiShotOptimizationConfig as k, type MultiShotOptimizationResult as l, type MultiShotRun as m, type MultiShotRunInput as n, type MultiShotRunner as o, type MultiShotScore as p, type MultiShotScorer as q, type MultiShotSplit as r, type MultiShotTrace as s, type MultiShotTrialResult as t, type MultiShotVariant as u, type PromptEvolutionConfig as v, type PromptEvolutionEvent as w, type PromptEvolutionResult as x, type ReflectionProposal as y, type ScoreAdapter as z };

package/dist/openapi.json

@@ -2,7 +2,7 @@
   "openapi": "3.1.0",
   "info": {
     "title": "@tangle-network/agent-eval — wire protocol",
-    "version": "0.20.11",
+    "version": "0.21.0",
     "description": "HTTP and stdio RPC interface to agent-eval. The TypeScript runtime is the source of truth; this spec is the contract that cross-language clients (Python, Rust, Go) generate from.\n\nWire-protocol version: 1.0.0. Bumps on breaking changes to request/response schemas.",
     "contact": {
       "name": "Tangle Network",