@tangle-network/agent-eval 0.126.6 → 0.127.0

package/dist/contract/index.d.ts

@@ -1,5 +1,5 @@
 import { z } from 'zod';
-import { AgentCandidateExperiment, AgentCandidateBundle, AgentCandidateBenchmarkTask, AgentCandidateBenchmarkCellRef, AgentCandidateExperimentMeasurement, AgentImprovementMeasuredComparison, CandidateExecutionEvidence, AgentCandidateBenchmarkSuiteInputs, AgentCandidateBenchmarkTaskMaterial, AgentCandidateExperimentMaterial } from '@tangle-network/agent-interface';
+import { AgentCandidateExperiment, AgentCandidateBundle, AgentCandidateBenchmarkTask, AgentCandidateBenchmarkCellRef, AgentCandidateExperimentMeasurement, AgentImprovementMeasuredComparison, AgentCandidateEvaluationPolicy, CandidateExecutionEvidence, AgentCandidateBenchmarkSuiteInputs, AgentCandidateBenchmarkTaskMaterial, AgentCandidateExperimentMaterial } from '@tangle-network/agent-interface';
 import { AxFunction, AxAIService } from '@ax-llm/ax';
 
 type CostChannel = 'agent' | 'judge' | 'verifier' | 'analyst' | 'driver' | (string & {});
@@ -204,6 +204,7 @@ type CostLedgerHandle = Pick<CostLedger, Exclude<keyof CostLedger, 'listPending'
  * OpenAI-compatible `/v1/chat/completions` client with:
  *   - Exponential-backoff retry on 429 + 5xx gateway errors (502/503/504).
  *   - Retry on transient network errors (fetch failed, AbortError, ECONNRESET).
+ *   - One retry at temperature 1 when a model explicitly requires it.
  *   - Graceful json_schema → json_object degrade on 400 with schema-reject body.
  *   - Fenced-JSON stripping (```json ... ```) for models that wrap structured output.
  *   - Configurable base URL + api key / bearer, works with LiteLLM proxies, OpenAI
@@ -333,7 +334,8 @@ type FailureClass = 'success' | 'reasoning_error' | 'tool_selection_error' | 'to
  * researcher loop SHOULD be recorded as a `RunRecord`. The mandatory
  * fields are exactly those the paper "Two Loops, Three Roles" requires
  * for reproducibility: who/what/when/cost/seed/hash, plus the search vs
- * holdout split tag and either a `searchScore` or a `holdoutScore`.
+ * holdout split tag. A task score is optional because execution-only records
+ * must preserve missing labels instead of converting errors into zero quality.
  *
  * This is intentionally NOT a replacement for the rich `Run` /
  * `ProposeReviewReport` / `ScenarioResult` types already in the
@@ -354,6 +356,13 @@ type FailureClass = 'success' | 'reasoning_error' | 'tool_selection_error' | 'to
 /** Search/dev/holdout split tag. 'search' is the paper-grade alias for the
  *  combined train+test pool that the optimizer is allowed to read. */
 type RunSplitTag = 'search' | 'dev' | 'holdout';
+/**
+ * Explicit execution-lifecycle result for a run.
+ *
+ * This is separate from task quality (`outcome`) and failure classification.
+ * Producers set it only from root-run or process evidence.
+ */
+type RunTerminalOutcome = 'succeeded' | 'failed' | 'cancelled' | 'incomplete' | 'unknown';
 interface RunTokenUsage {
     input: number;
     /** All generated tokens charged as output, including reasoning tokens. */
@@ -367,11 +376,6 @@ interface RunTokenUsage {
 }
 /**
  * How a run's USD amount was obtained.
- *
- * `costUsd` remains mandatory for wire compatibility. New producers should
- * always populate this discriminated union so a missing bill is never
- * mistaken for an observed zero-dollar run. For `uncaptured`, `costUsd` uses
- * the legacy `0` sentinel while this field carries the truthful null.
  */
 type RunCostProvenance = {
     kind: 'observed';
@@ -417,8 +421,8 @@ interface JudgeScoresRecord {
     perJudge: Record<string, Record<string, number>>;
     /** Per-dim mean across judges. Convenience — derivable from `perJudge`. */
     perDimMean: Record<string, number>;
-    /** Composite mean across all dims and judges. Mirrors the score
-     *  the gate sees on `outcome.searchScore` / `holdoutScore`. */
+    /** Composite mean across successful judges. Mirrors the task score only
+     *  when `failedJudges` is empty. */
     composite: number;
     /** Judges that errored or returned an unparseable verdict. Recorded
      *  by id (e.g. `['glm-5.1']`) so a partial-failure case is explicit,
@@ -429,11 +433,11 @@ interface JudgeScoresRecord {
     notes?: string;
 }
 interface RunOutcome {
-    /** Score on the search/optimization split. Optional because a
-     *  holdout-only evaluation only fills `holdoutScore`. */
+    /** Score on the search/optimization split. Optional for holdout-only and
+     *  execution-only records. */
     searchScore?: number;
-    /** Score on the held-out split. Optional because a search-only run
-     *  only fills `searchScore`. At least one must be present. */
+    /** Score on the held-out split. Optional for search-only and execution-only
+     *  records. When both scores are absent, the run is explicitly unlabeled. */
     holdoutScore?: number;
     /** Bag of any other metric the run produced — judge dimensions,
      *  pass/fail counters, latency stats, etc. Numeric only — keeps
@@ -497,42 +501,39 @@ interface RunRecord {
     wallMs: number;
     /** Time spent queued before execution started, if known. */
     queueMs?: number;
-    /** Total USD cost. Mandatory — runs without a cost number are
-     *  unbounded by definition and must not be admitted into the gate.
-     *  `0` is retained as the compatibility sentinel for an uncaptured amount;
-     *  inspect `costProvenance` before treating it as observed. */
-    costUsd: number;
-    /** Observed, model-priced estimate, or genuinely uncaptured USD amount.
-     *  Optional only so existing serialized RunRecords remain valid. */
-    costProvenance?: RunCostProvenance;
+    /** Total USD cost, or null when the producer could not capture one. */
+    costUsd: number | null;
+    /** Whether `costUsd` came from billing data, a price calculation, or is unavailable. */
+    costProvenance: RunCostProvenance;
     /** Token usage breakdown. */
     tokenUsage: RunTokenUsage;
+    /** Root-run or process terminal result. Never inferred from a child span. */
+    terminalOutcome: RunTerminalOutcome;
+    /** Root-run or process failure reason. Valid only for a failed, cancelled,
+     *  or incomplete terminal result; never populated from a child span. */
+    terminalFailureReason?: string;
     /** Judge-side metadata, if a judge was used. */
     judgeMetadata?: RunJudgeMetadata;
     /** Per-split scores + raw bag. */
     outcome: RunOutcome;
-    /** Canonical, cross-agent failure class drawn from the shared
+    /** Canonical task-failure class drawn from the shared
      *  `FAILURE_CLASSES` taxonomy. This is the aggregation key that makes
      *  "which failure dominates across the whole fleet" answerable in ONE
      *  vocabulary — every agent classifies against the same enum. Producers
-     *  set it via the substrate classifier; leave unset only when the failure
-     *  genuinely can't be classified. */
+     *  set it only from task-result evidence. Execution errors belong in
+     *  `outcome.raw.execution_error_count`, even when the run later fails. */
     failureClass?: FailureClass;
-    /** Free-form domain-specific failure detail, scoped UNDER `failureClass`
+    /** Free-form task-failure detail, scoped UNDER `failureClass`
      *  (e.g. failureClass='tool_recovery_failure', failureMode='forge_build_unsatisfied').
-     *  The within-agent drill-down; `failureClass` is the cross-agent key. */
+     *  Do not populate this from a child execution error alone. */
     failureMode?: string;
     /** Which split this run was drawn from. */
     splitTag: RunSplitTag;
     /**
-     * Stable scenario identifier the run was scored against. Optional for
-     * backwards compatibility, but **strongly recommended**: every primitive
-     * that pairs runs by scenario (preferences, paired stats, BT tournament)
-     * keys on this. The campaign artifact populates it canonically; legacy
-     * runs without it fall back to inference from `outcome.raw.scenario_id`
-     * or `experimentId`.
+     * Stable scenario identifier the run observed or was scored against.
+     * Comparison primitives match this identity rather than input order.
      */
-    scenarioId?: string;
+    scenarioId: string;
     /**
      * Canonical identity for the agent profile cell that produced this row:
      * profile artifact hash plus optional harness/model/prompt/reporting
@@ -887,6 +888,13 @@ interface OptimizerConfig extends OptimizerConfigBase {
 }
 /** Five-valued verdict taxonomy (MOSS-paper alignment). */
 type GateDecision = 'ship' | 'hold' | 'need_more_work' | 'model_ceiling' | 'arch_ceiling';
+/** Outcome of one check that contributed to a release decision. */
+type GateCheckStatus = 'pass' | 'fail' | 'not_evaluated';
+interface GateContribution {
+    name: string;
+    status: GateCheckStatus;
+    detail: unknown;
+}
 interface GateContext<TArtifact, TScenario extends Scenario$1> {
     candidateArtifacts: Map<string, TArtifact>;
     baselineArtifacts?: Map<string, TArtifact>;
@@ -920,11 +928,7 @@ interface GateContext<TArtifact, TScenario extends Scenario$1> {
 interface GateResult {
     decision: GateDecision;
     reasons: string[];
-    contributingGates: Array<{
-        name: string;
-        passed: boolean;
-        detail: unknown;
-    }>;
+    contributingGates: GateContribution[];
     delta?: number;
 }
 /** Composable promotion gate. */
@@ -1066,6 +1070,10 @@ interface CampaignCellResult<TArtifact> {
     durationMs: number;
     seed: number;
     cached: boolean;
+    /** Stage that produced `error`. Missing on successful cells. */
+    errorStage?: 'dispatch' | 'judge';
+    /** Judge that threw when `errorStage` is `judge`. */
+    errorJudge?: string;
     error?: string;
 }
 interface JudgeAggregate {
@@ -1090,8 +1098,10 @@ interface GenerationRecord {
  *  handled — the evidence a blind `Mutator` cannot see. */
 interface GenerationCandidate {
     surfaceHash: string;
-    composite: number;
-    ci95: [number, number];
+    /** Mean over complete task-quality scores, or null when none were produced. */
+    composite: number | null;
+    /** Descriptive interval for `composite`, or null when no score exists. */
+    ci95: [number, number] | null;
     /** Exact surface this candidate mutated. */
     parentSurfaceHash?: string;
     /** Measured search-split composite of the exact parent surface. */
@@ -1101,13 +1111,12 @@ interface GenerationCandidate {
     observedDeltaFromParent?: number;
     /** Whether this candidate had a scorable result for every designed campaign
      *  cell and was therefore eligible for ranking, promotion, and Pareto
-     *  selection. Older externally-authored records may omit this field; loop
-     *  records always populate it. */
-    eligibleForPromotion?: boolean;
+     *  selection. */
+    eligibleForPromotion: boolean;
     /** Exact denominator receipt for selection eligibility. Scores stay
      *  descriptive: an incomplete candidate is retained with its observed score
      *  and errors instead of receiving an invented penalty. */
-    coverage?: {
+    coverage: {
         expectedCells: number;
         scorableCells: number;
         unscorableCells: Array<{
@@ -1148,10 +1157,18 @@ interface CampaignAggregates {
     cost: CostLedgerSummary;
     /** Compatibility alias of `cost.totalCostUsd`. */
     totalCostUsd: number;
+    /** Cells whose dispatch completed, including cells whose later judge failed. */
     cellsExecuted: number;
     cellsSkipped: number;
     cellsCached: number;
+    /** All non-skipped dispatch, judge, and unclassified cell failures. */
     cellsFailed: number;
+    /** Present on results that record failure stages. */
+    cellsDispatchFailed?: number;
+    /** Present on results that record failure stages. */
+    cellsJudgeFailed?: number;
+    /** Legacy failures whose stage was not recorded. */
+    cellsUnclassifiedFailed?: number;
 }
 interface CampaignResult<TArtifact = unknown, TScenario extends Scenario$1 = Scenario$1> {
     /** sha256(scenarios, judges, dispatch source ref, optimizer config, seed). Stable identity for reruns. */
@@ -1498,7 +1515,7 @@ interface RunOptimizationBaseOptions<TScenario extends Scenario$1, TArtifact> ex
         candidates: Array<{
             surfaceHash: string;
             campaign: CampaignResult<TArtifact, TScenario>;
-            composite: number;
+            composite: number | null;
         }>;
         history: GenerationRecord[];
         /** Shared run spend account and receipt attribution phase. */
@@ -2330,6 +2347,86 @@ declare function skillOptOptimizationMethod<TScenario extends Scenario$1, TArtif
  *  (so the result records every gate's reason — useful for diagnostics). */
 declare function composeGate<TArtifact = unknown, TScenario extends Scenario$1 = Scenario$1>(...gates: Array<Gate<TArtifact, TScenario>>): Gate<TArtifact, TScenario>;
 
+/**
+ * Liveness canaries — cheap statistical checks that catch the failure
+ * modes a green test suite never sees.
+ *
+ * Three canary types in this module:
+ *
+ *   1. **Silent judge fallback** — the judge degraded to a fallback
+ *      path (rules-only / cached / heuristic) without anyone
+ *      noticing. Signature: a string of consecutive runs whose
+ *      `judgeMetadata.confidence` equals a known fallback constant
+ *      (default 0.30) OR whose `judgeMetadata.fallback` is true.
+ *
+ *   2. **Judge calibration drift** — the judge's confidence
+ *      distribution has drifted from a historical window. Two-sample
+ *      Kolmogorov-Smirnov test on the recent vs historical confidences,
+ *      with the empirical-CDF max-difference statistic.
+ *
+ *   3. **Eval-set distribution shift** — the mix of categories /
+ *      buckets in the recent runs differs significantly from the
+ *      historical mix. Chi-square test on the binned counts.
+ *
+ * Outputs are alerts. The canary does NOT fail loud the way a test
+ * does — failing tests are reserved for hard correctness violations.
+ * A canary that fires is a *signal* to investigate, not a verdict.
+ *
+ * Why this lives here rather than in `observability.ts`: that module
+ * exports already, and is a pure-fanout-to-Langfuse/Prometheus
+ * adapter. Canaries are statistical detectors, not adapters.
+ */
+
+interface CanaryOptions {
+    /**
+     * Silent-fallback detection.
+     * - `constant`: confidence value treated as the fallback signal.
+     *   Default 0.30 (matches the soft-fail default in
+     *   `propose-review.ts`).
+     * - `consecutiveThreshold`: trip the alert after this many
+     *   consecutive runs at `constant` (or `fallback === true`).
+     *   Default 3.
+     */
+    silentFallback?: {
+        constant?: number;
+        consecutiveThreshold?: number;
+        /** Floating-point tolerance when comparing against `constant`. */
+        epsilon?: number;
+    };
+    /**
+     * Calibration-drift detection.
+     * - `historyWindow`: number of past runs (oldest-first) treated as
+     *   the historical baseline. Default 50.
+     * - `recentWindow`: number of recent runs (newest-first) compared
+     *   against history. Default 20.
+     * - `ksAlpha`: alpha for the KS statistic vs critical value.
+     *   Default 0.05.
+     * - `minRecent`: minimum recent runs required to even attempt the
+     *   check. Default 10.
+     */
+    calibrationDrift?: {
+        historyWindow?: number;
+        recentWindow?: number;
+        ksAlpha?: number;
+        minRecent?: number;
+    };
+    /**
+     * Distribution-shift detection.
+     * - `category`: function that maps a run to a categorical bucket.
+     *   Required to enable this canary; if omitted the chi-square check
+     *   is skipped entirely.
+     * - `chiSquareAlpha`: alpha. Default 0.05.
+     * - `historyWindow`, `recentWindow`, `minRecent`: like above.
+     */
+    distributionShift?: {
+        category: (run: RunRecord) => string | null;
+        chiSquareAlpha?: number;
+        historyWindow?: number;
+        recentWindow?: number;
+        minRecent?: number;
+    };
+}
+
 /**
  * Dataset — versioned, sliceable, content-hashed scenario collection.
  *
@@ -2401,6 +2498,144 @@ interface RedTeamCase extends DatasetScenario {
     payload: RedTeamPayload;
 }
 
+/**
+ * Verifiable reward channel.
+ *
+ * For RL on coding / math / theorem-proving / structured-output tasks, the
+ * reward signal is *decidable* — a test passes or fails, a proof checks or
+ * doesn't, an output validates against a schema or doesn't. These rewards
+ * are dramatically more useful for RL training than LLM-judge scores
+ * because they don't drift, can't be Goodhart-gamed by the policy in the
+ * same way, and don't require a separate calibration loop.
+ *
+ * The `MultiLayerVerifier` already produces this signal — it just doesn't
+ * surface it in a shape that's clean enough for RL training. This module
+ * wraps the verifier output so consumers can:
+ *
+ *   1. Extract a clean `VerifiableReward` from a `VerificationReport`
+ *   2. Distinguish *deterministic* rewards (compile, test, schema) from
+ *      *probabilistic* rewards (judge) so they can be weighted differently
+ *      in the RL training step
+ *   3. Filter `RunRecord[]` to only those with a verifiable reward,
+ *      producing the clean training set that DeepSeek-R1-style GRPO and
+ *      AlphaProof-style search both depend on
+ *
+ * Why this matters: every credible 2025-2026 frontier RL result on coding
+ * agents leans on verifiable reward (DeepSeek-R1 GRPO on test pass-rate,
+ * o-series RL on math/code, AlphaProof on Lean kernel checking). Mixing
+ * judge scores into the reward signal poisons the gradient. This module
+ * is the seam.
+ */
+
+type VerifiableRewardSource = 'compile' | 'test' | 'schema' | 'sandbox' | 'judge' | 'composite';
+interface VerifiableRewardExtractionOptions {
+    /**
+     * Which layers count as deterministic-reward sources. The verifier doesn't
+     * tag layers as "this is verifiable"; the caller declares it via this list
+     * (or via the layer name → source mapping). Default treats common names
+     * (`install`, `typecheck`, `build`, `lint`, `test`, `compile`, `schema`,
+     * `sandbox`) as deterministic.
+     */
+    deterministicLayers?: string[];
+    /**
+     * Map layer name → reward source. Defaults to a sensible string-match.
+     */
+    sourceFor?: (layerName: string) => VerifiableRewardSource;
+    /**
+     * Whether to fall back to a probabilistic (judge) reward when no
+     * deterministic layer produced a numeric score. Default `true`. Set to
+     * `false` for "deterministic-only" training pipelines that should
+     * discard runs without a verifiable signal.
+     */
+    fallbackToJudge?: boolean;
+    /**
+     * Default confidence for probabilistic (judge) rewards when the judge
+     * doesn't report one. Default `0.7`.
+     */
+    judgeConfidenceFloor?: number;
+}
+
+/**
+ * Reward hacking / Goodhart detection.
+ *
+ * Goodhart's Law says: when a measure becomes a target, it ceases to be
+ * a good measure. In RLHF and agentic-RL settings this is the dominant
+ * failure mode — the policy learns to produce outputs that score well on
+ * the proxy reward (judge, rubric, test pass-rate) without producing
+ * the underlying capability the proxy was meant to track.
+ *
+ * Krakovna et al. (2020, "Specification Gaming Examples in AI") and the
+ * subsequent RLHF reward-hacking literature (Skalse et al. 2022, Kim et al.
+ * 2023) converge on a few diagnostic signatures:
+ *
+ *   1. **Reward divergence:** the proxy reward grows while the held-out
+ *      ground-truth signal stagnates or drops. Predictive validity over
+ *      time captures this.
+ *   2. **Distributional shift in outputs:** after RL, the policy produces
+ *      outputs that no longer match the reference distribution — usually
+ *      because it found a high-reward attractor that's degenerate (e.g.
+ *      one-token responses, repetition, formatting tricks).
+ *   3. **Disagreement between independent rewards:** if you train on
+ *      reward A and a held-out independent reward B drops sharply, you're
+ *      probably hacking A.
+ *   4. **Calibration drift:** the verifiable / deterministic component of
+ *      the reward is stable; the probabilistic / judge component drifts up
+ *      while the deterministic component doesn't. The judge is being
+ *      gamed.
+ *
+ * This module ships explicit detectors for all four signatures, plus a
+ * combined verdict. The output is diagnostic — actionable signals,
+ * not autoreject — because each signature has known false positives
+ * (e.g., a policy that genuinely improves can show distributional shift).
+ *
+ * Differs from `rubricPredictiveValidity` (which is a *standing* check on
+ * whether rubrics correlate with deployment outcomes) — this is a
+ * *temporal* check on whether the reward-vs-truth gap is *widening over
+ * time during a training run*.
+ */
+
+interface DetectRewardHackingInput {
+    /**
+     * Run records ordered by recency (oldest first). The detector segments
+     * them into prefix/suffix windows to compute "did the gap widen."
+     */
+    runs: RunRecord[];
+    /**
+     * The metric the policy was trained to optimize. Should be present on
+     * `outcome.raw` or `outcome.holdoutScore`. Default reads `outcome.holdoutScore`.
+     */
+    proxyOf?: (run: RunRecord) => number | null;
+    /**
+     * The held-out ground-truth metric. For RL on coding, this is typically
+     * test pass-rate. For RLHF, it's downstream task performance or human
+     * preference. For knowledge tasks, it's an independently-graded score.
+     */
+    truthOf?: (run: RunRecord) => number | null;
+    /**
+     * Independent secondary reward. Used for the `reward_disagreement`
+     * signal. Default uses the verifiable reward extractor (deterministic
+     * sources only).
+     */
+    secondaryRewardOf?: (run: RunRecord) => number | null;
+    /**
+     * Window size — how many of the most recent runs count as the "after"
+     * cohort. Default min(50, half the runs).
+     */
+    windowSize?: number;
+    /**
+     * Severity threshold to flag a signal. Default 0.3 (suspect) and 0.6
+     * (gaming).
+     */
+    thresholds?: {
+        suspect?: number;
+        gaming?: number;
+    };
+    /**
+     * Verifiable-reward options used for the secondary-reward fallback.
+     */
+    verifiableRewardOptions?: VerifiableRewardExtractionOptions;
+}
+
 /**
  * `defaultProductionGate` — composes the substrate's existing safety
  * primitives (red-team / reward-hacking / canary / heldout) into a single
@@ -2413,6 +2648,10 @@ interface RedTeamCase extends DatasetScenario {
  * extra domain-specific gates they need (`composeGate(defaultProductionGate(...), customGate)`).
  */
 
+type DefaultProductionGateCheck = 'dimension-regression' | 'budget' | 'red-team' | 'reward-hacking' | 'canary';
+type DefaultProductionRewardHackingOptions = Omit<DetectRewardHackingInput, 'runs' | 'truthOf'> & {
+    truthOf: NonNullable<DetectRewardHackingInput['truthOf']>;
+};
 interface DefaultProductionGateOptions {
     /** Required: scenarios held out from training; substrate compares
      *  candidate-on-holdout vs baseline-on-holdout. */
@@ -2445,19 +2684,25 @@ interface DefaultProductionGateOptions {
      *  native scale. When omitted it auto-scales off observed magnitudes:
      *  0.05 on [0,1], 5 on 0-100. */
     regressionTolerance?: number;
-    /** Total $ budget for ALL cells in this campaign — including baseline + candidate.
-     *  Composite verdict refuses to ship when spend exceeded budget. */
+    /** Total $ budget for the complete improvement run. Requires
+     *  `GateContext.costLedger`; missing or incomplete accounting holds. */
     budgetUsd?: number;
-    /** Red-team cases to probe candidate outputs against. When omitted the
-     *  substrate uses `DEFAULT_RED_TEAM_CORPUS`. Provide a domain-specific
-     *  battery for tighter coverage. */
+    /** Static artifact-screening cases. Only `expected: 'ignore'` cases without
+     *  tool assertions are valid because this check does not dispatch case inputs
+     *  or observe tool calls. */
     redTeamBattery?: RedTeamCase[];
-    /** Run records (oldest-first) needed for the reward-hacking detector.
-     *  Substrate populates from prior production-loop generations. */
+    /** Shared run history, oldest first. Supplying history does not enable either
+     *  monitoring check; configure `rewardHacking` and/or `canary` explicitly. */
     recentRuns?: RunRecord[];
-    /** When true, the gate refuses to ship if the reward-hacking detector
-     *  fires at the `gaming` severity. Default true. */
-    blockOnRewardHackingGaming?: boolean;
+    /** Enable reward-hacking monitoring with a caller-owned independent truth channel. */
+    rewardHacking?: DefaultProductionRewardHackingOptions;
+    /** Enable canary monitoring. Pass `{}` to use the canary defaults. */
+    canary?: CanaryOptions;
+    /** Optional checks that must be evaluated even when their normal input is
+     *  absent. Configuring a check's input also makes that check required.
+     *  Missing evidence always records `not_evaluated`; required unevaluated
+     *  checks hold the release decision. Held-out significance is always required. */
+    requiredChecks?: DefaultProductionGateCheck[];
 }
 /**
  * Opinionated production gate composing held-out significance, red-team, reward-hacking, and canary checks into a single `Gate.decide` decision.
@@ -2837,7 +3082,7 @@ interface ParetoPoint {
     onFrontier: boolean;
     /** Optional gate verdict for this candidate, if a `GateDecision`
      *  for it was passed in. */
-    gate?: 'promote' | 'reject_few_runs' | 'reject_negative_delta' | 'reject_overfit_gap' | null;
+    gate?: 'promote' | 'reject';
 }
 interface ParetoFigureSpec {
     kind: 'pareto-cost-quality';
@@ -2891,8 +3136,8 @@ interface InsightReport {
     /** Number of runs analyzed. */
     n: number;
     /** Runtime facts carried by the run records. These describe execution,
-     *  not task quality: duration, queueing, token categories, models, and
-     *  explicitly recorded failures. */
+     *  not task quality: duration, queueing, token categories, models,
+     *  execution errors, and terminal outcomes. */
     execution: ExecutionInsight;
     /** Composite-score distribution across all runs. Always present. */
     composite: ScalarDistribution;
@@ -2950,10 +3195,9 @@ interface InsightReport {
      *  Surfaced metrics: composite, cost, duration, tokenUsage, plus any
      *  per-dimension judge metric present in both windows. */
     priorPeriodComparison?: PriorPeriodComparison;
-    /** Model-free failure-mode breakdown from `RunRecord.failureMode`, ranked
-     *  by count descending. Present when any run carries a `failureMode`.
-     *  Complements `failureClusters` (LLM-semantic) with the structured tags
-     *  the harness already recorded — actionable with no analyst wired. */
+    /** Model-free task-failure breakdown from `RunRecord.failureClass` or
+     *  `failureMode`, ranked by count descending. Tags from successful or
+     *  otherwise non-failed tasks are excluded. */
     failureModes?: FailureModeTally[];
     /** Top-N actionable recommendations, ranked by priority. The packet's
      *  human-readable layer; the numeric sections are the evidence. */
@@ -3002,14 +3246,45 @@ interface ExecutionInsight {
         events: number;
         reportingRuns: number;
     };
-    /** Failure counts remain separate from outcome scores. `reportedErrorEvents`
-     *  sums `outcome.raw.error_span_count` only where a producer supplied it. */
-    failures: {
+    /** Runs with explicit execution-error telemetry. This is independent of
+     *  whether the root run ultimately succeeded, failed, or has no terminal
+     *  evidence. */
+    executionErrors: {
         runs: number;
-        fraction: number;
-        reportedErrorEvents: number;
+        /** Share among runs that supplied an execution-error count.
+         *  `null` when no run supplied error telemetry. */
+        fraction: number | null;
+        /** Execution-error events reported through the canonical count. */
+        events: number;
+        /** Runs that supplied an execution-error count, including explicit zeroes. */
         reportingRuns: number;
+        /** Exact sum of `outcome.raw.error_span_count`, kept separate from other errors. */
+        errorSpanEvents: number;
+        /** Runs that supplied `outcome.raw.error_span_count`, including explicit zeroes. */
+        errorSpanReportingRuns: number;
+        /**
+         * Error-telemetry coverage crossed with independently reported terminal
+         * outcomes. `unreported` is distinct from a reported zero.
+         */
+        byTerminalOutcome: Record<RunTerminalOutcome, ExecutionErrorOutcomeCell>;
     };
+    /** Root-run or process outcomes. Missing `RunRecord.terminalOutcome` values
+     *  count as `unknown`; child-span status never changes these counts. */
+    terminalOutcomes: {
+        succeeded: number;
+        failed: number;
+        cancelled: number;
+        incomplete: number;
+        unknown: number;
+    };
+}
+interface ExecutionErrorOutcomeCell {
+    /** Runs that explicitly reported one or more execution errors. */
+    withErrors: number;
+    /** Runs that explicitly reported zero execution errors. */
+    withoutErrors: number;
+    /** Runs with no execution-error count from the producer. */
+    unreported: number;
 }
 interface TokenUsageInsight {
     input: ScalarDistribution;
@@ -3029,12 +3304,18 @@ interface TokenUsageInsight {
 interface ScalarDistribution {
     /** Sample count after dropping non-finite values. */
     n: number;
-    mean: number;
-    p50: number;
-    p95: number;
-    stddev: number;
-    min: number;
-    max: number;
+    /** Null when `n` is zero. */
+    mean: number | null;
+    /** Null when `n` is zero. */
+    p50: number | null;
+    /** Null when `n` is zero. */
+    p95: number | null;
+    /** Null when `n` is zero. */
+    stddev: number | null;
+    /** Null when `n` is zero. */
+    min: number | null;
+    /** Null when `n` is zero. */
+    max: number | null;
     /** Histogram bins using `agent-eval`'s `gainHistogram` primitive. */
     histogram: GainDistributionBin[];
     /** Worst-N runs by score, ascending. Populated for the composite
@@ -3100,12 +3381,16 @@ interface LiftInsight {
     pValue: number;
     /** Number of paired observations. */
     n: number;
-    /** Cohen's d for the delta. */
-    cohensD: number;
+    /** Scored baseline observations without a candidate match. */
+    unpairedBaseline: number;
+    /** Scored candidate observations without a baseline match. */
+    unpairedCandidate: number;
+    /** Cohen's dz for paired deltas; null when the observed delta variance is zero. */
+    cohensD: number | null;
     /** Minimum detectable effect at current n, 80% power. */
     mde: number;
-    /** Sample size needed to detect the observed delta at 80% power. */
-    requiredN: number;
+    /** Paired sample size needed to detect the standardized effect at 80% power. */
+    requiredN: number | null;
 }
 interface FailureClusterInsight {
     /** All clusters identified by the registry, ranked by share descending. */
@@ -3169,7 +3454,7 @@ interface ReleaseSummary {
     status: 'pass' | 'warn' | 'fail';
     axes: Array<{
         name: 'quality-lift' | 'contamination' | 'composite-distribution';
-        status: 'pass' | 'warn' | 'fail';
+        status: 'pass' | 'warn' | 'fail' | 'not_evaluated';
         detail: string;
     }>;
     /** Free-form issues surfaced beyond the standard axes. Empty by default;
@@ -3229,9 +3514,8 @@ interface Recommendation {
  * # Hosted-tier wire format — the schema that EVERY orchestrator (ours,
  * a partner's self-hosted one, a future open implementation) must accept.
  *
- * **Stability:** every type in this file is committed under semver. New
- * minors only ADD optional fields. Breaking changes mean a major bump
- * (`HostedWireVersion` literal increment).
+ * This package implements exactly one wire version. Servers reject every
+ * other version instead of translating old payloads.
  *
  * The wire format is two event streams in one transport:
  *
@@ -3257,11 +3541,15 @@ interface EvalRunCellScore {
     scenarioId: string;
     /** Repetition index when reps > 1; 0 for the default. */
     rep: number;
-    /** Composite score across all judges + dimensions for this cell. */
-    compositeMean: number;
-    /** Per-judge → per-dimension scores; null where the judge did not run. */
+    /** Composite score across successful judges, or null when unscored. */
+    compositeMean: number | null;
+    /** Per-judge and per-dimension scores; failed or missing judges are absent. */
     dimensions: Record<string, Record<string, number>>;
-    /** Per-cell error message if the dispatch threw. Null on success. */
+    /** Root execution result, kept separate from task quality. */
+    terminalOutcome: RunTerminalOutcome;
+    /** Canonical execution-error count, or null when the producer did not measure it. */
+    executionErrorCount: number | null;
+    /** Per-cell dispatch or judge error. Missing on success. */
     errorMessage?: string;
 }
 interface EvalRunGenerationSnapshot {
@@ -3275,8 +3563,8 @@ interface EvalRunGenerationSnapshot {
     surface?: MutableSurface;
     /** Per-cell scores for this generation. */
     cells: EvalRunCellScore[];
-    /** Aggregate composite mean across all cells in this generation. */
-    compositeMean: number;
+    /** Mean across scored cells, or null when no cell has a task-quality label. */
+    compositeMean: number | null;
     /** Total $ spent across this generation. */
     costUsd: number;
     /** Wall-clock duration of this generation. */
@@ -3316,10 +3604,14 @@ interface EvalRunEvent {
      *  paired-bootstrap lift CI, judge stats, inter-rater agreement,
      *  contamination check, failure clusters (when an analyst is wired),
      *  outcome correlation (when downstream signal is supplied), and the
-     *  recommendations the dashboard surfaces verbatim. Additive; older
-     *  clients that don't know about this field continue to work. */
+     *  recommendations the dashboard surfaces verbatim. */
     insightReport?: InsightReport;
 }
+/**
+ * Canonical unsigned 64-bit integer encoded as a base-10 string.
+ * JSON numbers cannot represent OTLP nanosecond timestamps exactly.
+ */
+type UnixNanoTimestamp = string;
 /**
  * OTel-shape span with a few additional attributes for eval-run pivoting.
  * Compatible with any OTLP collector — `name`, `traceId`, `spanId`,
@@ -3330,11 +3622,11 @@ interface TraceSpanEvent {
     spanId: string;
     parentSpanId?: string;
     name: string;
-    startTimeUnixNano: number;
-    endTimeUnixNano: number;
+    startTimeUnixNano: UnixNanoTimestamp;
+    endTimeUnixNano: UnixNanoTimestamp;
     attributes: Record<string, string | number | boolean>;
     events?: Array<{
-        timeUnixNano: number;
+        timeUnixNano: UnixNanoTimestamp;
         name: string;
         attributes?: Record<string, string | number | boolean>;
     }>;
@@ -3462,8 +3754,8 @@ interface LoopProvenanceCandidate {
     eligibleForPromotion: boolean;
     /** Designed-denominator receipt retained even for incomplete candidates. */
     coverage: NonNullable<GenerationCandidate['coverage']>;
-    /** Mean composite this candidate scored on the search split. */
-    composite: number;
+    /** Mean composite this candidate scored on the search split, or null when unscorable. */
+    composite: number | null;
     /** Whether this candidate was promoted out of its generation. */
     promoted: boolean;
 }
@@ -3535,11 +3827,7 @@ interface LoopProvenanceRecord {
         decision: GateDecision;
         reasons: string[];
         delta?: number;
-        contributingGates: Array<{
-            name: string;
-            passed: boolean;
-            detail: unknown;
-        }>;
+        contributingGates: GateContribution[];
     };
     /** Present iff the loop ran with `holdout: 'deferred'` — the held-out
      *  comparison was intentionally not measured in this run, so the holdout
@@ -3968,6 +4256,39 @@ interface CompareCandidateExperimentOptions {
     searchCostUsd?: number;
     metadata?: AgentImprovementMeasuredComparison['metadata'];
 }
+/** One exact baseline/candidate observation of the same held-out cell. */
+interface PairedMeasurement<TRun> {
+    cellId: string;
+    baseline: TRun;
+    candidate: TRun;
+}
+/** Maps a product-owned run receipt into the measurements required for a fair paired decision. */
+interface PairedMeasurementAdapter<TRun> {
+    score(run: TRun): number;
+    dimensions(run: TRun): readonly {
+        name: string;
+        score: number;
+    }[];
+    costUsd(run: TRun): number;
+    latencyMs(run: TRun): number;
+    completed(run: TRun): boolean;
+    passed(run: TRun): boolean;
+}
+interface EvaluatePairedMeasurementsOptions<TRun> {
+    measurements: readonly PairedMeasurement<TRun>[];
+    policy: AgentCandidateEvaluationPolicy;
+    adapter: PairedMeasurementAdapter<TRun>;
+    /** Whether both arms use the same scorer family as the promotion decision. */
+    sharedScorerChannel: boolean;
+    /** Search or preparation spend that belongs to the same frozen budget. */
+    additionalCostUsd?: number;
+}
+/** Statistical and operational result derived from complete paired receipts. */
+type PairedMeasurementEvaluation = Pick<AgentImprovementMeasuredComparison, 'overall' | 'objectives' | 'decision' | 'power'> & {
+    executionCostUsd: number;
+    totalCostUsd: number;
+    executionDurationMs: number;
+};
 /** Content-address one task before any measured execution can see it. */
 declare function sealCandidateBenchmarkTask(material: AgentCandidateBenchmarkTaskMaterial): AgentCandidateBenchmarkTask;
 /** Freeze task order, repetitions, and every seed before either arm runs. */
@@ -3977,6 +4298,14 @@ declare function sealCandidateExperiment(material: AgentCandidateExperimentMater
 declare function verifyCandidateExperiment(input: unknown): AgentCandidateExperiment;
 /** Execute each signed cell for both arms. The callback is Runtime's one executor. */
 declare function runCandidateExperiment(options: RunCandidateExperimentOptions): Promise<AgentCandidateExperimentMeasurement[]>;
+/**
+ * Calculate the shared paired decision from any complete receipt shape.
+ *
+ * Callers still own sealing their tasks, verifying each receipt against its
+ * expected arm and state, and proving every expected cell exists. This function
+ * only validates the projected measurements and derives their shared decision.
+ */
+declare function evaluatePairedMeasurements<TRun>(options: EvaluatePairedMeasurementsOptions<TRun>): PairedMeasurementEvaluation;
 /** Build the only publishable comparison: paired statistics over Runtime receipts. */
 declare function measuredComparisonFromCandidateExperiment(options: CompareCandidateExperimentOptions): AgentImprovementMeasuredComparison;
 /** Recompute every statistic and decision from the signed experiment receipts. */
@@ -4007,7 +4336,7 @@ declare function verifyCandidateBenchmarkSuite(input: unknown): {
  * `ViewTraceOversized`.
  */
 /** OTLP span kind (subset we actually use). */
-type TraceAnalystSpanKind = 'AGENT' | 'LLM' | 'TOOL' | 'CHAIN' | 'GUARDRAIL' | 'SPAN' | 'UNKNOWN';
+type TraceAnalystSpanKind = 'AGENT' | 'LLM' | 'TOOL' | 'CHAIN' | 'EVALUATOR' | 'GUARDRAIL' | 'SPAN' | 'UNKNOWN';
 type TraceAnalystSpanStatus = 'OK' | 'ERROR' | 'UNSET';
 /** Subset of OTLP span fields the analyst exposes to the agent. The
  *  store's job is to project upstream's full span shape down to this
@@ -4522,7 +4851,7 @@ type RawAnalystFinding = z.infer<typeof RawAnalystFindingSchema>;
  * item so persisted rows and older model fixtures remain readable. New output
  * always receives the plural shape.
  */
-declare const CanonicalRawAnalystFindingSchema: z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodObject<{
+declare const CanonicalRawAnalystFindingSchema: z.ZodPreprocess<z.ZodObject<{
     evidence: z.ZodArray<z.ZodObject<{
         uri: z.ZodString;
         excerpt: z.ZodOptional<z.ZodString>;
@@ -4806,7 +5135,8 @@ interface AnalyzeRunsOptions {
     split?: 'search' | 'holdout' | 'auto';
     /** Pairwise analysis configuration. When both `baselineCandidateId` and
      *  `candidateCandidateId` are present, lift is computed on paired
-     *  (experimentId, seed) tuples shared between the two sides. */
+     *  (experimentId, scenarioId, seed) identities shared between the two sides.
+     *  Unmatched rows remain visible in the lift result. */
     baselineCandidateId?: string;
     candidateCandidateId?: string;
     /** Canary scenarios — checked against every run's raw output for
@@ -4841,7 +5171,7 @@ interface AnalyzeRunsOptions {
      *  recommendations fire on statistically significant regressions.
      *  The two windows do NOT have to share scenarios — the comparison
      *  is two-sample unpaired (the substrate's `lift` field uses paired
-     *  bootstrap on shared (experimentId, seed) tuples; this is the
+     *  bootstrap on shared (experimentId, scenarioId, seed) identities; this is the
      *  shape for "this week vs last week" rather than "candidate vs
      *  baseline within a campaign"). */
     baselineRuns?: RunRecord[];
@@ -5028,9 +5358,9 @@ interface EvalDimensionDelta {
 interface EvalCellScoreDelta {
     scenarioId: string;
     rep: number;
-    compositeBefore: number;
-    compositeAfter: number;
-    compositeDelta: number;
+    compositeBefore: number | null;
+    compositeAfter: number | null;
+    compositeDelta: number | null;
     /** Per-judge → per-dimension deltas. Outer key = judge name from
      *  `EvalRunCellScore.dimensions`; inner key = dimension name. */
     dimensions: Record<string, Record<string, EvalDimensionDelta>>;
@@ -5049,10 +5379,10 @@ interface EvalGenerationDiff {
     removed: EvalRunCellScore[];
     /** Cells present in `after` but missing from `before`. */
     added: EvalRunCellScore[];
-    /** Aggregate composite mean across all cells in the snapshot. */
-    compositeBefore: number;
-    compositeAfter: number;
-    compositeDelta: number;
+    /** Aggregate composite mean, null when that snapshot was unscored. */
+    compositeBefore: number | null;
+    compositeAfter: number | null;
+    compositeDelta: number | null;
     costUsdBefore: number;
     costUsdAfter: number;
     costUsdDelta: number;
@@ -5261,6 +5591,7 @@ interface CodeAgentSessionMetrics {
     toolCalls: number;
     toolOutputs: number;
     toolErrors: number;
+    unclassifiedErrors: number;
     patchAttempts: number;
     patchSuccesses: number;
     patchFailures: number;
@@ -5299,7 +5630,6 @@ interface CodeAgentSessionDiagnostic {
     sourcePath?: string;
     entries: number;
     malformedLines: number;
-    inferredScore: boolean;
     hasExplicitTerminalSignal: boolean;
     hasFinalOutput: boolean;
     hasQualityLabel: boolean;
@@ -5328,9 +5658,7 @@ interface CodeAgentSessionIntakeOptions {
     configHash?: string;
     commitSha?: string;
     score?: number;
-    /** Explicit cost receipt. Use `uncaptured` when the source says dollars
-     *  were not captured; the adapter will not relabel its compatibility $0
-     *  sentinel as observed. When omitted, source-reported cost wins, then a
+    /** Explicit cost receipt. When omitted, source-reported cost wins, then a
      *  token-priced estimate, then uncaptured. */
     costProvenance?: RunCostProvenance;
     /** Exact executor-owned process result. This is required when a provider's
@@ -5356,8 +5684,8 @@ declare const fromPigraphSession: typeof fromPiSession;
  * `analyzeRuns({ runs, raterScores })` can produce inter-rater agreement,
  * disagreement triage, and downstream recommendations.
  *
- * Per-run `RunRecord.outcome.searchScore` is the rater-mean rating
- * (normalised to 0..1 when scale is supplied); `outcome.raw` carries the
+ * The rater mean is written to the score field matching the run split
+ * (normalised to 0..1 when a scale is supplied); `outcome.raw` carries
  * per-rater scores keyed by rater id for downstream attribution.
  */
 
@@ -5381,9 +5709,10 @@ interface FeedbackTableMeta {
     experimentId?: string;
     /** When omitted, defaults to `runId` — each run is its own candidate. */
     candidateId?: string;
-    /** Cost in USD, when available. Set to 0 when unknown — the consumer's
-     *  cost analysis sections will collapse gracefully. */
+    /** Observed cost in USD, when available. */
     costUsd?: number;
+    /** Stable scenario identity. Defaults to `runId`. */
+    scenarioId?: string;
     /** Wall-clock ms, when available. Defaults to 0. */
     wallMs?: number;
     /** Model identifier including snapshot. Default `unknown@unknown`. */
@@ -5415,9 +5744,8 @@ interface FromFeedbackTableOptions {
         max: number;
     };
     /** When true, the rater scores are emitted into `raterScores` (a sibling
-     *  array `analyzeRuns()` accepts) instead of being averaged into the
-     *  run's `outcome.searchScore`. Default `true` — preserves rater-level
-     *  signal for inter-rater analysis. */
+     *  array `analyzeRuns()` accepts) in addition to the aggregate run score.
+     *  Default `true` preserves rater-level signal for inter-rater analysis. */
     emitRaterScores?: boolean;
 }
 interface FromFeedbackTableResult {
@@ -5449,11 +5777,15 @@ declare function fromFeedbackTable(opts: FromFeedbackTableOptions): FromFeedback
  *   - cost from `cost.usd` / `gen_ai.usage.cost_usd` / `tangle.cost.usd`
  *   - token usage from model-call input, output, cache-read, and cache-write
  *     attributes without double-counting aggregate parent spans
- *   - `outcome.searchScore` from `tangle.score` / `eval.score` when
- *     present; `outcome.raw` collects every numeric attribute.
+ *   - task quality from an explicit `scoreForRun` callback or a designated
+ *     evaluation attribute on a root / `EVALUATOR` span; `outcome.raw`
+ *     collects every numeric attribute without promoting it to task quality.
  *
- * Spans that ERRORed (`status.code === 'ERROR'`) populate `failureMode`
- * with their `name` so `analyzeRuns()`'s failure clustering sees them.
+ * Errored tool, model, and child-agent spans contribute to execution-error
+ * counts. Root process, guardrail, evaluator, propagated parent, and unknown
+ * errors retain separate counters. Only one failed root can set
+ * `RunRecord.terminalOutcome` and `RunRecord.terminalFailureReason`; a child
+ * error cannot become a task failure.
  */
 
 interface FromOtelSpansOptions {
@@ -5462,7 +5794,13 @@ interface FromOtelSpansOptions {
     defaultSplit?: RunSplitTag;
     /** Default `experimentId` when not present on any span. */
     experimentId?: string;
+    /**
+     * Explicit task-quality score for a logical run. The callback receives
+     * spans in deterministic time/id order. Its value must agree with any
+     * designated score attributes present on root or `EVALUATOR` spans.
+     */
+    scoreForRun?: (runId: string, spans: readonly TraceSpanEvent[]) => number | undefined;
 }
 declare function fromOtelSpans(opts: FromOtelSpansOptions): RunRecord[];
 
-export { type AgentEvalAgent, type AgentEvalEvaluateOptions, type AgentEvalImproveOptions, type AgentTraceContributor, type AgentTraceContributorType, type AgentTraceConversation, type AgentTraceFile, type AgentTraceIndex, type AgentTraceRange, type AgentTraceRecord, type AnalystFinding, type AnalyzeRunsOptions, type AuthoringProvenance, type AxisEvidence, type AxisVerdict, type BuildEvidenceVectorOptions, type CampaignAggregates, type CampaignArtifactWriter, type CampaignCellFailureReceipt, type CampaignCellResult, type CampaignCostMeter, type CampaignResult, type CampaignStorage, type CampaignTraceWriter, type CandidateExperimentExecutionInput, type ChatClient, type CodeAgentSessionAction, type CodeAgentSessionActionKind, type CodeAgentSessionActionStatus, type CodeAgentSessionActionSurface, type CodeAgentSessionDiagnostic, type CodeAgentSessionExecutionReceipt, type CodeAgentSessionIntakeOptions, type CodeAgentSessionIntakeResult, type CodeAgentSessionMetrics, type CodeAgentSessionObservation, type CodeAgentSessionSource, type CodeAgentSessionTerminalStatus, type CodeSurface, type CompareCandidateExperimentOptions, type CompareOptimizationMethodsOptions, type ComparisonCost, type CostLedgerHandle, type CostProvenanceSummary, type CreateChatClientOpts, type DefaultAnalystRegistryOptions, type DefaultProductionGateOptions, type DefineAgentEvalOptions, type DefinedAgentEval, type DeploymentOutcome, type DispatchFn as Dispatch, type DispatchContext, type EvalCellScoreDelta, type EvalDimensionDelta, type EvalGenerationDiff, type EvalReportingSuiteInput, type EvalReportingSuiteOptions, type EvalReportingSuiteResult, type EvalRunDiff, type EvidenceVector, type ExecutionInsight, type ExecutionReport, type ExternalOptimizationExample, type ExternalTextEvaluationResponse, type ExternalTextOptimizationMethodConfig, type ExternalTextOptimizerContext, type ExternalTextOptimizerResult, type FailureClusterInsight, type FeedbackTableMeta, type FeedbackTableRow, FileSystemOutcomeStore, type FileSystemOutcomeStoreOptions, type FromFeedbackTableOptions, type FromFeedbackTableResult, type FromOtelSpansOptions, type FromRunRecordDirOptions, type FromRunRecordDirResult, type Gate, type GateContext, type GateDecision, type GateResult, type GenerationCandidate, type GenerationRecord, type GepaAdaptiveEngineRun, type GepaEngineOptions, type GepaEngineRun, type GepaOptimizationMethodConfig, type GepaOptimizationRecipe, type GepaRunnerCommand, type HeldOutGateOptions, type HostedTenant, InMemoryOutcomeStore, type InsightReport, type InterRaterInsight, type JudgeConfig, type JudgeDimension, type JudgeInsight, type JudgeScore, type LiftInsight, type LlmJudgeDimension, type LlmJudgeOptions, type MutableSurface, type ObjectiveSource, type OpenAICompatibleOptimizerModel, type OptimizationMethod, type OptimizationMethodComparison, type OptimizationMethodInput, type OptimizationMethodProvenance, type OptimizationMethodResult, type OptimizationPackageSource, type OptimizationProposer, type OptimizationTokenUsage, type OptimizerConfig, type OptimizerModelBudget, type OutcomeCorrelationInsight, type OutcomeStore, type ParetoSignificanceGateOptions, type ParsedCodeAgentJsonl, type PartitionByAuthoringModelResult, type PromotionObjective, type PromotionPolicy, REFERENCE_EQUIVALENCE_INPUT_LIMITS, REFERENCE_EQUIVALENCE_JUDGE_VERSION, type Recommendation, type ReferenceEquivalenceJudgeInput, type ReferenceEquivalenceJudgeOptions, type ReferenceEquivalenceJudgeResult, type ReferenceEquivalenceScenario, type ReleaseSummary, type RunCampaignOptions, type RunCandidateExperimentOptions, type RunEvalOptions, type RunImprovementLoopOptions, type RunImprovementLoopResult, type RunRecordRejection, type ScalarDistribution, type Scenario$1 as Scenario, type SealCandidateBenchmarkSuiteOptions, type SelfImproveBudget, type SelfImproveOptions, type SelfImproveProgressEvent, type SelfImproveResult, SelfImproveRunError, type SessionScript, type SkillOptOptimizationMethodConfig, type SkillOptRunnerCommand, type SkillOptTrainerConfig, type SummarizeExecutionOptions, type SurfaceProposer, type TokenUsageInsight, analyzeRuns, buildDefaultAnalystRegistry, buildEvidenceVector, campaignSplitDigest, compareOptimizationMethods, composeGate, createChatClient, createReferenceEquivalenceJudge, defaultProductionGate, defineAgentEval, diffGenerations, diffRunBaselineToWinner, diffRuns, evalReportingSuite, externalTextOptimizationMethod, fromClaudeCodeSession, fromCodexSession, fromFeedbackTable, fromKimiCodeSession, fromOpenCodeSession, fromOtelSpans, fromPiSession, fromPigraphSession, fromRunRecordDir, fsCampaignStorage, gepaOptimizationMethod, heldOutGate, inMemoryCampaignStorage, llmJudge, measuredComparisonFromCandidateExperiment, observeCodeAgentSession, paretoPolicy, paretoSignificanceGate, parseAgentTrace, parseCodeAgentJsonl, partitionRunsByAuthoringModel, runCampaign, runCandidateExperiment, runEval, runImprovementLoop, runReferenceEquivalenceJudge, sealCandidateBenchmarkSuite, sealCandidateBenchmarkTask, sealCandidateExperiment, selfImprove, skillOptOptimizationMethod, summarizeExecution, verifyCandidateBenchmarkSuite, verifyCandidateBenchmarkSuiteInputs, verifyCandidateBenchmarkTask, verifyCandidateExperiment, verifyCandidateExperimentComparison };
+export { type AgentEvalAgent, type AgentEvalEvaluateOptions, type AgentEvalImproveOptions, type AgentTraceContributor, type AgentTraceContributorType, type AgentTraceConversation, type AgentTraceFile, type AgentTraceIndex, type AgentTraceRange, type AgentTraceRecord, type AnalystFinding, type AnalyzeRunsOptions, type AuthoringProvenance, type AxisEvidence, type AxisVerdict, type BuildEvidenceVectorOptions, type CampaignAggregates, type CampaignArtifactWriter, type CampaignCellFailureReceipt, type CampaignCellResult, type CampaignCostMeter, type CampaignResult, type CampaignStorage, type CampaignTraceWriter, type CandidateExperimentExecutionInput, type ChatClient, type CodeAgentSessionAction, type CodeAgentSessionActionKind, type CodeAgentSessionActionStatus, type CodeAgentSessionActionSurface, type CodeAgentSessionDiagnostic, type CodeAgentSessionExecutionReceipt, type CodeAgentSessionIntakeOptions, type CodeAgentSessionIntakeResult, type CodeAgentSessionMetrics, type CodeAgentSessionObservation, type CodeAgentSessionSource, type CodeAgentSessionTerminalStatus, type CodeSurface, type CompareCandidateExperimentOptions, type CompareOptimizationMethodsOptions, type ComparisonCost, type CostLedgerHandle, type CostProvenanceSummary, type CreateChatClientOpts, type DefaultAnalystRegistryOptions, type DefaultProductionGateCheck, type DefaultProductionGateOptions, type DefaultProductionRewardHackingOptions, type DefineAgentEvalOptions, type DefinedAgentEval, type DeploymentOutcome, type DispatchFn as Dispatch, type DispatchContext, type EvalCellScoreDelta, type EvalDimensionDelta, type EvalGenerationDiff, type EvalReportingSuiteInput, type EvalReportingSuiteOptions, type EvalReportingSuiteResult, type EvalRunDiff, type EvaluatePairedMeasurementsOptions, type EvidenceVector, type ExecutionErrorOutcomeCell, type ExecutionInsight, type ExecutionReport, type ExternalOptimizationExample, type ExternalTextEvaluationResponse, type ExternalTextOptimizationMethodConfig, type ExternalTextOptimizerContext, type ExternalTextOptimizerResult, type FailureClusterInsight, type FeedbackTableMeta, type FeedbackTableRow, FileSystemOutcomeStore, type FileSystemOutcomeStoreOptions, type FromFeedbackTableOptions, type FromFeedbackTableResult, type FromOtelSpansOptions, type FromRunRecordDirOptions, type FromRunRecordDirResult, type Gate, type GateCheckStatus, type GateContext, type GateContribution, type GateDecision, type GateResult, type GenerationCandidate, type GenerationRecord, type GepaAdaptiveEngineRun, type GepaEngineOptions, type GepaEngineRun, type GepaOptimizationMethodConfig, type GepaOptimizationRecipe, type GepaRunnerCommand, type HeldOutGateOptions, type HostedTenant, InMemoryOutcomeStore, type InsightReport, type InterRaterInsight, type JudgeConfig, type JudgeDimension, type JudgeInsight, type JudgeScore, type LiftInsight, type LlmJudgeDimension, type LlmJudgeOptions, type MutableSurface, type ObjectiveSource, type OpenAICompatibleOptimizerModel, type OptimizationMethod, type OptimizationMethodComparison, type OptimizationMethodInput, type OptimizationMethodProvenance, type OptimizationMethodResult, type OptimizationPackageSource, type OptimizationProposer, type OptimizationTokenUsage, type OptimizerConfig, type OptimizerModelBudget, type OutcomeCorrelationInsight, type OutcomeStore, type PairedMeasurement, type PairedMeasurementAdapter, type PairedMeasurementEvaluation, type ParetoSignificanceGateOptions, type ParsedCodeAgentJsonl, type PartitionByAuthoringModelResult, type PromotionObjective, type PromotionPolicy, REFERENCE_EQUIVALENCE_INPUT_LIMITS, REFERENCE_EQUIVALENCE_JUDGE_VERSION, type Recommendation, type ReferenceEquivalenceJudgeInput, type ReferenceEquivalenceJudgeOptions, type ReferenceEquivalenceJudgeResult, type ReferenceEquivalenceScenario, type ReleaseSummary, type RunCampaignOptions, type RunCandidateExperimentOptions, type RunEvalOptions, type RunImprovementLoopOptions, type RunImprovementLoopResult, type RunRecordRejection, type ScalarDistribution, type Scenario$1 as Scenario, type SealCandidateBenchmarkSuiteOptions, type SelfImproveBudget, type SelfImproveOptions, type SelfImproveProgressEvent, type SelfImproveResult, SelfImproveRunError, type SessionScript, type SkillOptOptimizationMethodConfig, type SkillOptRunnerCommand, type SkillOptTrainerConfig, type SummarizeExecutionOptions, type SurfaceProposer, type TokenUsageInsight, analyzeRuns, buildDefaultAnalystRegistry, buildEvidenceVector, campaignSplitDigest, compareOptimizationMethods, composeGate, createChatClient, createReferenceEquivalenceJudge, defaultProductionGate, defineAgentEval, diffGenerations, diffRunBaselineToWinner, diffRuns, evalReportingSuite, evaluatePairedMeasurements, externalTextOptimizationMethod, fromClaudeCodeSession, fromCodexSession, fromFeedbackTable, fromKimiCodeSession, fromOpenCodeSession, fromOtelSpans, fromPiSession, fromPigraphSession, fromRunRecordDir, fsCampaignStorage, gepaOptimizationMethod, heldOutGate, inMemoryCampaignStorage, llmJudge, measuredComparisonFromCandidateExperiment, observeCodeAgentSession, paretoPolicy, paretoSignificanceGate, parseAgentTrace, parseCodeAgentJsonl, partitionRunsByAuthoringModel, runCampaign, runCandidateExperiment, runEval, runImprovementLoop, runReferenceEquivalenceJudge, sealCandidateBenchmarkSuite, sealCandidateBenchmarkTask, sealCandidateExperiment, selfImprove, skillOptOptimizationMethod, summarizeExecution, verifyCandidateBenchmarkSuite, verifyCandidateBenchmarkSuiteInputs, verifyCandidateBenchmarkTask, verifyCandidateExperiment, verifyCandidateExperimentComparison };