@tangle-network/agent-eval 0.126.6 → 0.127.0

package/dist/index.d.ts

@@ -1492,6 +1492,7 @@ declare function providerFromBaseUrl(baseUrl: string): string;
  * OpenAI-compatible `/v1/chat/completions` client with:
  *   - Exponential-backoff retry on 429 + 5xx gateway errors (502/503/504).
  *   - Retry on transient network errors (fetch failed, AbortError, ECONNRESET).
+ *   - One retry at temperature 1 when a model explicitly requires it.
  *   - Graceful json_schema → json_object degrade on 400 with schema-reject body.
  *   - Fenced-JSON stripping (```json ... ```) for models that wrap structured output.
  *   - Configurable base URL + api key / bearer, works with LiteLLM proxies, OpenAI
@@ -1905,7 +1906,7 @@ declare function createChatClient(opts: CreateChatClientOpts): ChatClient;
  * `ViewTraceOversized`.
  */
 /** OTLP span kind (subset we actually use). */
-type TraceAnalystSpanKind = 'AGENT' | 'LLM' | 'TOOL' | 'CHAIN' | 'GUARDRAIL' | 'SPAN' | 'UNKNOWN';
+type TraceAnalystSpanKind = 'AGENT' | 'LLM' | 'TOOL' | 'CHAIN' | 'EVALUATOR' | 'GUARDRAIL' | 'SPAN' | 'UNKNOWN';
 type TraceAnalystSpanStatus = 'OK' | 'ERROR' | 'UNSET';
 /** Subset of OTLP span fields the analyst exposes to the agent. The
  *  store's job is to project upstream's full span shape down to this
@@ -2144,7 +2145,8 @@ interface TraceAnalysisStore {
  * researcher loop SHOULD be recorded as a `RunRecord`. The mandatory
  * fields are exactly those the paper "Two Loops, Three Roles" requires
  * for reproducibility: who/what/when/cost/seed/hash, plus the search vs
- * holdout split tag and either a `searchScore` or a `holdoutScore`.
+ * holdout split tag. A task score is optional because execution-only records
+ * must preserve missing labels instead of converting errors into zero quality.
  *
  * This is intentionally NOT a replacement for the rich `Run` /
  * `ProposeReviewReport` / `ScenarioResult` types already in the
@@ -2165,6 +2167,13 @@ interface TraceAnalysisStore {
 /** Search/dev/holdout split tag. 'search' is the paper-grade alias for the
  *  combined train+test pool that the optimizer is allowed to read. */
 type RunSplitTag = 'search' | 'dev' | 'holdout';
+/**
+ * Explicit execution-lifecycle result for a run.
+ *
+ * This is separate from task quality (`outcome`) and failure classification.
+ * Producers set it only from root-run or process evidence.
+ */
+type RunTerminalOutcome = 'succeeded' | 'failed' | 'cancelled' | 'incomplete' | 'unknown';
 interface RunTokenUsage {
     input: number;
     /** All generated tokens charged as output, including reasoning tokens. */
@@ -2178,11 +2187,6 @@ interface RunTokenUsage {
 }
 /**
  * How a run's USD amount was obtained.
- *
- * `costUsd` remains mandatory for wire compatibility. New producers should
- * always populate this discriminated union so a missing bill is never
- * mistaken for an observed zero-dollar run. For `uncaptured`, `costUsd` uses
- * the legacy `0` sentinel while this field carries the truthful null.
  */
 type RunCostProvenance = {
     kind: 'observed';
@@ -2228,8 +2232,8 @@ interface JudgeScoresRecord {
     perJudge: Record<string, Record<string, number>>;
     /** Per-dim mean across judges. Convenience — derivable from `perJudge`. */
     perDimMean: Record<string, number>;
-    /** Composite mean across all dims and judges. Mirrors the score
-     *  the gate sees on `outcome.searchScore` / `holdoutScore`. */
+    /** Composite mean across successful judges. Mirrors the task score only
+     *  when `failedJudges` is empty. */
     composite: number;
     /** Judges that errored or returned an unparseable verdict. Recorded
      *  by id (e.g. `['glm-5.1']`) so a partial-failure case is explicit,
@@ -2240,11 +2244,11 @@ interface JudgeScoresRecord {
     notes?: string;
 }
 interface RunOutcome {
-    /** Score on the search/optimization split. Optional because a
-     *  holdout-only evaluation only fills `holdoutScore`. */
+    /** Score on the search/optimization split. Optional for holdout-only and
+     *  execution-only records. */
     searchScore?: number;
-    /** Score on the held-out split. Optional because a search-only run
-     *  only fills `searchScore`. At least one must be present. */
+    /** Score on the held-out split. Optional for search-only and execution-only
+     *  records. When both scores are absent, the run is explicitly unlabeled. */
     holdoutScore?: number;
     /** Bag of any other metric the run produced — judge dimensions,
      *  pass/fail counters, latency stats, etc. Numeric only — keeps
@@ -2308,42 +2312,39 @@ interface RunRecord {
     wallMs: number;
     /** Time spent queued before execution started, if known. */
     queueMs?: number;
-    /** Total USD cost. Mandatory — runs without a cost number are
-     *  unbounded by definition and must not be admitted into the gate.
-     *  `0` is retained as the compatibility sentinel for an uncaptured amount;
-     *  inspect `costProvenance` before treating it as observed. */
-    costUsd: number;
-    /** Observed, model-priced estimate, or genuinely uncaptured USD amount.
-     *  Optional only so existing serialized RunRecords remain valid. */
-    costProvenance?: RunCostProvenance;
+    /** Total USD cost, or null when the producer could not capture one. */
+    costUsd: number | null;
+    /** Whether `costUsd` came from billing data, a price calculation, or is unavailable. */
+    costProvenance: RunCostProvenance;
     /** Token usage breakdown. */
     tokenUsage: RunTokenUsage;
+    /** Root-run or process terminal result. Never inferred from a child span. */
+    terminalOutcome: RunTerminalOutcome;
+    /** Root-run or process failure reason. Valid only for a failed, cancelled,
+     *  or incomplete terminal result; never populated from a child span. */
+    terminalFailureReason?: string;
     /** Judge-side metadata, if a judge was used. */
     judgeMetadata?: RunJudgeMetadata;
     /** Per-split scores + raw bag. */
     outcome: RunOutcome;
-    /** Canonical, cross-agent failure class drawn from the shared
+    /** Canonical task-failure class drawn from the shared
      *  `FAILURE_CLASSES` taxonomy. This is the aggregation key that makes
      *  "which failure dominates across the whole fleet" answerable in ONE
      *  vocabulary — every agent classifies against the same enum. Producers
-     *  set it via the substrate classifier; leave unset only when the failure
-     *  genuinely can't be classified. */
+     *  set it only from task-result evidence. Execution errors belong in
+     *  `outcome.raw.execution_error_count`, even when the run later fails. */
     failureClass?: FailureClass;
-    /** Free-form domain-specific failure detail, scoped UNDER `failureClass`
+    /** Free-form task-failure detail, scoped UNDER `failureClass`
      *  (e.g. failureClass='tool_recovery_failure', failureMode='forge_build_unsatisfied').
-     *  The within-agent drill-down; `failureClass` is the cross-agent key. */
+     *  Do not populate this from a child execution error alone. */
     failureMode?: string;
     /** Which split this run was drawn from. */
     splitTag: RunSplitTag;
     /**
-     * Stable scenario identifier the run was scored against. Optional for
-     * backwards compatibility, but **strongly recommended**: every primitive
-     * that pairs runs by scenario (preferences, paired stats, BT tournament)
-     * keys on this. The campaign artifact populates it canonically; legacy
-     * runs without it fall back to inference from `outcome.raw.scenario_id`
-     * or `experimentId`.
+     * Stable scenario identifier the run observed or was scored against.
+     * Comparison primitives match this identity rather than input order.
      */
-    scenarioId?: string;
+    scenarioId: string;
     /**
      * Canonical identity for the agent profile cell that produced this row:
      * profile artifact hash plus optional harness/model/prompt/reporting
@@ -2353,6 +2354,8 @@ interface RunRecord {
      */
     agentProfile?: AgentProfileCell;
 }
+/** Return task quality, preferring held-out evidence when both scores exist. */
+declare function runTaskScore(record: RunRecord): number | undefined;
 declare class RunRecordValidationError extends ValidationError {
     readonly path: string;
     constructor(message: string, path?: string);
@@ -2363,15 +2366,6 @@ declare class RunRecordValidationError extends ValidationError {
  * `RunRecord` on success — the validator does not coerce.
  */
 declare function validateRunRecord(input: unknown): RunRecord;
-/**
- * Resolve provenance for both new and legacy records.
- *
- * Legacy producers sometimes set `outcome.raw.cost_estimated = 1`. A positive
- * unlabeled amount is treated as observed, matching the historical contract.
- * Zero without an explicit label is conservatively uncaptured: claiming an
- * observed $0 would be stronger than the serialized evidence supports.
- */
-declare function resolveRunCostProvenance(run: Pick<RunRecord, 'costUsd' | 'costProvenance' | 'outcome'>): RunCostProvenance;
 /** Boolean validator — convenience for filtering arrays. */
 declare function isRunRecord(input: unknown): input is RunRecord;
 /** Non-throwing validator — returns a discriminated union. */
@@ -3036,7 +3030,7 @@ type RawAnalystFinding = z.infer<typeof RawAnalystFindingSchema>;
  * item so persisted rows and older model fixtures remain readable. New output
  * always receives the plural shape.
  */
-declare const CanonicalRawAnalystFindingSchema: z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodObject<{
+declare const CanonicalRawAnalystFindingSchema: z.ZodPreprocess<z.ZodObject<{
     evidence: z.ZodArray<z.ZodObject<{
         uri: z.ZodString;
         excerpt: z.ZodOptional<z.ZodString>;
@@ -4794,7 +4788,7 @@ declare function acquisitionPlansForKnowledgeGaps(gaps: KnowledgeRequirement[]):
  *     specific promotion path (still useful for replay-style evals).
  */
 
-type HeldOutGateRejectionCode = 'few_runs' | 'negative_delta' | 'overfit_gap' | 'cost_ceiling';
+type HeldOutGateRejectionCode = 'few_runs' | 'missing_split_scores' | 'missing_cost' | 'negative_delta' | 'overfit_gap' | 'cost_ceiling';
 interface HeldOutGateConfig {
     /** Minimum number of paired (candidate, baseline) holdout observations
      *  required before the gate will even consider promoting. Default 3. */
@@ -4825,39 +4819,43 @@ interface HeldOutGateConfig {
      *
      * This exists because "we ship the better prompt" is only an honest
      * pitch when the better prompt also fits a customer-stated budget.
-     * Cost is read from `RunRecord.costUsd` (already mandatory on every
-     * run) so no new schema is required.
+     * Cost is read from `RunRecord.costUsd`; a null amount rejects a
+     * configured cost check because the limit cannot be proven.
      */
     costPerTaskCeiling?: number;
 }
 interface GateEvidence {
     /** Number of paired (candidate, baseline) holdout observations used. */
     productiveRuns: number;
-    /** Median of (candidate − baseline) paired holdout deltas. */
-    medianPairedDelta: number;
-    /** Bootstrap CI on the median paired holdout delta. */
+    /** Candidate holdout rows with no baseline row at the same work identity. */
+    unpairedCandidateRuns: number;
+    /** Baseline holdout rows with no candidate row at the same work identity. */
+    unpairedBaselineRuns: number;
+    /** Median of paired holdout deltas, or null when there are no pairs. */
+    medianPairedDelta: number | null;
+    /** Bootstrap CI on the median paired holdout delta, if computed. */
     pairedCI: {
         low: number;
         high: number;
-    };
-    /** Wilcoxon signed-rank p-value on the paired holdout deltas. */
-    pairedPValue: number;
-    /** Mean candidate score on the search split (NaN if none). */
-    searchScore: number;
-    /** Mean candidate score on the holdout split (NaN if none). */
-    holdoutScore: number;
-    /** Candidate (search − holdout) gap. */
-    overfitGap: number;
-    /** Baseline (search − holdout) gap. */
-    baselineOverfitGap: number;
+    } | null;
+    /** Wilcoxon signed-rank p-value, if computed. */
+    pairedPValue: number | null;
+    /** Mean candidate score on the search split, or null when absent. */
+    searchScore: number | null;
+    /** Mean candidate score on the holdout split, or null when absent. */
+    holdoutScore: number | null;
+    /** Candidate (search − holdout) gap, or null when either side is absent. */
+    overfitGap: number | null;
+    /** Baseline (search − holdout) gap, or null when either side is absent. */
+    baselineOverfitGap: number | null;
     /** Median per-task USD cost across the candidate's runs. Recorded
      *  even when no `costPerTaskCeiling` is configured so downstream
      *  dashboards (intelligence.tangle.tools) can render \$/task per
      *  generation regardless of gating policy. */
-    medianCandidateCost: number;
+    medianCandidateCost: number | null;
     /** Median per-task USD cost across the baseline runs, for
      *  symmetric reporting. */
-    medianBaselineCost: number;
+    medianBaselineCost: number | null;
 }
 interface GateDecision$1 {
     /** Final promote/no-promote verdict. */
@@ -4888,10 +4886,9 @@ declare class HeldOutGate {
     private readonly seed?;
     private readonly costPerTaskCeiling?;
     constructor(config: HeldOutGateConfig);
-    /** Decide whether `candidate` should replace `baseline`. Pairing
-     *  is by (experimentId, seed) — identical experiment + seed pairs
-     *  the candidate run with the matching baseline run. Pairs without
-     *  a holdout score on both sides are dropped. */
+    /** Decide whether `candidate` should replace `baseline`.
+     *  Pairing is by `(experimentId, scenarioId, seed)`.
+     *  Missing or duplicate identities throw instead of comparing by position. */
     evaluate(candidate: RunRecord[], baseline: RunRecord[]): GateDecision$1;
 }
 
@@ -4929,7 +4926,7 @@ interface ActionableSideInfo {
     metadata?: Record<string, unknown>;
 }
 type ReleaseConfidenceStatus = 'pass' | 'warn' | 'fail';
-type ReleaseConfidenceAxisName = 'corpus' | 'quality' | 'generalization' | 'diagnostics' | 'efficiency';
+type ReleaseConfidenceAxisName = 'corpus' | 'quality' | 'reliability' | 'generalization' | 'diagnostics' | 'efficiency';
 interface ReleaseTraceEvidence {
     scenarioId: string;
     candidateId?: string;
@@ -4976,7 +4973,7 @@ interface ReleaseConfidenceInput {
 interface ReleaseConfidenceAxis {
     name: ReleaseConfidenceAxisName;
     status: ReleaseConfidenceStatus;
-    score: number;
+    score: number | null;
     detail: string;
 }
 interface ReleaseConfidenceIssue {
@@ -4987,15 +4984,25 @@ interface ReleaseConfidenceIssue {
 }
 interface ReleaseConfidenceMetrics {
     scenarioCount: number;
+    /** Search rows with a finite search score. */
     searchRuns: number;
+    /** Holdout rows with a finite holdout score. */
     holdoutRuns: number;
-    passRate: number;
-    meanScore: number;
-    searchMeanScore: number;
-    holdoutMeanScore: number;
-    overfitGap: number;
-    meanCostUsd: number;
-    p95WallMs: number;
+    /** Runs with neither a split-matched score nor an explicit task failure. */
+    unscoredRuns: number;
+    /** Run rows, or trace rows when no runs exist, with no classified terminal result. */
+    unclassifiedTerminalRuns: number;
+    /** Run rows, or trace rows when no runs exist, that ended unsuccessfully. */
+    terminalFailureRuns: number;
+    /** Success fraction when every run or fallback trace row has a classified result. */
+    reliabilityRate: number | null;
+    passRate: number | null;
+    meanScore: number | null;
+    searchMeanScore: number | null;
+    holdoutMeanScore: number | null;
+    overfitGap: number | null;
+    meanCostUsd: number | null;
+    p95WallMs: number | null;
     failedRows: number;
     failuresWithAsi: number;
     singleShotTraces: number;
@@ -5343,6 +5350,15 @@ declare function wilcoxonSignedRank(before: number[], after: number[]): {
  * Rule of thumb: |d| < 0.2 negligible, 0.2–0.5 small, 0.5–0.8 medium, > 0.8 large.
  */
 declare function cohensD(a: number[], b: number[]): number;
+/**
+ * Cohen's dz for paired observations: mean(after - before) divided by the
+ * sample standard deviation of those within-pair deltas.
+ *
+ * Returns null when fewer than two pairs exist or a non-zero constant delta
+ * has zero observed variance. In that case the standardized effect is
+ * undefined, not an arbitrarily large finite number.
+ */
+declare function pairedCohensDz(before: number[], after: number[]): number | null;
 type CliffsMagnitude = 'negligible' | 'small' | 'medium' | 'large';
 /**
  * Cliff's delta — a non-parametric effect size for two independent samples.
@@ -5486,6 +5502,16 @@ declare function requiredSampleSize(opts: {
     power?: number;
     twoSided?: boolean;
 }): number;
+/**
+ * Required number of paired observations for a target Cohen's dz.
+ * Unlike the independent-groups formula, this has no two-arm factor of two.
+ */
+declare function requiredPairedSampleSize(opts: {
+    effect: number;
+    alpha?: number;
+    power?: number;
+    twoSided?: boolean;
+}): number;
 /**
  * Minimum detectable paired effect (standardised units) for a target paired
  * sample size: d_min = (z_{1-α/2} + z_β) / sqrt(n_paired). Multiply by
@@ -5881,10 +5907,10 @@ interface PairedMetricDelta {
     n: number;
     /** Pairs where at least one side does not carry the metric. */
     nMissing: number;
-    /** Median paired delta; NaN when `n === 0` (no data ≠ measured zero). */
-    medianDelta: number;
-    /** Mean paired delta; NaN when `n === 0`. */
-    meanDelta: number;
+    /** Median paired delta, or null when `n === 0`. */
+    medianDelta: number | null;
+    /** Mean paired delta, or null when `n === 0`. */
+    meanDelta: number | null;
     /** Bootstrap CI on the paired delta (`pairedBootstrap`); null when
      *  `n === 0` — a zero-width [0, 0] interval on no data would read as a
      *  measured tight null. */
@@ -5926,6 +5952,25 @@ interface PairedArmsComparison {
  * absent" would misreport it as missing coverage.
  */
 declare function comparePairedArms(rows: readonly PairedArmRow[], opts: ComparePairedArmsOptions): PairedArmsComparison;
+interface MatchedRunRecordPair {
+    pairKey: string;
+    repKey: string;
+    baseline: RunRecord;
+    treatment: RunRecord;
+}
+interface PairRunRecordsResult {
+    pairs: MatchedRunRecordPair[];
+    unpairedBaseline: RunRecord[];
+    unpairedTreatment: RunRecord[];
+}
+/**
+ * Pair two RunRecord arms by the identity of the evaluated work:
+ * `(experimentId, scenarioId, seed)`.
+ *
+ * Falling back to array order, candidate id, or experiment id can compare
+ * different tasks and fabricate lift. Duplicate identities throw.
+ */
+declare function pairRunRecords(baselineRuns: readonly RunRecord[], treatmentRuns: readonly RunRecord[]): PairRunRecordsResult;
 
 type PrReviewSource = 'drew' | 'donovan' | 'shady' | 'codex' | 'claude-code' | 'gpt-5.5-high' | 'claude-opus-4.7-high' | 'kimi' | 'opencode' | (string & {});
 type PrReviewSeverity = 'critical' | 'high' | 'medium' | 'low' | 'nit';
@@ -6085,18 +6130,10 @@ declare function printDriverSummary(results: DriverResult[]): void;
  * flag: a gated line must never export as a positive training example.
  */
 declare const ROLLOUT_SCHEMA = "tangle.rollout.v1";
-/** @deprecated alias kept for consumers of the pre-unification constant name. */
-declare const ROLLOUT_FORMAT = "tangle.rollout.v1";
 /** `agent` = a solo evaluation run (no multi-agent topology). */
 type RolloutRole = 'agent' | 'supervisor' | 'worker' | 'proposer' | 'judge' | 'analyst';
-/**
- * Split vocabulary follows `RunRecord.splitTag` ('search' is the pool the
- * optimizer may read — the trainable split), extended with the ledger's
- * 'canary'. 'train' is a legacy alias for 'search' emitted by
- * pre-unification ledgers; it validates and counts as trainable, but new
- * producers must emit 'search'.
- */
-type RolloutSplit = 'search' | 'dev' | 'holdout' | 'canary' | 'train';
+/** Split vocabulary follows `RunRecord.splitTag`, extended with `canary`. */
+type RolloutSplit = 'search' | 'dev' | 'holdout' | 'canary';
 declare function isTrainableSplit(split: RolloutSplit): boolean;
 /** 'mint' = joined live from RunRecord + trace by `mintRolloutRows`. */
 type RolloutCapture = 'mint' | 'settle-time' | 'backfill';
@@ -6188,10 +6225,9 @@ interface RolloutOutcome {
     /**
      * Anti-Goodhart flag from `RunRecord.outcome.realness.gated`: the run
      * faked its success signal. Reward is forced to 0 at mint time and the
-     * line never qualifies for SFT. Optional on the wire (absent = false)
-     * so pre-unification ledgers stay readable.
+     * line never qualifies for SFT.
      */
-    realness_gated?: boolean;
+    realness_gated: boolean;
 }
 interface RolloutCostBlock {
     usd: number | null;
@@ -6220,11 +6256,10 @@ interface RolloutLine {
     /** Spawning invocation within the same episode (worker → supervisor). */
     parent_rollout_id: string | null;
     run_id: string;
-    /** Logical experiment grouping from `RunRecord.experimentId`. Optional on
-     *  the wire (pre-unification ledgers lack it); null = not recorded. */
-    experiment_id?: string | null;
+    /** Logical experiment grouping from `RunRecord.experimentId`; null = not recorded. */
+    experiment_id: string | null;
     /** Stable candidate identity from `RunRecord.candidateId`; null = not recorded. */
-    candidate_id?: string | null;
+    candidate_id: string | null;
     /** Improvement-loop generation (-1 = baseline); null = not an improvement loop. */
     generation: number | null;
     /** Improvement-loop candidate index (-1 = baseline); null = not an improvement loop. */
@@ -6259,10 +6294,13 @@ declare function isRolloutLine(value: unknown): value is RolloutLine;
  * labels, no joins.
  */
 
-interface SftExportOptions {
-    /** Export only lines with reward ≥ this (default 1 = clean successes only). */
-    minReward?: number;
+interface TrainingExportOptions {
+    /** Include held-out evaluation data in training output. Default false. */
+    allowHeldOutTrainingData?: boolean;
+    /** Require reward to be strictly greater than this value. Default 0. */
+    minimumQualityExclusive?: number;
 }
+type SftExportOptions = TrainingExportOptions;
 interface SftRow {
     messages: ChatMessage[];
     metadata: {
@@ -6276,7 +6314,7 @@ interface SftRow {
 /**
  * Supervised fine-tune rows: the completed conversation of each qualifying
  * line. Fail-closed filters: trainable split only (never holdout/canary),
- * reward ≥ minReward, realness-gated lines never qualify, gap lines carry
+ * positive reward, realness-gated lines never qualify, gap lines carry
  * no trainable content.
  */
 declare function toSftRows(lines: RolloutLine[], options?: SftExportOptions): SftRow[];
@@ -6294,13 +6332,9 @@ interface RewardRow {
     };
 }
 /**
- * Reward-labeled rows: every line with a scalar reward, success or
- * failure. Failures are signal here — only the realness-gate zeroing
- * (applied at mint time) touches the reward, never filtering. Lines with
- * no verdict (reward null) are excluded: an unlabeled example is a gap,
- * not a zero.
+ * Reward-labeled rows for completed, positive-quality training runs.
  */
-declare function toRewardRows(lines: RolloutLine[]): RewardRow[];
+declare function toRewardRows(lines: RolloutLine[], options?: TrainingExportOptions): RewardRow[];
 declare function toJsonl(rows: ReadonlyArray<unknown>): string;
 
 /**
@@ -6350,12 +6384,14 @@ declare function rolloutReward(record: RunRecord): {
 /**
  * Join RunRecords with their traces into canonical rollout lines. Records
  * without spans are emitted as labeled gap lines and reported in
- * `missingTraces` — a capture gap is a finding, not a silent omission.
+ * `missingTraces`. Execution-only records without a task score are rejected
+ * because a missing training label is not a zero reward.
  */
 declare function mintRolloutRows(records: RunRecord[], store: TraceStore, options?: MintRolloutOptions): Promise<MintRolloutResult>;
 
 interface RunEvidenceMetadata {
     experimentId: string;
+    scenarioId: string;
     candidateId: string;
     seed: number;
     model: string;
@@ -6364,6 +6400,7 @@ interface RunEvidenceMetadata {
     commitSha: string;
     splitTag: RunSplitTag;
     tokenUsage: RunTokenUsage;
+    costProvenance: RunRecord['costProvenance'];
     queueMs?: number;
     judgeMetadata?: RunRecord['judgeMetadata'];
     raw?: Record<string, number>;
@@ -7002,6 +7039,19 @@ declare function applyLlmSpanOtlpAttributes(attributes: Record<string, unknown>,
 /** Canonical OpenInference-over-OTLP attribute vocabulary used at the trace boundary. */
 
 type ToolSpanOtlpInput = Pick<ToolSpan, 'toolName' | 'args' | 'argsCaptured' | 'result' | 'latencyMs'>;
+type OtlpSpanRole = 'AGENT' | 'CHAIN' | 'EVALUATOR' | 'GUARDRAIL' | 'LLM' | 'SPAN' | 'TOOL' | 'UNKNOWN';
+interface OtlpSpanRoleInput {
+    name: string;
+    attributes: Record<string, unknown>;
+    kind?: string | null;
+}
+/**
+ * Classify a span once for both measurement and error accounting.
+ * An explicit OpenInference kind wins; untyped spans use the same tool and
+ * model signals in online and offline intake.
+ */
+declare function classifyOtlpSpanRole(input: OtlpSpanRoleInput): OtlpSpanRole;
+declare function isOtlpModelCall(input: OtlpSpanRoleInput): boolean;
 declare function applyToolSpanOtlpAttributes(attributes: Record<string, unknown>, span: ToolSpanOtlpInput): void;
 declare function traceSpanKindToOpenInferenceKind(kind: string): string;
 
@@ -7399,13 +7449,17 @@ declare function firstStringAttr(attrs: Record<string, unknown>, keys: readonly
  *     `opts.priceUsdPerToken` from the aggregated tokens; else 0 with a
  *     loud `raw.cost_unpriced = 1` marker so a missing price is visible, not
  *     a silent zero folded into a gate.
- *   - failureMode: the first `STATUS_CODE_ERROR` span's normalized status
- *     message (carries the real failure signature, not a generic class).
+ *   - terminalFailureReason: the failed root's normalized status message,
+ *     when one unambiguous root supplies terminal failure evidence.
+ *   - terminalOutcome: reduced from root-span status only. Child tool errors
+ *     remain visible in `error_span_count` and `execution_error_count` without
+ *     changing the run outcome. Root, guardrail, evaluator, propagated, and
+ *     unknown errors retain separate counters.
  *   - model: the dominant LLM model in the trace (snapshot-padded to satisfy
  *     `validateRunRecord` when the trace's model is a bare alias).
  *   - outcome score: `opts.scoreForTrace` (AppWorld `world.evaluate()` →
- *     TGC/SGC) when supplied; else 1 when the trace had no error span, 0
- *     when it did — a defensible default the caller can override.
+ *     TGC/SGC) when supplied. Traces without an external task-quality signal
+ *     remain unlabeled; execution errors never become a task score.
  *   - prompt / completion: carried into `raw` as token-count signals and,
  *     when the first/last LLM span exposes `input.value` / `output.value`,
  *     the verbatim text is preserved on the optional `promptText` /
@@ -7461,9 +7515,9 @@ interface OtlpToRunRecordsOptions {
      * Score for a produced run's outcome (AppWorld `world.evaluate()` →
      * TGC/SGC, or
      * any [0,1] task-success signal). Keyed by the logical run id when
-     * `logicalRunIdForTrace` is supplied, otherwise by `trace_id`; falls through to
-     * the error-derived default (1 = no error span, 0 = had one) when the map
-     * has no entry or the function returns undefined.
+     * `logicalRunIdForTrace` is supplied, otherwise by `trace_id`. When the map
+     * has no entry or the function returns undefined, the record remains
+     * unlabeled.
      */
     scoreForTrace?: (runId: string, span: TraceAggregate) => number | undefined;
     /**
@@ -7495,6 +7549,12 @@ interface TraceAggregate {
     toolSpanCount: number;
     agentSpanCount: number;
     errorSpanCount: number;
+    executionErrorCount: number;
+    processErrorCount: number;
+    guardrailErrorCount: number;
+    judgeErrorCount: number;
+    propagatedErrorCount: number;
+    unclassifiedErrorCount: number;
     tokenUsage: RunTokenUsage;
     /** First error span's normalized status message, if any. */
     firstErrorMessage?: string;
@@ -7502,6 +7562,8 @@ interface TraceAggregate {
     startTime: string;
     endTime: string;
     wallMs: number;
+    /** Root-span terminal result. Child span errors do not change this value. */
+    terminalOutcome: RunTerminalOutcome;
 }
 /**
  * Parse + aggregate an OTLP traces.jsonl string into validated
@@ -8425,14 +8487,17 @@ interface JudgeScore {
 }
 /** Five-valued verdict taxonomy (MOSS-paper alignment). */
 type GateDecision = 'ship' | 'hold' | 'need_more_work' | 'model_ceiling' | 'arch_ceiling';
+/** Outcome of one check that contributed to a release decision. */
+type GateCheckStatus = 'pass' | 'fail' | 'not_evaluated';
+interface GateContribution {
+    name: string;
+    status: GateCheckStatus;
+    detail: unknown;
+}
 interface GateResult {
     decision: GateDecision;
     reasons: string[];
-    contributingGates: Array<{
-        name: string;
-        passed: boolean;
-        detail: unknown;
-    }>;
+    contributingGates: GateContribution[];
     delta?: number;
 }
 /** Scoped trace writer handed to each dispatch — every span
@@ -8491,6 +8556,10 @@ interface CampaignCellResult<TArtifact> {
     durationMs: number;
     seed: number;
     cached: boolean;
+    /** Stage that produced `error`. Missing on successful cells. */
+    errorStage?: 'dispatch' | 'judge';
+    /** Judge that threw when `errorStage` is `judge`. */
+    errorJudge?: string;
     error?: string;
 }
 interface JudgeAggregate {
@@ -8515,8 +8584,10 @@ interface GenerationRecord {
  *  handled — the evidence a blind `Mutator` cannot see. */
 interface GenerationCandidate {
     surfaceHash: string;
-    composite: number;
-    ci95: [number, number];
+    /** Mean over complete task-quality scores, or null when none were produced. */
+    composite: number | null;
+    /** Descriptive interval for `composite`, or null when no score exists. */
+    ci95: [number, number] | null;
     /** Exact surface this candidate mutated. */
     parentSurfaceHash?: string;
     /** Measured search-split composite of the exact parent surface. */
@@ -8526,13 +8597,12 @@ interface GenerationCandidate {
     observedDeltaFromParent?: number;
     /** Whether this candidate had a scorable result for every designed campaign
      *  cell and was therefore eligible for ranking, promotion, and Pareto
-     *  selection. Older externally-authored records may omit this field; loop
-     *  records always populate it. */
-    eligibleForPromotion?: boolean;
+     *  selection. */
+    eligibleForPromotion: boolean;
     /** Exact denominator receipt for selection eligibility. Scores stay
      *  descriptive: an incomplete candidate is retained with its observed score
      *  and errors instead of receiving an invented penalty. */
-    coverage?: {
+    coverage: {
         expectedCells: number;
         scorableCells: number;
         unscorableCells: Array<{
@@ -8573,10 +8643,18 @@ interface CampaignAggregates {
     cost: CostLedgerSummary;
     /** Compatibility alias of `cost.totalCostUsd`. */
     totalCostUsd: number;
+    /** Cells whose dispatch completed, including cells whose later judge failed. */
     cellsExecuted: number;
     cellsSkipped: number;
     cellsCached: number;
+    /** All non-skipped dispatch, judge, and unclassified cell failures. */
     cellsFailed: number;
+    /** Present on results that record failure stages. */
+    cellsDispatchFailed?: number;
+    /** Present on results that record failure stages. */
+    cellsJudgeFailed?: number;
+    /** Legacy failures whose stage was not recorded. */
+    cellsUnclassifiedFailed?: number;
 }
 interface CampaignResult<TArtifact = unknown, TScenario extends Scenario = Scenario> {
     /** sha256(scenarios, judges, dispatch source ref, optimizer config, seed). Stable identity for reruns. */
@@ -8698,7 +8776,7 @@ interface FailureClusterReport {
  */
 
 interface SummaryTableOptions {
-    /** Comparator candidate id. Wilcoxon + Cohen's d are computed
+    /** Comparator candidate id. Wilcoxon + paired Cohen's dz are computed
      *  versus this candidate. Required for paired stats columns. */
     comparator?: string;
     /** Which split to read scores from. Default 'holdout'. */
@@ -8714,10 +8792,16 @@ interface SummaryTableRow {
     mean: number;
     ciLow: number;
     ciHigh: number;
-    /** BH-adjusted q-value vs comparator. NaN if no comparator. */
-    qValue: number;
-    /** Cohen's d vs comparator. NaN if no comparator. */
-    cohensD: number;
+    /** BH-adjusted q-value vs comparator, or null when unavailable. */
+    qValue: number | null;
+    /** Paired Cohen's dz vs comparator, or null when the paired variance is zero. */
+    cohensD: number | null;
+    /** Matched observations used for paired comparison, or null on the comparator row. */
+    pairedN: number | null;
+    /** Candidate observations without a comparator match. */
+    unpairedCandidateN: number | null;
+    /** Comparator observations without a candidate match. */
+    unpairedComparatorN: number | null;
 }
 interface SummaryTable {
     rows: SummaryTableRow[];
@@ -8729,7 +8813,7 @@ interface SummaryTable {
 /**
  * Table 1 helper. Buckets runs by `candidateId`, computes mean +
  * bootstrap CI on the chosen split, and (when a comparator is given)
- * BH-adjusted Wilcoxon p + Cohen's d versus that comparator.
+ * BH-adjusted Wilcoxon p + paired Cohen's dz versus that comparator.
  */
 declare function summaryTable(runs: RunRecord[], opts?: SummaryTableOptions): SummaryTable;
 interface ParetoPoint {
@@ -8745,7 +8829,7 @@ interface ParetoPoint {
     onFrontier: boolean;
     /** Optional gate verdict for this candidate, if a `GateDecision`
      *  for it was passed in. */
-    gate?: 'promote' | 'reject_few_runs' | 'reject_negative_delta' | 'reject_overfit_gap' | null;
+    gate?: 'promote' | 'reject';
 }
 interface ParetoFigureSpec {
     kind: 'pareto-cost-quality';
@@ -8780,12 +8864,16 @@ interface GainDistributionFigureSpec {
     split: 'search' | 'holdout';
     /** Number of pairs used. */
     n: number;
+    /** Candidate rows without a comparator match. */
+    unpairedCandidateN: number;
+    /** Comparator rows without a candidate match. */
+    unpairedComparatorN: number;
     bins: GainDistributionBin[];
-    median: number;
+    median: number | null;
     ci: {
         low: number;
         high: number;
-    };
+    } | null;
 }
 interface GainDistributionOptions {
     /** Number of histogram bins. Default 11 (so the centre is exact at 0). */
@@ -8882,8 +8970,8 @@ interface ResearchReportCandidate {
     mean: number;
     ciLow: number;
     ciHigh: number;
-    qValue: number;
-    cohensD: number;
+    qValue: number | null;
+    cohensD: number | null;
     meanDeltaVsComparator: number | null;
     pairedN: number;
     medianGain: number | null;
@@ -8893,10 +8981,9 @@ interface ResearchReportCandidate {
         high: number;
     } | null;
     /**
-     * Bayesian-bootstrap-style posterior summaries on the paired delta. Computed
-     * from the same resamples that produce the gain CI; interpretable as
-     * "fraction of resamples in which the candidate beats the comparator on
-     * matched pairs."
+     * Bayesian-bootstrap posterior summaries on the paired mean delta.
+     * Dirichlet(1, ..., 1) weights represent uncertainty over the empirical
+     * distribution of matched deltas.
      */
     prGreaterThanZero: number | null;
     prInRope: number | null;
@@ -8964,7 +9051,7 @@ interface ResearchReport {
  *   - `summaryTable`         marginal stats with BH-FDR-adjusted q-values
  *   - `paretoChart`           cost-vs-quality frontier with gate overlay
  *   - `gainHistogram`         per-candidate paired-delta distribution
- *   - paired posterior (this file): bootstrap CI on median, Pr(Δ>0),
+ *   - paired posterior (this file): bootstrap CI on median, Bayesian-bootstrap Pr(Δ>0),
  *                              Pr(Δ∈ROPE), MDE at the configured power
  *
  * Decisions are made on paired evidence — never on marginal means alone —
@@ -9008,7 +9095,8 @@ interface AnalyzeRunsOptions {
     split?: 'search' | 'holdout' | 'auto';
     /** Pairwise analysis configuration. When both `baselineCandidateId` and
      *  `candidateCandidateId` are present, lift is computed on paired
-     *  (experimentId, seed) tuples shared between the two sides. */
+     *  (experimentId, scenarioId, seed) identities shared between the two sides.
+     *  Unmatched rows remain visible in the lift result. */
     baselineCandidateId?: string;
     candidateCandidateId?: string;
     /** Canary scenarios — checked against every run's raw output for
@@ -9043,7 +9131,7 @@ interface AnalyzeRunsOptions {
      *  recommendations fire on statistically significant regressions.
      *  The two windows do NOT have to share scenarios — the comparison
      *  is two-sample unpaired (the substrate's `lift` field uses paired
-     *  bootstrap on shared (experimentId, seed) tuples; this is the
+     *  bootstrap on shared (experimentId, scenarioId, seed) identities; this is the
      *  shape for "this week vs last week" rather than "candidate vs
      *  baseline within a campaign"). */
     baselineRuns?: RunRecord[];
@@ -11242,9 +11330,9 @@ interface CostSummary {
  */
 
 /** The score the query/compare layer ranks on: holdout when present (the
- *  gated number), else search. Throws when a record carries neither — a
- *  RunRecord is invalid without at least one, but a hand-built object might. */
-declare function runScore(record: RunRecord): number;
+ *  gated number), else search. Execution-only records are valid RunRecords,
+ *  but cannot participate in score-ranked queries. */
+declare function runScore(record: RunRecord): number | undefined;
 interface RunRecordFilter {
     experimentId?: string;
     candidateId?: string;
@@ -13513,6 +13601,8 @@ interface Finding {
 interface LayerResult {
     layer: string;
     status: LayerStatus;
+    /** Origin of an `error` or `timeout`. Defaults to `execution`. */
+    errorSource?: 'execution' | 'judge';
     /** 0..1 score, optional — layers that don't produce a numeric score omit. */
     score?: number;
     durationMs: number;
@@ -13541,6 +13631,8 @@ interface VerifyContext<Env = unknown> {
 }
 interface Layer<Env = unknown> {
     name: string;
+    /** Origin assigned when this layer errors or times out. Defaults to `execution`. */
+    errorSource?: 'execution' | 'judge';
     /** Stages that must have `status: 'pass'` before this layer runs. */
     dependsOn?: string[];
     /**
@@ -13568,22 +13660,29 @@ interface VerifyOptions<Env = unknown> {
     /** Called with each layer result as it completes. */
     onLayer?: (result: LayerResult) => void;
 }
-/** Extends the substrate verdict spine: `valid` = `allPass` and `score` =
- *  `blendedScore` — derived where the report is aggregated, so spine
- *  consumers (drivers, gates) read this report without an adapter. */
+/** Extends the substrate verdict spine: `valid` = `allPass`; `score` is the
+ * complete task score or 0 when the configured scoring panel was incomplete. */
 interface VerificationReport extends DefaultVerdict {
     layers: LayerResult[];
     passCount: number;
     failCount: number;
     skippedCount: number;
     errorCount: number;
-    /** True iff at least one scored layer ran AND every scored layer passed. */
+    /** True iff the configured scoring panel completed and every layer passed. */
     allPass: boolean;
     /**
-     * Weighted mean of `score` across contributing layers. 0 when no layers
-     * contributed. See {@link Layer.failContributesToScore} for fail semantics.
+     * Diagnostic weighted mean across contributing layers. This may represent a
+     * partial panel. It is 0 when no layer contributed.
      */
     blendedScore: number;
+    /**
+     * Complete task-quality measurement.
+     * Present when at least one layer produced a valid score, every other layer
+     * completed successfully or contributed an explicit scored failure, and no
+     * result is missing because of a failure, skip, error, or timeout.
+     * Use this field, not `blendedScore`, when creating task labels.
+     */
+    taskScore?: number;
     durationMs: number;
     startedAt: string;
     finishedAt: string;
@@ -14825,6 +14924,115 @@ interface CampaignStorage {
     append?(path: string, content: string, expectedBytes: number): number | undefined;
 }
 
+/**
+ * Liveness canaries — cheap statistical checks that catch the failure
+ * modes a green test suite never sees.
+ *
+ * Three canary types in this module:
+ *
+ *   1. **Silent judge fallback** — the judge degraded to a fallback
+ *      path (rules-only / cached / heuristic) without anyone
+ *      noticing. Signature: a string of consecutive runs whose
+ *      `judgeMetadata.confidence` equals a known fallback constant
+ *      (default 0.30) OR whose `judgeMetadata.fallback` is true.
+ *
+ *   2. **Judge calibration drift** — the judge's confidence
+ *      distribution has drifted from a historical window. Two-sample
+ *      Kolmogorov-Smirnov test on the recent vs historical confidences,
+ *      with the empirical-CDF max-difference statistic.
+ *
+ *   3. **Eval-set distribution shift** — the mix of categories /
+ *      buckets in the recent runs differs significantly from the
+ *      historical mix. Chi-square test on the binned counts.
+ *
+ * Outputs are alerts. The canary does NOT fail loud the way a test
+ * does — failing tests are reserved for hard correctness violations.
+ * A canary that fires is a *signal* to investigate, not a verdict.
+ *
+ * Why this lives here rather than in `observability.ts`: that module
+ * exports already, and is a pure-fanout-to-Langfuse/Prometheus
+ * adapter. Canaries are statistical detectors, not adapters.
+ */
+
+type CanaryKind = 'silent_judge_fallback' | 'judge_calibration_drift' | 'distribution_shift';
+type CanarySeverity = 'info' | 'warn' | 'error';
+interface CanaryAlert {
+    kind: CanaryKind;
+    severity: CanarySeverity;
+    message: string;
+    /** Numbers that informed the decision — drop straight into a
+     *  dashboard / paper figure. */
+    evidence: Record<string, unknown>;
+}
+interface CanaryReport {
+    alerts: CanaryAlert[];
+    /** Per-kind summary count. */
+    counts: Record<CanaryKind, number>;
+    /** Whether each enabled detector had enough observations to run. */
+    evaluations: CanaryEvaluation[];
+}
+interface CanaryEvaluation {
+    kind: CanaryKind;
+    status: 'evaluated' | 'not_evaluated';
+    observations: number;
+    reason?: string;
+}
+interface CanaryOptions {
+    /**
+     * Silent-fallback detection.
+     * - `constant`: confidence value treated as the fallback signal.
+     *   Default 0.30 (matches the soft-fail default in
+     *   `propose-review.ts`).
+     * - `consecutiveThreshold`: trip the alert after this many
+     *   consecutive runs at `constant` (or `fallback === true`).
+     *   Default 3.
+     */
+    silentFallback?: {
+        constant?: number;
+        consecutiveThreshold?: number;
+        /** Floating-point tolerance when comparing against `constant`. */
+        epsilon?: number;
+    };
+    /**
+     * Calibration-drift detection.
+     * - `historyWindow`: number of past runs (oldest-first) treated as
+     *   the historical baseline. Default 50.
+     * - `recentWindow`: number of recent runs (newest-first) compared
+     *   against history. Default 20.
+     * - `ksAlpha`: alpha for the KS statistic vs critical value.
+     *   Default 0.05.
+     * - `minRecent`: minimum recent runs required to even attempt the
+     *   check. Default 10.
+     */
+    calibrationDrift?: {
+        historyWindow?: number;
+        recentWindow?: number;
+        ksAlpha?: number;
+        minRecent?: number;
+    };
+    /**
+     * Distribution-shift detection.
+     * - `category`: function that maps a run to a categorical bucket.
+     *   Required to enable this canary; if omitted the chi-square check
+     *   is skipped entirely.
+     * - `chiSquareAlpha`: alpha. Default 0.05.
+     * - `historyWindow`, `recentWindow`, `minRecent`: like above.
+     */
+    distributionShift?: {
+        category: (run: RunRecord) => string | null;
+        chiSquareAlpha?: number;
+        historyWindow?: number;
+        recentWindow?: number;
+        minRecent?: number;
+    };
+}
+/**
+ * Run all configured canaries against a chronological run list.
+ * Runs MUST be sorted oldest-to-newest by the caller — the order of
+ * the input is used to define "recent" vs "historical" windows.
+ */
+declare function runCanaries(runs: RunRecord[], opts?: CanaryOptions): CanaryReport;
+
 interface BenchmarkRunOptions<TPayload = unknown, TArtifact = string> {
     adapter: BenchmarkAdapter<BenchmarkDatasetItem<TPayload>, TPayload, TArtifact>;
     respond: BenchmarkResponder<TPayload, TArtifact>;
@@ -15046,107 +15254,6 @@ declare namespace index$1 {
   export { index$1_BENCHMARK_SPLIT_SEED as BENCHMARK_SPLIT_SEED, type index$1_BenchmarkAdapter as BenchmarkAdapter, type index$1_BenchmarkDatasetItem as BenchmarkDatasetItem, type index$1_BenchmarkDistribution as BenchmarkDistribution, type index$1_BenchmarkEvaluation as BenchmarkEvaluation, type index$1_BenchmarkFamily as BenchmarkFamily, type index$1_BenchmarkMetricCalibrationOptions as BenchmarkMetricCalibrationOptions, type index$1_BenchmarkMetricCalibrationResult as BenchmarkMetricCalibrationResult, type index$1_BenchmarkReport as BenchmarkReport, type index$1_BenchmarkResponder as BenchmarkResponder, type index$1_BenchmarkRunOptions as BenchmarkRunOptions, type index$1_BenchmarkRunResult as BenchmarkRunResult, type index$1_BenchmarkScenario as BenchmarkScenario, type index$1_BenchmarkSliceSummary as BenchmarkSliceSummary, type index$1_BenchmarkSource as BenchmarkSource, type index$1_BenchmarkTaskKind as BenchmarkTaskKind, type index$1_BuildStandardRetrievalItemsOptions as BuildStandardRetrievalItemsOptions, type index$1_RetrievalIdAdapterOptions as RetrievalIdAdapterOptions, type index$1_StandardRetrievalArtifact as StandardRetrievalArtifact, type index$1_StandardRetrievalDocument as StandardRetrievalDocument, type index$1_StandardRetrievalEvaluationOptions as StandardRetrievalEvaluationOptions, type index$1_StandardRetrievalPayload as StandardRetrievalPayload, type index$1_StandardRetrievalQrel as StandardRetrievalQrel, type index$1_StandardRetrievalQuery as StandardRetrievalQuery, type index$1_StandardRetrievalResult as StandardRetrievalResult, index$1_buildStandardRetrievalItems as buildStandardRetrievalItems, index$1_calibrateBenchmarkMetric as calibrateBenchmarkMetric, index$1_createRetrievalIdBenchmarkAdapter as createRetrievalIdBenchmarkAdapter, index$1_deterministicSplit as deterministicSplit, index$1_evaluateStandardRetrieval as evaluateStandardRetrieval, index$1_normalizeRetrievedDocumentIds as normalizeRetrievedDocumentIds, index$1_parseBeirCorpusJsonl as parseBeirCorpusJsonl, index$1_parseBeirQueriesJsonl as parseBeirQueriesJsonl, index$1_parseJsonlRows as parseJsonlRows, index$1_parseQrels as parseQrels, index$1_parseTsvRows as parseTsvRows, index$1_renderBenchmarkReportMarkdown as renderBenchmarkReportMarkdown, index$1_retrievalMetricsAtCutoff as retrievalMetricsAtCutoff, index$2 as routing, index$1_runBenchmarkAdapter as runBenchmarkAdapter, index$1_summarizeBenchmarkCampaign as summarizeBenchmarkCampaign };
 }
 
-/**
- * Liveness canaries — cheap statistical checks that catch the failure
- * modes a green test suite never sees.
- *
- * Three canary types in this module:
- *
- *   1. **Silent judge fallback** — the judge degraded to a fallback
- *      path (rules-only / cached / heuristic) without anyone
- *      noticing. Signature: a string of consecutive runs whose
- *      `judgeMetadata.confidence` equals a known fallback constant
- *      (default 0.30) OR whose `judgeMetadata.fallback` is true.
- *
- *   2. **Judge calibration drift** — the judge's confidence
- *      distribution has drifted from a historical window. Two-sample
- *      Kolmogorov-Smirnov test on the recent vs historical confidences,
- *      with the empirical-CDF max-difference statistic.
- *
- *   3. **Eval-set distribution shift** — the mix of categories /
- *      buckets in the recent runs differs significantly from the
- *      historical mix. Chi-square test on the binned counts.
- *
- * Outputs are alerts. The canary does NOT fail loud the way a test
- * does — failing tests are reserved for hard correctness violations.
- * A canary that fires is a *signal* to investigate, not a verdict.
- *
- * Why this lives here rather than in `observability.ts`: that module
- * exports already, and is a pure-fanout-to-Langfuse/Prometheus
- * adapter. Canaries are statistical detectors, not adapters.
- */
-
-type CanaryKind = 'silent_judge_fallback' | 'judge_calibration_drift' | 'distribution_shift';
-type CanarySeverity = 'info' | 'warn' | 'error';
-interface CanaryAlert {
-    kind: CanaryKind;
-    severity: CanarySeverity;
-    message: string;
-    /** Numbers that informed the decision — drop straight into a
-     *  dashboard / paper figure. */
-    evidence: Record<string, unknown>;
-}
-interface CanaryReport {
-    alerts: CanaryAlert[];
-    /** Per-kind summary count. */
-    counts: Record<CanaryKind, number>;
-}
-interface CanaryOptions {
-    /**
-     * Silent-fallback detection.
-     * - `constant`: confidence value treated as the fallback signal.
-     *   Default 0.30 (matches the soft-fail default in
-     *   `propose-review.ts`).
-     * - `consecutiveThreshold`: trip the alert after this many
-     *   consecutive runs at `constant` (or `fallback === true`).
-     *   Default 3.
-     */
-    silentFallback?: {
-        constant?: number;
-        consecutiveThreshold?: number;
-        /** Floating-point tolerance when comparing against `constant`. */
-        epsilon?: number;
-    };
-    /**
-     * Calibration-drift detection.
-     * - `historyWindow`: number of past runs (oldest-first) treated as
-     *   the historical baseline. Default 50.
-     * - `recentWindow`: number of recent runs (newest-first) compared
-     *   against history. Default 20.
-     * - `ksAlpha`: alpha for the KS statistic vs critical value.
-     *   Default 0.05.
-     * - `minRecent`: minimum recent runs required to even attempt the
-     *   check. Default 10.
-     */
-    calibrationDrift?: {
-        historyWindow?: number;
-        recentWindow?: number;
-        ksAlpha?: number;
-        minRecent?: number;
-    };
-    /**
-     * Distribution-shift detection.
-     * - `category`: function that maps a run to a categorical bucket.
-     *   Required to enable this canary; if omitted the chi-square check
-     *   is skipped entirely.
-     * - `chiSquareAlpha`: alpha. Default 0.05.
-     * - `historyWindow`, `recentWindow`, `minRecent`: like above.
-     */
-    distributionShift?: {
-        category: (run: RunRecord) => string | null;
-        chiSquareAlpha?: number;
-        historyWindow?: number;
-        recentWindow?: number;
-        minRecent?: number;
-    };
-}
-/**
- * Run all configured canaries against a chronological run list.
- * Runs MUST be sorted oldest-to-newest by the caller — the order of
- * the input is used to define "recent" vs "historical" windows.
- */
-declare function runCanaries(runs: RunRecord[], opts?: CanaryOptions): CanaryReport;
-
 /**
  * concurrency — small primitives the evolution loop needs.
  *
@@ -15410,8 +15517,10 @@ interface CampaignRunOutcome {
     pass: boolean;
     /** Score for the run on its split. Maps to `searchScore` or `holdoutScore`. */
     score: number;
-    /** Mandatory cost in USD. Use 0 + raw.cost_unknown=1 only if truly unknown. */
-    costUsd: number;
+    /** Cost in USD, or null when the runner could not capture it. */
+    costUsd: number | null;
+    /** Source of the cost amount. */
+    costProvenance: RunCostProvenance;
     tokenUsage: RunTokenUsage;
     /** Snapshot model id (e.g. `claude-sonnet-4-6@2025-04-15`). */
     model: string;
@@ -17065,4 +17174,4 @@ type CachedJudge<TArtifact, TScenario extends Scenario = Scenario> = JudgeConfig
  */
 declare function cachedJudge<TArtifact, TScenario extends Scenario = Scenario>(judge: JudgeConfig<TArtifact, TScenario>, store: VerdictCacheStore, options: CachedJudgeOptions): CachedJudge<TArtifact, TScenario>;
 
-export { AGENT_PROFILE_KINDS, ATTESTATION_ALGORITHM, type ActionExecutionPolicy, type ActionPolicyDecision, type ActionableSideInfo, type ActiveLearningOptions, type AdapterRun, AgentDriver, type AgentDriverConfig, AgentEvalError, type AgentEvalErrorCode, type AgentInterfaceProfileLike, type AgentProfileCell, type AgentProfileCellInput, type AgentProfileCellSchemaVersion, AgentProfileCellValidationError, type AgentProfileDimensionValue, type AgentProfileHarness, type AgentProfileJson, type AgentProfileJsonObject, type AgentProfileKind, type AgentProfileRuntimeReceipt, type AgentProfileSource, type AgentProfileSourceInput, type AlignmentOp, type Analyst, type AnalystContext, type AnalystCost, type AnalystFinding, type AnalystHooks, type AnalystInputKind, AnalystRegistry, type AnalystRegistryOptions, type AnalystRequirements, type AnalystRunEvent, type AnalystRunInputs, type AnalystRunResult, type AnalystRunSummary, type AnalystSeverity, type AnalystUsageReceipt, type AnalyzeTracesInput, type AnalyzeTracesOptions, type AnalyzeTracesResult, type AnalyzeTracesTurnSnapshot, type AntiSlopConfig, type AntiSlopIssue, type AntiSlopReport, type Artifact$1 as Artifact, type ArtifactCheck, type Artifact as ArtifactCheckArtifact, type ArtifactEventLike, type ArtifactResult, type ArtifactValidator, type AsiSeverity, type AssertCapabilityHeadroomOptions, type AssertCrossFamilyOptions, type AssertSingleBackendOptions, type AttestationProvenance, type AttestationVerification, type AttestedReport, type AutoPrClient, AxGepaSteeringOptimizer, type AxSteeringOptimizerConfig, BENCHMARK_SPLIT_SEED, type BackendDescriptor, BackendIntegrityError, type BackendIntegrityReport, type BaselineOptions, type BaselineReport, BehaviorAssertion, type BehavioralMetrics, type BehavioralTokenSequence, type BenchmarkAdapter, type BenchmarkDatasetItem, type BenchmarkEvaluation, type BenchmarkFamily, type BenchmarkReport$1 as BenchmarkReport, type BenchmarkResponder, BenchmarkRunner, type BenchmarkRunnerConfig, type BenchmarkScenario, type BenchmarkSource, type BenchmarkTaskKind, type BisectOptions, type BisectResult, type BisectStep, type BlendWeights, type BootstrapOptions, type BootstrapResult, BudgetBreachError, BudgetGuard, type BudgetLedgerEntry, type BudgetPolicy, type BudgetSpec, CODING_HARNESSES, type CachedJudge, type CachedJudgeOptions, type CalibrationResult, CallExpectation, CallbackResearcher, type CallbackResearcherOptions, type CampaignFactoryParams, type CampaignIntegrityPolicy, type CampaignRunContext, type CampaignRunOutcome, type CampaignRunner, type CampaignScenario, type CampaignVariant, type CanaryAlert, type CanaryKind, type CanaryLeak, type CanaryOptions, type CanaryReport, type CanarySeverity, type CandidateComparison, type CandidateScenario, type CandidateScore, type CanonicalRawAnalystFinding, type CapabilityHeadroomOptions, type CapabilityHeadroomResult, type CaptureFetchContext, type CaptureFetchOptions, CaptureIntegrityError, type CausalAttributionReport, type CellVerdict, type ChannelRollup, type ChatCallOpts, type ChatClient, type ChatMessage, type ChatRequest, type ChatResponse, type ChatToolCall, type ChatTransport, type CheckResult, type CliBridgeTransportOpts, type CliffsMagnitude, type ClusterBootstrapInterval, type ClusterSignFlipAlternative, type ClusterSignFlipResult, type ClusteredBinaryCluster, type ClusteredMatchedPair, type ClusteredPairedBinaryOptions, type ClusteredPairedBinaryResult, type ClusteredPairedBinaryStatistics, type CollectedArtifacts, type CommandRunner, type ComparePairedArmsOptions, type CompletionCriterion, type CompletionRequirement, type CompletionVerdict, type ConceptComplexity, type ConceptFinding, type ConceptSpec, type ConceptWeightStrategy, ConfigError, type ContinuityCheck, type ContinuityCheckResult, type ContinuityReport, type ContinuitySnapshotPair, type ContinuousAgreement, type ContinuousAgreementOptions, type ContinuousCalibrationResult, type ContractCheckResult, type ContractJudgeOptions, type ContractMetric, type ContractReport, type ContractRule, type ContractRuleKind, type ContractSpan, type ContractVerdict, type ContractViolation, type ControlActionFailureMode, type ControlActionOutcome, type ControlBudget, type ControlContext, type ControlDecision, type ControlEvalResult, type ControlRunResult, type ControlRunToRunRecordOptions, type ControlRuntimeConfig, type ControlRuntimeError, type ControlSeverity, type ControlStep, type ControlStopPolicies, ConvergenceTracker, type CorpusAgreementOptions, type CorpusAgreementPerDimension, type CorpusAgreementReport, type CorpusScoreRecord, type CorrectnessChecker, CostAccountingIncompleteError, CostCallConflictError, CostCeilingReachedError, type CostChannel, type CostEntry, CostLedger, type CostLedgerEntry, type CostLedgerFilter, type CostLedgerHandle, type CostLedgerOptions, type CostLedgerPersistence, CostLedgerPersistenceError, type CostLedgerSummary, type CostReceipt, CostReceiptCaptureError, type CostReceiptInput, type CostReport, CostReservationExceededError, type CostResult, type CostSummary, CostTracker, type CostUsage, type CounterfactualContext, type CounterfactualMutation, type CounterfactualResult, type CounterfactualRunner, type CreateAnalystAiConfig, type CreateChatClientOpts, type CreateDefaultReviewerOptions, type CreateExperimentInput, type CreateSandboxPoolOpts, type CreateTraceAnalystKindOpts, CrossFamilyError, type CrossTraceDiff, type CrossTraceDiffOptions, type CustomTokenPricing, DEFAULT_AGENT_SLOS, DEFAULT_COMPLEXITY_WEIGHTS, DEFAULT_RULES as DEFAULT_FAILURE_RULES, DEFAULT_FINDERS, DEFAULT_HARNESS_OBJECTIVES, DEFAULT_MUTATION_PRIMITIVES, DEFAULT_MUTATORS, DEFAULT_PR_REVIEW_SCORE_WEIGHTS, DEFAULT_REDACTION_RULES, DEFAULT_RED_TEAM_CORPUS, DEFAULT_RUN_SCORE_WEIGHTS, DEFAULT_SEVERITY_WEIGHTS, DEFAULT_TRACE_ANALYST_BUDGETS, DEFAULT_TRACE_ANALYST_KINDS, type DataAcquisitionPlan, Dataset, type DatasetDifficulty, type DatasetManifest, type DatasetOverview, type DatasetProvenance, type DatasetScenario, type DatasetSplit, type DecideNextUserTurnOpts, type DefaultAnalystRegistryOptions, type DefaultVerdict, type DeployFamily, type DeployGateLayerInput, type DeployRunResult, type DeployRunner, type DescriptionLengthCandidate, type DescriptionLengthConfig, type DescriptionLengthDecision, type DescriptionLengthEvidence, DescriptionLengthGate, type DescriptionLengthRejectionCode, type DetectorEvent, type DetectorSeverity, type DetectorSignal, type DiffPolicy, type DiffScorecardOptions, type DirEntry, type DirectProviderTransportOpts, type Direction, type DiscoverPersonasOptions, type DiscoveredPersona, DockerSandboxDriver, type DriverResult, type DriverState, DualAgentBench, type DualAgentBenchConfig, type DualAgentReport, type DualAgentRound, type DualAgentScenario, type DualAgentScenarioResult, type EProcess, type EProcessOptions, type EProcessState, type EProcessStep, ERROR_COUNT_PATTERNS, type EnsembleAggregate, type EnsembleJudgeOptions, type ErrorCluster, type ErrorCountPattern, type ErrorStreakOptions, type EvalCampaignOptions, type EvalCampaignResult, type EvalResult, type EvalToolDef, EvalTraceStore, type EventFilter, type EventKind, type EvidenceRef, type EvolutionRound, type ExecutorConfig, type Expectation, type Experiment, type ExperimentPlan, type ExperimentProvenance, type ExperimentRep, type ExperimentResult, type ExperimentStats, type ExperimentStore, ExperimentTracker, type ExperimentTrackerOptions, type ExperimentVerdict, type ExportableSpan, type ExportedRewardModel, type ExtractOptions, type ExtractResult, type ExtractUsageFromSseOptions, type ExtractedUsage, FAILURE_CLASSES, FAILURE_MODE_KIND_SPEC, type FactorContribution, type FactorialCell, type FailedRun, type FailureClass, type FailureClassification, type FailureContext, type FailureMode, type FailureRule, type FeedbackArtifactType, type FeedbackAttempt, type FeedbackLabel, type FeedbackLabelKind, type FeedbackLabelSource, type FeedbackOptimizerRow, type FeedbackOutcome, type FeedbackPattern, type FeedbackReplayAdapter, type FeedbackReplayResult, type FeedbackSeverity, type FeedbackSplitPolicy, type FeedbackTask, type FeedbackTrajectory, type FeedbackTrajectoryFilter, type FeedbackTrajectoryStore, type FieldDestination, type FileChange, FileSystemFeedbackTrajectoryStore, FileSystemRawProviderSink, type FileSystemRawProviderSinkOptions, FileSystemTraceStore, type FileSystemTraceStoreOptions, type Finding, type FindingSubject, type FindingSubjectKind, type FindingsDiff, FindingsStore, type FlattenOtlpOptions, type FlowAction, type FlowLayerEnv, type FlowLayerFactoryInput, type FlowRunner, type FlowRunnerStepResult, type FlowSpec, type FlowStep, type GainDistributionBin, type GainDistributionFigureSpec, type GainDistributionOptions, type GateDecision$1 as GateDecision, type GateEvidence, type GenericSpan, type GhCliClientOptions, type GoldenItem, type GoldenSeverity, type GoldenSpec, HARNESS_NATIVE_MODEL, type HarnessAdapter, type HarnessConfig, type HarnessExperimentConfig, type HarnessExperimentResult, type HarnessIntervention, type HarnessRunRequest, type HarnessRunResult, type HarnessScenario, type HarnessSelection, type HarnessVariant, type HarnessVariantReport, type HeadroomClass, type HeadroomInput, HeldOutGate, type HeldOutGateConfig, type HeldOutGateRejectionCode, type HeldOutPartition, type HiddenCriteriaGrader, type HiddenGradeResult, type HiddenLeak, HoldoutAuditor, HoldoutLockedError, type HttpGithubClientOptions, type HypothesisManifest, type HypothesisResult, IMPROVEMENT_KIND_SPEC, INPUT_VALUE, INTENT_MATCH_JUDGE_VERSION, type ImageData, type ImprovementThresholds, type ImprovementVerdictResult, InMemoryFeedbackTrajectoryStore, InMemoryRawProviderSink, type InMemoryRawProviderSinkOptions, InMemoryTraceStore, InMemoryWorkspaceInspector, type InferenceScorer, type InspectorContext, type IntentMatchInput, type IntentMatchOptions, type IntentMatchResult, type InteractionContribution, type InterimReleaseConfidence, type InterimReleaseConfidenceInput, type JudgeConfig$1 as JudgeConfig, JudgeError, type JudgeFamily, type JudgeFleetOptions, type JudgeFn, type JudgeInput, JudgeParseError, type JudgeReplayGateArgs, type JudgeReplayResult, type JudgeRetryOutcome, type JudgeRetryPolicy, type JudgeRubric, JudgeRunner, type JudgeScore$1 as JudgeScore, type JudgeScoreInput, type JudgeScoresRecord, type JudgeSpan, type JudgeVerdict, KNOWLEDGE_GAP_KIND_SPEC, KNOWLEDGE_POISONING_KIND_SPEC, type KeywordConceptSpec, type KeywordCoverageFinding, type KeywordCoverageOptions, type KeywordCoverageResult, type KnowledgeAcquisitionMode, type KnowledgeBundle, type KnowledgeFallbackPolicy, type KnowledgeFreshness, type KnowledgeImportance, type KnowledgeReadinessReport, type KnowledgeRecommendedAction, type KnowledgeRequirement, type KnowledgeRequirementCategory, type KnowledgeResponsibleSurface, type KnowledgeSensitivity, LLM_CACHED_TOKENS, LLM_CACHED_TOKEN_ATTR_KEYS, LLM_CACHE_WRITE_TOKENS, LLM_CACHE_WRITE_TOKEN_ATTR_KEYS, LLM_CONTEXT_TOKENS, LLM_COST_ATTR_KEYS, LLM_COST_USD, LLM_INPUT_TOKENS, LLM_INPUT_TOKEN_ATTR_KEYS, LLM_MODEL_ATTR_KEYS, LLM_MODEL_NAME, LLM_OUTPUT_TOKENS, LLM_OUTPUT_TOKEN_ATTR_KEYS, LLM_REASONING_TOKENS, LLM_REASONING_TOKEN_ATTR_KEYS, type LangfuseEnvelope, type LangfuseGeneration, type LangfuseScore, type Layer, type LayerResult, type LayerStatus, type LeaderboardOptions, type LeaderboardRow, type LiveProofArtifact, type LiveProofConfig, type LiveProofContext, type LiveProofResult, LlmCallError, type LlmCallMetadata, type LlmCallRequest, type LlmCallResult, LlmClient, type LlmClientOptions, type LlmCorrectnessCheckerOpts, type LlmJsonCall, type LlmJudgeDimension, type LlmJudgeOptions, type LlmMessage, LlmResponseError, type LlmReviewerConfig, LlmRouteAssertionError, type LlmRouteRequirements, type LlmSpan, type LlmSpanOtlpInput, type LlmUsage, LockedJsonlAppender, MODEL_PRICING, type MakeEvalToolsConfig, type MatchResult, type MatchedPair, type MatcherResult, type MaximumCharge, type McNemarResult, type Measured, type MeasurementPolicy, type MergeOptions, type Message, type MetricSamples, type MetricVerdict, MetricsCollector, type MintRolloutOptions, type MintRolloutResult, type MockTransportOpts, type ModelCostRollup, type ModelPreflight, type ModelSeats, ModelsUnreachableError, type MuffledFinder, type MuffledFinding, MultiLayerVerifier, type MultiToolchainLayerConfig, type Mutator, Mutex, type NoLeakOptions, type NoProgressOptions, NoopRawProviderSink, NoopResearcher, NotFoundError, OPENINFERENCE_SPAN_KIND, OTEL_AGENT_EVAL_SCOPE, OUTPUT_VALUE, type Objective, type Oracle, type OracleObservation, type OracleReport, type OracleResult, type OrthogonalityInput, type OrthogonalityResult, type OtelExportConfig, type OtelExporter, type OtelPipelineHandle, type OtelPipelineOptions, type OtlpExport, OtlpFileTraceStore, type OtlpFileTraceStoreOptions, type OtlpFlatLine, type OtlpResourceSpans, type OtlpSpan, type OtlpToRunRecordsOptions, type OtlpTraceRunRecord, type PaidCallResult, type PairArmsOptions, type PairArmsResult, type PairedArmRow, type PairedArmsComparison, type PairedBootstrapOptions, type PairedBootstrapResult, type PairedCorrectness, type PairedEvalueOptions, type PairedEvalueSequence, type PairedEvalueStep, type PairedMetricDelta, type PairedSignTestResult, PairwiseSteeringOptimizer, type ParaphraseRobustnessScenarioInput, type ParaphraseRobustnessScenarioResult, type ParetoFigureSpec, type ParetoPoint, type ParetoResult, type PartitionHeldOutOptions, type PendingCostCall, type PendingCostCallView, type PersistedFinding, type PersonaConfig, type PersonaRigor, type Playbook, type PlaybookEntry, type PoolSlot, type PositionalBiasResult, type PrReviewAuditCase, type PrReviewBenchmarkSummary, type PrReviewComment, type PrReviewMatchedFinding, type PrReviewOutcome, type PrReviewReferenceFinding, type PrReviewScore, type PrReviewScoreWeights, type PrReviewSeverity, type PrReviewSource, type PreferenceMemoryEntry, type PreflightModelsOptions, type PreflightOutcome, type ProducedProposal, type ProducedState, type ProductBenchmarkArm, type ProductBenchmarkArtifactPaths, type ProductBenchmarkBudgets, type ProductBenchmarkExportOptions, type ProductBenchmarkExportResult, type ProductBenchmarkManifest, type ProductBenchmarkProfileRef, type ProductBenchmarkRecord, type ProductBenchmarkRepoRef, type ProductBenchmarkRunInput, type ProductBenchmarkScenario, type ProductBenchmarkSingleRunExportOptions, type ProductBenchmarkSplit, type ProductBenchmarkSubstrateVersions, type ProductBenchmarkValidationReport, ProductClient, type ProductClientConfig, type ProfileAxisSpec, type ProjectRuntimeTrajectoryEvidenceOptions, type ProjectedOtlpSpan, type PromptHandle, PromptRegistry, type ProportionInterval, type ProposalEventLike, type ProposeAutomatedPullRequestInput, type ProposeAutomatedPullRequestResult, type ProposeFn, type ProposeInput, type ProposeOutput, type ProposeReviewConfig, type ProposeReviewControlAction, type ProposeReviewControlConfig, type ProposeReviewControlResult, type ProposeReviewControlState, type ProposeReviewReport, type ProposeReviewShot, type ProposedSideEffect, type ProvenanceReader, type ProviderRedactor, type QueryTracesPage, REDACTION_VERSION, REFERENCE_EQUIVALENCE_INPUT_LIMITS, REFERENCE_EQUIVALENCE_JUDGE_VERSION, RESEARCH_REPORT_HARD_PAIR_FLOOR, ROLLOUT_FORMAT, ROLLOUT_SCHEMA, RUN_COST_ATTR_KEYS, type RawAnalystEvidence, type RawAnalystFinding, type RawProviderDirection, type RawProviderEvent, type RawProviderSink, type RawProviderSinkFilter, type RecordRunsOptions, type RedTeamCase, type RedTeamCategory, type RedTeamFinding, type RedTeamPayload, type RedTeamReport, type RedactionReport, type RedactionRule, type ReferenceEquivalenceJudgeInput, type ReferenceEquivalenceJudgeOptions, type ReferenceEquivalenceJudgeResult, type ReferenceEquivalenceScenario, type ReferenceMatchResult, type ReferenceReplayAdapter, type ReferenceReplayAdapterFn, type ReferenceReplayAdapterLike, type ReferenceReplayAggregate, type ReferenceReplayCandidate, type ReferenceReplayCase, type ReferenceReplayCaseRun, type ReferenceReplayExecutionScenario, type ReferenceReplayItem, type ReferenceReplayMatch, type ReferenceReplayMatchStrategy, type ReferenceReplayMatcher, type ReferenceReplayPromotionDecision, type ReferenceReplayPromotionPolicy, type ReferenceReplayRun, type ReferenceReplayRunContext, type ReferenceReplayRunOptions, type ReferenceReplayRunStore, type ReferenceReplayScenario, type ReferenceReplayScenarioScore, type ReferenceReplayScore, type ReferenceReplayScoreOptions, type ReferenceReplaySplit, type ReferenceReplaySplitComparison, type ReferenceReplaySteeringRowsOptions, type ReflectionContext, type ReflectionProposal, type RegistryRunOpts, type ReleaseConfidenceAxis, type ReleaseConfidenceAxisName, type ReleaseConfidenceInput, type ReleaseConfidenceIssue, type ReleaseConfidenceMetrics, type ReleaseConfidenceScorecard, type ReleaseConfidenceStatus, type ReleaseConfidenceThresholds, type ReleaseTraceEvidence, type RenderReleaseReportOptions, type RepeatedActionOptions, ReplayCache, type ReplayCacheEntry, ReplayCacheMissError, type ReplayCacheStats, ReplayError, type ReplayFetchOptions, type RepoRef, type RequirementCheck, type ResearchReport, type ResearchReportCandidate, type ResearchReportDecision, type ResearchReportMethodology, type ResearchReportOptions, type ResearchReportRecommendation, type Researcher, type RetrievalSpan, type Review, type ReviewFn, type ReviewInput, type ReviewMemoryEntry, type ReviewMemoryStore, type ReviewerMemoryEntry, type ReviewerOutput, type ReviewerPromptInput, type ReviewerSoftFailDefaults, type ReviewerVerificationSummary, type RewardRow, type RiskDifferenceResult, type RobustnessResult, type RolloutCapture, type RolloutLine, type RolloutRole, type RolloutScrubber, type RolloutSplit, type RolloutStep, type RouteMap, type RoutedField, type RouterTransportOpts, type RubricDimension, type Run, type RunCommandInput, type RunCommandResult, type RunCompleteHook, type RunCompleteHookContext, type RunCostProvenance, RunCritic, type RunCriticOptions, type RunEvidenceMetadata, type RunFilter, RunIntegrityError, type RunIntegrityExpectations, type RunIntegrityIssue, type RunIntegrityIssueCode, type RunIntegrityReport, type RunJudgeMetadata, type RunLayer, type RunOutcome, type RunPaidCallInput, type RunRecord, type RunRecordBackend, type RunRecordFilter, RunRecordValidationError, type RunScore, type RunScoreWeights, type RunSplitTag, type RunStatus, type RunTokenUsage, type RunTrace, type RuntimeEventLike, type RuntimeResolution, type RuntimeTrajectoryEvidenceProjection, type RuntimeTrajectoryEvidenceSummary, type RuntimeTrajectoryHookEvent, type RuntimeTrajectoryRecord, type RuntimeTrajectoryRunRecord, SEMANTIC_CONCEPT_JUDGE_VERSION, SKILL_USAGE_ANALYST, SPAN_KIND_ATTR_KEYS, SUPERVISOR_RUN_SCHEMA, type SandboxDriver, SandboxHarness, type SandboxHarnessResult, type SandboxJudgeKind, type SandboxJudgeResult, type SandboxJudgeSpec, type SandboxPool, type SandboxResult, type SandboxSdkTransportOpts, type SandboxSpan, type SatisfiedBy, type ScanOptions, type Scenario$1 as Scenario, type ScenarioCost, type ScenarioFile, ScenarioRegistry, type ScenarioResult, type ScoreKnowledgeReadinessOptions, type Scorecard, type ScorecardCell, type ScorecardCellDiff, type ScorecardDiff, type ScorecardEntry, type ScorecardLogLine, type ScoredTarget, type SearchSpanResult, type SearchTraceResult, type SeatName, type SeatPresetName, SeatUnsetError, type SelfPlayOptions, type SelfPlayProposer, type SelfPlayScorer, type SelfPreferenceResult, type SemanticConceptJudgeInput, type SemanticConceptJudgeOptions, type SemanticConceptJudgeResult, type SequentialDecision, type SerializedRegex, type SeriesConvergenceOptions, type SeriesConvergenceResult, type Severity, type SftExportOptions, type SftRow, type SignTestAlternative, type SignedManifest, type SignedManifestAlgo, type SingleBackendDivergence, SingleBackendError, type SingleBackendField, type SingleBackendReport, SkillUsageAnalyst, type SliceOptions, type Slo, type SloCheckResult, type SloComparator, type SloReport, type SloSeverity, type SlopCategory, type SlotFactory, type SourceLimits, type Span, type SpanBase, type SpanFilter, type SpanHandle, type SpanKind, type SpanMatchRecord, SpanNotFoundError, type SpanPredicate, type SpanStatus, type SseUsageMode, type SteeringBundle, type SteeringChange, type SteeringDelta, type SteeringOptimizationResult, type SteeringOptimizationRow, type SteeringOptimizationSelector, type SteeringOptimizerBackend, type SteeringOptimizerConfig, type SteeringRolePrompt, type StepAttribution, type StopDecision, type StreamingDetector, type SuboptimalCode, type SuboptimalSignal, SubprocessSandboxDriver, type SubprocessSandboxDriverOptions, type SummaryTable, type SummaryTableOptions, type SummaryTableRow, type SupervisorRunReader, type SupervisorRunReport, type SupervisorRunRollup, type SupervisorRunSources, type SupervisorRunTree, type SynthesisReason, type SynthesisTarget, TOOL_ARGS_CAPTURED, TOOL_LATENCY_MS, TOOL_NAME, TOOL_NAME_ATTR_KEYS, TRACE_ANALYST_ACTOR_DESCRIPTION, TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION, TRACE_ANALYST_TRUNCATION_MARKER_PREFIX, TRACE_SCHEMA_VERSION, type TaskGold, type TaskHeadroom, type TestGradedRunOptions, type TestGradedRunResult, type TestGradedScenario, type TestOutputParser, type TestResult, type TextMatcher, type ThresholdContract, TokenCounter, type TokenSpec, type ToolCallEventLike, type ToolDef, type ToolMatcher, type ToolSpan, type ToolSpanOtlpInput, type ToolStats, type ToolUseMetrics, type ToolUseOptions, type TraceAggregate, type TraceAnalysisStore, type TraceAnalystByteBudgets, type TraceAnalystFilters, type TraceAnalystGolden, type TraceAnalystHookOptions, type TraceAnalystKindSpec, type TraceAnalystSpan, type TraceAnalystSpanKind, type TraceAnalystSpanStatus, type TraceAnalystTraceSummary, type TraceContract, TraceContractBuilder, TraceEmitter, type TraceEmitterOptions, type TraceEvent, TraceFileMissingError, type TraceInsightContext, type TraceInsightFinding, type TraceInsightPanelRole, type TraceInsightPromptInput, type TraceInsightQualityGate, type TraceInsightQuestion, type TraceInsightReadiness, type TraceInsightSuite, type TraceInsightTask, TraceNotFoundError, type TraceStore, type TraceStoreSource, type TraceStoreToOtlpOptions, type TracedAnalystOptions, type TracedJudgeOptions, type TracesToOtlpResult, type Trajectory, type TrajectoryStep, type TreatmentClass, type TreatmentGate, type TreatmentGateInput, type TreatmentGateOptions, type TrialTrace, type Turn, type TurnMetrics, type TurnResult, UI_FINDING_SEVERITIES, UI_LENSES, UNIVERSAL_FINDERS, type UiFinding, type UiFindingScreenshot, type UiFindingSeverity, type UiLens, type Unavailable, type UserQuestion, type ValidationContext, ValidationError, type ValidationIssue, type ValidationResult, type VerbosityBiasResult, type Verdict, type VerdictCacheStats, type VerdictCacheStore, type Verification, VerificationError, type VerificationReport, type VerifyContext, type VerifyFn, type VerifyOptions, type ViewSpansResult, type ViewTraceOversized, type ViewTraceResult, type VisualDiffOptions, type VisualDiffResult, type ViteDeployRunnerInput, type WeightedCompositeInput, type WeightedCompositeResult, type WorkerDriverContext, type WorkflowTopology, type WorkspaceAssertion, type WorkspaceAssertionResult, type WorkspaceInspector, type WorkspaceSnapshot, type WranglerDeployRunnerInput, acquisitionPlansForKnowledgeGaps, adversarialJudge, agentProfileCellHashMaterial, agentProfileCellKey, agentProfileHash, agentProfileId, agentProfileModelId, agentVisibleFields, aggregateJudgeVerdicts, aggregateLlm, aggregatePrReviewScore, aggregateRunScore, allCriticalPassed, analyzeAntiSlop, analyzeSeries, analyzeSupervisorRun, analyzeSupervisorRunSources, analyzeTraces, appendScorecard, applyLlmSpanOtlpAttributes, applyToolSpanOtlpAttributes, argHash, asNumber, asString, assertCapabilityHeadroom, assertCrossFamily, assertLlmRoute, assertModelsServed, assertNoHiddenLeak, assertProductBenchmarkRun, assertRealAgentReceipts, assertRealBackend, assertReleaseConfidence, assertRolloutLine, assertRunAgentProfileCell, assertRunCaptured, assertSingleBackend, assignFeedbackSplit, assignHeldOutTag, attachCostToReport, attest, attributeCounterfactuals, backoffMs, deterministicSplit as benchmarkDeterministicSplit, index$1 as benchmarks, benjaminiHochberg, bisect, blendHeldout, blockingKnowledgeEval, bonferroni, bootstrapCi, buildAgentInterfaceProfileCell, buildAgentProfileCell, buildDefaultAnalystRegistry, buildDriverSystemPrompt, buildProductBenchmarkManifest, buildReflectionPrompt, buildReviewerPrompt, buildTraceAnalystTools, buildTraceInsightContext, buildTraceInsightPrompt, buildTrajectory, buildWorkerDriverSystemPrompt, byteLengthRange, cachedJudge, calibrateJudge, calibrateJudgeContinuous, callLlm, callLlmJson, canaryLeakView, canonicalJson, canonicalize, capabilityHeadroom, captureFetchToRawSink, causalAttribution, checkBehavioralCanary, checkCanaries, checkSlos, checkTraceContracts, clamp01, classifyFailure, classifyTreatment, claudeCodeSupervisorRunReader, cliffsDelta, clusteredPairedBinary, codeExecutionJudge, cohensD, coherenceJudge, collectionPreserved, commentsForSource, commitBisect, comparePairedArms, compareReferenceReplay, compareToBaseline, compilerJudge, completionVerdict, composeParsers, composeValidators, computeExperimentStats, computeFindingId, computeToolUseMetrics, computeTraceMetrics, confidenceInterval, containsAll, contentHash, contextInputTokens, continuousAgreement, contractJudge, controlFailureClassFromVerification, controlRunToFeedbackTrajectory, controlRunToRunRecord, convertTraceStoresToOtlp, corpusInterRaterAgreement, corpusInterRaterAgreementFromJudgeScores, costForTokenPricing, costForUsage, costReceiptFromLlm, costReceiptFromLlmError, costReport, createAnalystAi, createAntiSlopJudge, createChatClient, createCustomJudge, createDefaultReviewer, createDomainExpertJudge, createFeedbackTrajectory, createIntentMatchJudge, createLlmCorrectnessChecker, createLlmReviewer, createOtelExporter, createOtelTracingStore, createReferenceEquivalenceJudge, createReplayFetch, createSandboxPool, createSemanticConceptJudge, createTokenRecallChecker, createTraceAnalystKind, crossTraceDiff, crowdingDistance, dataDescriptionBits, decideNextUserTurn, decideReferenceReplayPromotion, decideReferenceReplayRunPromotion, defaultBlendWeights, defaultIsMaterial, defaultJudges, defaultProviderRedactor, defaultReferenceReplayMatcher, defaultTraceInsightPanel, deployGateLayer, describeTraceInsightScope, diffFindings, diffScorecard, discoverPersonas, distillPlaybook, domainEvidencePattern, dominates, eProcess, ensembleJudge, errorStreakDetector, estimateCost, estimateTokens, evaluateActionPolicy, evaluateContract, evaluateHypothesis, evaluateInterimReleaseConfidence, evaluateOracles, evaluateReleaseConfidence, evaluateTraceContract, executeScenario, expandProfileAxes, expectAgent, exportProductBenchmark, exportProductBenchmarkRuns, exportRewardModel, exportRunAsOtlp, extractAssetUrls, extractErrorCount, extractOtlpAttributes, extractProducedState, extractUsage, extractUsageFromResponse, extractUsageFromSse, feedbackTrajectoriesToDatasetScenarios, feedbackTrajectoriesToOptimizerRows, feedbackTrajectoryToDatasetScenario, feedbackTrajectoryToOptimizerRow, fileContains, fileExists, fileExperimentStore, fileVerdictCache, findAutoMatchNoExpectation, findConstructorCwdDropped, findFallbackToPass, findLiteralTruePass, findProductBenchmarkArtifacts, findSkipCountsAsPass, firstNumberAttr, firstStringAttr, flattenOtlpExportToNdjson, flowLayer, fnv1a32, formatBenchmarkReport, formatDriverReport, formatFindings, formatScorecardDiff, gainHistogram, gateTreatmentApplied, gateTreatmentFromMetrics, gateTreatmentFromSpans, gateTreatmentFromToolSpans, ghCliClient, gitProvenanceReader, precision as goldenPrecision, gradeOnHidden, gradeSemanticStatus, groupBy, groupRunsByAgentProfileCell, harnessAxisOf, hasCapturedToolArgs, hashContent, hashJson, hashScenarios, hashToUnit, hiddenGrade, holm, htmlContainsElement, httpGithubClient, improvementVerdict, inMemoryExperimentStore, inMemoryReferenceReplayStore, inMemoryReviewStore, inMemoryRunRecordBackend, inMemoryVerdictCache, inferDomainKeywords, inferOtlpKind, interRaterReliability, interpretCliffs, iqr, isHiddenDestination, isJudgeSpan, isLlmSpan, isModelPriced, isOtelConfigured, isRetrievalSpan, isRolloutLine, isRunRecord, isSandboxSpan, isToolSpan, isTrainableSplit, isTransientLlmError, isUnavailable, iterateRawCalls, jestTestParser, jsonHasKeys, jsonShape, jsonlReferenceReplayStore, jsonlReviewStore, jsonlRunRecordBackend, judgeFamily, judgeReplayGate, judgeSpans, keyPreserved, knowledgeReadinessTracePayload, leaderboard, linterJudge, llmJudge, llmSpanFromProvider, llmSpans, loadScorecard, loadScorerFromGrader, localCommandRunner, lowercaseMutator, makeEvalTools, makeFinding, mannWhitneyU, mapConcurrent, matchGoldens, matchSpan, maximumChargeForLlmRequest, mcnemar, mcnemarPower, mcnemarRequiredN, mergeLayerResults, mergeSteeringBundle, mintRolloutRows, modelDescriptionBits, modelHasSnapshot, modelPriceKey, mulberry32, multiToolchainLayer, noProgressDetector, normalizeScores, notBlocked, objectiveEval, observeAll, otelRunCompleteHook, otlpRowsToRunRecords, otlpRowsToTraceRunRecords, otlpToRunRecords, otlpToTraceRunRecords, pairArms, pairedBootstrap, pairedEvalueSequence, pairedMde, pairedRiskDifference, pairedSignTest, pairedTTest, paraphraseRobustness, paraphraseRobustnessScenarios, paretoChart, paretoFrontier, paretoFrontierWithCrowding, parseCorrectnessResponse, parseFeedbackTrajectoriesJsonl, parseReflectionResponse, parseRunRecordSafe, parseRuntimeTrajectoryHookEvent, partialCredit, partitionHeldOut, passAtK, passOrthogonality, pearsonR, pixelDeltaRatio, planTraceInsightQuestions, politenessPrefixMutator, positionalBias, preflightModels, printDriverSummary, probeLlm, productBenchmarkIntegrityFailures, productBenchmarkMutableSurfaces, productBenchmarkRepoIdentity, productBenchmarkSplits, index as profile, projectOtlpFlatLine, projectRuntimeTrajectoryEvidence, promptBisect, proposeSynthesisTargets, providerFromBaseUrl, pytestTestParser, ranks, readClaudeCodeSupervisorRun, readOtlpStatus, readProductBenchmarkManifest, readProductBenchmarkRecords, recordRuns, recordRunsToScorecard, redTeamDataset, redTeamReport, redactString, redactValue, referenceReplayRunsToSteeringRows, referenceReplayScenarioToRunScore, regexMatch, regexMatches, renderMarkdownReport, renderPlaybookMarkdown, renderPreferenceMemoryMarkdown, renderPriorFindings, renderReleaseReport, renderSteeringText, renderSupervisorRunHeadline, renderSupervisorRunMarkdown, renderUpstreamFindings, repeatedActionDetector, replayFeedbackTrajectories, replayFeedbackTrajectory, replayScorerOverCorpus, replayTraceThroughJudge, requireAgentProfileCell, requiredSampleSize, researchReport, resolveModelPricing, resolveRunCostProvenance, resolveSeat, rolloutReward, rollupSupervisorRuns, roundTripRunRecord, routeFields, rowCount, rowWhere, runAgentControlLoop, runAssertions, runBehavioralCanaries, runCanaries, runCounterfactual, runE2EWorkflow, runEvalCampaign, runExpectations, runFailureClass, runHarnessExperiment, runIntentMatchJudge, runJudgeFleet, runKeywordCoverageJudge, runKeywordCoverageJudgeUrl, runLiveProof, runProposeReview, runProposeReviewAsControlLoop, runRecordToProductBenchmarkRecord, runReferenceEquivalenceJudge, runReferenceReplay, runScore, runSelfPlay, runSemanticConceptJudge, runTestGradedScenario, runsForScenario, scalarScore, scanForMuffledGates, scoreContinuity, scoreFromEvals, scoreKnowledgeReadiness, scorePrReviewComments, scorePrReviewSource, scoreRedTeamOutput, scoreReferenceReplay, scoreTraceInsightReadiness, seatPresets, securityJudge, selectHarnessVariant, selfPreference, sentenceReorderMutator, serializeFeedbackTrajectoriesJsonl, showMeasured, signManifest, spearmanR, statusAdvanced, stopOnNoProgress, stopOnRepeatedAction, stringField, stripFencedJson, subjectiveEval, summarizeAgentReceiptIntegrity, summarizeBackendIntegrity, summarizeHarnessResults, summarizePrReviewBenchmark, summarizePreferenceMemory, summaryTable, supervisorRunRolloutLines, testJudge, textInSnapshot, throwIfRunIncomplete, toAgentProfileJson, toJsonl, toLangfuseEnvelope, toOpenAiTool, toPrometheusText, toRewardRows, toSftRows, tokenizeDomainWords, toolNamesForRun, toolSpans, traceAnalystFunctionGroup, traceAnalystOnRunComplete, traceContract, traceJudge, traceJudgeEnsemble, traceSpanKindToOpenInferenceKind, tracedAnalyzeTraces, typoMutator, urlContains, userQuestionsForKnowledgeGaps, validateAgentProfileCell, validateProductBenchmarkManifest, validateProductBenchmarkRecord, validateProductBenchmarkRun, validateRolloutLine, validateRunRecord, verbosityBias, verifyAgentProfileCell, verifyAttestation, verifyCompletion, verifyManifest, visualDiff, viteDeployRunner, vitestTestParser, weightedComposite, weightedMean, weightedRecall, welchsTTest, whitespaceCollapseMutator, wilcoxonSignedRank, wilson, withAssignedFeedbackSplit, withHeldoutBlend, withJudgeRetry, withOtelPipeline, wranglerDeployRunner, writeSupervisorRunReport };
+export { AGENT_PROFILE_KINDS, ATTESTATION_ALGORITHM, type ActionExecutionPolicy, type ActionPolicyDecision, type ActionableSideInfo, type ActiveLearningOptions, type AdapterRun, AgentDriver, type AgentDriverConfig, AgentEvalError, type AgentEvalErrorCode, type AgentInterfaceProfileLike, type AgentProfileCell, type AgentProfileCellInput, type AgentProfileCellSchemaVersion, AgentProfileCellValidationError, type AgentProfileDimensionValue, type AgentProfileHarness, type AgentProfileJson, type AgentProfileJsonObject, type AgentProfileKind, type AgentProfileRuntimeReceipt, type AgentProfileSource, type AgentProfileSourceInput, type AlignmentOp, type Analyst, type AnalystContext, type AnalystCost, type AnalystFinding, type AnalystHooks, type AnalystInputKind, AnalystRegistry, type AnalystRegistryOptions, type AnalystRequirements, type AnalystRunEvent, type AnalystRunInputs, type AnalystRunResult, type AnalystRunSummary, type AnalystSeverity, type AnalystUsageReceipt, type AnalyzeTracesInput, type AnalyzeTracesOptions, type AnalyzeTracesResult, type AnalyzeTracesTurnSnapshot, type AntiSlopConfig, type AntiSlopIssue, type AntiSlopReport, type Artifact$1 as Artifact, type ArtifactCheck, type Artifact as ArtifactCheckArtifact, type ArtifactEventLike, type ArtifactResult, type ArtifactValidator, type AsiSeverity, type AssertCapabilityHeadroomOptions, type AssertCrossFamilyOptions, type AssertSingleBackendOptions, type AttestationProvenance, type AttestationVerification, type AttestedReport, type AutoPrClient, AxGepaSteeringOptimizer, type AxSteeringOptimizerConfig, BENCHMARK_SPLIT_SEED, type BackendDescriptor, BackendIntegrityError, type BackendIntegrityReport, type BaselineOptions, type BaselineReport, BehaviorAssertion, type BehavioralMetrics, type BehavioralTokenSequence, type BenchmarkAdapter, type BenchmarkDatasetItem, type BenchmarkEvaluation, type BenchmarkFamily, type BenchmarkReport$1 as BenchmarkReport, type BenchmarkResponder, BenchmarkRunner, type BenchmarkRunnerConfig, type BenchmarkScenario, type BenchmarkSource, type BenchmarkTaskKind, type BisectOptions, type BisectResult, type BisectStep, type BlendWeights, type BootstrapOptions, type BootstrapResult, BudgetBreachError, BudgetGuard, type BudgetLedgerEntry, type BudgetPolicy, type BudgetSpec, CODING_HARNESSES, type CachedJudge, type CachedJudgeOptions, type CalibrationResult, CallExpectation, CallbackResearcher, type CallbackResearcherOptions, type CampaignFactoryParams, type CampaignIntegrityPolicy, type CampaignRunContext, type CampaignRunOutcome, type CampaignRunner, type CampaignScenario, type CampaignVariant, type CanaryAlert, type CanaryEvaluation, type CanaryKind, type CanaryLeak, type CanaryOptions, type CanaryReport, type CanarySeverity, type CandidateComparison, type CandidateScenario, type CandidateScore, type CanonicalRawAnalystFinding, type CapabilityHeadroomOptions, type CapabilityHeadroomResult, type CaptureFetchContext, type CaptureFetchOptions, CaptureIntegrityError, type CausalAttributionReport, type CellVerdict, type ChannelRollup, type ChatCallOpts, type ChatClient, type ChatMessage, type ChatRequest, type ChatResponse, type ChatToolCall, type ChatTransport, type CheckResult, type CliBridgeTransportOpts, type CliffsMagnitude, type ClusterBootstrapInterval, type ClusterSignFlipAlternative, type ClusterSignFlipResult, type ClusteredBinaryCluster, type ClusteredMatchedPair, type ClusteredPairedBinaryOptions, type ClusteredPairedBinaryResult, type ClusteredPairedBinaryStatistics, type CollectedArtifacts, type CommandRunner, type ComparePairedArmsOptions, type CompletionCriterion, type CompletionRequirement, type CompletionVerdict, type ConceptComplexity, type ConceptFinding, type ConceptSpec, type ConceptWeightStrategy, ConfigError, type ContinuityCheck, type ContinuityCheckResult, type ContinuityReport, type ContinuitySnapshotPair, type ContinuousAgreement, type ContinuousAgreementOptions, type ContinuousCalibrationResult, type ContractCheckResult, type ContractJudgeOptions, type ContractMetric, type ContractReport, type ContractRule, type ContractRuleKind, type ContractSpan, type ContractVerdict, type ContractViolation, type ControlActionFailureMode, type ControlActionOutcome, type ControlBudget, type ControlContext, type ControlDecision, type ControlEvalResult, type ControlRunResult, type ControlRunToRunRecordOptions, type ControlRuntimeConfig, type ControlRuntimeError, type ControlSeverity, type ControlStep, type ControlStopPolicies, ConvergenceTracker, type CorpusAgreementOptions, type CorpusAgreementPerDimension, type CorpusAgreementReport, type CorpusScoreRecord, type CorrectnessChecker, CostAccountingIncompleteError, CostCallConflictError, CostCeilingReachedError, type CostChannel, type CostEntry, CostLedger, type CostLedgerEntry, type CostLedgerFilter, type CostLedgerHandle, type CostLedgerOptions, type CostLedgerPersistence, CostLedgerPersistenceError, type CostLedgerSummary, type CostReceipt, CostReceiptCaptureError, type CostReceiptInput, type CostReport, CostReservationExceededError, type CostResult, type CostSummary, CostTracker, type CostUsage, type CounterfactualContext, type CounterfactualMutation, type CounterfactualResult, type CounterfactualRunner, type CreateAnalystAiConfig, type CreateChatClientOpts, type CreateDefaultReviewerOptions, type CreateExperimentInput, type CreateSandboxPoolOpts, type CreateTraceAnalystKindOpts, CrossFamilyError, type CrossTraceDiff, type CrossTraceDiffOptions, type CustomTokenPricing, DEFAULT_AGENT_SLOS, DEFAULT_COMPLEXITY_WEIGHTS, DEFAULT_RULES as DEFAULT_FAILURE_RULES, DEFAULT_FINDERS, DEFAULT_HARNESS_OBJECTIVES, DEFAULT_MUTATION_PRIMITIVES, DEFAULT_MUTATORS, DEFAULT_PR_REVIEW_SCORE_WEIGHTS, DEFAULT_REDACTION_RULES, DEFAULT_RED_TEAM_CORPUS, DEFAULT_RUN_SCORE_WEIGHTS, DEFAULT_SEVERITY_WEIGHTS, DEFAULT_TRACE_ANALYST_BUDGETS, DEFAULT_TRACE_ANALYST_KINDS, type DataAcquisitionPlan, Dataset, type DatasetDifficulty, type DatasetManifest, type DatasetOverview, type DatasetProvenance, type DatasetScenario, type DatasetSplit, type DecideNextUserTurnOpts, type DefaultAnalystRegistryOptions, type DefaultVerdict, type DeployFamily, type DeployGateLayerInput, type DeployRunResult, type DeployRunner, type DescriptionLengthCandidate, type DescriptionLengthConfig, type DescriptionLengthDecision, type DescriptionLengthEvidence, DescriptionLengthGate, type DescriptionLengthRejectionCode, type DetectorEvent, type DetectorSeverity, type DetectorSignal, type DiffPolicy, type DiffScorecardOptions, type DirEntry, type DirectProviderTransportOpts, type Direction, type DiscoverPersonasOptions, type DiscoveredPersona, DockerSandboxDriver, type DriverResult, type DriverState, DualAgentBench, type DualAgentBenchConfig, type DualAgentReport, type DualAgentRound, type DualAgentScenario, type DualAgentScenarioResult, type EProcess, type EProcessOptions, type EProcessState, type EProcessStep, ERROR_COUNT_PATTERNS, type EnsembleAggregate, type EnsembleJudgeOptions, type ErrorCluster, type ErrorCountPattern, type ErrorStreakOptions, type EvalCampaignOptions, type EvalCampaignResult, type EvalResult, type EvalToolDef, EvalTraceStore, type EventFilter, type EventKind, type EvidenceRef, type EvolutionRound, type ExecutorConfig, type Expectation, type Experiment, type ExperimentPlan, type ExperimentProvenance, type ExperimentRep, type ExperimentResult, type ExperimentStats, type ExperimentStore, ExperimentTracker, type ExperimentTrackerOptions, type ExperimentVerdict, type ExportableSpan, type ExportedRewardModel, type ExtractOptions, type ExtractResult, type ExtractUsageFromSseOptions, type ExtractedUsage, FAILURE_CLASSES, FAILURE_MODE_KIND_SPEC, type FactorContribution, type FactorialCell, type FailedRun, type FailureClass, type FailureClassification, type FailureContext, type FailureMode, type FailureRule, type FeedbackArtifactType, type FeedbackAttempt, type FeedbackLabel, type FeedbackLabelKind, type FeedbackLabelSource, type FeedbackOptimizerRow, type FeedbackOutcome, type FeedbackPattern, type FeedbackReplayAdapter, type FeedbackReplayResult, type FeedbackSeverity, type FeedbackSplitPolicy, type FeedbackTask, type FeedbackTrajectory, type FeedbackTrajectoryFilter, type FeedbackTrajectoryStore, type FieldDestination, type FileChange, FileSystemFeedbackTrajectoryStore, FileSystemRawProviderSink, type FileSystemRawProviderSinkOptions, FileSystemTraceStore, type FileSystemTraceStoreOptions, type Finding, type FindingSubject, type FindingSubjectKind, type FindingsDiff, FindingsStore, type FlattenOtlpOptions, type FlowAction, type FlowLayerEnv, type FlowLayerFactoryInput, type FlowRunner, type FlowRunnerStepResult, type FlowSpec, type FlowStep, type GainDistributionBin, type GainDistributionFigureSpec, type GainDistributionOptions, type GateDecision$1 as GateDecision, type GateEvidence, type GenericSpan, type GhCliClientOptions, type GoldenItem, type GoldenSeverity, type GoldenSpec, HARNESS_NATIVE_MODEL, type HarnessAdapter, type HarnessConfig, type HarnessExperimentConfig, type HarnessExperimentResult, type HarnessIntervention, type HarnessRunRequest, type HarnessRunResult, type HarnessScenario, type HarnessSelection, type HarnessVariant, type HarnessVariantReport, type HeadroomClass, type HeadroomInput, HeldOutGate, type HeldOutGateConfig, type HeldOutGateRejectionCode, type HeldOutPartition, type HiddenCriteriaGrader, type HiddenGradeResult, type HiddenLeak, HoldoutAuditor, HoldoutLockedError, type HttpGithubClientOptions, type HypothesisManifest, type HypothesisResult, IMPROVEMENT_KIND_SPEC, INPUT_VALUE, INTENT_MATCH_JUDGE_VERSION, type ImageData, type ImprovementThresholds, type ImprovementVerdictResult, InMemoryFeedbackTrajectoryStore, InMemoryRawProviderSink, type InMemoryRawProviderSinkOptions, InMemoryTraceStore, InMemoryWorkspaceInspector, type InferenceScorer, type InspectorContext, type IntentMatchInput, type IntentMatchOptions, type IntentMatchResult, type InteractionContribution, type InterimReleaseConfidence, type InterimReleaseConfidenceInput, type JudgeConfig$1 as JudgeConfig, JudgeError, type JudgeFamily, type JudgeFleetOptions, type JudgeFn, type JudgeInput, JudgeParseError, type JudgeReplayGateArgs, type JudgeReplayResult, type JudgeRetryOutcome, type JudgeRetryPolicy, type JudgeRubric, JudgeRunner, type JudgeScore$1 as JudgeScore, type JudgeScoreInput, type JudgeScoresRecord, type JudgeSpan, type JudgeVerdict, KNOWLEDGE_GAP_KIND_SPEC, KNOWLEDGE_POISONING_KIND_SPEC, type KeywordConceptSpec, type KeywordCoverageFinding, type KeywordCoverageOptions, type KeywordCoverageResult, type KnowledgeAcquisitionMode, type KnowledgeBundle, type KnowledgeFallbackPolicy, type KnowledgeFreshness, type KnowledgeImportance, type KnowledgeReadinessReport, type KnowledgeRecommendedAction, type KnowledgeRequirement, type KnowledgeRequirementCategory, type KnowledgeResponsibleSurface, type KnowledgeSensitivity, LLM_CACHED_TOKENS, LLM_CACHED_TOKEN_ATTR_KEYS, LLM_CACHE_WRITE_TOKENS, LLM_CACHE_WRITE_TOKEN_ATTR_KEYS, LLM_CONTEXT_TOKENS, LLM_COST_ATTR_KEYS, LLM_COST_USD, LLM_INPUT_TOKENS, LLM_INPUT_TOKEN_ATTR_KEYS, LLM_MODEL_ATTR_KEYS, LLM_MODEL_NAME, LLM_OUTPUT_TOKENS, LLM_OUTPUT_TOKEN_ATTR_KEYS, LLM_REASONING_TOKENS, LLM_REASONING_TOKEN_ATTR_KEYS, type LangfuseEnvelope, type LangfuseGeneration, type LangfuseScore, type Layer, type LayerResult, type LayerStatus, type LeaderboardOptions, type LeaderboardRow, type LiveProofArtifact, type LiveProofConfig, type LiveProofContext, type LiveProofResult, LlmCallError, type LlmCallMetadata, type LlmCallRequest, type LlmCallResult, LlmClient, type LlmClientOptions, type LlmCorrectnessCheckerOpts, type LlmJsonCall, type LlmJudgeDimension, type LlmJudgeOptions, type LlmMessage, LlmResponseError, type LlmReviewerConfig, LlmRouteAssertionError, type LlmRouteRequirements, type LlmSpan, type LlmSpanOtlpInput, type LlmUsage, LockedJsonlAppender, MODEL_PRICING, type MakeEvalToolsConfig, type MatchResult, type MatchedPair, type MatchedRunRecordPair, type MatcherResult, type MaximumCharge, type McNemarResult, type Measured, type MeasurementPolicy, type MergeOptions, type Message, type MetricSamples, type MetricVerdict, MetricsCollector, type MintRolloutOptions, type MintRolloutResult, type MockTransportOpts, type ModelCostRollup, type ModelPreflight, type ModelSeats, ModelsUnreachableError, type MuffledFinder, type MuffledFinding, MultiLayerVerifier, type MultiToolchainLayerConfig, type Mutator, Mutex, type NoLeakOptions, type NoProgressOptions, NoopRawProviderSink, NoopResearcher, NotFoundError, OPENINFERENCE_SPAN_KIND, OTEL_AGENT_EVAL_SCOPE, OUTPUT_VALUE, type Objective, type Oracle, type OracleObservation, type OracleReport, type OracleResult, type OrthogonalityInput, type OrthogonalityResult, type OtelExportConfig, type OtelExporter, type OtelPipelineHandle, type OtelPipelineOptions, type OtlpExport, OtlpFileTraceStore, type OtlpFileTraceStoreOptions, type OtlpFlatLine, type OtlpResourceSpans, type OtlpSpan, type OtlpSpanRole, type OtlpSpanRoleInput, type OtlpToRunRecordsOptions, type OtlpTraceRunRecord, type PaidCallResult, type PairArmsOptions, type PairArmsResult, type PairRunRecordsResult, type PairedArmRow, type PairedArmsComparison, type PairedBootstrapOptions, type PairedBootstrapResult, type PairedCorrectness, type PairedEvalueOptions, type PairedEvalueSequence, type PairedEvalueStep, type PairedMetricDelta, type PairedSignTestResult, PairwiseSteeringOptimizer, type ParaphraseRobustnessScenarioInput, type ParaphraseRobustnessScenarioResult, type ParetoFigureSpec, type ParetoPoint, type ParetoResult, type PartitionHeldOutOptions, type PendingCostCall, type PendingCostCallView, type PersistedFinding, type PersonaConfig, type PersonaRigor, type Playbook, type PlaybookEntry, type PoolSlot, type PositionalBiasResult, type PrReviewAuditCase, type PrReviewBenchmarkSummary, type PrReviewComment, type PrReviewMatchedFinding, type PrReviewOutcome, type PrReviewReferenceFinding, type PrReviewScore, type PrReviewScoreWeights, type PrReviewSeverity, type PrReviewSource, type PreferenceMemoryEntry, type PreflightModelsOptions, type PreflightOutcome, type ProducedProposal, type ProducedState, type ProductBenchmarkArm, type ProductBenchmarkArtifactPaths, type ProductBenchmarkBudgets, type ProductBenchmarkExportOptions, type ProductBenchmarkExportResult, type ProductBenchmarkManifest, type ProductBenchmarkProfileRef, type ProductBenchmarkRecord, type ProductBenchmarkRepoRef, type ProductBenchmarkRunInput, type ProductBenchmarkScenario, type ProductBenchmarkSingleRunExportOptions, type ProductBenchmarkSplit, type ProductBenchmarkSubstrateVersions, type ProductBenchmarkValidationReport, ProductClient, type ProductClientConfig, type ProfileAxisSpec, type ProjectRuntimeTrajectoryEvidenceOptions, type ProjectedOtlpSpan, type PromptHandle, PromptRegistry, type ProportionInterval, type ProposalEventLike, type ProposeAutomatedPullRequestInput, type ProposeAutomatedPullRequestResult, type ProposeFn, type ProposeInput, type ProposeOutput, type ProposeReviewConfig, type ProposeReviewControlAction, type ProposeReviewControlConfig, type ProposeReviewControlResult, type ProposeReviewControlState, type ProposeReviewReport, type ProposeReviewShot, type ProposedSideEffect, type ProvenanceReader, type ProviderRedactor, type QueryTracesPage, REDACTION_VERSION, REFERENCE_EQUIVALENCE_INPUT_LIMITS, REFERENCE_EQUIVALENCE_JUDGE_VERSION, RESEARCH_REPORT_HARD_PAIR_FLOOR, ROLLOUT_SCHEMA, RUN_COST_ATTR_KEYS, type RawAnalystEvidence, type RawAnalystFinding, type RawProviderDirection, type RawProviderEvent, type RawProviderSink, type RawProviderSinkFilter, type RecordRunsOptions, type RedTeamCase, type RedTeamCategory, type RedTeamFinding, type RedTeamPayload, type RedTeamReport, type RedactionReport, type RedactionRule, type ReferenceEquivalenceJudgeInput, type ReferenceEquivalenceJudgeOptions, type ReferenceEquivalenceJudgeResult, type ReferenceEquivalenceScenario, type ReferenceMatchResult, type ReferenceReplayAdapter, type ReferenceReplayAdapterFn, type ReferenceReplayAdapterLike, type ReferenceReplayAggregate, type ReferenceReplayCandidate, type ReferenceReplayCase, type ReferenceReplayCaseRun, type ReferenceReplayExecutionScenario, type ReferenceReplayItem, type ReferenceReplayMatch, type ReferenceReplayMatchStrategy, type ReferenceReplayMatcher, type ReferenceReplayPromotionDecision, type ReferenceReplayPromotionPolicy, type ReferenceReplayRun, type ReferenceReplayRunContext, type ReferenceReplayRunOptions, type ReferenceReplayRunStore, type ReferenceReplayScenario, type ReferenceReplayScenarioScore, type ReferenceReplayScore, type ReferenceReplayScoreOptions, type ReferenceReplaySplit, type ReferenceReplaySplitComparison, type ReferenceReplaySteeringRowsOptions, type ReflectionContext, type ReflectionProposal, type RegistryRunOpts, type ReleaseConfidenceAxis, type ReleaseConfidenceAxisName, type ReleaseConfidenceInput, type ReleaseConfidenceIssue, type ReleaseConfidenceMetrics, type ReleaseConfidenceScorecard, type ReleaseConfidenceStatus, type ReleaseConfidenceThresholds, type ReleaseTraceEvidence, type RenderReleaseReportOptions, type RepeatedActionOptions, ReplayCache, type ReplayCacheEntry, ReplayCacheMissError, type ReplayCacheStats, ReplayError, type ReplayFetchOptions, type RepoRef, type RequirementCheck, type ResearchReport, type ResearchReportCandidate, type ResearchReportDecision, type ResearchReportMethodology, type ResearchReportOptions, type ResearchReportRecommendation, type Researcher, type RetrievalSpan, type Review, type ReviewFn, type ReviewInput, type ReviewMemoryEntry, type ReviewMemoryStore, type ReviewerMemoryEntry, type ReviewerOutput, type ReviewerPromptInput, type ReviewerSoftFailDefaults, type ReviewerVerificationSummary, type RewardRow, type RiskDifferenceResult, type RobustnessResult, type RolloutCapture, type RolloutLine, type RolloutRole, type RolloutScrubber, type RolloutSplit, type RolloutStep, type RouteMap, type RoutedField, type RouterTransportOpts, type RubricDimension, type Run, type RunCommandInput, type RunCommandResult, type RunCompleteHook, type RunCompleteHookContext, type RunCostProvenance, RunCritic, type RunCriticOptions, type RunEvidenceMetadata, type RunFilter, RunIntegrityError, type RunIntegrityExpectations, type RunIntegrityIssue, type RunIntegrityIssueCode, type RunIntegrityReport, type RunJudgeMetadata, type RunLayer, type RunOutcome, type RunPaidCallInput, type RunRecord, type RunRecordBackend, type RunRecordFilter, RunRecordValidationError, type RunScore, type RunScoreWeights, type RunSplitTag, type RunStatus, type RunTerminalOutcome, type RunTokenUsage, type RunTrace, type RuntimeEventLike, type RuntimeResolution, type RuntimeTrajectoryEvidenceProjection, type RuntimeTrajectoryEvidenceSummary, type RuntimeTrajectoryHookEvent, type RuntimeTrajectoryRecord, type RuntimeTrajectoryRunRecord, SEMANTIC_CONCEPT_JUDGE_VERSION, SKILL_USAGE_ANALYST, SPAN_KIND_ATTR_KEYS, SUPERVISOR_RUN_SCHEMA, type SandboxDriver, SandboxHarness, type SandboxHarnessResult, type SandboxJudgeKind, type SandboxJudgeResult, type SandboxJudgeSpec, type SandboxPool, type SandboxResult, type SandboxSdkTransportOpts, type SandboxSpan, type SatisfiedBy, type ScanOptions, type Scenario$1 as Scenario, type ScenarioCost, type ScenarioFile, ScenarioRegistry, type ScenarioResult, type ScoreKnowledgeReadinessOptions, type Scorecard, type ScorecardCell, type ScorecardCellDiff, type ScorecardDiff, type ScorecardEntry, type ScorecardLogLine, type ScoredTarget, type SearchSpanResult, type SearchTraceResult, type SeatName, type SeatPresetName, SeatUnsetError, type SelfPlayOptions, type SelfPlayProposer, type SelfPlayScorer, type SelfPreferenceResult, type SemanticConceptJudgeInput, type SemanticConceptJudgeOptions, type SemanticConceptJudgeResult, type SequentialDecision, type SerializedRegex, type SeriesConvergenceOptions, type SeriesConvergenceResult, type Severity, type SftExportOptions, type SftRow, type SignTestAlternative, type SignedManifest, type SignedManifestAlgo, type SingleBackendDivergence, SingleBackendError, type SingleBackendField, type SingleBackendReport, SkillUsageAnalyst, type SliceOptions, type Slo, type SloCheckResult, type SloComparator, type SloReport, type SloSeverity, type SlopCategory, type SlotFactory, type SourceLimits, type Span, type SpanBase, type SpanFilter, type SpanHandle, type SpanKind, type SpanMatchRecord, SpanNotFoundError, type SpanPredicate, type SpanStatus, type SseUsageMode, type SteeringBundle, type SteeringChange, type SteeringDelta, type SteeringOptimizationResult, type SteeringOptimizationRow, type SteeringOptimizationSelector, type SteeringOptimizerBackend, type SteeringOptimizerConfig, type SteeringRolePrompt, type StepAttribution, type StopDecision, type StreamingDetector, type SuboptimalCode, type SuboptimalSignal, SubprocessSandboxDriver, type SubprocessSandboxDriverOptions, type SummaryTable, type SummaryTableOptions, type SummaryTableRow, type SupervisorRunReader, type SupervisorRunReport, type SupervisorRunRollup, type SupervisorRunSources, type SupervisorRunTree, type SynthesisReason, type SynthesisTarget, TOOL_ARGS_CAPTURED, TOOL_LATENCY_MS, TOOL_NAME, TOOL_NAME_ATTR_KEYS, TRACE_ANALYST_ACTOR_DESCRIPTION, TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION, TRACE_ANALYST_TRUNCATION_MARKER_PREFIX, TRACE_SCHEMA_VERSION, type TaskGold, type TaskHeadroom, type TestGradedRunOptions, type TestGradedRunResult, type TestGradedScenario, type TestOutputParser, type TestResult, type TextMatcher, type ThresholdContract, TokenCounter, type TokenSpec, type ToolCallEventLike, type ToolDef, type ToolMatcher, type ToolSpan, type ToolSpanOtlpInput, type ToolStats, type ToolUseMetrics, type ToolUseOptions, type TraceAggregate, type TraceAnalysisStore, type TraceAnalystByteBudgets, type TraceAnalystFilters, type TraceAnalystGolden, type TraceAnalystHookOptions, type TraceAnalystKindSpec, type TraceAnalystSpan, type TraceAnalystSpanKind, type TraceAnalystSpanStatus, type TraceAnalystTraceSummary, type TraceContract, TraceContractBuilder, TraceEmitter, type TraceEmitterOptions, type TraceEvent, TraceFileMissingError, type TraceInsightContext, type TraceInsightFinding, type TraceInsightPanelRole, type TraceInsightPromptInput, type TraceInsightQualityGate, type TraceInsightQuestion, type TraceInsightReadiness, type TraceInsightSuite, type TraceInsightTask, TraceNotFoundError, type TraceStore, type TraceStoreSource, type TraceStoreToOtlpOptions, type TracedAnalystOptions, type TracedJudgeOptions, type TracesToOtlpResult, type Trajectory, type TrajectoryStep, type TreatmentClass, type TreatmentGate, type TreatmentGateInput, type TreatmentGateOptions, type TrialTrace, type Turn, type TurnMetrics, type TurnResult, UI_FINDING_SEVERITIES, UI_LENSES, UNIVERSAL_FINDERS, type UiFinding, type UiFindingScreenshot, type UiFindingSeverity, type UiLens, type Unavailable, type UserQuestion, type ValidationContext, ValidationError, type ValidationIssue, type ValidationResult, type VerbosityBiasResult, type Verdict, type VerdictCacheStats, type VerdictCacheStore, type Verification, VerificationError, type VerificationReport, type VerifyContext, type VerifyFn, type VerifyOptions, type ViewSpansResult, type ViewTraceOversized, type ViewTraceResult, type VisualDiffOptions, type VisualDiffResult, type ViteDeployRunnerInput, type WeightedCompositeInput, type WeightedCompositeResult, type WorkerDriverContext, type WorkflowTopology, type WorkspaceAssertion, type WorkspaceAssertionResult, type WorkspaceInspector, type WorkspaceSnapshot, type WranglerDeployRunnerInput, acquisitionPlansForKnowledgeGaps, adversarialJudge, agentProfileCellHashMaterial, agentProfileCellKey, agentProfileHash, agentProfileId, agentProfileModelId, agentVisibleFields, aggregateJudgeVerdicts, aggregateLlm, aggregatePrReviewScore, aggregateRunScore, allCriticalPassed, analyzeAntiSlop, analyzeSeries, analyzeSupervisorRun, analyzeSupervisorRunSources, analyzeTraces, appendScorecard, applyLlmSpanOtlpAttributes, applyToolSpanOtlpAttributes, argHash, asNumber, asString, assertCapabilityHeadroom, assertCrossFamily, assertLlmRoute, assertModelsServed, assertNoHiddenLeak, assertProductBenchmarkRun, assertRealAgentReceipts, assertRealBackend, assertReleaseConfidence, assertRolloutLine, assertRunAgentProfileCell, assertRunCaptured, assertSingleBackend, assignFeedbackSplit, assignHeldOutTag, attachCostToReport, attest, attributeCounterfactuals, backoffMs, deterministicSplit as benchmarkDeterministicSplit, index$1 as benchmarks, benjaminiHochberg, bisect, blendHeldout, blockingKnowledgeEval, bonferroni, bootstrapCi, buildAgentInterfaceProfileCell, buildAgentProfileCell, buildDefaultAnalystRegistry, buildDriverSystemPrompt, buildProductBenchmarkManifest, buildReflectionPrompt, buildReviewerPrompt, buildTraceAnalystTools, buildTraceInsightContext, buildTraceInsightPrompt, buildTrajectory, buildWorkerDriverSystemPrompt, byteLengthRange, cachedJudge, calibrateJudge, calibrateJudgeContinuous, callLlm, callLlmJson, canaryLeakView, canonicalJson, canonicalize, capabilityHeadroom, captureFetchToRawSink, causalAttribution, checkBehavioralCanary, checkCanaries, checkSlos, checkTraceContracts, clamp01, classifyFailure, classifyOtlpSpanRole, classifyTreatment, claudeCodeSupervisorRunReader, cliffsDelta, clusteredPairedBinary, codeExecutionJudge, cohensD, coherenceJudge, collectionPreserved, commentsForSource, commitBisect, comparePairedArms, compareReferenceReplay, compareToBaseline, compilerJudge, completionVerdict, composeParsers, composeValidators, computeExperimentStats, computeFindingId, computeToolUseMetrics, computeTraceMetrics, confidenceInterval, containsAll, contentHash, contextInputTokens, continuousAgreement, contractJudge, controlFailureClassFromVerification, controlRunToFeedbackTrajectory, controlRunToRunRecord, convertTraceStoresToOtlp, corpusInterRaterAgreement, corpusInterRaterAgreementFromJudgeScores, costForTokenPricing, costForUsage, costReceiptFromLlm, costReceiptFromLlmError, costReport, createAnalystAi, createAntiSlopJudge, createChatClient, createCustomJudge, createDefaultReviewer, createDomainExpertJudge, createFeedbackTrajectory, createIntentMatchJudge, createLlmCorrectnessChecker, createLlmReviewer, createOtelExporter, createOtelTracingStore, createReferenceEquivalenceJudge, createReplayFetch, createSandboxPool, createSemanticConceptJudge, createTokenRecallChecker, createTraceAnalystKind, crossTraceDiff, crowdingDistance, dataDescriptionBits, decideNextUserTurn, decideReferenceReplayPromotion, decideReferenceReplayRunPromotion, defaultBlendWeights, defaultIsMaterial, defaultJudges, defaultProviderRedactor, defaultReferenceReplayMatcher, defaultTraceInsightPanel, deployGateLayer, describeTraceInsightScope, diffFindings, diffScorecard, discoverPersonas, distillPlaybook, domainEvidencePattern, dominates, eProcess, ensembleJudge, errorStreakDetector, estimateCost, estimateTokens, evaluateActionPolicy, evaluateContract, evaluateHypothesis, evaluateInterimReleaseConfidence, evaluateOracles, evaluateReleaseConfidence, evaluateTraceContract, executeScenario, expandProfileAxes, expectAgent, exportProductBenchmark, exportProductBenchmarkRuns, exportRewardModel, exportRunAsOtlp, extractAssetUrls, extractErrorCount, extractOtlpAttributes, extractProducedState, extractUsage, extractUsageFromResponse, extractUsageFromSse, feedbackTrajectoriesToDatasetScenarios, feedbackTrajectoriesToOptimizerRows, feedbackTrajectoryToDatasetScenario, feedbackTrajectoryToOptimizerRow, fileContains, fileExists, fileExperimentStore, fileVerdictCache, findAutoMatchNoExpectation, findConstructorCwdDropped, findFallbackToPass, findLiteralTruePass, findProductBenchmarkArtifacts, findSkipCountsAsPass, firstNumberAttr, firstStringAttr, flattenOtlpExportToNdjson, flowLayer, fnv1a32, formatBenchmarkReport, formatDriverReport, formatFindings, formatScorecardDiff, gainHistogram, gateTreatmentApplied, gateTreatmentFromMetrics, gateTreatmentFromSpans, gateTreatmentFromToolSpans, ghCliClient, gitProvenanceReader, precision as goldenPrecision, gradeOnHidden, gradeSemanticStatus, groupBy, groupRunsByAgentProfileCell, harnessAxisOf, hasCapturedToolArgs, hashContent, hashJson, hashScenarios, hashToUnit, hiddenGrade, holm, htmlContainsElement, httpGithubClient, improvementVerdict, inMemoryExperimentStore, inMemoryReferenceReplayStore, inMemoryReviewStore, inMemoryRunRecordBackend, inMemoryVerdictCache, inferDomainKeywords, inferOtlpKind, interRaterReliability, interpretCliffs, iqr, isHiddenDestination, isJudgeSpan, isLlmSpan, isModelPriced, isOtelConfigured, isOtlpModelCall, isRetrievalSpan, isRolloutLine, isRunRecord, isSandboxSpan, isToolSpan, isTrainableSplit, isTransientLlmError, isUnavailable, iterateRawCalls, jestTestParser, jsonHasKeys, jsonShape, jsonlReferenceReplayStore, jsonlReviewStore, jsonlRunRecordBackend, judgeFamily, judgeReplayGate, judgeSpans, keyPreserved, knowledgeReadinessTracePayload, leaderboard, linterJudge, llmJudge, llmSpanFromProvider, llmSpans, loadScorecard, loadScorerFromGrader, localCommandRunner, lowercaseMutator, makeEvalTools, makeFinding, mannWhitneyU, mapConcurrent, matchGoldens, matchSpan, maximumChargeForLlmRequest, mcnemar, mcnemarPower, mcnemarRequiredN, mergeLayerResults, mergeSteeringBundle, mintRolloutRows, modelDescriptionBits, modelHasSnapshot, modelPriceKey, mulberry32, multiToolchainLayer, noProgressDetector, normalizeScores, notBlocked, objectiveEval, observeAll, otelRunCompleteHook, otlpRowsToRunRecords, otlpRowsToTraceRunRecords, otlpToRunRecords, otlpToTraceRunRecords, pairArms, pairRunRecords, pairedBootstrap, pairedCohensDz, pairedEvalueSequence, pairedMde, pairedRiskDifference, pairedSignTest, pairedTTest, paraphraseRobustness, paraphraseRobustnessScenarios, paretoChart, paretoFrontier, paretoFrontierWithCrowding, parseCorrectnessResponse, parseFeedbackTrajectoriesJsonl, parseReflectionResponse, parseRunRecordSafe, parseRuntimeTrajectoryHookEvent, partialCredit, partitionHeldOut, passAtK, passOrthogonality, pearsonR, pixelDeltaRatio, planTraceInsightQuestions, politenessPrefixMutator, positionalBias, preflightModels, printDriverSummary, probeLlm, productBenchmarkIntegrityFailures, productBenchmarkMutableSurfaces, productBenchmarkRepoIdentity, productBenchmarkSplits, index as profile, projectOtlpFlatLine, projectRuntimeTrajectoryEvidence, promptBisect, proposeSynthesisTargets, providerFromBaseUrl, pytestTestParser, ranks, readClaudeCodeSupervisorRun, readOtlpStatus, readProductBenchmarkManifest, readProductBenchmarkRecords, recordRuns, recordRunsToScorecard, redTeamDataset, redTeamReport, redactString, redactValue, referenceReplayRunsToSteeringRows, referenceReplayScenarioToRunScore, regexMatch, regexMatches, renderMarkdownReport, renderPlaybookMarkdown, renderPreferenceMemoryMarkdown, renderPriorFindings, renderReleaseReport, renderSteeringText, renderSupervisorRunHeadline, renderSupervisorRunMarkdown, renderUpstreamFindings, repeatedActionDetector, replayFeedbackTrajectories, replayFeedbackTrajectory, replayScorerOverCorpus, replayTraceThroughJudge, requireAgentProfileCell, requiredPairedSampleSize, requiredSampleSize, researchReport, resolveModelPricing, resolveSeat, rolloutReward, rollupSupervisorRuns, roundTripRunRecord, routeFields, rowCount, rowWhere, runAgentControlLoop, runAssertions, runBehavioralCanaries, runCanaries, runCounterfactual, runE2EWorkflow, runEvalCampaign, runExpectations, runFailureClass, runHarnessExperiment, runIntentMatchJudge, runJudgeFleet, runKeywordCoverageJudge, runKeywordCoverageJudgeUrl, runLiveProof, runProposeReview, runProposeReviewAsControlLoop, runRecordToProductBenchmarkRecord, runReferenceEquivalenceJudge, runReferenceReplay, runScore, runSelfPlay, runSemanticConceptJudge, runTaskScore, runTestGradedScenario, runsForScenario, scalarScore, scanForMuffledGates, scoreContinuity, scoreFromEvals, scoreKnowledgeReadiness, scorePrReviewComments, scorePrReviewSource, scoreRedTeamOutput, scoreReferenceReplay, scoreTraceInsightReadiness, seatPresets, securityJudge, selectHarnessVariant, selfPreference, sentenceReorderMutator, serializeFeedbackTrajectoriesJsonl, showMeasured, signManifest, spearmanR, statusAdvanced, stopOnNoProgress, stopOnRepeatedAction, stringField, stripFencedJson, subjectiveEval, summarizeAgentReceiptIntegrity, summarizeBackendIntegrity, summarizeHarnessResults, summarizePrReviewBenchmark, summarizePreferenceMemory, summaryTable, supervisorRunRolloutLines, testJudge, textInSnapshot, throwIfRunIncomplete, toAgentProfileJson, toJsonl, toLangfuseEnvelope, toOpenAiTool, toPrometheusText, toRewardRows, toSftRows, tokenizeDomainWords, toolNamesForRun, toolSpans, traceAnalystFunctionGroup, traceAnalystOnRunComplete, traceContract, traceJudge, traceJudgeEnsemble, traceSpanKindToOpenInferenceKind, tracedAnalyzeTraces, typoMutator, urlContains, userQuestionsForKnowledgeGaps, validateAgentProfileCell, validateProductBenchmarkManifest, validateProductBenchmarkRecord, validateProductBenchmarkRun, validateRolloutLine, validateRunRecord, verbosityBias, verifyAgentProfileCell, verifyAttestation, verifyCompletion, verifyManifest, visualDiff, viteDeployRunner, vitestTestParser, weightedComposite, weightedMean, weightedRecall, welchsTTest, whitespaceCollapseMutator, wilcoxonSignedRank, wilson, withAssignedFeedbackSplit, withHeldoutBlend, withJudgeRetry, withOtelPipeline, wranglerDeployRunner, writeSupervisorRunReport };