@tangle-network/agent-eval 0.40.5 → 0.42.0

package/dist/sequential-DdV5ShjT.d.ts

@@ -0,0 +1,561 @@
+import { R as RunRecord } from './run-record-BGY6bHRh.js';
+import { F as FailureClusterReport } from './failure-cluster-Cw65_5FY.js';
+
+/**
+ * HeldOutGate — first-class held-out paired-delta promotion gate.
+ *
+ * Encodes the "honesty override" pattern that lived inline in
+ * `~/webb/redteam/scripts/agent-eval-autoresearch.ts:138–171`.
+ * The optimizer's best-guess is one thing; what we should actually
+ * ship is another. The gate is the line between them.
+ *
+ * A candidate is promoted iff ALL three pass:
+ *
+ *   1. **Productive runs**: the candidate has at least
+ *      `minProductiveRuns` paired observations on items where BOTH
+ *      candidate and baseline produced a real (non-silent) score.
+ *   2. **Paired delta**: the lower bound of the bootstrap CI on the
+ *      median per-item delta (candidate − baseline) on the HOLDOUT
+ *      split is strictly greater than `pairedDeltaThreshold`.
+ *   3. **Overfit gap**: the candidate's gap between search-split
+ *      score and holdout-split score is no worse (more positive)
+ *      than the baseline's gap by more than `overfitGapThreshold`.
+ *      "Better on search, worse on holdout" is the canonical
+ *      overfit pattern; this catches it.
+ *
+ * The decision carries a machine-readable `rejectionCode` plus an
+ * `evidence` block with every number the gate looked at, so the
+ * downstream researcher / paper / dashboard can re-derive the
+ * verdict without re-running.
+ *
+ * See also:
+ *   - `src/statistics.ts` for `pairedBootstrap` + `wilcoxonSignedRank`
+ *   - `src/run-record.ts` for the input row schema
+ *   - `src/reference-replay.ts` for the older, reference-replay-
+ *     specific promotion path (still useful for replay-style evals).
+ */
+
+type HeldOutGateRejectionCode = 'few_runs' | 'negative_delta' | 'overfit_gap';
+interface HeldOutGateConfig {
+    /** Minimum number of paired (candidate, baseline) holdout observations
+     *  required before the gate will even consider promoting. Default 3. */
+    minProductiveRuns?: number;
+    /** The bootstrap-CI lower bound on the median paired holdout delta
+     *  must exceed this to promote. Default 0. */
+    pairedDeltaThreshold?: number;
+    /** Maximum allowed worsening of (search − holdout) gap relative to
+     *  baseline. Default 0.15 (i.e. candidate may overfit by up to 15
+     *  absolute score points more than baseline before rejection). */
+    overfitGapThreshold?: number;
+    /** Stable label of the baseline candidate. Required — paper-grade
+     *  evaluation never compares two unlabelled candidates. */
+    baselineKey: string;
+    /** Confidence level for the bootstrap CI. Default 0.95. */
+    confidence?: number;
+    /** Bootstrap resamples. Default 2000. */
+    bootstrapResamples?: number;
+    /** Optional deterministic seed for the bootstrap. Default undefined
+     *  (Math.random). */
+    seed?: number;
+}
+interface GateEvidence {
+    /** Number of paired (candidate, baseline) holdout observations used. */
+    productiveRuns: number;
+    /** Median of (candidate − baseline) paired holdout deltas. */
+    medianPairedDelta: number;
+    /** Bootstrap CI on the median paired holdout delta. */
+    pairedCI: {
+        low: number;
+        high: number;
+    };
+    /** Wilcoxon signed-rank p-value on the paired holdout deltas. */
+    pairedPValue: number;
+    /** Mean candidate score on the search split (NaN if none). */
+    searchScore: number;
+    /** Mean candidate score on the holdout split (NaN if none). */
+    holdoutScore: number;
+    /** Candidate (search − holdout) gap. */
+    overfitGap: number;
+    /** Baseline (search − holdout) gap. */
+    baselineOverfitGap: number;
+}
+interface GateDecision {
+    /** Final promote/no-promote verdict. */
+    promote: boolean;
+    /** The candidate that was evaluated. */
+    candidateId: string;
+    /** The baseline it was compared against. */
+    baselineId: string;
+    /** Every number the gate looked at, for audit + paper export. */
+    evidence: GateEvidence;
+    /** Human-readable reason. */
+    reason: string;
+    /** Machine-readable rejection code, or null on promote. */
+    rejectionCode: HeldOutGateRejectionCode | null;
+}
+/**
+ * Held-out paired-delta promotion gate. Construct once with config,
+ * call `evaluate(candidateRuns, baselineRuns)` per (candidate,
+ * baseline) pair. Stateless across calls.
+ */
+declare class HeldOutGate {
+    private readonly minProductiveRuns;
+    private readonly pairedDeltaThreshold;
+    private readonly overfitGapThreshold;
+    private readonly baselineKey;
+    private readonly confidence;
+    private readonly resamples;
+    private readonly seed?;
+    constructor(config: HeldOutGateConfig);
+    /** Decide whether `candidate` should replace `baseline`. Pairing
+     *  is by (experimentId, seed) — identical experiment + seed pairs
+     *  the candidate run with the matching baseline run. Pairs without
+     *  a holdout score on both sides are dropped. */
+    evaluate(candidate: RunRecord[], baseline: RunRecord[]): GateDecision;
+}
+
+/**
+ * Reporting helpers — production summaries and paper-quality figures — sit alongside `reporter.ts` rather
+ * than replacing it.
+ *
+ * Three artefacts:
+ *
+ *   - `summaryTable`           Markdown table of per-candidate means,
+ *                            95% bootstrap CIs, BH-adjusted Wilcoxon
+ *                            p-values, and Cohen's d versus a
+ *                            comparator candidate.
+ *   - `paretoChart`         Abstract spec for a cost vs quality
+ *                            scatter, with gate decisions overlaid.
+ *                            Returns numbers + labels — caller
+ *                            chooses the plotting library.
+ *   - `gainHistogram`
+ *                            Per-item paired holdout deltas as a
+ *                            histogram spec (bins + counts + median +
+ *                            CI). Same "data, not images" contract.
+ *
+ * The figure types are PlotSpecs — JSON-friendly, library-agnostic.
+ * They aren't React components and they aren't PNGs; they are
+ * what you'd hand to vega-lite, plotly, matplotlib, or your own
+ * Canvas renderer to draw the actual figure.
+ */
+
+interface SummaryTableOptions {
+    /** Comparator candidate id. Wilcoxon + Cohen's d are computed
+     *  versus this candidate. Required for paired stats columns. */
+    comparator?: string;
+    /** Which split to read scores from. Default 'holdout'. */
+    split?: 'search' | 'holdout';
+    /** Confidence level for the bootstrap CI on the mean. Default 0.95. */
+    confidence?: number;
+    /** FDR for BH adjustment of the comparison p-values. Default 0.05. */
+    fdr?: number;
+}
+interface SummaryTableRow {
+    candidateId: string;
+    n: number;
+    mean: number;
+    ciLow: number;
+    ciHigh: number;
+    /** BH-adjusted q-value vs comparator. NaN if no comparator. */
+    qValue: number;
+    /** Cohen's d vs comparator. NaN if no comparator. */
+    cohensD: number;
+}
+interface SummaryTable {
+    rows: SummaryTableRow[];
+    comparator: string | null;
+    split: 'search' | 'holdout';
+    /** Pre-rendered markdown — drop into a paper or PR. */
+    markdown: string;
+}
+/**
+ * Table 1 helper. Buckets runs by `candidateId`, computes mean +
+ * bootstrap CI on the chosen split, and (when a comparator is given)
+ * BH-adjusted Wilcoxon p + Cohen's d versus that comparator.
+ */
+declare function summaryTable(runs: RunRecord[], opts?: SummaryTableOptions): SummaryTable;
+interface ParetoPoint {
+    candidateId: string;
+    /** Mean USD cost per run on the chosen split. */
+    cost: number;
+    /** Mean score on the chosen split. */
+    quality: number;
+    /** Number of runs that informed this point. */
+    n: number;
+    /** Whether this candidate is on the Pareto frontier — high
+     *  quality, low cost, no dominator. */
+    onFrontier: boolean;
+    /** Optional gate verdict for this candidate, if a `GateDecision`
+     *  for it was passed in. */
+    gate?: 'promote' | 'reject_few_runs' | 'reject_negative_delta' | 'reject_overfit_gap' | null;
+}
+interface ParetoFigureSpec {
+    kind: 'pareto-cost-quality';
+    split: 'search' | 'holdout';
+    points: ParetoPoint[];
+    axes: {
+        x: 'costUsd';
+        y: 'score';
+    };
+}
+/**
+ * Cost vs quality scatter spec. `gateDecisions` is keyed by
+ * candidate id; if present, every point picks up the gate verdict
+ * for overlay.
+ */
+declare function paretoChart(runs: RunRecord[], opts?: {
+    split?: 'search' | 'holdout';
+    gateDecisions?: Record<string, GateDecision>;
+}): ParetoFigureSpec;
+interface GainDistributionBin {
+    /** Inclusive lower edge. */
+    lo: number;
+    /** Exclusive upper edge (or inclusive if it's the last bin). */
+    hi: number;
+    /** Number of pairs whose delta lands in this bin. */
+    count: number;
+}
+interface GainDistributionFigureSpec {
+    kind: 'gain-distribution';
+    candidateId: string;
+    comparator: string;
+    split: 'search' | 'holdout';
+    /** Number of pairs used. */
+    n: number;
+    bins: GainDistributionBin[];
+    median: number;
+    ci: {
+        low: number;
+        high: number;
+    };
+}
+interface GainDistributionOptions {
+    /** Number of histogram bins. Default 11 (so the centre is exact at 0). */
+    bins?: number;
+    /** Which split to use. Default 'holdout'. */
+    split?: 'search' | 'holdout';
+    /** Confidence level for the CI. Default 0.95. */
+    confidence?: number;
+    /** Bootstrap resamples. Default 2000. */
+    resamples?: number;
+    /** Deterministic seed. */
+    seed?: number;
+}
+/**
+ * Held-out improvement distribution: per-pair delta (candidate −
+ * comparator), histogrammed. Includes the bootstrap CI on the median
+ * delta — same primitive the promotion gate uses.
+ */
+declare function gainHistogram(runs: RunRecord[], candidateId: string, comparator: string, opts?: GainDistributionOptions): GainDistributionFigureSpec;
+type ResearchReportDecision = 'promote' | 'hold' | 'reject' | 'equivalent' | 'needs_more_data';
+/**
+ * Hard floor below which a paired comparison is treated as uninformative
+ * regardless of `minPairs`. Mirrors the lower limit on Wilcoxon signed-rank
+ * exact tables; below this the test has no power to separate effect sizes.
+ */
+declare const RESEARCH_REPORT_HARD_PAIR_FLOOR = 6;
+interface ResearchReportOptions {
+    /** Human-readable report title. */
+    title?: string;
+    /** Comparator candidate id. Required for statistical decision guidance. */
+    comparator?: string;
+    /** Which split to use for the primary decision. Default 'holdout'. */
+    split?: 'search' | 'holdout';
+    /** Confidence level used by lower-level report helpers. Default 0.95. */
+    confidence?: number;
+    /** FDR threshold for q-values. Default 0.05. */
+    fdr?: number;
+    /**
+     * Soft floor on paired observations before issuing a directional
+     * promote / reject. Below this we report `needs_more_data` and surface the
+     * minimum detectable effect at the current N. Default 20 — chosen so the
+     * Wilcoxon signed-rank approximation is reasonable and so the paired
+     * bootstrap CI has non-degenerate coverage. Hard floor is enforced at
+     * `RESEARCH_REPORT_HARD_PAIR_FLOOR` (6) regardless of this value.
+     */
+    minPairs?: number;
+    /**
+     * Region of Practical Equivalence on the paired delta. When a candidate's
+     * paired-delta CI is fully contained in `[low, high]`, the decision is
+     * `equivalent` rather than `hold`. Sourced from the domain owner — there is
+     * no statistically-defensible default.
+     */
+    rope?: {
+        low: number;
+        high: number;
+    };
+    /**
+     * Power for the minimum detectable effect (MDE) reported on each candidate.
+     * Default 0.8.
+     */
+    mdePower?: number;
+    /**
+     * Two-sided alpha for the MDE. Default matches `fdr` so the reported MDE
+     * lines up with the test the report actually runs.
+     */
+    mdeAlpha?: number;
+    /** Optional held-out gate decisions keyed by candidate id. */
+    gateDecisions?: Record<string, GateDecision>;
+    /** Optional failure clusters from failureClusterView. */
+    failureClusters?: FailureClusterReport;
+    /** Build gain histograms for these candidates. Defaults to all non-comparator candidates. */
+    candidateIds?: string[];
+    /** Deterministic bootstrap seed passed to gainHistogram and the posterior helper. */
+    seed?: number;
+    /** Report timestamp. Defaults to current time. */
+    generatedAt?: string;
+    /**
+     * Hash of a preregistered protocol (e.g. `signManifest({...}).contentHash`).
+     * Embedded verbatim in the report so the analysis can be cited as the
+     * preregistered one rather than a post-hoc fishing expedition.
+     */
+    preregistrationHash?: string;
+}
+interface ResearchReportRecommendation {
+    decision: ResearchReportDecision;
+    candidateId: string | null;
+    rationale: string[];
+    risks: string[];
+    nextActions: string[];
+}
+interface ResearchReportCandidate {
+    candidateId: string;
+    n: number;
+    mean: number;
+    ciLow: number;
+    ciHigh: number;
+    qValue: number;
+    cohensD: number;
+    meanDeltaVsComparator: number | null;
+    pairedN: number;
+    medianGain: number | null;
+    meanGain: number | null;
+    gainCi: {
+        low: number;
+        high: number;
+    } | null;
+    /**
+     * Bayesian-bootstrap-style posterior summaries on the paired delta. Computed
+     * from the same resamples that produce the gain CI; interpretable as
+     * "fraction of resamples in which the candidate beats the comparator on
+     * matched pairs."
+     */
+    prGreaterThanZero: number | null;
+    prInRope: number | null;
+    /**
+     * Minimum detectable effect (in score units) at the candidate's paired N,
+     * the configured power, and the configured alpha. Standardised by the
+     * observed paired-delta SD and inverted via `requiredSampleSize`. Reported
+     * for every candidate so a `needs_more_data` verdict is actionable.
+     */
+    mde: number | null;
+    onParetoFrontier: boolean;
+    gate?: ParetoPoint['gate'];
+    decision: ResearchReportDecision;
+    decisionReason: string;
+}
+interface ResearchReportMethodology {
+    /**
+     * Plain-language assumptions the report depends on. Read these first when
+     * deciding whether the verdict is load-bearing for a launch decision.
+     */
+    assumptions: string[];
+    /** Tests and estimators the verdict was computed from. */
+    methods: string[];
+    /** Alternatives the author considered and why this report didn't take them. */
+    alternatives: string[];
+    /** Failure modes — when this report should NOT drive a decision. */
+    whenNotToApply: string[];
+    /** Citations for the methodological choices above. */
+    citations: string[];
+}
+interface ResearchReport {
+    kind: 'agent-eval-research-report';
+    title: string;
+    generatedAt: string;
+    split: 'search' | 'holdout';
+    comparator: string | null;
+    /**
+     * SHA-256 over the canonicalised set of `(runId, candidateId, split)` triples
+     * the report was computed from, plus the comparator and split. Stable across
+     * key insertion order; recomputable by the reader to verify provenance.
+     */
+    runFingerprint: string;
+    preregistrationHash: string | null;
+    rope: {
+        low: number;
+        high: number;
+    } | null;
+    executiveSummary: string[];
+    recommendation: ResearchReportRecommendation;
+    candidates: ResearchReportCandidate[];
+    summary: SummaryTable;
+    charts: {
+        pareto: ParetoFigureSpec;
+        gains: GainDistributionFigureSpec[];
+    };
+    methodology: ResearchReportMethodology;
+    failureClusters?: FailureClusterReport;
+    markdown: string;
+    html: string;
+}
+/**
+ * Executive research report for CPO / AI-lead / launch-review consumption.
+ *
+ * Composes:
+ *   - `summaryTable`         marginal stats with BH-FDR-adjusted q-values
+ *   - `paretoChart`           cost-vs-quality frontier with gate overlay
+ *   - `gainHistogram`         per-candidate paired-delta distribution
+ *   - paired posterior (this file): bootstrap CI on median, Pr(Δ>0),
+ *                              Pr(Δ∈ROPE), MDE at the configured power
+ *
+ * Decisions are made on paired evidence — never on marginal means alone —
+ * and respect any held-out gate decision the caller passes through. The
+ * report embeds a SHA-256 fingerprint of the input run set and, optionally,
+ * the hash of a preregistered protocol so a downstream reader can verify
+ * provenance and that the analysis was the preregistered one.
+ *
+ * Async because the fingerprint uses Web Crypto via `hashJson`; deterministic
+ * for any fixed `runs`, `seed`, and ROPE.
+ */
+declare function researchReport(runs: RunRecord[], opts?: ResearchReportOptions): Promise<ResearchReport>;
+
+/**
+ * Always-valid sequential evaluation.
+ *
+ * `researchReport` assumes a single pre-specified analysis. Real
+ * consumers run campaigns weekly / nightly / per-PR; each new run silently
+ * inflates the false-discovery rate, because the BH-FDR guarantee is for
+ * the *first* look, not the 47th. Without time-uniform inference,
+ * launch-decision teams either (a) don't peek, which forfeits the cost
+ * advantage of stop-when-decisive, or (b) peek and pretend they didn't,
+ * which forfeits scientific validity.
+ *
+ * This module ships **e-value-based confidence sequences** for paired
+ * bounded outcomes. The methodology is the predictable plug-in betting
+ * martingale of Waudby-Smith & Ramdas (2024) — provably valid at *any*
+ * stopping time. Concretely:
+ *
+ *   For paired deltas D_1, D_2, … ∈ [-c, c] with the null H_0: E[D] ≤ 0,
+ *   a betting fraction λ_i is chosen using only D_{1..i-1} (predictable
+ *   plug-in), and the running e-value is
+ *
+ *     E_t = ∏_{i=1}^{t} (1 + λ_i · D_i)
+ *
+ *   E_t is a non-negative martingale under H_0 with E[E_t] ≤ 1, so by
+ *   Ville's inequality, P(∃ t : E_t ≥ 1/α) ≤ α — we can reject the null
+ *   at any time without inflating the type-I error.
+ *
+ * Combined with `runEvalCampaign`, every consumer running rolling
+ * campaigns gains the ability to ship the moment evidence is decisive,
+ * stop-early on dead-on-arrival variants, and accumulate evidence across
+ * partial runs without spending the FDR budget. No new sweep is wasted.
+ *
+ * References:
+ *   - Howard, S. R., Ramdas, A., McAuliffe, J., Sekhon, J. (2021).
+ *     Time-uniform, nonparametric, nonasymptotic confidence sequences.
+ *     Annals of Statistics, 49(2), 1055–1080.
+ *   - Waudby-Smith, I., Ramdas, A. (2024). Estimating means of bounded
+ *     random variables by betting. JRSS B, 86(1), 1–27.
+ */
+type SequentialDecision = 'promote_now' | 'continue' | 'reject_now' | 'equivalent';
+interface PairedEvalueOptions {
+    /**
+     * Bound on |delta|. Default 1 (matching most score scales). Must satisfy
+     * c > 0; deltas outside [-c, c] are clipped with a warning attached to
+     * the return value.
+     */
+    bound?: number;
+    /** Target Type-I error. Default 0.05. */
+    alpha?: number;
+    /**
+     * Region of Practical Equivalence on the *mean* paired delta. When
+     * supplied, the verdict can return `'equivalent'` once the running
+     * confidence sequence on the mean is fully contained in [low, high].
+     */
+    rope?: {
+        low: number;
+        high: number;
+    };
+    /** Initial bet shrinkage (0 < scale ≤ 1). Default 0.5 — empirically robust. */
+    initialBetShrinkage?: number;
+}
+interface PairedEvalueStep {
+    /** 1-indexed observation count. */
+    t: number;
+    delta: number;
+    /** Running e-value E_t = ∏ (1 + λ_i · D_i). */
+    evalue: number;
+    /** Time-uniform p-value at stopping time t. */
+    pValue: number;
+    /** Lower bound of the empirical Bernstein confidence sequence at level 1-α. */
+    csLow: number;
+    csHigh: number;
+    /** Verdict at this stopping time. */
+    decision: SequentialDecision;
+}
+interface PairedEvalueSequence {
+    steps: PairedEvalueStep[];
+    /** The decision at the final step. */
+    finalDecision: SequentialDecision;
+    /** Index (1-based) at which a non-`continue` decision first fired, or null. */
+    decisionFiredAt: number | null;
+    /** True if any deltas were clipped to [-bound, bound]. */
+    clipped: boolean;
+}
+/**
+ * Run the paired e-value sequence over an in-order delta stream.
+ *
+ * Use for *streaming* / interim analyses: pass the deltas you have so
+ * far, get the verdict at every prefix length. The decision is
+ * monotone-stable in the sense that once `'reject_now'` or `'promote_now'`
+ * fires, the verdict at later steps remains decisive (the e-value is a
+ * non-negative martingale; once it crosses the threshold, it's crossed).
+ */
+declare function pairedEvalueSequence(deltas: number[], opts?: PairedEvalueOptions): PairedEvalueSequence;
+interface InterimReleaseConfidenceInput {
+    /**
+     * One delta series per candidate (paired deltas vs comparator). Order
+     * within a series is the order the campaigns were run.
+     */
+    deltaSeries: Array<{
+        candidateId: string;
+        deltas: number[];
+    }>;
+    alpha?: number;
+    bound?: number;
+    rope?: {
+        low: number;
+        high: number;
+    };
+}
+interface InterimReleaseConfidence {
+    candidates: Array<{
+        candidateId: string;
+        decision: SequentialDecision;
+        decisionFiredAt: number | null;
+        finalEvalue: number;
+        finalPValue: number;
+        pairs: number;
+        csLow: number;
+        csHigh: number;
+    }>;
+    /**
+     * Campaign-level recommendation: pick the strongest 'promote_now', else
+     * 'continue' if any candidate is still live, else 'reject_now' if every
+     * candidate is dead, else 'equivalent'.
+     */
+    recommendation: {
+        decision: SequentialDecision;
+        candidateId: string | null;
+    };
+}
+/**
+ * Run interim sequential analyses across many candidates at once,
+ * preserving the time-uniform α guarantee for each candidate's series and
+ * synthesising a campaign-level recommendation. Designed to be called on
+ * every campaign tick — the recommendation is anytime-valid.
+ */
+declare function evaluateInterimReleaseConfidence(input: InterimReleaseConfidenceInput): InterimReleaseConfidence;
+
+export { type GainDistributionBin as G, HeldOutGate as H, type InterimReleaseConfidence as I, type PairedEvalueOptions as P, RESEARCH_REPORT_HARD_PAIR_FLOOR as R, type SequentialDecision as S, type GainDistributionFigureSpec as a, type GainDistributionOptions as b, type InterimReleaseConfidenceInput as c, type PairedEvalueSequence as d, type PairedEvalueStep as e, type ParetoFigureSpec as f, type ParetoPoint as g, type ResearchReport as h, type ResearchReportCandidate as i, type ResearchReportDecision as j, type ResearchReportMethodology as k, type ResearchReportOptions as l, type ResearchReportRecommendation as m, type SummaryTable as n, type SummaryTableOptions as o, type SummaryTableRow as p, evaluateInterimReleaseConfidence as q, gainHistogram as r, pairedEvalueSequence as s, paretoChart as t, researchReport as u, summaryTable as v, type GateDecision as w, type GateEvidence as x, type HeldOutGateConfig as y, type HeldOutGateRejectionCode as z };

package/dist/traces.d.ts

@@ -1,8 +1,9 @@
 import { N as NotFoundError, R as ReplayError } from './errors-mje_cKOs.js';
-import { R as RawProviderSink, f as RawProviderEvent } from './integrity-DYR5gWlb.js';
-export { F as FileSystemRawProviderSink, c as FileSystemRawProviderSinkOptions, I as InMemoryRawProviderSink, d as InMemoryRawProviderSinkOptions, N as NoopRawProviderSink, P as ProviderRedactor, e as RawProviderDirection, g as RawProviderSinkFilter, h as RunIntegrityError, a as RunIntegrityExpectations, i as RunIntegrityIssue, j as RunIntegrityIssueCode, b as RunIntegrityReport, k as assertRunCaptured, l as defaultProviderRedactor, p as providerFromBaseUrl, t as throwIfRunIncomplete } from './integrity-DYR5gWlb.js';
+import { R as RawProviderSink, d as RawProviderEvent } from './raw-provider-sink-C46HDghv.js';
+export { F as FileSystemRawProviderSink, a as FileSystemRawProviderSinkOptions, I as InMemoryRawProviderSink, b as InMemoryRawProviderSinkOptions, N as NoopRawProviderSink, P as ProviderRedactor, c as RawProviderDirection, e as RawProviderSinkFilter, f as defaultProviderRedactor, p as providerFromBaseUrl } from './raw-provider-sink-C46HDghv.js';
 import { R as RunCompleteHook, a as RunCompleteHookContext } from './emitter-DP_cSSiw.js';
 export { S as SpanHandle, T as TraceEmitter, b as TraceEmitterOptions, l as llmSpanFromProvider } from './emitter-DP_cSSiw.js';
+export { b as RunIntegrityError, R as RunIntegrityExpectations, c as RunIntegrityIssue, d as RunIntegrityIssueCode, a as RunIntegrityReport, e as assertRunCaptured, t as throwIfRunIncomplete } from './integrity-CTDhR1Sg.js';
 import { T as TraceStore } from './store-Db2Bv8Cf.js';
 export { A as Artifact, B as BudgetLedgerEntry, g as BudgetSpec, i as EventFilter, E as EventKind, j as FAILURE_CLASSES, F as FailureClass, k as FileSystemTraceStore, l as FileSystemTraceStoreOptions, G as GenericSpan, I as InMemoryTraceStore, J as JudgeSpan, L as LlmSpan, M as Message, e as RetrievalSpan, R as Run, h as RunFilter, m as RunLayer, c as RunOutcome, n as RunStatus, f as SandboxSpan, S as Span, o as SpanBase, p as SpanFilter, d as SpanKind, q as SpanStatus, r as TRACE_SCHEMA_VERSION, a as ToolSpan, b as TraceEvent, s as isJudgeSpan, t as isLlmSpan, u as isRetrievalSpan, v as isSandboxSpan, w as isToolSpan } from './store-Db2Bv8Cf.js';
 export { a as aggregateLlm, b as argHash, g as groupBy, j as judgeSpans, l as llmSpans, r as runFailureClass, c as runsForScenario, t as toolSpans } from './query-DODUYdPg.js';

package/dist/traces.js

@@ -63,6 +63,11 @@ import {
   assertRunCaptured,
   throwIfRunIncomplete
 } from "./chunk-UBPIXOC4.js";
+import {
+  TraceEmitter,
+  llmSpanFromProvider
+} from "./chunk-TVVP3ZZQ.js";
+import "./chunk-VSMTAMNK.js";
 import {
   FileSystemRawProviderSink,
   InMemoryRawProviderSink,
@@ -70,11 +75,6 @@ import {
   defaultProviderRedactor,
   providerFromBaseUrl
 } from "./chunk-PC4UYEBM.js";
-import {
-  TraceEmitter,
-  llmSpanFromProvider
-} from "./chunk-TVVP3ZZQ.js";
-import "./chunk-VSMTAMNK.js";
 import "./chunk-QYJT52YW.js";
 import "./chunk-PZ5AY32C.js";
 export {