@tangle-network/agent-eval 0.117.1 → 0.118.0

package/dist/summary-report-C5bKFfm-.d.ts

@@ -1,445 +0,0 @@
-import { R as RunRecord } from './run-record-BDH49H2E.js';
-import { F as FailureClusterReport } from './failure-cluster-DOAcSJ87.js';
-
-/**
- * HeldOutGate — first-class held-out paired-delta promotion gate.
- *
- * Encodes the "honesty override" pattern that lived inline in
- * `~/webb/redteam/scripts/agent-eval-autoresearch.ts:138–171`.
- * The optimizer's best-guess is one thing; what we should actually
- * ship is another. The gate is the line between them.
- *
- * A candidate is promoted iff ALL three pass:
- *
- *   1. **Productive runs**: the candidate has at least
- *      `minProductiveRuns` paired observations on items where BOTH
- *      candidate and baseline produced a real (non-silent) score.
- *   2. **Paired delta**: the lower bound of the bootstrap CI on the
- *      median per-item delta (candidate − baseline) on the HOLDOUT
- *      split is strictly greater than `pairedDeltaThreshold`.
- *   3. **Overfit gap**: the candidate's gap between search-split
- *      score and holdout-split score is no worse (more positive)
- *      than the baseline's gap by more than `overfitGapThreshold`.
- *      "Better on search, worse on holdout" is the canonical
- *      overfit pattern; this catches it.
- *
- * The decision carries a machine-readable `rejectionCode` plus an
- * `evidence` block with every number the gate looked at, so the
- * downstream researcher / paper / dashboard can re-derive the
- * verdict without re-running.
- *
- * See also:
- *   - `src/statistics.ts` for `pairedBootstrap` + `wilcoxonSignedRank`
- *   - `src/run-record.ts` for the input row schema
- *   - `src/reference-replay.ts` for the older, reference-replay-
- *     specific promotion path (still useful for replay-style evals).
- */
-
-type HeldOutGateRejectionCode = 'few_runs' | 'negative_delta' | 'overfit_gap' | 'cost_ceiling';
-interface HeldOutGateConfig {
-    /** Minimum number of paired (candidate, baseline) holdout observations
-     *  required before the gate will even consider promoting. Default 3. */
-    minProductiveRuns?: number;
-    /** The bootstrap-CI lower bound on the median paired holdout delta
-     *  must exceed this to promote. Default 0. */
-    pairedDeltaThreshold?: number;
-    /** Maximum allowed worsening of (search − holdout) gap relative to
-     *  baseline. Default 0.15 (i.e. candidate may overfit by up to 15
-     *  absolute score points more than baseline before rejection). */
-    overfitGapThreshold?: number;
-    /** Stable label of the baseline candidate. Required — paper-grade
-     *  evaluation never compares two unlabelled candidates. */
-    baselineKey: string;
-    /** Confidence level for the bootstrap CI. Default 0.95. */
-    confidence?: number;
-    /** Bootstrap resamples. Default 2000. */
-    bootstrapResamples?: number;
-    /** Optional deterministic seed for the bootstrap. Default undefined
-     *  (Math.random). */
-    seed?: number;
-    /**
-     * Hard ceiling on the candidate's median per-task USD cost. When the
-     * candidate clears the quality gates (paired-delta + overfit-gap) but
-     * its median cost exceeds this number, the gate rejects with
-     * `cost_ceiling`. Default `undefined` = no cost ceiling, behaving
-     * exactly like the pre-cost gate.
-     *
-     * This exists because "we ship the better prompt" is only an honest
-     * pitch when the better prompt also fits a customer-stated budget.
-     * Cost is read from `RunRecord.costUsd` (already mandatory on every
-     * run) so no new schema is required.
-     */
-    costPerTaskCeiling?: number;
-}
-interface GateEvidence {
-    /** Number of paired (candidate, baseline) holdout observations used. */
-    productiveRuns: number;
-    /** Median of (candidate − baseline) paired holdout deltas. */
-    medianPairedDelta: number;
-    /** Bootstrap CI on the median paired holdout delta. */
-    pairedCI: {
-        low: number;
-        high: number;
-    };
-    /** Wilcoxon signed-rank p-value on the paired holdout deltas. */
-    pairedPValue: number;
-    /** Mean candidate score on the search split (NaN if none). */
-    searchScore: number;
-    /** Mean candidate score on the holdout split (NaN if none). */
-    holdoutScore: number;
-    /** Candidate (search − holdout) gap. */
-    overfitGap: number;
-    /** Baseline (search − holdout) gap. */
-    baselineOverfitGap: number;
-    /** Median per-task USD cost across the candidate's runs. Recorded
-     *  even when no `costPerTaskCeiling` is configured so downstream
-     *  dashboards (intelligence.tangle.tools) can render \$/task per
-     *  generation regardless of gating policy. */
-    medianCandidateCost: number;
-    /** Median per-task USD cost across the baseline runs, for
-     *  symmetric reporting. */
-    medianBaselineCost: number;
-}
-interface GateDecision {
-    /** Final promote/no-promote verdict. */
-    promote: boolean;
-    /** The candidate that was evaluated. */
-    candidateId: string;
-    /** The baseline it was compared against. */
-    baselineId: string;
-    /** Every number the gate looked at, for audit + paper export. */
-    evidence: GateEvidence;
-    /** Human-readable reason. */
-    reason: string;
-    /** Machine-readable rejection code, or null on promote. */
-    rejectionCode: HeldOutGateRejectionCode | null;
-}
-/**
- * Held-out paired-delta promotion gate. Construct once with config,
- * call `evaluate(candidateRuns, baselineRuns)` per (candidate,
- * baseline) pair. Stateless across calls.
- */
-declare class HeldOutGate {
-    private readonly minProductiveRuns;
-    private readonly pairedDeltaThreshold;
-    private readonly overfitGapThreshold;
-    private readonly baselineKey;
-    private readonly confidence;
-    private readonly resamples;
-    private readonly seed?;
-    private readonly costPerTaskCeiling?;
-    constructor(config: HeldOutGateConfig);
-    /** Decide whether `candidate` should replace `baseline`. Pairing
-     *  is by (experimentId, seed) — identical experiment + seed pairs
-     *  the candidate run with the matching baseline run. Pairs without
-     *  a holdout score on both sides are dropped. */
-    evaluate(candidate: RunRecord[], baseline: RunRecord[]): GateDecision;
-}
-
-/**
- * Reporting helpers — production summaries and paper-quality figures — sit alongside `reporter.ts` rather
- * than replacing it.
- *
- * Three artefacts:
- *
- *   - `summaryTable`           Markdown table of per-candidate means,
- *                            95% bootstrap CIs, BH-adjusted Wilcoxon
- *                            p-values, and Cohen's d versus a
- *                            comparator candidate.
- *   - `paretoChart`         Abstract spec for a cost vs quality
- *                            scatter, with gate decisions overlaid.
- *                            Returns numbers + labels — caller
- *                            chooses the plotting library.
- *   - `gainHistogram`
- *                            Per-item paired holdout deltas as a
- *                            histogram spec (bins + counts + median +
- *                            CI). Same "data, not images" contract.
- *
- * The figure types are PlotSpecs — JSON-friendly, library-agnostic.
- * They aren't React components and they aren't PNGs; they are
- * what you'd hand to vega-lite, plotly, matplotlib, or your own
- * Canvas renderer to draw the actual figure.
- */
-
-interface SummaryTableOptions {
-    /** Comparator candidate id. Wilcoxon + Cohen's d are computed
-     *  versus this candidate. Required for paired stats columns. */
-    comparator?: string;
-    /** Which split to read scores from. Default 'holdout'. */
-    split?: 'search' | 'holdout';
-    /** Confidence level for the bootstrap CI on the mean. Default 0.95. */
-    confidence?: number;
-    /** FDR for BH adjustment of the comparison p-values. Default 0.05. */
-    fdr?: number;
-}
-interface SummaryTableRow {
-    candidateId: string;
-    n: number;
-    mean: number;
-    ciLow: number;
-    ciHigh: number;
-    /** BH-adjusted q-value vs comparator. NaN if no comparator. */
-    qValue: number;
-    /** Cohen's d vs comparator. NaN if no comparator. */
-    cohensD: number;
-}
-interface SummaryTable {
-    rows: SummaryTableRow[];
-    comparator: string | null;
-    split: 'search' | 'holdout';
-    /** Pre-rendered markdown — drop into a paper or PR. */
-    markdown: string;
-}
-/**
- * Table 1 helper. Buckets runs by `candidateId`, computes mean +
- * bootstrap CI on the chosen split, and (when a comparator is given)
- * BH-adjusted Wilcoxon p + Cohen's d versus that comparator.
- */
-declare function summaryTable(runs: RunRecord[], opts?: SummaryTableOptions): SummaryTable;
-interface ParetoPoint {
-    candidateId: string;
-    /** Mean USD cost per run on the chosen split. */
-    cost: number;
-    /** Mean score on the chosen split. */
-    quality: number;
-    /** Number of runs that informed this point. */
-    n: number;
-    /** Whether this candidate is on the Pareto frontier — high
-     *  quality, low cost, no dominator. */
-    onFrontier: boolean;
-    /** Optional gate verdict for this candidate, if a `GateDecision`
-     *  for it was passed in. */
-    gate?: 'promote' | 'reject_few_runs' | 'reject_negative_delta' | 'reject_overfit_gap' | null;
-}
-interface ParetoFigureSpec {
-    kind: 'pareto-cost-quality';
-    split: 'search' | 'holdout';
-    points: ParetoPoint[];
-    axes: {
-        x: 'costUsd';
-        y: 'score';
-    };
-}
-/**
- * Cost vs quality scatter spec. `gateDecisions` is keyed by
- * candidate id; if present, every point picks up the gate verdict
- * for overlay.
- */
-declare function paretoChart(runs: RunRecord[], opts?: {
-    split?: 'search' | 'holdout';
-    gateDecisions?: Record<string, GateDecision>;
-}): ParetoFigureSpec;
-interface GainDistributionBin {
-    /** Inclusive lower edge. */
-    lo: number;
-    /** Exclusive upper edge (or inclusive if it's the last bin). */
-    hi: number;
-    /** Number of pairs whose delta lands in this bin. */
-    count: number;
-}
-interface GainDistributionFigureSpec {
-    kind: 'gain-distribution';
-    candidateId: string;
-    comparator: string;
-    split: 'search' | 'holdout';
-    /** Number of pairs used. */
-    n: number;
-    bins: GainDistributionBin[];
-    median: number;
-    ci: {
-        low: number;
-        high: number;
-    };
-}
-interface GainDistributionOptions {
-    /** Number of histogram bins. Default 11 (so the centre is exact at 0). */
-    bins?: number;
-    /** Which split to use. Default 'holdout'. */
-    split?: 'search' | 'holdout';
-    /** Confidence level for the CI. Default 0.95. */
-    confidence?: number;
-    /** Bootstrap resamples. Default 2000. */
-    resamples?: number;
-    /** Deterministic seed. */
-    seed?: number;
-}
-/**
- * Held-out improvement distribution: per-pair delta (candidate −
- * comparator), histogrammed. Includes the bootstrap CI on the median
- * delta — same primitive the promotion gate uses.
- */
-declare function gainHistogram(runs: RunRecord[], candidateId: string, comparator: string, opts?: GainDistributionOptions): GainDistributionFigureSpec;
-type ResearchReportDecision = 'promote' | 'hold' | 'reject' | 'equivalent' | 'needs_more_data';
-/**
- * Hard floor below which a paired comparison is treated as uninformative
- * regardless of `minPairs`. Mirrors the lower limit on Wilcoxon signed-rank
- * exact tables; below this the test has no power to separate effect sizes.
- */
-declare const RESEARCH_REPORT_HARD_PAIR_FLOOR = 6;
-interface ResearchReportOptions {
-    /** Human-readable report title. */
-    title?: string;
-    /** Comparator candidate id. Required for statistical decision guidance. */
-    comparator?: string;
-    /** Which split to use for the primary decision. Default 'holdout'. */
-    split?: 'search' | 'holdout';
-    /** Confidence level used by lower-level report helpers. Default 0.95. */
-    confidence?: number;
-    /** FDR threshold for q-values. Default 0.05. */
-    fdr?: number;
-    /**
-     * Soft floor on paired observations before issuing a directional
-     * promote / reject. Below this we report `needs_more_data` and surface the
-     * minimum detectable effect at the current N. Default 20 — chosen so the
-     * Wilcoxon signed-rank approximation is reasonable and so the paired
-     * bootstrap CI has non-degenerate coverage. Hard floor is enforced at
-     * `RESEARCH_REPORT_HARD_PAIR_FLOOR` (6) regardless of this value.
-     */
-    minPairs?: number;
-    /**
-     * Region of Practical Equivalence on the paired delta. When a candidate's
-     * paired-delta CI is fully contained in `[low, high]`, the decision is
-     * `equivalent` rather than `hold`. Sourced from the domain owner — there is
-     * no statistically-defensible default.
-     */
-    rope?: {
-        low: number;
-        high: number;
-    };
-    /**
-     * Power for the minimum detectable effect (MDE) reported on each candidate.
-     * Default 0.8.
-     */
-    mdePower?: number;
-    /**
-     * Two-sided alpha for the MDE. Default matches `fdr` so the reported MDE
-     * lines up with the test the report actually runs.
-     */
-    mdeAlpha?: number;
-    /** Optional held-out gate decisions keyed by candidate id. */
-    gateDecisions?: Record<string, GateDecision>;
-    /** Optional failure clusters from failureClusterView. */
-    failureClusters?: FailureClusterReport;
-    /** Build gain histograms for these candidates. Defaults to all non-comparator candidates. */
-    candidateIds?: string[];
-    /** Deterministic bootstrap seed passed to gainHistogram and the posterior helper. */
-    seed?: number;
-    /** Report timestamp. Defaults to current time. */
-    generatedAt?: string;
-    /**
-     * Hash of a preregistered protocol (e.g. `signManifest({...}).contentHash`).
-     * Embedded verbatim in the report so the analysis can be cited as the
-     * preregistered one rather than a post-hoc fishing expedition.
-     */
-    preregistrationHash?: string;
-}
-interface ResearchReportRecommendation {
-    decision: ResearchReportDecision;
-    candidateId: string | null;
-    rationale: string[];
-    risks: string[];
-    nextActions: string[];
-}
-interface ResearchReportCandidate {
-    candidateId: string;
-    n: number;
-    mean: number;
-    ciLow: number;
-    ciHigh: number;
-    qValue: number;
-    cohensD: number;
-    meanDeltaVsComparator: number | null;
-    pairedN: number;
-    medianGain: number | null;
-    meanGain: number | null;
-    gainCi: {
-        low: number;
-        high: number;
-    } | null;
-    /**
-     * Bayesian-bootstrap-style posterior summaries on the paired delta. Computed
-     * from the same resamples that produce the gain CI; interpretable as
-     * "fraction of resamples in which the candidate beats the comparator on
-     * matched pairs."
-     */
-    prGreaterThanZero: number | null;
-    prInRope: number | null;
-    /**
-     * Minimum detectable effect (in score units) at the candidate's paired N,
-     * the configured power, and the configured alpha. Standardised by the
-     * observed paired-delta SD and inverted via `requiredSampleSize`. Reported
-     * for every candidate so a `needs_more_data` verdict is actionable.
-     */
-    mde: number | null;
-    onParetoFrontier: boolean;
-    gate?: ParetoPoint['gate'];
-    decision: ResearchReportDecision;
-    decisionReason: string;
-}
-interface ResearchReportMethodology {
-    /**
-     * Plain-language assumptions the report depends on. Read these first when
-     * deciding whether the verdict is load-bearing for a launch decision.
-     */
-    assumptions: string[];
-    /** Tests and estimators the verdict was computed from. */
-    methods: string[];
-    /** Alternatives the author considered and why this report didn't take them. */
-    alternatives: string[];
-    /** Failure modes — when this report should NOT drive a decision. */
-    whenNotToApply: string[];
-    /** Citations for the methodological choices above. */
-    citations: string[];
-}
-interface ResearchReport {
-    kind: 'agent-eval-research-report';
-    title: string;
-    generatedAt: string;
-    split: 'search' | 'holdout';
-    comparator: string | null;
-    /**
-     * SHA-256 over the canonicalised set of `(runId, candidateId, split)` triples
-     * the report was computed from, plus the comparator and split. Stable across
-     * key insertion order; recomputable by the reader to verify provenance.
-     */
-    runFingerprint: string;
-    preregistrationHash: string | null;
-    rope: {
-        low: number;
-        high: number;
-    } | null;
-    executiveSummary: string[];
-    recommendation: ResearchReportRecommendation;
-    candidates: ResearchReportCandidate[];
-    summary: SummaryTable;
-    charts: {
-        pareto: ParetoFigureSpec;
-        gains: GainDistributionFigureSpec[];
-    };
-    methodology: ResearchReportMethodology;
-    failureClusters?: FailureClusterReport;
-    markdown: string;
-    html: string;
-}
-/**
- * Executive research report for CPO / AI-lead / launch-review consumption.
- *
- * Composes:
- *   - `summaryTable`         marginal stats with BH-FDR-adjusted q-values
- *   - `paretoChart`           cost-vs-quality frontier with gate overlay
- *   - `gainHistogram`         per-candidate paired-delta distribution
- *   - paired posterior (this file): bootstrap CI on median, Pr(Δ>0),
- *                              Pr(Δ∈ROPE), MDE at the configured power
- *
- * Decisions are made on paired evidence — never on marginal means alone —
- * and respect any held-out gate decision the caller passes through. The
- * report embeds a SHA-256 fingerprint of the input run set and, optionally,
- * the hash of a preregistered protocol so a downstream reader can verify
- * provenance and that the analysis was the preregistered one.
- *
- * Async because the fingerprint uses Web Crypto via `hashJson`; deterministic
- * for any fixed `runs`, `seed`, and ROPE.
- */
-declare function researchReport(runs: RunRecord[], opts?: ResearchReportOptions): Promise<ResearchReport>;
-
-export { type GainDistributionBin as G, HeldOutGate as H, type ParetoFigureSpec as P, RESEARCH_REPORT_HARD_PAIR_FLOOR as R, type SummaryTable as S, type GainDistributionFigureSpec as a, type GainDistributionOptions as b, type ParetoPoint as c, type ResearchReport as d, type ResearchReportCandidate as e, type ResearchReportDecision as f, type ResearchReportMethodology as g, type ResearchReportOptions as h, type ResearchReportRecommendation as i, type SummaryTableOptions as j, type SummaryTableRow as k, gainHistogram as l, type GateDecision as m, type GateEvidence as n, type HeldOutGateConfig as o, paretoChart as p, type HeldOutGateRejectionCode as q, researchReport as r, summaryTable as s };

package/dist/test-graded-scenario-B0ybnPY7.d.ts

@@ -1,166 +0,0 @@
-import { T as TraceEmitter } from './emitter-CjD7vUwv.js';
-import { R as Run, F as FailureClass } from './schema-B3Q3l9Z_.js';
-import { T as TraceStore } from './store-DGqD0Pyo.js';
-
-/**
- * SandboxHarness — executes a scenario in an isolated environment and
- * emits a rich SandboxSpan into the trace.
- *
- * Two built-in drivers:
- *   - `SubprocessSandboxDriver` — spawn in a local cwd with env vars.
- *     Fast, no dependencies, fine for unit tests and most CI gates.
- *   - `DockerSandboxDriver` — lifted from tangle-router's sandbox path;
- *     shells out to `docker run`. Stronger isolation, slower startup.
- *
- * Consumers implement `SandboxDriver` for custom backends (Firecracker,
- * Cloudflare sandbox product, etc.). The harness doesn't care which.
- */
-
-interface HarnessConfig {
-    /** Setup command (e.g. "pnpm install"). Non-zero exit fails the run. */
-    setupCommand?: string;
-    /** Run command (e.g. "pnpm build"). */
-    runCommand?: string;
-    /** Test command (e.g. "pnpm test --run"). Drives the test count + pass count. */
-    testCommand?: string;
-    /** Absolute cwd for the subprocess driver. Ignored by docker driver. */
-    cwd?: string;
-    /** Max wall-clock per phase in ms. Default 10 minutes. */
-    timeoutMs?: number;
-    /**
-     * Cap on captured stdout+stderr bytes per phase. A runaway process can
-     * otherwise grow the in-memory buffer without bound. Once hit, further
-     * output is dropped and `outputTruncated` is set. Default 16 MiB.
-     */
-    maxOutputBytes?: number;
-    /** Image for the docker driver. */
-    image?: string;
-    /** Extra env vars (validated; shell-escaped). */
-    env?: Record<string, string>;
-    /** Parser for the test output — maps stdout/stderr/exit code → pass count. */
-    testParser?: TestOutputParser;
-}
-interface TestOutputParser {
-    id: string;
-    parse(stdout: string, stderr: string, exitCode: number): {
-        testsTotal: number;
-        testsPassed: number;
-    } | undefined;
-}
-interface SandboxResult {
-    phase: 'setup' | 'run' | 'test';
-    exitCode: number;
-    stdout: string;
-    stderr: string;
-    wallMs: number;
-    testsTotal?: number;
-    testsPassed?: number;
-    /**
-     * True when the process was killed because it exceeded `timeoutMs`. A
-     * SIGKILLed child can still close with exit code 0; callers MUST treat
-     * a timed-out phase as a hard failure regardless of `exitCode`, never
-     * as a pass. `undefined`/`false` means the process completed on its own.
-     */
-    killedByTimeout?: boolean;
-    /**
-     * True when captured stdout/stderr hit `maxOutputBytes` and further
-     * output was dropped. The result is still returned (the process was
-     * not killed for this), but downstream parsers see truncated text.
-     */
-    outputTruncated?: boolean;
-}
-interface SandboxDriver {
-    id: string;
-    exec(phase: SandboxResult['phase'], command: string, config: HarnessConfig): Promise<SandboxResult>;
-}
-/** Vitest default summary line: "Tests  X passed | Y failed". */
-declare const vitestTestParser: TestOutputParser;
-/** Pytest default: "collected N items" + " X passed, Y failed". */
-declare const pytestTestParser: TestOutputParser;
-/** Jest: "Tests: X passed, Y total" (and optional failed). */
-declare const jestTestParser: TestOutputParser;
-/** Composite parser — tries a list of parsers in order. */
-declare function composeParsers(...parsers: TestOutputParser[]): TestOutputParser;
-interface SubprocessSandboxDriverOptions {
-    /**
-     * Default cwd for all `exec` calls. Used when the per-call `HarnessConfig`
-     * does not set its own `cwd`. Lets callers bind the driver to a working
-     * directory once instead of spreading cwd into every harness config —
-     * useful when the harness config is constructed far from the call site
-     * (e.g. starter-foundry's promoter passes a static HarnessConfig per
-     * family taxonomy but needs a per-run composed-scaffold cwd).
-     */
-    cwd?: string;
-    /**
-     * Default env merged into every `exec` call's env (per-call `HarnessConfig.env`
-     * still wins on key collision). Same ergonomic rationale as `cwd` above.
-     */
-    env?: Record<string, string>;
-}
-declare class SubprocessSandboxDriver implements SandboxDriver {
-    id: string;
-    private defaultCwd?;
-    private defaultEnv?;
-    constructor(options?: SubprocessSandboxDriverOptions);
-    exec(phase: SandboxResult['phase'], command: string, config: HarnessConfig): Promise<SandboxResult>;
-}
-declare class DockerSandboxDriver implements SandboxDriver {
-    id: string;
-    exec(phase: SandboxResult['phase'], command: string, config: HarnessConfig): Promise<SandboxResult>;
-}
-interface SandboxHarnessResult {
-    passed: boolean;
-    setup?: SandboxResult;
-    run?: SandboxResult;
-    test?: SandboxResult;
-    totalWallMs: number;
-    /** Final score — 0 when no tests; otherwise testsPassed/testsTotal. */
-    score: number;
-}
-declare class SandboxHarness {
-    private driver;
-    constructor(driver?: SandboxDriver);
-    run(config: HarnessConfig, emitter: TraceEmitter): Promise<SandboxHarnessResult>;
-}
-
-/**
- * TestGradedScenario — a scenario whose score comes from a test suite.
- *
- * This is the SWE-bench pattern generalized. The scenario ships:
- *   - fixture data (setup instructions)
- *   - a test command the harness runs
- *   - optional assertion overrides
- *
- * The runner emits a run, delegates to SandboxHarness, records the
- * outcome, and returns a structured verdict. Consumers bind their own
- * agent execution to this contract.
- */
-
-interface TestGradedScenario {
-    id: string;
-    description?: string;
-    harness: HarnessConfig;
-    /** Optional pass threshold in 0..1 (default 1.0 = all tests must pass). */
-    passThreshold?: number;
-    /** Provenance for dataset tracking. */
-    datasetVersion?: string;
-    /** Free-form tags (difficulty, category, etc.). */
-    tags?: Record<string, string>;
-}
-interface TestGradedRunOptions {
-    variantId?: string;
-    driver?: SandboxDriver;
-    /** Metadata recorded on the Run (codeSha, promptSha, modelFingerprint, seed). */
-    provenance?: Pick<Run, 'codeSha' | 'promptSha' | 'modelFingerprint' | 'seed' | 'envFingerprint'>;
-}
-interface TestGradedRunResult {
-    runId: string;
-    scenario: TestGradedScenario;
-    harness: SandboxHarnessResult;
-    pass: boolean;
-    score: number;
-    failureClass?: FailureClass;
-}
-declare function runTestGradedScenario(scenario: TestGradedScenario, store: TraceStore, options?: TestGradedRunOptions): Promise<TestGradedRunResult>;
-
-export { DockerSandboxDriver as D, type HarnessConfig as H, type SandboxDriver as S, type TestGradedScenario as T, type SandboxHarnessResult as a, type TestGradedRunResult as b, SandboxHarness as c, type SandboxResult as d, SubprocessSandboxDriver as e, type SubprocessSandboxDriverOptions as f, type TestGradedRunOptions as g, type TestOutputParser as h, composeParsers as i, jestTestParser as j, pytestTestParser as p, runTestGradedScenario as r, vitestTestParser as v };