@tangle-network/agent-eval 0.104.0 → 0.105.0

package/dist/index.d.ts

@@ -23,7 +23,8 @@ export { A as ArtifactCheckArtifact, d as ArtifactEventLike, e as ArtifactValida
 export { DataAcquisitionPlan, KnowledgeAcquisitionMode, KnowledgeBundle, KnowledgeFallbackPolicy, KnowledgeFreshness, KnowledgeImportance, KnowledgeReadinessReport, KnowledgeRecommendedAction, KnowledgeRequirement, KnowledgeRequirementCategory, KnowledgeResponsibleSurface, KnowledgeSensitivity, ScoreKnowledgeReadinessOptions, UserQuestion, acquisitionPlansForKnowledgeGaps, blockingKnowledgeEval, knowledgeReadinessTracePayload, scoreKnowledgeReadiness, userQuestionsForKnowledgeGaps } from './knowledge/index.js';
 import { h as ReleaseConfidenceThresholds, f as ReleaseConfidenceScorecard } from './release-report-DeJpsBiA.js';
 export { A as ActionableSideInfo, o as AsiSeverity, B as BootstrapOptions, a as BootstrapResult, J as JudgeReplayGateArgs, R as ReleaseConfidenceAxis, b as ReleaseConfidenceAxisName, c as ReleaseConfidenceInput, d as ReleaseConfidenceIssue, e as ReleaseConfidenceMetrics, g as ReleaseConfidenceStatus, i as ReleaseTraceEvidence, j as RenderReleaseReportOptions, V as Verdict, k as assertReleaseConfidence, l as bootstrapCi, m as evaluateReleaseConfidence, n as judgeReplayGate, r as renderReleaseReport } from './release-report-DeJpsBiA.js';
-export { c as CliffsMagnitude, d as CorpusAgreementOptions, e as CorpusAgreementPerDimension, C as CorpusAgreementReport, f as CorpusScoreRecord, g as EProcess, h as EProcessOptions, E as EProcessState, i as EProcessStep, M as McNemarResult, P as PairedBootstrapOptions, a as PairedBootstrapResult, j as ProportionInterval, R as RiskDifferenceResult, W as WeightedCompositeInput, k as WeightedCompositeResult, b as benjaminiHochberg, l as bonferroni, m as cliffsDelta, n as cohensD, o as confidenceInterval, q as corpusInterRaterAgreement, r as corpusInterRaterAgreementFromJudgeScores, s as eProcess, t as interRaterReliability, u as interpretCliffs, v as mannWhitneyU, x as mcnemar, y as mcnemarPower, z as mcnemarRequiredN, A as mulberry32, B as normalizeScores, p as pairedBootstrap, D as pairedMde, F as pairedRiskDifference, G as pairedTTest, H as partialCredit, I as passAtK, J as pearsonR, K as ranks, L as requiredSampleSize, N as spearmanR, O as weightedComposite, Q as weightedMean, w as wilcoxonSignedRank, S as wilson } from './statistics-D88peojY.js';
+import { P as PairedBootstrapOptions, M as McNemarResult, R as RiskDifferenceResult, a as PairedBootstrapResult } from './statistics-D88peojY.js';
+export { c as CliffsMagnitude, d as CorpusAgreementOptions, e as CorpusAgreementPerDimension, C as CorpusAgreementReport, f as CorpusScoreRecord, g as EProcess, h as EProcessOptions, E as EProcessState, i as EProcessStep, j as ProportionInterval, W as WeightedCompositeInput, k as WeightedCompositeResult, b as benjaminiHochberg, l as bonferroni, m as cliffsDelta, n as cohensD, o as confidenceInterval, q as corpusInterRaterAgreement, r as corpusInterRaterAgreementFromJudgeScores, s as eProcess, t as interRaterReliability, u as interpretCliffs, v as mannWhitneyU, x as mcnemar, y as mcnemarPower, z as mcnemarRequiredN, A as mulberry32, B as normalizeScores, p as pairedBootstrap, D as pairedMde, F as pairedRiskDifference, G as pairedTTest, H as partialCredit, I as passAtK, J as pearsonR, K as ranks, L as requiredSampleSize, N as spearmanR, O as weightedComposite, Q as weightedMean, w as wilcoxonSignedRank, S as wilson } from './statistics-D88peojY.js';
 import { OtelExporter, OtelExportConfig } from './traces.js';
 export { CaptureFetchContext, CaptureFetchOptions, ExportableSpan, ExtractedUsage, FlattenOtlpOptions, LLM_CACHED_TOKENS, LLM_CACHED_TOKEN_ATTR_KEYS, LLM_COST_ATTR_KEYS, LLM_COST_USD, LLM_INPUT_TOKENS, LLM_INPUT_TOKEN_ATTR_KEYS, LLM_MODEL_ATTR_KEYS, LLM_MODEL_NAME, LLM_OUTPUT_TOKENS, LLM_OUTPUT_TOKEN_ATTR_KEYS, OPENINFERENCE_SPAN_KIND, OTEL_AGENT_EVAL_SCOPE, OtlpExport, OtlpFileTraceStore, OtlpFileTraceStoreOptions, OtlpFlatLine, OtlpResourceSpans, OtlpSpan, OtlpToRunRecordsOptions, OtlpTraceRunRecord, ProjectedOtlpSpan, ReplayCache, ReplayCacheEntry, ReplayCacheMissError, ReplayCacheStats, ReplayFetchOptions, SPAN_KIND_ATTR_KEYS, SpanNotFoundError, TOOL_NAME, TOOL_NAME_ATTR_KEYS, TRACE_ANALYST_ACTOR_DESCRIPTION, TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION, TRACE_ANALYST_SUBAGENT_DESCRIPTION, TraceAggregate, TraceAnalystHookOptions, TraceFileMissingError, TraceInsightContext, TraceInsightFinding, TraceInsightPanelRole, TraceInsightPromptInput, TraceInsightQualityGate, TraceInsightQuestion, TraceInsightReadiness, TraceInsightSuite, TraceInsightTask, TraceNotFoundError, TraceStoreSource, TraceStoreToOtlpOptions, TracesToOtlpResult, asNumber, asString, buildTraceAnalystTools, buildTraceInsightContext, buildTraceInsightPrompt, captureFetchToRawSink, convertTraceStoresToOtlp, createOtelExporter, createOtelTracingStore, createReplayFetch, defaultTraceInsightPanel, describeTraceInsightScope, domainEvidencePattern, exportRunAsOtlp, extractOtlpAttributes, extractUsage, extractUsageFromResponse, extractUsageFromSse, firstNumberAttr, firstStringAttr, flattenOtlpExportToNdjson, inferDomainKeywords, inferOtlpKind, iterateRawCalls, otelRunCompleteHook, otlpToRunRecords, otlpToTraceRunRecords, planTraceInsightQuestions, projectOtlpFlatLine, readOtlpStatus, scoreTraceInsightReadiness, stringField, tokenizeDomainWords, traceAnalystFunctionGroup, traceAnalystOnRunComplete, traceSpanKindToOpenInferenceKind } from './traces.js';
 import { a as AnalyzeTracesInput, A as AnalyzeTracesOptions, b as AnalyzeTracesResult } from './analyst-C8HHvfJp.js';
@@ -217,6 +218,98 @@ declare class BenchmarkRunner {
     run(scenarios?: Scenario[]): Promise<BenchmarkReport>;
 }
 
+/**
+ * Capability-headroom gate — "can this task set even SEE the capability
+ * you're about to A/B?"
+ *
+ * A capability A/B (grant one arm a tool, a skill, extra context — anything)
+ * can only detect the capability on tasks the capability-ABSENT baseline
+ * FAILS. A task the baseline already passes is saturated: the treatment arm
+ * has no headroom to demonstrate anything, and averaging saturated tasks into
+ * the comparison dilutes a real effect toward a false null. Calibrate before
+ * measuring: run the baseline arm first, classify every task's headroom, and
+ * refuse to run the comparison when no task has a gap.
+ *
+ * Fail-closed on unknown outcomes: a baseline run whose outcome could not be
+ * determined (infra loss, unverified, missing) NEVER counts as a failure —
+ * treating "couldn't tell" as "failed" would manufacture headroom out of
+ * telemetry gaps and launder infra noise into a treatment effect.
+ *
+ * Domain-free by construction: the capability under test never appears here.
+ * Callers decide what the baseline arm lacks; this module only classifies
+ * baseline outcomes per task.
+ */
+/** One baseline-arm (capability-absent) observation of one task. */
+interface HeadroomInput {
+    /** Stable task identity; multiple rows per task are multiple baseline reps. */
+    taskId: string;
+    /** Baseline outcome. 'unknown' = the outcome could not be determined —
+     *  distinct from 'fail' and never counted as one. */
+    baselineOutcome: 'pass' | 'fail' | 'unknown';
+}
+/** Headroom classification for one task.
+ *  - 'gap': the baseline fails it — the comparison can see the capability here.
+ *  - 'saturated': the baseline already passes it — no headroom.
+ *  - 'unknown': every baseline outcome is unknown — fail-closed, not a gap. */
+type HeadroomClass = 'gap' | 'saturated' | 'unknown';
+interface TaskHeadroom {
+    taskId: string;
+    /** Total baseline observations for this task (known + unknown). */
+    n: number;
+    /** Observations with a known ('pass' | 'fail') outcome — the evidence the
+     *  classification actually rests on. A large `n` with a small `nKnown`
+     *  means the headroom verdict is thin, not well-replicated. */
+    nKnown: number;
+    /** Pass rate over KNOWN outcomes only; NaN when every outcome is unknown
+     *  (no data ≠ measured zero). */
+    baselinePassRate: number;
+    headroom: HeadroomClass;
+}
+interface CapabilityHeadroomOptions {
+    /** A task counts as 'gap' when its baseline pass rate is ≤ this. Default 0
+     *  (the baseline must fail every known rep). Must be in [0, 1) — a
+     *  threshold of 1 would classify a fully-saturated task as a gap. */
+    keepThreshold?: number;
+}
+interface CapabilityHeadroomResult {
+    /** Per-task classification, sorted by `taskId`. */
+    tasks: TaskHeadroom[];
+    summary: {
+        tasksWithGap: number;
+        tasksSaturated: number;
+        tasksUnknown: number;
+        /** Total unknown-outcome reps across ALL tasks (including tasks that
+         *  still classified via their known reps) — the size of the telemetry
+         *  hole behind the classifications. */
+        repsUnknown: number;
+    };
+}
+/**
+ * Classify every task's baseline headroom from capability-absent baseline
+ * outcomes.
+ *
+ * A task is 'gap' when it has ≥ 1 known outcome and its pass rate over known
+ * outcomes is ≤ `keepThreshold`; 'saturated' when the rate exceeds the
+ * threshold; 'unknown' when every outcome is unknown. Unknown reps never
+ * count toward the rate in either direction (fail-closed — see the module
+ * doc), and evidence strength stays visible: each task reports `nKnown`
+ * alongside `n`, and the summary totals the unknown reps in `repsUnknown`.
+ * Throws on empty input, an out-of-range `keepThreshold`, or an
+ * unrecognized outcome value (rows are often projected from untyped records).
+ */
+declare function capabilityHeadroom(rows: readonly HeadroomInput[], opts?: CapabilityHeadroomOptions): CapabilityHeadroomResult;
+interface AssertCapabilityHeadroomOptions {
+    /** Minimum tasks classified 'gap' for the comparison to proceed. Default 1. */
+    minTasksWithGap?: number;
+}
+/**
+ * The go/no-go guard: throws when fewer than `minTasksWithGap` tasks have
+ * baseline headroom — i.e. the benchmark cannot see the capability, so any
+ * comparison run on it would measure noise. Call between the baseline
+ * calibration run and the treatment run.
+ */
+declare function assertCapabilityHeadroom(result: CapabilityHeadroomResult, opts?: AssertCapabilityHeadroomOptions): void;
+
 declare class ProductClient {
     private baseUrl;
     private routes;
@@ -864,6 +957,159 @@ declare class MetricsCollector {
     getConvergenceCurve(): number[];
 }
 
+/**
+ * Matched-pair arm comparison — "did the treatment arm beat the baseline arm
+ * on the SAME work items?"
+ *
+ * An arm A/B over run records is only trustworthy when it is PAIRED: the same
+ * task/scenario/seed evaluated under both arms, compared item-by-item, so
+ * inter-item difficulty variance cancels instead of masquerading as an arm
+ * effect. This module owns the two error-prone steps every consumer otherwise
+ * hand-rolls:
+ *
+ *   1. Pairing — matching rows across arms by `pairKey` (and by `repKey`
+ *      within multi-rep items), with leftovers REPORTED rather than silently
+ *      dropped (a silently unbalanced pairing biases every paired statistic
+ *      downstream). Pairing never keys on outcome content: matching reps by
+ *      their outcomes deflates discordant-pair counts and makes McNemar
+ *      anti-conservative, so reps pair only by (`pairKey`, `repKey`) identity.
+ *   2. Composition — feeding the matched pairs to the correct paired
+ *      estimators that already live in `statistics`: `mcnemar` +
+ *      `pairedRiskDifference` for pass/fail, `pairedBootstrap` +
+ *      `wilcoxonSignedRank` for continuous metrics. No statistic is
+ *      re-implemented here.
+ *
+ * The row shape is deliberately structural — callers project a `RunRecord`
+ * (or any record) into `{ pairKey, arm, pass?, metrics? }`. Arm names are
+ * caller-supplied parameters; the module ships no domain literal.
+ */
+
+/** One arm observation of one work item. Structural on purpose: callers
+ *  project their own record type (e.g. a `RunRecord`) into this shape. */
+interface PairedArmRow {
+    /** Matching key — rows sharing a `pairKey` across both arms form pairs
+     *  (typically the task/scenario/seed identity). */
+    pairKey: string;
+    /** Rep identity within a `pairKey` (e.g. a seed or rep number). Required on
+     *  every row of a `pairKey` that has more than one rep in either arm; reps
+     *  then pair only on exact (`pairKey`, `repKey`) match, never on outcome
+     *  content. Optional when each arm has at most one rep of the item. */
+    repKey?: string;
+    /** Arm label this row was produced under. */
+    arm: string;
+    /** Binary outcome; omit when the comparison has no pass/fail notion. */
+    pass?: boolean;
+    /** Named numeric measurements (score, cost, latency, …). */
+    metrics?: Record<string, number>;
+}
+interface PairArmsOptions {
+    /** Arm treated as the control side of every pair. */
+    baselineArm: string;
+    /** Arm treated as the treatment side of every pair. */
+    treatmentArm: string;
+}
+/** One matched (baseline, treatment) observation of the same work item. */
+interface MatchedPair {
+    pairKey: string;
+    /** 0-based position of this pair within its `pairKey`, ordered by sorted
+     *  `repKey` (always 0 for a single-rep item). The rep identity itself is on
+     *  the rows (`baseline.repKey` / `treatment.repKey`). */
+    repIndex: number;
+    baseline: PairedArmRow;
+    treatment: PairedArmRow;
+}
+interface PairArmsResult {
+    /** Matched pairs, ordered by (`pairKey`, `repIndex`). */
+    pairs: MatchedPair[];
+    /** Baseline rows left without a treatment counterpart — reported, never
+     *  silently dropped. */
+    unpairedBaseline: PairedArmRow[];
+    /** Treatment rows left without a baseline counterpart. */
+    unpairedTreatment: PairedArmRow[];
+}
+/**
+ * Match rows across two arms into (baseline, treatment) pairs by `pairKey`.
+ *
+ * A `pairKey` with at most one row per arm pairs directly, no `repKey`
+ * needed. A `pairKey` with multiple reps in either arm requires `repKey` on
+ * every one of its rows, and reps pair only on exact (`pairKey`, `repKey`)
+ * match — pairing is keyed purely on row identity, never on outcome content
+ * (outcome-keyed matching deflates discordant counts and biases McNemar), and
+ * is therefore independent of input order. Reps whose `repKey` has no
+ * counterpart in the other arm, and items present in only one arm, land in
+ * the unpaired lists — reported, never truncated.
+ *
+ * Fail-loud: throws when either named arm has zero rows (an unknown arm
+ * name would otherwise read as "everything unpaired"), when the two arm
+ * names are equal, when a multi-rep `pairKey` has a row without `repKey`, or
+ * when a (`pairKey`, arm) group repeats a `repKey` (the match would be
+ * ambiguous).
+ */
+declare function pairArms(rows: readonly PairedArmRow[], opts: PairArmsOptions): PairArmsResult;
+/** Paired pass/fail comparison over the pairs where BOTH sides carry `pass`. */
+interface PairedCorrectness {
+    /** Discordant pairs where the treatment passed and the baseline failed. */
+    b10: number;
+    /** Discordant pairs where the baseline passed and the treatment failed. */
+    b01: number;
+    /** Exact McNemar significance over the paired outcomes (`b === b10`, `c === b01`). */
+    mcnemar: McNemarResult;
+    /** Paired effect size: p(treatment) − p(baseline) with a paired-variance CI. */
+    riskDifference: RiskDifferenceResult;
+}
+/** Paired delta summary for one named metric (delta = treatment − baseline). */
+interface PairedMetricDelta {
+    name: string;
+    /** Pairs where BOTH sides carry a finite value for this metric. */
+    n: number;
+    /** Pairs where at least one side does not carry the metric. */
+    nMissing: number;
+    /** Median paired delta; NaN when `n === 0` (no data ≠ measured zero). */
+    medianDelta: number;
+    /** Mean paired delta; NaN when `n === 0`. */
+    meanDelta: number;
+    /** Bootstrap CI on the paired delta (`pairedBootstrap`); null when
+     *  `n === 0` — a zero-width [0, 0] interval on no data would read as a
+     *  measured tight null. */
+    bootstrapCi: PairedBootstrapResult | null;
+    /** Wilcoxon signed-rank test on the paired deltas; null when `n === 0`. */
+    wilcoxon: {
+        w: number;
+        p: number;
+    } | null;
+}
+interface ComparePairedArmsOptions extends PairArmsOptions {
+    /** Metrics to compare. Default: every metric name observed on any matched
+     *  pair, sorted. A name that appears on no pair is still reported (with
+     *  `n = 0`) so a misspelled metric is visible instead of vanishing. */
+    metricNames?: string[];
+    /** Passed through to `pairedBootstrap` — set `seed` for reproducible CIs. */
+    bootstrap?: PairedBootstrapOptions;
+}
+interface PairedArmsComparison {
+    nPairs: number;
+    nUnpairedBaseline: number;
+    nUnpairedTreatment: number;
+    /** null when no matched pair carries `pass` on both sides — a pass/fail
+     *  verdict over rows that never measured pass/fail would be fabricated. */
+    correctness: PairedCorrectness | null;
+    metricDeltas: PairedMetricDelta[];
+}
+/**
+ * Full matched-pair arm comparison: pair via {@link pairArms}, then compose
+ * the paired estimators from `statistics` over the matched pairs.
+ *
+ * Correctness uses only the pairs where both sides carry `pass` (`mcnemar.n`
+ * is that subset's size); each metric uses only the pairs where both sides
+ * carry a finite value for it, with the remainder counted in `nMissing`.
+ * Deltas are treatment − baseline throughout.
+ *
+ * Fail-loud: inherits {@link pairArms}'s unknown-arm throw, and throws on a
+ * non-finite metric value — silently treating corrupt telemetry as "metric
+ * absent" would misreport it as missing coverage.
+ */
+declare function comparePairedArms(rows: readonly PairedArmRow[], opts: ComparePairedArmsOptions): PairedArmsComparison;
+
 type PrReviewSource = 'drew' | 'donovan' | 'shady' | 'codex' | 'claude-code' | 'gpt-5.5-high' | 'claude-opus-4.7-high' | 'kimi' | 'opencode' | (string & {});
 type PrReviewSeverity = 'critical' | 'high' | 'medium' | 'low' | 'nit';
 type PrReviewOutcome = 'accepted' | 'fixed' | 'rejected' | 'duplicate' | 'noise' | 'unknown';
@@ -6087,4 +6333,4 @@ type CachedJudge<TArtifact, TScenario extends Scenario$1 = Scenario$1> = JudgeCo
  */
 declare function cachedJudge<TArtifact, TScenario extends Scenario$1 = Scenario$1>(judge: JudgeConfig<TArtifact, TScenario>, store: VerdictCacheStore, options: CachedJudgeOptions): CachedJudge<TArtifact, TScenario>;
 
-export { ATTESTATION_ALGORITHM, type ActiveLearningOptions, type AdapterRun, AgentDriver, type AgentDriverConfig, AgentEvalError, type AgreementResult, type AlignmentOp, AnalyzeTracesInput, AnalyzeTracesOptions, AnalyzeTracesResult, type AntiSlopConfig, type AntiSlopIssue, type AntiSlopReport, type AssertCrossFamilyOptions, type AssertSingleBackendOptions, type AttestationProvenance, type AttestationVerification, type AttestedReport, type AutoPrClient, AxGepaSteeringOptimizer, type AxSteeringOptimizerConfig, type BackendDescriptor, BaselineReport, BehaviorAssertion, BehavioralMetrics, BenchmarkReport, BenchmarkRunner, BenchmarkRunnerConfig, type BisectOptions, type BisectResult, type BisectStep, type BlendWeights, BudgetBreachError, BudgetGuard, BudgetLedgerEntry, BudgetSpec, type BuildAgreementJudgeOptions, CODING_HARNESSES, type CachedJudge, type CachedJudgeOptions, CallExpectation, type CanaryAlert, type CanaryKind, type CanaryLeak, type CanaryOptions, type CanaryReport, type CanarySeverity, type CandidateComparison, type CandidateScenario, type CausalAttributionReport, type CellVerdict, ChannelRollup, ChatRequest, CheckResult, CollectedArtifacts, type CommandRunner, type CompareLabels, CompletionCriterion, ConfigError, type ContinuityCheck, type ContinuityCheckResult, type ContinuityReport, type ContinuitySnapshotPair, type ContractCheckResult, type ContractJudgeOptions, type ContractMetric, type ContractReport, type ContractRule, type ContractRuleKind, type ContractSpan, type ContractVerdict, type ContractViolation, ConvergenceTracker, CorrectnessChecker, type CostEntry, CostLedger, type CostReport, type CostSummary, CostTracker, CreateChatClientOpts, type CreateDefaultReviewerOptions, type CreateExperimentInput, type CreateSandboxPoolOpts, CrossFamilyError, type CrossTraceDiff, type CrossTraceDiffOptions, DEFAULT_AGENT_SLOS, DEFAULT_FINDERS, DEFAULT_MUTATION_PRIMITIVES, DEFAULT_MUTATORS, DEFAULT_PR_REVIEW_SCORE_WEIGHTS, DEFAULT_SEVERITY_WEIGHTS, Dataset, DatasetScenario, type DecideNextUserTurnOpts, DefaultVerdict, type DeployFamily, type DeployGateLayerInput, type DeployRunResult, type DeployRunner, type DescriptionLengthCandidate, type DescriptionLengthConfig, type DescriptionLengthDecision, type DescriptionLengthEvidence, DescriptionLengthGate, type DescriptionLengthRejectionCode, type DetectorEvent, type DetectorSeverity, type DetectorSignal, type DiffScorecardOptions, type DirEntry, type DiscoverPersonasOptions, type DiscoveredPersona, DriverResult, DriverState, DualAgentBench, type DualAgentBenchConfig, type DualAgentReport, type DualAgentRound, type DualAgentScenario, type DualAgentScenarioResult, ERROR_COUNT_PATTERNS, type EnsembleAggregate, type EnsembleJudgeOptions, type ErrorCountPattern, type ErrorStreakOptions, type EvalToolDef, EvalTraceStore, type EvolutionRound, type ExecutorConfig, type Expectation, type Experiment, type ExperimentProvenance, type ExperimentRep, type ExperimentStats, type ExperimentStore, ExperimentTracker, type ExperimentTrackerOptions, type ExperimentVerdict, type ExportedRewardModel, type ExtractOptions, type ExtractResult, type FactorContribution, type FactorialCell, FailureClass, FeedbackLabel, FeedbackTrajectory, FeedbackTrajectoryStore, type FieldAgreementSpec, type FieldDestination, type FileChange, type FlowAction, type FlowLayerEnv, type FlowLayerFactoryInput, type FlowRunner, type FlowRunnerStepResult, type FlowSpec, type FlowStep, type GhCliClientOptions, type GoldScenario, type GoldSplit, type GoldenSeverity, type GoldenSpec, HARNESS_NATIVE_MODEL, HarnessConfig, type HeldOutPartition, type HiddenCriteriaGrader, type HiddenGradeResult, type HiddenLeak, HoldoutAuditor, type HttpGithubClientOptions, INTENT_MATCH_JUDGE_VERSION, type ImageData, type ImprovementThresholds, type ImprovementVerdictResult, InMemoryWorkspaceInspector, type InferenceScorer, type InspectorContext, type IntentMatchInput, type IntentMatchOptions, type IntentMatchResult, type InteractionContribution, JudgeError, type JudgeFamily, type JudgeFleetOptions, JudgeFn, JudgeParseError, type JudgeReplayResult, type JudgeRetryOutcome, type JudgeRetryPolicy, JudgeRunner, type JudgeScoreInput, type JudgeVerdict, type KeywordConceptSpec, type KeywordCoverageFinding, type KeywordCoverageOptions, type KeywordCoverageResult, type LangfuseEnvelope, type LangfuseGeneration, type LangfuseScore, Layer, LayerResult, type LeaderboardOptions, type LeaderboardRow, type LiveProofArtifact, type LiveProofConfig, type LiveProofContext, type LiveProofResult, LlmClientOptions, LlmSpan, MODEL_PRICING, type MakeEvalToolsConfig, type MatchResult, type MatcherResult, type MergeOptions, MetricsCollector, type ModelCostRollup, type ModelPreflight, type ModelSeats, ModelsUnreachableError, type MuffledFinder, type MuffledFinding, type MultiToolchainLayerConfig, type Mutator, Mutex, type NoLeakOptions, type NoProgressOptions, type Oracle, type OracleObservation, type OracleReport, type OracleResult, type OrthogonalityInput, type OrthogonalityResult, OtelExportConfig, OtelExporter, type OtelPipelineHandle, type OtelPipelineOptions, PairwiseSteeringOptimizer, type ParaphraseRobustnessScenarioInput, type ParaphraseRobustnessScenarioResult, type ParseStudentLabel, type PartitionHeldOutOptions, PersonaConfig, type Playbook, type PlaybookEntry, type PoolSlot, type PrReviewAuditCase, type PrReviewBenchmarkSummary, type PrReviewComment, type PrReviewMatchedFinding, type PrReviewOutcome, type PrReviewReferenceFinding, type PrReviewScore, type PrReviewScoreWeights, type PrReviewSeverity, type PrReviewSource, type PreflightModelsOptions, type PreflightOutcome, ProductClient, ProductClientConfig, type ProfileAxisSpec, type PromptHandle, PromptRegistry, type ProposeAutomatedPullRequestInput, type ProposeAutomatedPullRequestResult, type ProvenanceReader, type RecordRunsOptions, type ReferenceMatchResult, type ReferenceReplayAdapter, type ReferenceReplayAdapterFn, type ReferenceReplayAdapterLike, type ReferenceReplayAggregate, type ReferenceReplayCandidate, type ReferenceReplayCase, type ReferenceReplayCaseRun, type ReferenceReplayExecutionScenario, type ReferenceReplayItem, type ReferenceReplayMatch, type ReferenceReplayMatchStrategy, type ReferenceReplayMatcher, type ReferenceReplayPromotionDecision, type ReferenceReplayPromotionPolicy, type ReferenceReplayRun, type ReferenceReplayRunContext, type ReferenceReplayRunOptions, type ReferenceReplayRunStore, type ReferenceReplayScenario, type ReferenceReplayScenarioScore, type ReferenceReplayScore, type ReferenceReplayScoreOptions, type ReferenceReplaySplit, type ReferenceReplaySplitComparison, type ReferenceReplaySteeringRowsOptions, type ReflectionContext, type ReflectionProposal, ReleaseConfidenceScorecard, ReleaseConfidenceThresholds, type RenderStudentPrompt, type RepeatedActionOptions, type RepoRef, type ReviewerMemoryEntry, type ReviewerOutput, type ReviewerPromptInput, type ReviewerSoftFailDefaults, type ReviewerVerificationSummary, type RobustnessResult, type RoutedField, Run, type RunCommandInput, type RunCommandResult, type RunDistillationOptions, type RunDistillationResult, RunFilter, RunRecord, type RunRecordBackend, type RunRecordFilter, RunScore, RunScoreWeights, RunSplitTag, SandboxDriver, SandboxHarnessResult, type SandboxJudgeKind, type SandboxJudgeResult, type SandboxJudgeSpec, type SandboxPool, type ScanOptions, Scenario, type ScenarioCost, ScenarioFile, ScenarioRegistry, ScenarioResult, type Scorecard, type ScorecardCell, type ScorecardCellDiff, type ScorecardDiff, type ScorecardEntry, type ScorecardLogLine, type ScoredTarget, type SeatName, type SeatPresetName, SeatUnsetError, type SelfPlayOptions, type SelfPlayProposer, type SelfPlayScorer, type SerializedRegex, Severity, type SingleBackendDivergence, SingleBackendError, type SingleBackendField, type SingleBackendReport, type Slo, type SloCheckResult, type SloComparator, type SloReport, type SloSeverity, type SlopCategory, type SlotFactory, type SpanPredicate, type SplitGoldOptions, SteeringBundle, type SteeringOptimizationResult, type SteeringOptimizationRow, type SteeringOptimizationSelector, type SteeringOptimizerBackend, type SteeringOptimizerConfig, type StepAttribution, type StreamingDetector, type SynthesisReason, type SynthesisTarget, TestResult, type TextMatcher, type ThresholdContract, TokenCounter, type TokenSpec, type ToolMatcher, ToolSpan, TraceAnalystSpan, type TraceContract, TraceContractBuilder, TraceEmitter, TraceStore, type TracedAnalystOptions, type TracedJudgeOptions, Trajectory, TrajectoryStep, type TreatmentClass, type TreatmentGate, type TreatmentGateInput, type TreatmentGateOptions, type TrialTrace, TurnMetrics, UI_FINDING_SEVERITIES, UI_LENSES, UNIVERSAL_FINDERS, type UiFinding, type UiFindingScreenshot, type UiFindingSeverity, type UiLens, type VerdictCacheStats, type VerdictCacheStore, VerifyContext, type VisualDiffOptions, type VisualDiffResult, type ViteDeployRunnerInput, type WorkerDriverContext, type WorkspaceAssertion, type WorkspaceAssertionResult, type WorkspaceInspector, type WorkspaceSnapshot, type WranglerDeployRunnerInput, adversarialJudge, agentProfileHash, agentProfileId, agentProfileModelId, agentVisibleFields, aggregateJudgeVerdicts, aggregatePrReviewScore, analyzeAntiSlop, appendScorecard, assertCrossFamily, assertModelsServed, assertNoHiddenLeak, assertSingleBackend, assignHeldOutTag, attachCostToReport, attest, bisect, blendHeldout, buildAgreementJudge, buildDriverSystemPrompt, buildReflectionPrompt, buildReviewerPrompt, buildWorkerDriverSystemPrompt, cachedJudge, canaryLeakView, canonicalJson, causalAttribution, checkBehavioralCanary, checkCanaries, checkSlos, checkTraceContracts, classifyTreatment, codeExecutionJudge, coherenceJudge, collectionPreserved, commentsForSource, commitBisect, compareReferenceReplay, compilerJudge, computeExperimentStats, contentHash, contractJudge, costReport, createAntiSlopJudge, createCustomJudge, createDefaultReviewer, createDomainExpertJudge, createIntentMatchJudge, createSandboxPool, crossTraceDiff, dataDescriptionBits, decideNextUserTurn, decideReferenceReplayPromotion, decideReferenceReplayRunPromotion, defaultBlendWeights, defaultJudges, defaultParseStudentLabel, defaultReferenceReplayMatcher, defaultRenderStudentPrompt, deployGateLayer, diffScorecard, discoverPersonas, distillPlaybook, ensembleJudge, errorStreakDetector, estimateCost, estimateTokens, evaluateContract, evaluateOracles, evaluateTraceContract, executeScenario, expandProfileAxes, expectAgent, exportRewardModel, extractAssetUrls, extractErrorCount, fieldAgreement, fileContains, fileExists, fileExperimentStore, fileVerdictCache, findAutoMatchNoExpectation, findConstructorCwdDropped, findFallbackToPass, findLiteralTruePass, findSkipCountsAsPass, flowLayer, fnv1a32, formatBenchmarkReport, formatDriverReport, formatFindings, formatScorecardDiff, gateTreatmentApplied, gateTreatmentFromMetrics, gateTreatmentFromSpans, gateTreatmentFromToolSpans, ghCliClient, gitProvenanceReader, precision as goldenPrecision, gradeOnHidden, harnessAxisOf, hashContent, hashToUnit, hiddenGrade, htmlContainsElement, httpGithubClient, improvementVerdict, inMemoryExperimentStore, inMemoryReferenceReplayStore, inMemoryRunRecordBackend, inMemoryVerdictCache, isHiddenDestination, isModelPriced, isOtelConfigured, jsonShape, jsonlReferenceReplayStore, jsonlRunRecordBackend, judgeFamily, keyPreserved, leaderboard, linterJudge, loadGoldScenarios, loadScorecard, loadScorerFromGrader, localCommandRunner, lowercaseMutator, makeEvalTools, matchGoldens, matchSpan, mergeLayerResults, modelDescriptionBits, multiToolchainLayer, noProgressDetector, notBlocked, observeAll, paraphraseRobustness, paraphraseRobustnessScenarios, parseGoldJsonl, parseReflectionResponse, partitionHeldOut, passOrthogonality, pixelDeltaRatio, politenessPrefixMutator, preflightModels, printDriverSummary, index as profile, promptBisect, proposeSynthesisTargets, recordRuns, recordRunsToScorecard, referenceReplayRunsToSteeringRows, referenceReplayScenarioToRunScore, regexMatches, renderMarkdownReport, renderPlaybookMarkdown, repeatedActionDetector, replayScorerOverCorpus, replayTraceThroughJudge, resolveModelPricing, resolveSeat, routeFields, rowCount, rowWhere, runAssertions, runBehavioralCanaries, runCanaries, runDistillation, runE2EWorkflow, runExpectations, runIntentMatchJudge, runJudgeFleet, runKeywordCoverageJudge, runKeywordCoverageJudgeUrl, runLiveProof, runReferenceReplay, runScore, runSelfPlay, scanForMuffledGates, scoreContinuity, scorePrReviewComments, scorePrReviewSource, scoreReferenceReplay, seatPresets, securityJudge, sentenceReorderMutator, splitGold, statusAdvanced, summarizePrReviewBenchmark, testJudge, textInSnapshot, toLangfuseEnvelope, toOpenAiTool, toPrometheusText, traceContract, traceJudge, traceJudgeEnsemble, tracedAnalyzeTraces, typoMutator, urlContains, verifyAttestation, visualDiff, viteDeployRunner, weightedRecall, whitespaceCollapseMutator, withHeldoutBlend, withJudgeRetry, withOtelPipeline, wranglerDeployRunner };
+export { ATTESTATION_ALGORITHM, type ActiveLearningOptions, type AdapterRun, AgentDriver, type AgentDriverConfig, AgentEvalError, type AgreementResult, type AlignmentOp, AnalyzeTracesInput, AnalyzeTracesOptions, AnalyzeTracesResult, type AntiSlopConfig, type AntiSlopIssue, type AntiSlopReport, type AssertCapabilityHeadroomOptions, type AssertCrossFamilyOptions, type AssertSingleBackendOptions, type AttestationProvenance, type AttestationVerification, type AttestedReport, type AutoPrClient, AxGepaSteeringOptimizer, type AxSteeringOptimizerConfig, type BackendDescriptor, BaselineReport, BehaviorAssertion, BehavioralMetrics, BenchmarkReport, BenchmarkRunner, BenchmarkRunnerConfig, type BisectOptions, type BisectResult, type BisectStep, type BlendWeights, BudgetBreachError, BudgetGuard, BudgetLedgerEntry, BudgetSpec, type BuildAgreementJudgeOptions, CODING_HARNESSES, type CachedJudge, type CachedJudgeOptions, CallExpectation, type CanaryAlert, type CanaryKind, type CanaryLeak, type CanaryOptions, type CanaryReport, type CanarySeverity, type CandidateComparison, type CandidateScenario, type CapabilityHeadroomOptions, type CapabilityHeadroomResult, type CausalAttributionReport, type CellVerdict, ChannelRollup, ChatRequest, CheckResult, CollectedArtifacts, type CommandRunner, type CompareLabels, type ComparePairedArmsOptions, CompletionCriterion, ConfigError, type ContinuityCheck, type ContinuityCheckResult, type ContinuityReport, type ContinuitySnapshotPair, type ContractCheckResult, type ContractJudgeOptions, type ContractMetric, type ContractReport, type ContractRule, type ContractRuleKind, type ContractSpan, type ContractVerdict, type ContractViolation, ConvergenceTracker, CorrectnessChecker, type CostEntry, CostLedger, type CostReport, type CostSummary, CostTracker, CreateChatClientOpts, type CreateDefaultReviewerOptions, type CreateExperimentInput, type CreateSandboxPoolOpts, CrossFamilyError, type CrossTraceDiff, type CrossTraceDiffOptions, DEFAULT_AGENT_SLOS, DEFAULT_FINDERS, DEFAULT_MUTATION_PRIMITIVES, DEFAULT_MUTATORS, DEFAULT_PR_REVIEW_SCORE_WEIGHTS, DEFAULT_SEVERITY_WEIGHTS, Dataset, DatasetScenario, type DecideNextUserTurnOpts, DefaultVerdict, type DeployFamily, type DeployGateLayerInput, type DeployRunResult, type DeployRunner, type DescriptionLengthCandidate, type DescriptionLengthConfig, type DescriptionLengthDecision, type DescriptionLengthEvidence, DescriptionLengthGate, type DescriptionLengthRejectionCode, type DetectorEvent, type DetectorSeverity, type DetectorSignal, type DiffScorecardOptions, type DirEntry, type DiscoverPersonasOptions, type DiscoveredPersona, DriverResult, DriverState, DualAgentBench, type DualAgentBenchConfig, type DualAgentReport, type DualAgentRound, type DualAgentScenario, type DualAgentScenarioResult, ERROR_COUNT_PATTERNS, type EnsembleAggregate, type EnsembleJudgeOptions, type ErrorCountPattern, type ErrorStreakOptions, type EvalToolDef, EvalTraceStore, type EvolutionRound, type ExecutorConfig, type Expectation, type Experiment, type ExperimentProvenance, type ExperimentRep, type ExperimentStats, type ExperimentStore, ExperimentTracker, type ExperimentTrackerOptions, type ExperimentVerdict, type ExportedRewardModel, type ExtractOptions, type ExtractResult, type FactorContribution, type FactorialCell, FailureClass, FeedbackLabel, FeedbackTrajectory, FeedbackTrajectoryStore, type FieldAgreementSpec, type FieldDestination, type FileChange, type FlowAction, type FlowLayerEnv, type FlowLayerFactoryInput, type FlowRunner, type FlowRunnerStepResult, type FlowSpec, type FlowStep, type GhCliClientOptions, type GoldScenario, type GoldSplit, type GoldenSeverity, type GoldenSpec, HARNESS_NATIVE_MODEL, HarnessConfig, type HeadroomClass, type HeadroomInput, type HeldOutPartition, type HiddenCriteriaGrader, type HiddenGradeResult, type HiddenLeak, HoldoutAuditor, type HttpGithubClientOptions, INTENT_MATCH_JUDGE_VERSION, type ImageData, type ImprovementThresholds, type ImprovementVerdictResult, InMemoryWorkspaceInspector, type InferenceScorer, type InspectorContext, type IntentMatchInput, type IntentMatchOptions, type IntentMatchResult, type InteractionContribution, JudgeError, type JudgeFamily, type JudgeFleetOptions, JudgeFn, JudgeParseError, type JudgeReplayResult, type JudgeRetryOutcome, type JudgeRetryPolicy, JudgeRunner, type JudgeScoreInput, type JudgeVerdict, type KeywordConceptSpec, type KeywordCoverageFinding, type KeywordCoverageOptions, type KeywordCoverageResult, type LangfuseEnvelope, type LangfuseGeneration, type LangfuseScore, Layer, LayerResult, type LeaderboardOptions, type LeaderboardRow, type LiveProofArtifact, type LiveProofConfig, type LiveProofContext, type LiveProofResult, LlmClientOptions, LlmSpan, MODEL_PRICING, type MakeEvalToolsConfig, type MatchResult, type MatchedPair, type MatcherResult, McNemarResult, type MergeOptions, MetricsCollector, type ModelCostRollup, type ModelPreflight, type ModelSeats, ModelsUnreachableError, type MuffledFinder, type MuffledFinding, type MultiToolchainLayerConfig, type Mutator, Mutex, type NoLeakOptions, type NoProgressOptions, type Oracle, type OracleObservation, type OracleReport, type OracleResult, type OrthogonalityInput, type OrthogonalityResult, OtelExportConfig, OtelExporter, type OtelPipelineHandle, type OtelPipelineOptions, type PairArmsOptions, type PairArmsResult, type PairedArmRow, type PairedArmsComparison, PairedBootstrapOptions, PairedBootstrapResult, type PairedCorrectness, type PairedMetricDelta, PairwiseSteeringOptimizer, type ParaphraseRobustnessScenarioInput, type ParaphraseRobustnessScenarioResult, type ParseStudentLabel, type PartitionHeldOutOptions, PersonaConfig, type Playbook, type PlaybookEntry, type PoolSlot, type PrReviewAuditCase, type PrReviewBenchmarkSummary, type PrReviewComment, type PrReviewMatchedFinding, type PrReviewOutcome, type PrReviewReferenceFinding, type PrReviewScore, type PrReviewScoreWeights, type PrReviewSeverity, type PrReviewSource, type PreflightModelsOptions, type PreflightOutcome, ProductClient, ProductClientConfig, type ProfileAxisSpec, type PromptHandle, PromptRegistry, type ProposeAutomatedPullRequestInput, type ProposeAutomatedPullRequestResult, type ProvenanceReader, type RecordRunsOptions, type ReferenceMatchResult, type ReferenceReplayAdapter, type ReferenceReplayAdapterFn, type ReferenceReplayAdapterLike, type ReferenceReplayAggregate, type ReferenceReplayCandidate, type ReferenceReplayCase, type ReferenceReplayCaseRun, type ReferenceReplayExecutionScenario, type ReferenceReplayItem, type ReferenceReplayMatch, type ReferenceReplayMatchStrategy, type ReferenceReplayMatcher, type ReferenceReplayPromotionDecision, type ReferenceReplayPromotionPolicy, type ReferenceReplayRun, type ReferenceReplayRunContext, type ReferenceReplayRunOptions, type ReferenceReplayRunStore, type ReferenceReplayScenario, type ReferenceReplayScenarioScore, type ReferenceReplayScore, type ReferenceReplayScoreOptions, type ReferenceReplaySplit, type ReferenceReplaySplitComparison, type ReferenceReplaySteeringRowsOptions, type ReflectionContext, type ReflectionProposal, ReleaseConfidenceScorecard, ReleaseConfidenceThresholds, type RenderStudentPrompt, type RepeatedActionOptions, type RepoRef, type ReviewerMemoryEntry, type ReviewerOutput, type ReviewerPromptInput, type ReviewerSoftFailDefaults, type ReviewerVerificationSummary, RiskDifferenceResult, type RobustnessResult, type RoutedField, Run, type RunCommandInput, type RunCommandResult, type RunDistillationOptions, type RunDistillationResult, RunFilter, RunRecord, type RunRecordBackend, type RunRecordFilter, RunScore, RunScoreWeights, RunSplitTag, SandboxDriver, SandboxHarnessResult, type SandboxJudgeKind, type SandboxJudgeResult, type SandboxJudgeSpec, type SandboxPool, type ScanOptions, Scenario, type ScenarioCost, ScenarioFile, ScenarioRegistry, ScenarioResult, type Scorecard, type ScorecardCell, type ScorecardCellDiff, type ScorecardDiff, type ScorecardEntry, type ScorecardLogLine, type ScoredTarget, type SeatName, type SeatPresetName, SeatUnsetError, type SelfPlayOptions, type SelfPlayProposer, type SelfPlayScorer, type SerializedRegex, Severity, type SingleBackendDivergence, SingleBackendError, type SingleBackendField, type SingleBackendReport, type Slo, type SloCheckResult, type SloComparator, type SloReport, type SloSeverity, type SlopCategory, type SlotFactory, type SpanPredicate, type SplitGoldOptions, SteeringBundle, type SteeringOptimizationResult, type SteeringOptimizationRow, type SteeringOptimizationSelector, type SteeringOptimizerBackend, type SteeringOptimizerConfig, type StepAttribution, type StreamingDetector, type SynthesisReason, type SynthesisTarget, type TaskHeadroom, TestResult, type TextMatcher, type ThresholdContract, TokenCounter, type TokenSpec, type ToolMatcher, ToolSpan, TraceAnalystSpan, type TraceContract, TraceContractBuilder, TraceEmitter, TraceStore, type TracedAnalystOptions, type TracedJudgeOptions, Trajectory, TrajectoryStep, type TreatmentClass, type TreatmentGate, type TreatmentGateInput, type TreatmentGateOptions, type TrialTrace, TurnMetrics, UI_FINDING_SEVERITIES, UI_LENSES, UNIVERSAL_FINDERS, type UiFinding, type UiFindingScreenshot, type UiFindingSeverity, type UiLens, type VerdictCacheStats, type VerdictCacheStore, VerifyContext, type VisualDiffOptions, type VisualDiffResult, type ViteDeployRunnerInput, type WorkerDriverContext, type WorkspaceAssertion, type WorkspaceAssertionResult, type WorkspaceInspector, type WorkspaceSnapshot, type WranglerDeployRunnerInput, adversarialJudge, agentProfileHash, agentProfileId, agentProfileModelId, agentVisibleFields, aggregateJudgeVerdicts, aggregatePrReviewScore, analyzeAntiSlop, appendScorecard, assertCapabilityHeadroom, assertCrossFamily, assertModelsServed, assertNoHiddenLeak, assertSingleBackend, assignHeldOutTag, attachCostToReport, attest, bisect, blendHeldout, buildAgreementJudge, buildDriverSystemPrompt, buildReflectionPrompt, buildReviewerPrompt, buildWorkerDriverSystemPrompt, cachedJudge, canaryLeakView, canonicalJson, capabilityHeadroom, causalAttribution, checkBehavioralCanary, checkCanaries, checkSlos, checkTraceContracts, classifyTreatment, codeExecutionJudge, coherenceJudge, collectionPreserved, commentsForSource, commitBisect, comparePairedArms, compareReferenceReplay, compilerJudge, computeExperimentStats, contentHash, contractJudge, costReport, createAntiSlopJudge, createCustomJudge, createDefaultReviewer, createDomainExpertJudge, createIntentMatchJudge, createSandboxPool, crossTraceDiff, dataDescriptionBits, decideNextUserTurn, decideReferenceReplayPromotion, decideReferenceReplayRunPromotion, defaultBlendWeights, defaultJudges, defaultParseStudentLabel, defaultReferenceReplayMatcher, defaultRenderStudentPrompt, deployGateLayer, diffScorecard, discoverPersonas, distillPlaybook, ensembleJudge, errorStreakDetector, estimateCost, estimateTokens, evaluateContract, evaluateOracles, evaluateTraceContract, executeScenario, expandProfileAxes, expectAgent, exportRewardModel, extractAssetUrls, extractErrorCount, fieldAgreement, fileContains, fileExists, fileExperimentStore, fileVerdictCache, findAutoMatchNoExpectation, findConstructorCwdDropped, findFallbackToPass, findLiteralTruePass, findSkipCountsAsPass, flowLayer, fnv1a32, formatBenchmarkReport, formatDriverReport, formatFindings, formatScorecardDiff, gateTreatmentApplied, gateTreatmentFromMetrics, gateTreatmentFromSpans, gateTreatmentFromToolSpans, ghCliClient, gitProvenanceReader, precision as goldenPrecision, gradeOnHidden, harnessAxisOf, hashContent, hashToUnit, hiddenGrade, htmlContainsElement, httpGithubClient, improvementVerdict, inMemoryExperimentStore, inMemoryReferenceReplayStore, inMemoryRunRecordBackend, inMemoryVerdictCache, isHiddenDestination, isModelPriced, isOtelConfigured, jsonShape, jsonlReferenceReplayStore, jsonlRunRecordBackend, judgeFamily, keyPreserved, leaderboard, linterJudge, loadGoldScenarios, loadScorecard, loadScorerFromGrader, localCommandRunner, lowercaseMutator, makeEvalTools, matchGoldens, matchSpan, mergeLayerResults, modelDescriptionBits, multiToolchainLayer, noProgressDetector, notBlocked, observeAll, pairArms, paraphraseRobustness, paraphraseRobustnessScenarios, parseGoldJsonl, parseReflectionResponse, partitionHeldOut, passOrthogonality, pixelDeltaRatio, politenessPrefixMutator, preflightModels, printDriverSummary, index as profile, promptBisect, proposeSynthesisTargets, recordRuns, recordRunsToScorecard, referenceReplayRunsToSteeringRows, referenceReplayScenarioToRunScore, regexMatches, renderMarkdownReport, renderPlaybookMarkdown, repeatedActionDetector, replayScorerOverCorpus, replayTraceThroughJudge, resolveModelPricing, resolveSeat, routeFields, rowCount, rowWhere, runAssertions, runBehavioralCanaries, runCanaries, runDistillation, runE2EWorkflow, runExpectations, runIntentMatchJudge, runJudgeFleet, runKeywordCoverageJudge, runKeywordCoverageJudgeUrl, runLiveProof, runReferenceReplay, runScore, runSelfPlay, scanForMuffledGates, scoreContinuity, scorePrReviewComments, scorePrReviewSource, scoreReferenceReplay, seatPresets, securityJudge, sentenceReorderMutator, splitGold, statusAdvanced, summarizePrReviewBenchmark, testJudge, textInSnapshot, toLangfuseEnvelope, toOpenAiTool, toPrometheusText, traceContract, traceJudge, traceJudgeEnsemble, tracedAnalyzeTraces, typoMutator, urlContains, verifyAttestation, visualDiff, viteDeployRunner, weightedRecall, whitespaceCollapseMutator, withHeldoutBlend, withJudgeRetry, withOtelPipeline, wranglerDeployRunner };

package/dist/index.js

@@ -1069,6 +1069,77 @@ var BenchmarkRunner = class {
   }
 };
 
+// src/capability-headroom.ts
+function capabilityHeadroom(rows, opts = {}) {
+  const keepThreshold = opts.keepThreshold ?? 0;
+  if (!Number.isFinite(keepThreshold) || keepThreshold < 0 || keepThreshold >= 1) {
+    throw new ValidationError(
+      `capabilityHeadroom: keepThreshold must be in [0, 1), got ${keepThreshold}`
+    );
+  }
+  if (rows.length === 0) {
+    throw new ValidationError(
+      "capabilityHeadroom: no baseline rows supplied \u2014 run the capability-absent baseline arm first"
+    );
+  }
+  const byTask = /* @__PURE__ */ new Map();
+  for (const row of rows) {
+    if (row.baselineOutcome !== "pass" && row.baselineOutcome !== "fail" && row.baselineOutcome !== "unknown") {
+      throw new ValidationError(
+        `capabilityHeadroom: unrecognized baselineOutcome '${row.baselineOutcome}' for task '${row.taskId}' (expected 'pass' | 'fail' | 'unknown')`
+      );
+    }
+    const agg = byTask.get(row.taskId) ?? { n: 0, passes: 0, known: 0 };
+    agg.n++;
+    if (row.baselineOutcome !== "unknown") {
+      agg.known++;
+      if (row.baselineOutcome === "pass") agg.passes++;
+    }
+    byTask.set(row.taskId, agg);
+  }
+  const tasks = [...byTask.entries()].sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0).map(([taskId, agg]) => {
+    if (agg.known === 0) {
+      return {
+        taskId,
+        n: agg.n,
+        nKnown: 0,
+        baselinePassRate: Number.NaN,
+        headroom: "unknown"
+      };
+    }
+    const baselinePassRate = agg.passes / agg.known;
+    return {
+      taskId,
+      n: agg.n,
+      nKnown: agg.known,
+      baselinePassRate,
+      headroom: baselinePassRate <= keepThreshold ? "gap" : "saturated"
+    };
+  });
+  const summary = {
+    tasksWithGap: tasks.filter((t) => t.headroom === "gap").length,
+    tasksSaturated: tasks.filter((t) => t.headroom === "saturated").length,
+    tasksUnknown: tasks.filter((t) => t.headroom === "unknown").length,
+    repsUnknown: tasks.reduce((sum3, t) => sum3 + (t.n - t.nKnown), 0)
+  };
+  return { tasks, summary };
+}
+function assertCapabilityHeadroom(result, opts = {}) {
+  const minTasksWithGap = opts.minTasksWithGap ?? 1;
+  if (!Number.isInteger(minTasksWithGap) || minTasksWithGap < 1) {
+    throw new ValidationError(
+      `assertCapabilityHeadroom: minTasksWithGap must be an integer \u2265 1, got ${minTasksWithGap}`
+    );
+  }
+  const { tasksWithGap, tasksSaturated, tasksUnknown } = result.summary;
+  if (tasksWithGap < minTasksWithGap) {
+    const total = result.tasks.length;
+    throw new ValidationError(
+      `assertCapabilityHeadroom: only ${tasksWithGap} of ${total} task(s) have baseline headroom (need \u2265 ${minTasksWithGap}). ${tasksSaturated} saturated (baseline already passes), ${tasksUnknown} unknown (no determinable baseline outcome). A capability A/B on this task set cannot detect the capability \u2014 add tasks the baseline fails, or fix baseline outcome capture, before measuring.`
+    );
+  }
+}
+
 // src/client.ts
 var DEFAULT_REQUEST_TIMEOUT_MS = 3e4;
 var ProductClient = class {
@@ -2303,6 +2374,155 @@ function liveProofToReleaseTrace(config, trajectory, durationMs) {
   };
 }
 
+// src/paired-arms.ts
+function pairArms(rows, opts) {
+  const { baselineArm, treatmentArm } = opts;
+  if (baselineArm === treatmentArm) {
+    throw new ValidationError(
+      `pairArms: baselineArm and treatmentArm are both '${baselineArm}' \u2014 an arm cannot be compared to itself`
+    );
+  }
+  const byArm = /* @__PURE__ */ new Map();
+  const armsSeen = /* @__PURE__ */ new Set();
+  for (const row of rows) {
+    armsSeen.add(row.arm);
+    if (row.arm !== baselineArm && row.arm !== treatmentArm) continue;
+    const byKey = byArm.get(row.arm) ?? /* @__PURE__ */ new Map();
+    const group = byKey.get(row.pairKey) ?? [];
+    group.push(row);
+    byKey.set(row.pairKey, group);
+    byArm.set(row.arm, byKey);
+  }
+  for (const arm of [baselineArm, treatmentArm]) {
+    if (!byArm.has(arm)) {
+      const seen = [...armsSeen].sort().join(", ") || "<none>";
+      throw new ValidationError(`pairArms: no rows for arm '${arm}' (arms present: ${seen})`);
+    }
+  }
+  const baselineByKey = byArm.get(baselineArm);
+  const treatmentByKey = byArm.get(treatmentArm);
+  const allKeys = [.../* @__PURE__ */ new Set([...baselineByKey.keys(), ...treatmentByKey.keys()])].sort();
+  const pairs = [];
+  const unpairedBaseline = [];
+  const unpairedTreatment = [];
+  for (const pairKey2 of allKeys) {
+    const b = baselineByKey.get(pairKey2) ?? [];
+    const t = treatmentByKey.get(pairKey2) ?? [];
+    if (b.length <= 1 && t.length <= 1) {
+      if (b.length === 1 && t.length === 1) {
+        pairs.push({ pairKey: pairKey2, repIndex: 0, baseline: b[0], treatment: t[0] });
+      } else {
+        unpairedBaseline.push(...b);
+        unpairedTreatment.push(...t);
+      }
+      continue;
+    }
+    const bByRep = indexByRepKey(b, pairKey2, baselineArm);
+    const tByRep = indexByRepKey(t, pairKey2, treatmentArm);
+    const repKeys = [.../* @__PURE__ */ new Set([...bByRep.keys(), ...tByRep.keys()])].sort();
+    let repIndex = 0;
+    for (const repKey of repKeys) {
+      const baseline = bByRep.get(repKey);
+      const treatment = tByRep.get(repKey);
+      if (baseline !== void 0 && treatment !== void 0) {
+        pairs.push({ pairKey: pairKey2, repIndex: repIndex++, baseline, treatment });
+      } else if (baseline !== void 0) {
+        unpairedBaseline.push(baseline);
+      } else if (treatment !== void 0) {
+        unpairedTreatment.push(treatment);
+      }
+    }
+  }
+  return { pairs, unpairedBaseline, unpairedTreatment };
+}
+function indexByRepKey(group, pairKey2, arm) {
+  const byRep = /* @__PURE__ */ new Map();
+  for (const row of group) {
+    if (row.repKey === void 0) {
+      throw new ValidationError(
+        `pairArms: pairKey '${pairKey2}' has multiple reps in an arm, but a row in arm '${arm}' is missing repKey \u2014 multi-rep items require an explicit repKey on every row so reps pair by identity (pairing reps by outcome or by index would bias the paired statistics)`
+      );
+    }
+    if (byRep.has(row.repKey)) {
+      throw new ValidationError(
+        `pairArms: duplicate repKey '${row.repKey}' for pairKey '${pairKey2}' in arm '${arm}' \u2014 (pairKey, repKey) must uniquely identify a rep within an arm`
+      );
+    }
+    byRep.set(row.repKey, row);
+  }
+  return byRep;
+}
+function comparePairedArms(rows, opts) {
+  const { pairs, unpairedBaseline, unpairedTreatment } = pairArms(rows, opts);
+  let correctness = null;
+  const baselinePass = [];
+  const treatmentPass = [];
+  for (const pair of pairs) {
+    if (pair.baseline.pass === void 0 || pair.treatment.pass === void 0) continue;
+    baselinePass.push(pair.baseline.pass ? 1 : 0);
+    treatmentPass.push(pair.treatment.pass ? 1 : 0);
+  }
+  if (baselinePass.length > 0) {
+    const mc = mcnemar(baselinePass, treatmentPass);
+    correctness = {
+      b10: mc.b,
+      b01: mc.c,
+      mcnemar: mc,
+      riskDifference: pairedRiskDifference(baselinePass, treatmentPass)
+    };
+  }
+  const metricNames = opts.metricNames ?? [
+    ...new Set(
+      pairs.flatMap((p) => [
+        ...Object.keys(p.baseline.metrics ?? {}),
+        ...Object.keys(p.treatment.metrics ?? {})
+      ])
+    )
+  ].sort();
+  const metricDeltas = metricNames.map((name) => {
+    const before = [];
+    const after = [];
+    let nMissing = 0;
+    for (const pair of pairs) {
+      const b = metricValue(pair.baseline, name);
+      const t = metricValue(pair.treatment, name);
+      if (b === void 0 || t === void 0) {
+        nMissing++;
+        continue;
+      }
+      before.push(b);
+      after.push(t);
+    }
+    const bootstrapCi2 = before.length === 0 ? null : pairedBootstrap(before, after, opts.bootstrap);
+    return {
+      name,
+      n: before.length,
+      nMissing,
+      medianDelta: bootstrapCi2 === null ? Number.NaN : bootstrapCi2.median,
+      meanDelta: bootstrapCi2 === null ? Number.NaN : bootstrapCi2.mean,
+      bootstrapCi: bootstrapCi2,
+      wilcoxon: before.length === 0 ? null : wilcoxonSignedRank(before, after)
+    };
+  });
+  return {
+    nPairs: pairs.length,
+    nUnpairedBaseline: unpairedBaseline.length,
+    nUnpairedTreatment: unpairedTreatment.length,
+    correctness,
+    metricDeltas
+  };
+}
+function metricValue(row, name) {
+  const v = row.metrics?.[name];
+  if (v === void 0) return void 0;
+  if (!Number.isFinite(v)) {
+    throw new ValidationError(
+      `comparePairedArms: non-finite value for metric '${name}' on pairKey '${row.pairKey}' (arm '${row.arm}'): ${v}`
+    );
+  }
+  return v;
+}
+
 // src/pr-review-benchmark.ts
 var DEFAULT_PR_REVIEW_SCORE_WEIGHTS = {
   recall: 4,
@@ -10147,6 +10367,7 @@ export {
   argHash,
   asNumber,
   asString,
+  assertCapabilityHeadroom,
   assertCrossFamily,
   assertLlmRoute,
   assertModelsServed,
@@ -10194,6 +10415,7 @@ export {
   canaryLeakView,
   canonicalJson,
   canonicalize,
+  capabilityHeadroom,
   captureFetchToRawSink,
   causalAttribution,
   checkBehavioralCanary,
@@ -10212,6 +10434,7 @@ export {
   collectionPreserved,
   commentsForSource,
   commitBisect,
+  comparePairedArms,
   compareReferenceReplay,
   compareToBaseline,
   compilerJudge,
@@ -10417,6 +10640,7 @@ export {
   otelRunCompleteHook,
   otlpToRunRecords,
   otlpToTraceRunRecords,
+  pairArms,
   pairedBootstrap,
   pairedEvalueSequence,
   pairedMde,