@tangle-network/agent-eval 0.22.0 → 0.23.1

package/dist/{integrity-K2oVlF57.d.ts → integrity-Cr5YodSY.d.ts}

@@ -207,4 +207,4 @@ declare function assertRunCaptured(store: TraceStore, runId: string, expectation
 /** Strict mode: throws `RunIntegrityError` when the report isn't ok. */
 declare function throwIfRunIncomplete(report: RunIntegrityReport): void;
 
-export { FileSystemRawProviderSink as F, InMemoryRawProviderSink as I, NoopRawProviderSink as N, type ProviderRedactor as P, type RawProviderDirection as R, type FileSystemRawProviderSinkOptions as a, type InMemoryRawProviderSinkOptions as b, type RawProviderEvent as c, type RawProviderSink as d, type RawProviderSinkFilter as e, RunIntegrityError as f, type RunIntegrityExpectations as g, type RunIntegrityIssue as h, type RunIntegrityIssueCode as i, type RunIntegrityReport as j, assertRunCaptured as k, defaultProviderRedactor as l, providerFromBaseUrl as p, throwIfRunIncomplete as t };
+export { FileSystemRawProviderSink as F, InMemoryRawProviderSink as I, NoopRawProviderSink as N, type ProviderRedactor as P, type RawProviderSink as R, type RunIntegrityExpectations as a, type RunIntegrityReport as b, type FileSystemRawProviderSinkOptions as c, type InMemoryRawProviderSinkOptions as d, type RawProviderDirection as e, type RawProviderEvent as f, type RawProviderSinkFilter as g, RunIntegrityError as h, type RunIntegrityIssue as i, type RunIntegrityIssueCode as j, assertRunCaptured as k, defaultProviderRedactor as l, providerFromBaseUrl as p, throwIfRunIncomplete as t };

package/dist/openapi.json

@@ -2,7 +2,7 @@
   "openapi": "3.1.0",
   "info": {
     "title": "@tangle-network/agent-eval — wire protocol",
-    "version": "0.22.0",
+    "version": "0.23.1",
     "description": "HTTP and stdio RPC interface to agent-eval. The TypeScript runtime is the source of truth; this spec is the contract that cross-language clients (Python, Rust, Go) generate from.\n\nWire-protocol version: 1.0.0. Bumps on breaking changes to request/response schemas.",
     "contact": {
       "name": "Tangle Network",

package/dist/optimization.d.ts

@@ -1,8 +1,8 @@
-export { C as CallbackResearcher, a as CallbackResearcherOptions, b as CampaignFactoryParams, c as CampaignIntegrityPolicy, d as CampaignRunContext, e as CampaignRunOutcome, f as CampaignRunner, g as CampaignScenario, h as CampaignVariant, E as EvalCampaignOptions, i as EvalCampaignResult, j as ExperimentPlan, k as ExperimentResult, F as FailedRun, l as FailureMode, N as NoopResearcher, R as Researcher, S as SteeringChange, y as runEvalCampaign } from './optimization-UVDNKaO6.js';
-export { A as ActionableSideInfo, b as AsiSeverity, c as DEFAULT_MUTATION_PRIMITIVES, E as EvolvableVariant, m as GenerationReport, I as InMemoryTrialCache, p as MultiShotGateConfig, q as MultiShotGateResult, r as MultiShotMutateAdapter, s as MultiShotOptimizationConfig, t as MultiShotOptimizationResult, u as MultiShotRun, v as MultiShotRunInput, w as MultiShotRunner, x as MultiShotScore, y as MultiShotScorer, z as MultiShotSplit, B as MultiShotTrace, C as MultiShotTrialResult, J as MultiShotVariant, M as MutateAdapter, N as PromptEvolutionConfig, Q as PromptEvolutionEvent, R as PromptEvolutionResult, U as ReflectionContext, W as ReflectionProposal, a1 as ScenarioAggregate, a2 as ScoreAdapter, T as TrialCache, a as TrialResult, a6 as TrialTrace, V as VariantAggregate, a7 as buildReflectionPrompt, aa as defaultMultiShotObjectives, ah as parseReflectionResponse, aj as runMultiShotOptimization, ak as runPromptEvolution, an as trialTraceFromMultiShotTrial } from './summary-report-D4p7RlDu.js';
-export { n as FeedbackArtifactType, o as FeedbackAttempt, F as FeedbackLabel, p as FeedbackLabelKind, q as FeedbackLabelSource, r as FeedbackOptimizerRow, s as FeedbackOutcome, t as FeedbackReplayAdapter, u as FeedbackReplayResult, v as FeedbackSeverity, w as FeedbackSplitPolicy, x as FeedbackTask, b as FeedbackTrajectory, y as FeedbackTrajectoryFilter, a as FeedbackTrajectoryStore, z as FileSystemFeedbackTrajectoryStore, I as InMemoryFeedbackTrajectoryStore, P as PreferenceMemoryEntry, A as ProposedSideEffect, D as assignFeedbackSplit, E as controlRunToFeedbackTrajectory, G as createFeedbackTrajectory, H as feedbackTrajectoriesToDatasetScenarios, J as feedbackTrajectoriesToOptimizerRows, K as feedbackTrajectoryToDatasetScenario, L as feedbackTrajectoryToOptimizerRow, N as parseFeedbackTrajectoriesJsonl, O as renderPreferenceMemoryMarkdown, Q as replayFeedbackTrajectories, R as replayFeedbackTrajectory, U as serializeFeedbackTrajectoriesJsonl, Y as summarizePreferenceMemory, Z as withAssignedFeedbackSplit } from './feedback-trajectory-CB0A32o3.js';
-import './integrity-K2oVlF57.js';
+export { C as CallbackResearcher, a as CallbackResearcherOptions, b as CampaignFactoryParams, c as CampaignIntegrityPolicy, d as CampaignRunContext, e as CampaignRunOutcome, f as CampaignRunner, g as CampaignScenario, h as CampaignVariant, E as EvalCampaignOptions, i as EvalCampaignResult, j as ExperimentPlan, k as ExperimentResult, F as FailedRun, l as FailureMode, N as NoopResearcher, R as Researcher, S as SteeringChange, r as runEvalCampaign } from './eval-campaign-Ds5QljIh.js';
+export { A as ActionableSideInfo, a as AsiSeverity, D as DEFAULT_MUTATION_PRIMITIVES, E as EvolvableVariant, G as GenerationReport, I as InMemoryTrialCache, M as MultiShotGateConfig, b as MultiShotGateResult, c as MultiShotMutateAdapter, d as MultiShotOptimizationConfig, e as MultiShotOptimizationResult, f as MultiShotRun, g as MultiShotRunInput, h as MultiShotRunner, i as MultiShotScore, j as MultiShotScorer, k as MultiShotSplit, l as MultiShotTrace, m as MultiShotTrialResult, n as MultiShotVariant, o as MutateAdapter, P as PromptEvolutionConfig, p as PromptEvolutionEvent, q as PromptEvolutionResult, R as ReflectionContext, r as ReflectionProposal, S as ScenarioAggregate, s as ScoreAdapter, T as TrialCache, t as TrialResult, u as TrialTrace, V as VariantAggregate, v as buildReflectionPrompt, w as defaultMultiShotObjectives, x as parseReflectionResponse, y as runMultiShotOptimization, z as runPromptEvolution, B as trialTraceFromMultiShotTrial } from './summary-report-Ce1r4EYo.js';
+export { F as FeedbackArtifactType, a as FeedbackAttempt, b as FeedbackLabel, c as FeedbackLabelKind, d as FeedbackLabelSource, e as FeedbackOptimizerRow, f as FeedbackOutcome, g as FeedbackReplayAdapter, h as FeedbackReplayResult, i as FeedbackSeverity, j as FeedbackSplitPolicy, k as FeedbackTask, l as FeedbackTrajectory, m as FeedbackTrajectoryFilter, n as FeedbackTrajectoryStore, o as FileSystemFeedbackTrajectoryStore, I as InMemoryFeedbackTrajectoryStore, P as PreferenceMemoryEntry, p as ProposedSideEffect, q as assignFeedbackSplit, r as controlRunToFeedbackTrajectory, s as createFeedbackTrajectory, t as feedbackTrajectoriesToDatasetScenarios, u as feedbackTrajectoriesToOptimizerRows, v as feedbackTrajectoryToDatasetScenario, w as feedbackTrajectoryToOptimizerRow, x as parseFeedbackTrajectoriesJsonl, y as renderPreferenceMemoryMarkdown, z as replayFeedbackTrajectories, A as replayFeedbackTrajectory, B as serializeFeedbackTrajectoriesJsonl, C as summarizePreferenceMemory, D as withAssignedFeedbackSplit } from './feedback-trajectory-c43WGtTX.js';
+import './run-record-DNiOMBrZ.js';
+import './integrity-Cr5YodSY.js';
 import './store-u47QaJ9G.js';
 import './emitter-B2XqDKFU.js';
-import './run-record-CX_jcAyr.js';
 import './dataset-B9qvlm_o.js';

package/dist/optimization.js

@@ -19,21 +19,23 @@ import {
   renderPreferenceMemoryMarkdown,
   replayFeedbackTrajectories,
   replayFeedbackTrajectory,
-  runEvalCampaign,
   runMultiShotOptimization,
   runPromptEvolution,
   serializeFeedbackTrajectoriesJsonl,
   summarizePreferenceMemory,
   trialTraceFromMultiShotTrial,
   withAssignedFeedbackSplit
-} from "./chunk-USHQBPMH.js";
-import "./chunk-YUFXO3TU.js";
+} from "./chunk-VQQSPGSM.js";
+import "./chunk-QBW3YBTR.js";
+import {
+  runEvalCampaign
+} from "./chunk-EXGR4XEM.js";
+import "./chunk-KAO3Q65R.js";
 import "./chunk-IOXMGMHQ.js";
 import "./chunk-QUKKGHTZ.js";
+import "./chunk-SQQLHODJ.js";
 import "./chunk-5IIQKMD5.js";
 import "./chunk-6M774GY6.js";
-import "./chunk-KAO3Q65R.js";
-import "./chunk-SQQLHODJ.js";
 import "./chunk-PZ5AY32C.js";
 export {
   CallbackResearcher,

package/dist/reporting.d.ts

@@ -1,5 +1,295 @@
-export { B as BootstrapOptions, c as BootstrapResult, e as InterimReleaseConfidence, f as InterimReleaseConfidenceInput, J as JudgeReplayGateArgs, P as PairedBootstrapOptions, g as PairedBootstrapResult, h as PairedEvalueOptions, i as PairedEvalueSequence, j as PairedEvalueStep, k as ReleaseConfidenceAxis, l as ReleaseConfidenceAxisName, m as ReleaseConfidenceInput, n as ReleaseConfidenceIssue, o as ReleaseConfidenceMetrics, a as ReleaseConfidenceScorecard, p as ReleaseConfidenceStatus, R as ReleaseConfidenceThresholds, q as ReleaseTraceEvidence, r as RenderReleaseReportOptions, s as RubricOutcomePair, t as RubricPredictiveValidityInput, u as RubricPredictiveValidityReport, v as RubricRanking, S as SequentialDecision, V as Verdict, w as assertReleaseConfidence, x as bhAdjust, y as bootstrapCi, z as evaluateInterimReleaseConfidence, A as evaluateReleaseConfidence, C as judgeReplayGate, E as pairedBootstrap, G as pairedEvalueSequence, H as pairedWilcoxon, K as releaseTraceEvidenceFromMultiShotTrials, L as renderReleaseReport, M as rubricPredictiveValidity } from './reporting-B82RSv9C.js';
-export { G as GainDistributionBin, i as GainDistributionFigureSpec, j as GainDistributionOptions, K as ParetoFigureSpec, L as ParetoPoint, S as RESEARCH_REPORT_HARD_PAIR_FLOOR, X as ResearchReport, Y as ResearchReportCandidate, Z as ResearchReportDecision, _ as ResearchReportMethodology, $ as ResearchReportOptions, a0 as ResearchReportRecommendation, a3 as SummaryTable, a4 as SummaryTableOptions, a5 as SummaryTableRow, ad as gainHistogram, ae as paretoChart, ai as researchReport, am as summaryTable } from './summary-report-D4p7RlDu.js';
-import './dataset-B9qvlm_o.js';
-import './run-record-CX_jcAyr.js';
+import { D as DatasetSplit, c as DatasetManifest, a as DatasetScenario } from './dataset-B9qvlm_o.js';
+import { C as GateDecision, A as ActionableSideInfo, m as MultiShotTrialResult } from './summary-report-Ce1r4EYo.js';
+export { Y as GainDistributionBin, Z as GainDistributionFigureSpec, _ as GainDistributionOptions, a3 as ParetoFigureSpec, a4 as ParetoPoint, a5 as RESEARCH_REPORT_HARD_PAIR_FLOOR, H as ResearchReport, a6 as ResearchReportCandidate, a7 as ResearchReportDecision, a8 as ResearchReportMethodology, F as ResearchReportOptions, a9 as ResearchReportRecommendation, aa as SummaryTable, ab as SummaryTableOptions, ac as SummaryTableRow, ah as gainHistogram, ai as paretoChart, al as researchReport, an as summaryTable } from './summary-report-Ce1r4EYo.js';
+import { R as RunRecord, a as RunSplitTag } from './run-record-DNiOMBrZ.js';
+export { c as InterimReleaseConfidence, d as InterimReleaseConfidenceInput, P as PairedEvalueOptions, e as PairedEvalueSequence, f as PairedEvalueStep, R as RubricOutcomePair, g as RubricPredictiveValidityInput, h as RubricPredictiveValidityReport, i as RubricRanking, S as SequentialDecision, j as evaluateInterimReleaseConfidence, p as pairedEvalueSequence, r as rubricPredictiveValidity } from './sequential-DgU2mFsE.js';
 import './store-u47QaJ9G.js';
+
+/**
+ * Release confidence gate.
+ *
+ * This is the production-facing composition layer over the lower-level
+ * primitives:
+ *   - Dataset manifests prove corpus/version coverage.
+ *   - RunRecord rows prove reproducible search/holdout outcomes.
+ *   - Multi-shot trace evidence carries turn counts and ASI diagnostics.
+ *   - HeldOutGate decisions remain the paired promotion authority.
+ *
+ * The gate is intentionally pure and conservative. Missing declared evidence
+ * fails closed instead of being treated as a neutral zero.
+ */
+
+type ReleaseConfidenceStatus = 'pass' | 'warn' | 'fail';
+type ReleaseConfidenceAxisName = 'corpus' | 'quality' | 'generalization' | 'diagnostics' | 'efficiency';
+interface ReleaseTraceEvidence {
+    scenarioId: string;
+    candidateId?: string;
+    split?: RunSplitTag;
+    score?: number;
+    ok?: boolean;
+    turnCount?: number;
+    costUsd?: number;
+    durationMs?: number;
+    failureMode?: string;
+    asi?: ActionableSideInfo[];
+    metadata?: Record<string, unknown>;
+}
+interface ReleaseConfidenceThresholds {
+    /** Require a Dataset manifest or explicit scenarios. Default true. */
+    requireCorpus?: boolean;
+    minScenarioCount?: number;
+    minSearchRuns?: number;
+    minHoldoutRuns?: number;
+    /** Require at least one holdout scenario/run. Default true. */
+    requireHoldout?: boolean;
+    minPassRate?: number;
+    minMeanScore?: number;
+    /** Search mean may exceed holdout mean by at most this much. */
+    maxOverfitGap?: number;
+    maxMeanCostUsd?: number;
+    maxP95WallMs?: number;
+    /** Low-score/failed rows must carry ASI. Default true. */
+    requireAsiForFailures?: boolean;
+    /** Score below this is considered a failure for ASI coverage. Default 0.5. */
+    failureScoreThreshold?: number;
+}
+interface ReleaseConfidenceInput {
+    target: string;
+    candidateId?: string;
+    baselineId?: string;
+    dataset?: DatasetManifest;
+    scenarios?: readonly DatasetScenario[];
+    runs?: readonly RunRecord[];
+    traces?: readonly ReleaseTraceEvidence[];
+    gateDecision?: GateDecision | null;
+    thresholds?: ReleaseConfidenceThresholds;
+}
+interface ReleaseConfidenceAxis {
+    name: ReleaseConfidenceAxisName;
+    status: ReleaseConfidenceStatus;
+    score: number;
+    detail: string;
+}
+interface ReleaseConfidenceIssue {
+    axis: ReleaseConfidenceAxisName;
+    severity: 'critical' | 'warning';
+    code: string;
+    detail: string;
+}
+interface ReleaseConfidenceMetrics {
+    scenarioCount: number;
+    searchRuns: number;
+    holdoutRuns: number;
+    passRate: number;
+    meanScore: number;
+    searchMeanScore: number;
+    holdoutMeanScore: number;
+    overfitGap: number;
+    meanCostUsd: number;
+    p95WallMs: number;
+    failedRows: number;
+    failuresWithAsi: number;
+    singleShotTraces: number;
+    multiShotTraces: number;
+    splitCounts: Record<DatasetSplit, number>;
+    domainCounts: Record<string, number>;
+    failureModeCounts: Record<string, number>;
+    responsibleSurfaceCounts: Record<string, number>;
+}
+interface ReleaseConfidenceScorecard {
+    target: string;
+    candidateId: string | null;
+    baselineId: string | null;
+    status: ReleaseConfidenceStatus;
+    promote: boolean;
+    axes: ReleaseConfidenceAxis[];
+    issues: ReleaseConfidenceIssue[];
+    metrics: ReleaseConfidenceMetrics;
+    dataset: DatasetManifest | null;
+    gateDecision: GateDecision | null;
+    summary: string;
+}
+declare function releaseTraceEvidenceFromMultiShotTrials(trials: readonly MultiShotTrialResult[]): ReleaseTraceEvidence[];
+declare function evaluateReleaseConfidence(input: ReleaseConfidenceInput): ReleaseConfidenceScorecard;
+declare function assertReleaseConfidence(input: ReleaseConfidenceInput): ReleaseConfidenceScorecard;
+
+/**
+ * Paper-grade paired statistics for held-out promotion gates.
+ *
+ * The promotion gate (`HeldOutGate`) needs three things:
+ *
+ *   1. A bootstrap confidence interval on the per-item paired delta
+ *      (`pairedBootstrap`). Median delta is the headline number; the
+ *      CI lower bound is what the gate checks against `pairedDeltaThreshold`.
+ *   2. A non-parametric significance test on the paired deltas
+ *      (`pairedWilcoxon` — re-export of `wilcoxonSignedRank` under the
+ *      paper-style name).
+ *   3. False-discovery-rate correction across simultaneously-tested
+ *      candidate variants (`bhAdjust` — re-export of `benjaminiHochberg`).
+ *
+ * Why a separate file: every existing primitive lives in `statistics.ts`
+ * (general) or `power-analysis.ts` (correction). Paired-bootstrap is
+ * paired-only, paper-grade, and load-bearing for the promotion gate.
+ * Putting it next to `statistics.ts` would require editing that file;
+ * the brief forbids that. New file, new exports, no surface change.
+ */
+interface PairedBootstrapResult {
+    /** Number of paired observations (after dropping unequal lengths is rejected). */
+    n: number;
+    /** Median of paired deltas (after − before). */
+    median: number;
+    /** Mean of paired deltas. */
+    mean: number;
+    /** Lower bound of the bootstrap CI on the median delta. */
+    low: number;
+    /** Upper bound of the bootstrap CI on the median delta. */
+    high: number;
+    /** Confidence level used (e.g. 0.95). */
+    confidence: number;
+    /** Number of bootstrap resamples used. */
+    resamples: number;
+}
+interface PairedBootstrapOptions {
+    /** Confidence level. Default 0.95. */
+    confidence?: number;
+    /** Bootstrap resample count. Default 2000. */
+    resamples?: number;
+    /** Statistic to bootstrap. Default 'median'. */
+    statistic?: 'median' | 'mean';
+    /** Deterministic seed. If omitted, uses Math.random(). */
+    seed?: number;
+}
+/**
+ * Paired bootstrap on (after - before) deltas. Returns a CI on the
+ * chosen statistic (median by default). Pairs are resampled with
+ * replacement. The lower bound is what the promotion gate checks: if
+ * `low > pairedDeltaThreshold`, the gain is real at the chosen
+ * confidence level.
+ *
+ * Throws on unequal sample sizes — caller must align pairs upstream.
+ */
+declare function pairedBootstrap(before: number[], after: number[], opts?: PairedBootstrapOptions): PairedBootstrapResult;
+/**
+ * Paper-style alias for `wilcoxonSignedRank`. The signed-rank test on
+ * paired deltas is the standard non-parametric significance test for
+ * "candidate beats baseline on matched items." Use alongside the
+ * bootstrap CI: bootstrap gives effect size, Wilcoxon gives p.
+ */
+declare function pairedWilcoxon(before: number[], after: number[]): {
+    w: number;
+    p: number;
+};
+/**
+ * Paper-style alias for `benjaminiHochberg`. Use to correct p-values
+ * across multiple candidate-vs-baseline comparisons run in the same
+ * promotion sweep. Returns BH-adjusted q-values and significance at
+ * the requested FDR (default 0.05).
+ */
+declare function bhAdjust(pValues: number[], fdr?: number): {
+    qValues: number[];
+    significant: boolean[];
+};
+
+interface RenderReleaseReportOptions {
+    title?: string;
+    runs?: readonly RunRecord[];
+    comparator?: string;
+    traceAnalystFindings?: readonly string[];
+    nextActions?: readonly string[];
+}
+declare function renderReleaseReport(scorecard: ReleaseConfidenceScorecard, options?: RenderReleaseReportOptions): string;
+
+/**
+ * Bootstrap-CI promotion gate.
+ *
+ * In any iterative-improvement loop (GEPA, prompt evolution, dataset
+ * curation), the question is "did this generation actually improve, or are
+ * we celebrating noise?". With small N and noisy outcomes, point-estimate
+ * deltas lie. Bootstrap confidence intervals tell the operator whether the
+ * delta is real before code or prompts get promoted.
+ *
+ * This module is pure functions — no I/O, no model calls. Easy to unit-test
+ * and to compose into any verdict gate.
+ *
+ * Default gate:
+ *   - Bootstrap mean baseline vs candidate (1k resamples).
+ *   - Compute the delta distribution; pass if the lower CI bound > 0.
+ *   - Tunable confidence (default 95%) and resample count.
+ *
+ * Verdict semantics intentionally match the existing `experiments.jsonl`
+ * vocabulary:
+ *   - ADVANCE: candidate's CI lower bound > baseline mean (real win)
+ *   - KEEP:    overlap, but candidate point estimate >= baseline (neutral)
+ *   - REVERT:  candidate's CI upper bound < baseline mean (real regression)
+ *   - INCONCLUSIVE: not enough samples or CI straddles zero with no signal
+ */
+type Verdict = 'ADVANCE' | 'KEEP' | 'REVERT' | 'INCONCLUSIVE';
+interface BootstrapResult {
+    baselineMean: number;
+    candidateMean: number;
+    /** candidateMean - baselineMean, point estimate. */
+    delta: number;
+    /** Lower bound of the (1 - alpha) CI on the delta. */
+    ciLower: number;
+    /** Upper bound of the (1 - alpha) CI on the delta. */
+    ciUpper: number;
+    /** Number of bootstrap resamples used. */
+    iterations: number;
+    alpha: number;
+    verdict: Verdict;
+}
+interface BootstrapOptions {
+    /** Confidence level alpha (default 0.05 → 95% CI). */
+    alpha?: number;
+    /** Number of resamples (default 1000). */
+    iterations?: number;
+    /**
+     * Minimum total samples (baseline + candidate) below which we always
+     * return INCONCLUSIVE — bootstrap with too few samples is meaningless.
+     * Default 6 (combined).
+     */
+    minTotalSamples?: number;
+    /** RNG seed for reproducibility. Default: Math.random. */
+    seed?: number;
+}
+/**
+ * Compute the bootstrap CI on (candidateMean - baselineMean) and a verdict.
+ *
+ * Uses simple percentile bootstrap on the difference of resampled means.
+ * That's the standard non-parametric primitive — no distributional
+ * assumptions, robust to skew, easy to reason about.
+ */
+declare function bootstrapCi(baseline: number[], candidate: number[], options?: BootstrapOptions): BootstrapResult;
+/**
+ * Judge-replay promotion gate.
+ *
+ * The cheap inner-loop judge that drives an evolution run is by definition
+ * fast and noisy. When you're about to promote a winning variant to the
+ * canonical default, you want a STRONGER judge (a more expensive model, a
+ * human grader, a separately-trained reward model) to confirm the win
+ * generalises beyond the inner loop.
+ *
+ * This helper takes raw winner + baseline outputs, scores both through the
+ * stronger judge, and applies `bootstrapCi`. ADVANCE means the stronger
+ * judge agrees the winner is real with the configured confidence. Doesn't
+ * matter what shape your "output" is — pass a string, an object, anything
+ * the judge can read.
+ */
+interface JudgeReplayGateArgs<TOutput> {
+    baselineOutputs: TOutput[];
+    candidateOutputs: TOutput[];
+    /** Stronger judge — async to allow LLM calls. Return a 0..N scalar score. */
+    judge: (output: TOutput) => Promise<number> | number;
+    alpha?: number;
+    iterations?: number;
+    /** RNG seed for reproducibility. */
+    seed?: number;
+    /** Maximum concurrent judge calls. Default 4. */
+    judgeConcurrency?: number;
+}
+declare function judgeReplayGate<TOutput>(args: JudgeReplayGateArgs<TOutput>): Promise<BootstrapResult & {
+    baselineSamples: number;
+    candidateSamples: number;
+}>;
+
+export { type BootstrapOptions, type BootstrapResult, type JudgeReplayGateArgs, type PairedBootstrapOptions, type PairedBootstrapResult, type ReleaseConfidenceAxis, type ReleaseConfidenceAxisName, type ReleaseConfidenceInput, type ReleaseConfidenceIssue, type ReleaseConfidenceMetrics, type ReleaseConfidenceScorecard, type ReleaseConfidenceStatus, type ReleaseConfidenceThresholds, type ReleaseTraceEvidence, type RenderReleaseReportOptions, type Verdict, assertReleaseConfidence, bhAdjust, bootstrapCi, evaluateReleaseConfidence, judgeReplayGate, pairedBootstrap, pairedWilcoxon, releaseTraceEvidenceFromMultiShotTrials, renderReleaseReport };

package/dist/reporting.js

@@ -1,14 +1,16 @@
 import {
   assertReleaseConfidence,
   bootstrapCi,
-  evaluateInterimReleaseConfidence,
   evaluateReleaseConfidence,
   judgeReplayGate,
-  pairedEvalueSequence,
   releaseTraceEvidenceFromMultiShotTrials,
-  renderReleaseReport,
+  renderReleaseReport
+} from "./chunk-7EAUOUQS.js";
+import {
+  evaluateInterimReleaseConfidence,
+  pairedEvalueSequence,
   rubricPredictiveValidity
-} from "./chunk-UAND2LOT.js";
+} from "./chunk-AXHNWLIX.js";
 import {
   RESEARCH_REPORT_HARD_PAIR_FLOOR,
   bhAdjust,

package/dist/rl.d.ts

@@ -0,0 +1,8 @@
+export { A as AdaptationCurve, b as AdaptationPoint, c as AdaptationRunner, d as AdapterContext, e as AdversarialMutation, f as AdversarialScenario, g as AdversarialSearchOptions, h as AdversarialSearchReport, i as AnalyzeOptimizationResultOptions, j as AnalyzeOptimizationResultReport, B as BradleyTerryFit, k as BradleyTerryRating, l as BuildPairwiseFromCampaignInput, C as CellObservation, m as CompareCurvesResult, n as ComputeBestOfNOptions, o as ComputeBestOfNResult, p as ComputeCurve, q as ComputeCurveBudget, r as ComputeCurvePoint, s as ContaminationProbeInput, t as ContaminationProbeOptions, u as ContaminationProbeReport, v as CurriculumAllocation, D as DetectRewardHackingInput, w as DpoExportRow, x as DpoLookups, E as EloOptions, y as ExtractPreferencesOptions, z as ExtractStepRewardsOptions, G as GrpoExportRow, H as GrpoLookups, O as OffPolicyEstimate, J as OffPolicyOptions, K as OffPolicyTrajectory, P as PairwiseOutcome, N as ParetoPointInput, Q as PredictiveValidityResearcher, R as PredictiveValidityResearcherOptions, T as PreferenceExtractionReport, U as PreferenceStrategy, W as PreferenceTriple, X as PrmExportRow, Y as PrmLookups, Z as PrmTrainingTriple, _ as RLCampaignResult, $ as RewardHackingFinding, a0 as RewardHackingReport, a1 as RewardHackingSignal, a2 as RunAdaptationCurveOptions, a3 as RunComputeCurveOptions, a4 as RunRLCampaignOptions, a5 as RunwiseStepSummary, a6 as ScenarioPerturbation, a7 as ScenarioPerturbationKind, a8 as SelfConsistencyOptions, a9 as SelfConsistencyResult, aa as SftExportRow, ab as SftLookups, ac as StepReward, ad as StepRewardJsonlRow, ae as StepScorer, af as ThompsonCurriculumOptions, ag as VarianceCurriculumOptions, ah as VerifiableReward, ai as VerifiableRewardExtractionOptions, aj as VerifiableRewardSource, am as adversarialScenarioSearch, an as analyzeOptimizationResult, ao as applyEloUpdate, ap as bestOfN, aq as buildPairwiseFromCampaign, ar as compareAdaptationCurves, as as detectRewardHacking, at as doublyRobust, au as extractPreferences, av as extractStepRewards, aw as extractVerifiableReward, ax as extractVerifiableRewardsFromRecords, ay as filterDeterministicallyRewarded, az as firstPassK, aA as fitBradleyTerry, aC as injectIrrelevantClause, aD as inverseProbabilityWeighting, aE as observationsFromRunRecords, aF as offPolicyEstimateAll, b5 as paretoFrontier, aG as prmTrainingPairs, aH as renameVariables, aI as runAdaptationCurve, aJ as runComputeCurve, aK as runContaminationProbe, aL as runRLCampaign, aM as runwiseStepRewardSummary, aN as selfConsistency, aO as selfNormalizedImportanceWeighting, aP as shuffleOrder, aQ as stepRewardsToJsonl, aR as thompsonCurriculum, aS as toAnthropicFormat, aT as toDpoJsonl, aU as toDpoRows, aV as toGrpoJsonl, aW as toGrpoRows, aX as toPrmJsonl, aY as toPrmRows, aZ as toSftJsonl, a_ as toSftRows, a$ as toTRLFormat, b0 as trialToRunRecord, b1 as trialsToRunRecords, b2 as varianceBasedCurriculum, b3 as variantAggregateToRunRecord, b4 as verificationReportToRunRecord } from './index-ekBXweiQ.js';
+export { r as runEvalCampaign } from './eval-campaign-Ds5QljIh.js';
+import './summary-report-Ce1r4EYo.js';
+import './run-record-DNiOMBrZ.js';
+import './store-u47QaJ9G.js';
+import './sequential-DgU2mFsE.js';
+import './integrity-Cr5YodSY.js';
+import './emitter-B2XqDKFU.js';

package/dist/rl.js

@@ -0,0 +1,113 @@
+import {
+  PredictiveValidityResearcher,
+  adversarialScenarioSearch,
+  analyzeOptimizationResult,
+  applyEloUpdate,
+  bestOfN,
+  buildPairwiseFromCampaign,
+  compareAdaptationCurves,
+  detectRewardHacking,
+  doublyRobust,
+  extractPreferences,
+  extractStepRewards,
+  extractVerifiableReward,
+  extractVerifiableRewardsFromRecords,
+  filterDeterministicallyRewarded,
+  firstPassK,
+  fitBradleyTerry,
+  injectIrrelevantClause,
+  inverseProbabilityWeighting,
+  observationsFromRunRecords,
+  offPolicyEstimateAll,
+  paretoFrontier,
+  prmTrainingPairs,
+  renameVariables,
+  runAdaptationCurve,
+  runComputeCurve,
+  runContaminationProbe,
+  runRLCampaign,
+  runwiseStepRewardSummary,
+  selfConsistency,
+  selfNormalizedImportanceWeighting,
+  shuffleOrder,
+  stepRewardsToJsonl,
+  thompsonCurriculum,
+  toAnthropicFormat,
+  toDpoJsonl,
+  toDpoRows,
+  toGrpoJsonl,
+  toGrpoRows,
+  toPrmJsonl,
+  toPrmRows,
+  toSftJsonl,
+  toSftRows,
+  toTRLFormat,
+  trialToRunRecord,
+  trialsToRunRecords,
+  varianceBasedCurriculum,
+  variantAggregateToRunRecord,
+  verificationReportToRunRecord
+} from "./chunk-LZKIOBG2.js";
+import {
+  runEvalCampaign
+} from "./chunk-EXGR4XEM.js";
+import "./chunk-KAO3Q65R.js";
+import "./chunk-AXHNWLIX.js";
+import "./chunk-IOXMGMHQ.js";
+import "./chunk-QUKKGHTZ.js";
+import "./chunk-SQQLHODJ.js";
+import "./chunk-5IIQKMD5.js";
+import "./chunk-6M774GY6.js";
+import "./chunk-PZ5AY32C.js";
+export {
+  PredictiveValidityResearcher,
+  adversarialScenarioSearch,
+  analyzeOptimizationResult,
+  applyEloUpdate,
+  bestOfN,
+  buildPairwiseFromCampaign,
+  compareAdaptationCurves,
+  detectRewardHacking,
+  doublyRobust,
+  extractPreferences,
+  extractStepRewards,
+  extractVerifiableReward,
+  extractVerifiableRewardsFromRecords,
+  filterDeterministicallyRewarded,
+  firstPassK,
+  fitBradleyTerry,
+  injectIrrelevantClause,
+  inverseProbabilityWeighting,
+  observationsFromRunRecords,
+  offPolicyEstimateAll,
+  paretoFrontier,
+  prmTrainingPairs,
+  renameVariables,
+  runAdaptationCurve,
+  runComputeCurve,
+  runContaminationProbe,
+  runEvalCampaign,
+  runRLCampaign,
+  runwiseStepRewardSummary,
+  selfConsistency,
+  selfNormalizedImportanceWeighting,
+  shuffleOrder,
+  stepRewardsToJsonl,
+  thompsonCurriculum,
+  toAnthropicFormat,
+  toDpoJsonl,
+  toDpoRows,
+  toGrpoJsonl,
+  toGrpoRows,
+  toPrmJsonl,
+  toPrmRows,
+  toSftJsonl,
+  toSftRows,
+  toTRLFormat,
+  trialToRunRecord,
+  trialsToRunRecords,
+  varianceBasedCurriculum,
+  variantAggregateToRunRecord,
+  verificationReportToRunRecord
+};
+//# sourceMappingURL=rl.js.map

package/dist/rl.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/{run-record-CX_jcAyr.d.ts → run-record-DNiOMBrZ.d.ts}

@@ -107,6 +107,15 @@ interface RunRecord {
     failureMode?: string;
     /** Which split this run was drawn from. */
     splitTag: RunSplitTag;
+    /**
+     * Stable scenario identifier the run was scored against. Optional for
+     * backwards compatibility, but **strongly recommended**: every primitive
+     * that pairs runs by scenario (preferences, paired stats, BT tournament)
+     * keys on this. The campaign artifact populates it canonically; legacy
+     * runs without it fall back to inference from `outcome.raw.scenario_id`
+     * or `experimentId`.
+     */
+    scenarioId?: string;
 }
 declare class RunRecordValidationError extends Error {
     readonly path: string;
@@ -131,4 +140,4 @@ declare function parseRunRecordSafe(input: unknown): {
 /** Round-trip helper — `JSON.parse(JSON.stringify(record))` then validate. */
 declare function roundTripRunRecord(record: RunRecord): RunRecord;
 
-export { type RunSplitTag as R, type RunRecord as a, type RunJudgeMetadata as b, type RunOutcome as c, RunRecordValidationError as d, type RunTokenUsage as e, isRunRecord as i, parseRunRecordSafe as p, roundTripRunRecord as r, validateRunRecord as v };
+export { type RunRecord as R, type RunSplitTag as a, type RunTokenUsage as b, type RunJudgeMetadata as c, type RunOutcome as d, RunRecordValidationError as e, isRunRecord as i, parseRunRecordSafe as p, roundTripRunRecord as r, validateRunRecord as v };