@tangle-network/agent-eval 0.22.0 → 0.23.0

package/dist/sequential-DgU2mFsE.d.ts

@@ -0,0 +1,304 @@
+import { R as RunRecord } from './run-record-DNiOMBrZ.js';
+
+/**
+ * OutcomeStore — deployment outcomes attached to Run IDs.
+ *
+ * Outcomes arrive asynchronously from production telemetry after the
+ * eval run completed: user ratings, retention flags, conversion events,
+ * revenue, support-ticket rate, anything a product team can measure.
+ * The store is a peer to TraceStore — separate lifecycle, same runId
+ * foreign key.
+ *
+ * The whole point of this module is to make the meta-eval correlation
+ * question computable: `correlate(evalMetric, outcomeMetric) → r, ρ, n, CI`.
+ */
+interface DeploymentOutcome {
+    runId: string;
+    capturedAt: number;
+    /** Numeric outcomes keyed by name — retention_7d, csat, revenue_usd, etc. */
+    metrics: Record<string, number>;
+    /** Dimensions for stratified analysis — cohort, region, user_segment. */
+    labels?: Record<string, string>;
+    /** Free-form provenance (source system, pipeline version). */
+    source?: string;
+}
+interface OutcomeFilter {
+    runIds?: string[];
+    since?: number;
+    until?: number;
+    label?: {
+        key: string;
+        value: string;
+    };
+    source?: string;
+}
+interface OutcomeStore {
+    append(outcome: DeploymentOutcome): Promise<void>;
+    /** All outcomes attached to this run (a single run can have many — multiple
+     *  capture windows over deployment time). */
+    forRun(runId: string): Promise<DeploymentOutcome[]>;
+    list(filter?: OutcomeFilter): Promise<DeploymentOutcome[]>;
+}
+declare class InMemoryOutcomeStore implements OutcomeStore {
+    private items;
+    append(outcome: DeploymentOutcome): Promise<void>;
+    forRun(runId: string): Promise<DeploymentOutcome[]>;
+    list(filter?: OutcomeFilter): Promise<DeploymentOutcome[]>;
+}
+interface FileSystemOutcomeStoreOptions {
+    dir: string;
+    maxBytes?: number;
+}
+declare class FileSystemOutcomeStore implements OutcomeStore {
+    private dir;
+    private maxBytes;
+    private memo?;
+    private loaded;
+    constructor(options: FileSystemOutcomeStoreOptions);
+    private ensureDir;
+    append(outcome: DeploymentOutcome): Promise<void>;
+    private load;
+    forRun(runId: string): Promise<DeploymentOutcome[]>;
+    list(filter?: OutcomeFilter): Promise<DeploymentOutcome[]>;
+}
+
+/**
+ * Rubric predictive validity — does our eval rubric predict deployment
+ * outcomes?
+ *
+ * `correlationStudy` (already in this package) joins a `TraceStore` to an
+ * `OutcomeStore` and computes Pearson + Spearman + bootstrap CI for each
+ * (eval-metric, outcome-metric) pair. That answers "does X correlate with
+ * Y at all." `rubricPredictiveValidity` is the campaign-shaped wrapper
+ * around it: take a sequence of `RunRecord`s (the canonical campaign
+ * artifact) and a `DeploymentOutcomeStore`, join on `runId`, return a
+ * ranked verdict on every rubric whose dimension scores were captured in
+ * `outcome.raw`.
+ *
+ * The point — quoting the methodology doc — is that **without this loop
+ * every rubric is faith-based**. Once it's wired, you know which rubrics
+ * have earned their promotion power and which ones are decoration.
+ *
+ *   const validity = await rubricPredictiveValidity({
+ *     runs: lastQuarter,
+ *     outcomes: shipFlagOutcomeStore,
+ *     outcomeMetrics: ['revenue_lift', 'retention_30d', 'csat'],
+ *     rubrics: ['anti_slop', 'semantic_concept', 'tool_recovery'],
+ *   })
+ *   for (const r of validity.ranked) {
+ *     console.log(`${r.rubric} → ${r.bestOutcome}: ρ=${r.spearman.toFixed(2)}`)
+ *   }
+ *
+ * The function is intentionally read-only. Use the verdict to deprecate
+ * decorative rubrics, re-weight composite scores, or trigger a
+ * recalibration sweep when predictive validity drops below a threshold.
+ */
+
+interface RubricPredictiveValidityInput {
+    /**
+     * Canonical campaign output. Each record's `outcome.raw[<rubricId>]`
+     * provides the eval score; missing keys are silently skipped per pair.
+     */
+    runs: RunRecord[];
+    outcomes: OutcomeStore;
+    /**
+     * Outcome metric names to evaluate against. Each must appear in at
+     * least one `DeploymentOutcome.metrics` keyspace; pairs with too few
+     * joined samples are excluded from the result.
+     */
+    outcomeMetrics: string[];
+    /**
+     * Rubric ids to evaluate. Must appear as keys in `RunRecord.outcome.raw`.
+     * If omitted, every numeric key in `outcome.raw` across the run set is
+     * treated as a rubric.
+     */
+    rubrics?: string[];
+    /** Minimum joined-sample count before a pair is reported. Default 8. */
+    minSamples?: number;
+    /** Bootstrap resamples for CI. Default 500. */
+    bootstrapResamples?: number;
+    /** Random seed for the bootstrap (mulberry32). Default unset (Math.random). */
+    seed?: number;
+    /**
+     * Reduction when multiple outcomes attach to one runId. Default `'latest'`
+     * (most recently captured).
+     */
+    reduction?: 'latest' | 'mean' | 'max';
+}
+interface RubricOutcomePair {
+    rubric: string;
+    outcome: string;
+    n: number;
+    pearson: number;
+    spearman: number;
+    ci95: {
+        low: number;
+        high: number;
+    };
+    /**
+     * Verdict bucket. `load_bearing` ≥ 0.7, `informative` ≥ 0.4,
+     * `decorative` < 0.4 in absolute correlation. A negative correlation
+     * with a desired outcome is also `decorative` — actively misleading
+     * is worse than uninformative.
+     */
+    verdict: 'load_bearing' | 'informative' | 'decorative';
+}
+interface RubricRanking {
+    rubric: string;
+    /** Outcome metric this rubric correlated best with. */
+    bestOutcome: string;
+    spearman: number;
+    pearson: number;
+    n: number;
+    verdict: RubricOutcomePair['verdict'];
+}
+interface RubricPredictiveValidityReport {
+    pairs: RubricOutcomePair[];
+    /** Per-rubric best pair, sorted descending by |spearman|. */
+    ranked: RubricRanking[];
+    joinedSamples: number;
+    skippedRuns: number;
+    /** Rubrics that were declared but never produced a usable score. */
+    rubricsWithoutData: string[];
+}
+declare function rubricPredictiveValidity(input: RubricPredictiveValidityInput): Promise<RubricPredictiveValidityReport>;
+
+/**
+ * Always-valid sequential evaluation.
+ *
+ * `researchReport` (0.21+) assumes a single pre-specified analysis. Real
+ * consumers run campaigns weekly / nightly / per-PR; each new run silently
+ * inflates the false-discovery rate, because the BH-FDR guarantee was for
+ * the *first* look, not the 47th. Without time-uniform inference,
+ * launch-decision teams either (a) don't peek, which forfeits the cost
+ * advantage of stop-when-decisive, or (b) peek and pretend they didn't,
+ * which forfeits scientific validity.
+ *
+ * This module ships **e-value-based confidence sequences** for paired
+ * bounded outcomes. The methodology is the predictable plug-in betting
+ * martingale of Waudby-Smith & Ramdas (2024) — provably valid at *any*
+ * stopping time. Concretely:
+ *
+ *   For paired deltas D_1, D_2, … ∈ [-c, c] with the null H_0: E[D] ≤ 0,
+ *   a betting fraction λ_i is chosen using only D_{1..i-1} (predictable
+ *   plug-in), and the running e-value is
+ *
+ *     E_t = ∏_{i=1}^{t} (1 + λ_i · D_i)
+ *
+ *   E_t is a non-negative martingale under H_0 with E[E_t] ≤ 1, so by
+ *   Ville's inequality, P(∃ t : E_t ≥ 1/α) ≤ α — we can reject the null
+ *   at any time without inflating the type-I error.
+ *
+ * Combined with `runEvalCampaign`, every consumer running rolling
+ * campaigns gains the ability to ship the moment evidence is decisive,
+ * stop-early on dead-on-arrival variants, and accumulate evidence across
+ * partial runs without spending the FDR budget. No new sweep is wasted.
+ *
+ * References:
+ *   - Howard, S. R., Ramdas, A., McAuliffe, J., Sekhon, J. (2021).
+ *     Time-uniform, nonparametric, nonasymptotic confidence sequences.
+ *     Annals of Statistics, 49(2), 1055–1080.
+ *   - Waudby-Smith, I., Ramdas, A. (2024). Estimating means of bounded
+ *     random variables by betting. JRSS B, 86(1), 1–27.
+ */
+type SequentialDecision = 'promote_now' | 'continue' | 'reject_now' | 'equivalent';
+interface PairedEvalueOptions {
+    /**
+     * Bound on |delta|. Default 1 (matching most score scales). Must satisfy
+     * c > 0; deltas outside [-c, c] are clipped with a warning attached to
+     * the return value.
+     */
+    bound?: number;
+    /** Target Type-I error. Default 0.05. */
+    alpha?: number;
+    /**
+     * Region of Practical Equivalence on the *mean* paired delta. When
+     * supplied, the verdict can return `'equivalent'` once the running
+     * confidence sequence on the mean is fully contained in [low, high].
+     */
+    rope?: {
+        low: number;
+        high: number;
+    };
+    /** Initial bet shrinkage (0 < scale ≤ 1). Default 0.5 — empirically robust. */
+    initialBetShrinkage?: number;
+}
+interface PairedEvalueStep {
+    /** 1-indexed observation count. */
+    t: number;
+    delta: number;
+    /** Running e-value E_t = ∏ (1 + λ_i · D_i). */
+    evalue: number;
+    /** Time-uniform p-value at stopping time t. */
+    pValue: number;
+    /** Lower bound of the empirical Bernstein confidence sequence at level 1-α. */
+    csLow: number;
+    csHigh: number;
+    /** Verdict at this stopping time. */
+    decision: SequentialDecision;
+}
+interface PairedEvalueSequence {
+    steps: PairedEvalueStep[];
+    /** The decision at the final step. */
+    finalDecision: SequentialDecision;
+    /** Index (1-based) at which a non-`continue` decision first fired, or null. */
+    decisionFiredAt: number | null;
+    /** True if any deltas were clipped to [-bound, bound]. */
+    clipped: boolean;
+}
+/**
+ * Run the paired e-value sequence over an in-order delta stream.
+ *
+ * Use for *streaming* / interim analyses: pass the deltas you have so
+ * far, get the verdict at every prefix length. The decision is
+ * monotone-stable in the sense that once `'reject_now'` or `'promote_now'`
+ * fires, the verdict at later steps remains decisive (the e-value is a
+ * non-negative martingale; once it crosses the threshold, it's crossed).
+ */
+declare function pairedEvalueSequence(deltas: number[], opts?: PairedEvalueOptions): PairedEvalueSequence;
+interface InterimReleaseConfidenceInput {
+    /**
+     * One delta series per candidate (paired deltas vs comparator). Order
+     * within a series is the order the campaigns were run.
+     */
+    deltaSeries: Array<{
+        candidateId: string;
+        deltas: number[];
+    }>;
+    alpha?: number;
+    bound?: number;
+    rope?: {
+        low: number;
+        high: number;
+    };
+}
+interface InterimReleaseConfidence {
+    candidates: Array<{
+        candidateId: string;
+        decision: SequentialDecision;
+        decisionFiredAt: number | null;
+        finalEvalue: number;
+        finalPValue: number;
+        pairs: number;
+        csLow: number;
+        csHigh: number;
+    }>;
+    /**
+     * Campaign-level recommendation: pick the strongest 'promote_now', else
+     * 'continue' if any candidate is still live, else 'reject_now' if every
+     * candidate is dead, else 'equivalent'.
+     */
+    recommendation: {
+        decision: SequentialDecision;
+        candidateId: string | null;
+    };
+}
+/**
+ * Run interim sequential analyses across many candidates at once,
+ * preserving the time-uniform α guarantee for each candidate's series and
+ * synthesising a campaign-level recommendation. Designed to be called on
+ * every campaign tick — the recommendation is anytime-valid.
+ */
+declare function evaluateInterimReleaseConfidence(input: InterimReleaseConfidenceInput): InterimReleaseConfidence;
+
+export { type DeploymentOutcome as D, FileSystemOutcomeStore as F, InMemoryOutcomeStore as I, type OutcomeFilter as O, type PairedEvalueOptions as P, type RubricOutcomePair as R, type SequentialDecision as S, type OutcomeStore as a, type FileSystemOutcomeStoreOptions as b, type InterimReleaseConfidence as c, type InterimReleaseConfidenceInput as d, type PairedEvalueSequence as e, type PairedEvalueStep as f, type RubricPredictiveValidityInput as g, type RubricPredictiveValidityReport as h, type RubricRanking as i, evaluateInterimReleaseConfidence as j, pairedEvalueSequence as p, rubricPredictiveValidity as r };

package/dist/{summary-report-D4p7RlDu.d.ts → summary-report-Ce1r4EYo.d.ts}

@@ -1,4 +1,4 @@
-import { a as RunRecord, R as RunSplitTag } from './run-record-CX_jcAyr.js';
+import { R as RunRecord, a as RunSplitTag } from './run-record-DNiOMBrZ.js';
 import { a as Run, S as Span, f as TraceEvent, F as FailureClass, T as TraceStore } from './store-u47QaJ9G.js';
 
 /**
@@ -975,4 +975,4 @@ interface ResearchReport {
  */
 declare function researchReport(runs: RunRecord[], opts?: ResearchReportOptions): Promise<ResearchReport>;
 
-export { type ResearchReportOptions as $, type ActionableSideInfo as A, type MultiShotTrace as B, type MultiShotTrialResult as C, DEFAULT_RULES as D, type EvolvableVariant as E, type FailureClassification as F, type GainDistributionBin as G, HeldOutGate as H, InMemoryTrialCache as I, type MultiShotVariant as J, type ParetoFigureSpec as K, type ParetoPoint as L, type MutateAdapter as M, type PromptEvolutionConfig as N, type Objective as O, type ParetoResult as P, type PromptEvolutionEvent as Q, type PromptEvolutionResult as R, RESEARCH_REPORT_HARD_PAIR_FLOOR as S, type TrialCache as T, type ReflectionContext as U, type VariantAggregate as V, type ReflectionProposal as W, type ResearchReport as X, type ResearchReportCandidate as Y, type ResearchReportDecision as Z, type ResearchReportMethodology as _, type TrialResult as a, type ResearchReportRecommendation as a0, type ScenarioAggregate as a1, type ScoreAdapter as a2, type SummaryTable as a3, type SummaryTableOptions as a4, type SummaryTableRow as a5, type TrialTrace as a6, buildReflectionPrompt as a7, classifyFailure as a8, crowdingDistance as a9, defaultMultiShotObjectives as aa, dominates as ab, failureClusterView as ac, gainHistogram as ad, paretoChart as ae, paretoFrontier as af, paretoFrontierWithCrowding as ag, parseReflectionResponse as ah, researchReport as ai, runMultiShotOptimization as aj, runPromptEvolution as ak, scalarScore as al, summaryTable as am, trialTraceFromMultiShotTrial as an, type AsiSeverity as b, DEFAULT_MUTATION_PRIMITIVES as c, type Direction as d, type FailureCluster as e, type FailureClusterReport as f, type FailureContext as g, type FailureRule as h, type GainDistributionFigureSpec as i, type GainDistributionOptions as j, type GateDecision as k, type GateEvidence as l, type GenerationReport as m, type HeldOutGateConfig as n, type HeldOutGateRejectionCode as o, type MultiShotGateConfig as p, type MultiShotGateResult as q, type MultiShotMutateAdapter as r, type MultiShotOptimizationConfig as s, type MultiShotOptimizationResult as t, type MultiShotRun as u, type MultiShotRunInput as v, type MultiShotRunner as w, type MultiShotScore as x, type MultiShotScorer as y, type MultiShotSplit as z };
+export { type GateEvidence as $, type ActionableSideInfo as A, trialTraceFromMultiShotTrial as B, type GateDecision as C, DEFAULT_MUTATION_PRIMITIVES as D, type EvolvableVariant as E, type ResearchReportOptions as F, type GenerationReport as G, type ResearchReport as H, InMemoryTrialCache as I, type ParetoResult as J, DEFAULT_RULES as K, type Direction as L, type MultiShotGateConfig as M, type FailureClassification as N, type Objective as O, type PromptEvolutionConfig as P, type FailureCluster as Q, type ReflectionContext as R, type ScenarioAggregate as S, type TrialCache as T, type FailureClusterReport as U, type VariantAggregate as V, type FailureContext as W, type FailureRule as X, type GainDistributionBin as Y, type GainDistributionFigureSpec as Z, type GainDistributionOptions as _, type AsiSeverity as a, HeldOutGate as a0, type HeldOutGateConfig as a1, type HeldOutGateRejectionCode as a2, type ParetoFigureSpec as a3, type ParetoPoint as a4, RESEARCH_REPORT_HARD_PAIR_FLOOR as a5, type ResearchReportCandidate as a6, type ResearchReportDecision as a7, type ResearchReportMethodology as a8, type ResearchReportRecommendation as a9, type SummaryTable as aa, type SummaryTableOptions as ab, type SummaryTableRow as ac, classifyFailure as ad, crowdingDistance as ae, dominates as af, failureClusterView as ag, gainHistogram as ah, paretoChart as ai, paretoFrontier as aj, paretoFrontierWithCrowding as ak, researchReport as al, scalarScore as am, summaryTable as an, type MultiShotGateResult as b, type MultiShotMutateAdapter as c, type MultiShotOptimizationConfig as d, type MultiShotOptimizationResult as e, type MultiShotRun as f, type MultiShotRunInput as g, type MultiShotRunner as h, type MultiShotScore as i, type MultiShotScorer as j, type MultiShotSplit as k, type MultiShotTrace as l, type MultiShotTrialResult as m, type MultiShotVariant as n, type MutateAdapter as o, type PromptEvolutionEvent as p, type PromptEvolutionResult as q, type ReflectionProposal as r, type ScoreAdapter as s, type TrialResult as t, type TrialTrace as u, buildReflectionPrompt as v, defaultMultiShotObjectives as w, parseReflectionResponse as x, runMultiShotOptimization as y, runPromptEvolution as z };

package/dist/traces.d.ts

@@ -2,8 +2,8 @@ import { T as TraceStore, L as LlmSpan, J as JudgeSpan, a as Run, F as FailureCl
 export { A as Artifact, B as BudgetLedgerEntry, g as BudgetSpec, i as EventFilter, E as EventKind, j as FAILURE_CLASSES, k as FileSystemTraceStore, l as FileSystemTraceStoreOptions, G as GenericSpan, I as InMemoryTraceStore, M as Message, d as RetrievalSpan, h as RunFilter, m as RunLayer, R as RunOutcome, n as RunStatus, e as SandboxSpan, S as Span, o as SpanBase, p as SpanFilter, b as SpanKind, q as SpanStatus, r as TRACE_SCHEMA_VERSION, f as TraceEvent, s as isJudgeSpan, t as isLlmSpan, u as isRetrievalSpan, v as isSandboxSpan, w as isToolSpan } from './store-u47QaJ9G.js';
 import { a as RunCompleteHookContext, R as RunCompleteHook } from './emitter-B2XqDKFU.js';
 export { S as SpanHandle, T as TraceEmitter, b as TraceEmitterOptions, l as llmSpanFromProvider } from './emitter-B2XqDKFU.js';
-import { d as RawProviderSink, c as RawProviderEvent } from './integrity-K2oVlF57.js';
-export { F as FileSystemRawProviderSink, a as FileSystemRawProviderSinkOptions, I as InMemoryRawProviderSink, b as InMemoryRawProviderSinkOptions, N as NoopRawProviderSink, P as ProviderRedactor, R as RawProviderDirection, e as RawProviderSinkFilter, f as RunIntegrityError, g as RunIntegrityExpectations, h as RunIntegrityIssue, i as RunIntegrityIssueCode, j as RunIntegrityReport, k as assertRunCaptured, l as defaultProviderRedactor, p as providerFromBaseUrl, t as throwIfRunIncomplete } from './integrity-K2oVlF57.js';
+import { R as RawProviderSink, f as RawProviderEvent } from './integrity-Cr5YodSY.js';
+export { F as FileSystemRawProviderSink, c as FileSystemRawProviderSinkOptions, I as InMemoryRawProviderSink, d as InMemoryRawProviderSinkOptions, N as NoopRawProviderSink, P as ProviderRedactor, e as RawProviderDirection, g as RawProviderSinkFilter, h as RunIntegrityError, a as RunIntegrityExpectations, i as RunIntegrityIssue, j as RunIntegrityIssueCode, b as RunIntegrityReport, k as assertRunCaptured, l as defaultProviderRedactor, p as providerFromBaseUrl, t as throwIfRunIncomplete } from './integrity-Cr5YodSY.js';
 import { AxAIService, AxFunction } from '@ax-llm/ax';
 
 /**

package/dist/traces.js

@@ -54,11 +54,6 @@ import {
   assertRunCaptured,
   throwIfRunIncomplete
 } from "./chunk-QUKKGHTZ.js";
-import {
-  TraceEmitter,
-  llmSpanFromProvider
-} from "./chunk-5IIQKMD5.js";
-import "./chunk-6M774GY6.js";
 import {
   FileSystemRawProviderSink,
   InMemoryRawProviderSink,
@@ -66,6 +61,11 @@ import {
   defaultProviderRedactor,
   providerFromBaseUrl
 } from "./chunk-SQQLHODJ.js";
+import {
+  TraceEmitter,
+  llmSpanFromProvider
+} from "./chunk-5IIQKMD5.js";
+import "./chunk-6M774GY6.js";
 import "./chunk-PZ5AY32C.js";
 export {
   DEFAULT_REDACTION_RULES,

package/docs/auto-research-loop-end-to-end.md

@@ -0,0 +1,186 @@
+# Auto-research loop end-to-end
+
+This is the runnable composition pattern that closes the loop the package
+was originally designed for: capture-integrity → eval → preferences →
+mutation → improved candidate → repeat.
+
+There's no new orchestrator primitive that runs this for you (and we
+deliberately resisted shipping one — every consumer's loop has different
+invariants). What this doc gives you is **the integration recipe**: the
+imports, the wiring, and the explicit invariants every iteration must
+preserve.
+
+A working version of this recipe lives at
+[`examples/auto-research-with-agent-builder/`](../examples/auto-research-with-agent-builder/) —
+runnable, ~250 lines, demonstrates the score climbing across iterations.
+
+## The pattern
+
+```ts
+import {
+  runEvalCampaign,
+  analyzeOptimizationResult,
+  trialsToRunRecords,
+  PredictiveValidityResearcher,
+} from '@tangle-network/agent-eval'
+import { traceAnalystOnRunComplete } from '@tangle-network/agent-eval/traces'
+
+async function runAutoResearchLoop(opts: {
+  task: string
+  initialVariants: Variant[]
+  scenarios: Scenario[]
+  iterations: number
+  // The thing that turns a Variant into a scoreable artifact.
+  // For agent-builder this is `runForgeBuilderSim`; for tax-agent it's
+  // their domain runner; for the multi-shot prompt evolution case it's
+  // already wired inside `runPromptEvolution`.
+  candidateRunner: CandidateRunner<Variant>
+  // The thing that proposes the next variants given the analysis output.
+  // For prompt-only optimization, this is `reflective-mutation` against
+  // the top/bottom trials. For code+prompt, this is `createCompositeMutator`.
+  // For agent-builder, this can be a hand-rolled "edit the system prompt"
+  // function — the example shows one.
+  mutator: (champion: Variant, analysis: AnalysisReport) => Promise<Variant[]>
+  // Optional: outcome store for predictive validity. When present, the
+  // loop learns which scoring rubrics actually predict deployment outcomes
+  // and reweights the composite score accordingly.
+  outcomes?: { store: OutcomeStore; metrics: string[] }
+}): Promise<IterationReport[]> {
+  const reports: IterationReport[] = []
+  let variants = opts.initialVariants
+
+  // (Optional) standing researcher that drives rubric reweighting.
+  const researcher = opts.outcomes
+    ? new PredictiveValidityResearcher({
+        outcomes: opts.outcomes.store,
+        outcomeMetrics: opts.outcomes.metrics,
+      })
+    : null
+
+  for (let iter = 0; iter < opts.iterations; iter++) {
+    // 1. Capture-integrity-by-construction matrix run.
+    const campaign = await runEvalCampaign({
+      campaignId: `auto-research-iter-${iter}`,
+      commitSha: opts.task,
+      variants: variants.map((v) => ({ id: v.id, payload: v })),
+      scenarios: opts.scenarios,
+      seeds: [0, 1, 2],
+      llmOpts: { ... },
+      storeFactory: () => new InMemoryTraceStore(),
+      rawSinkFactory: () => new InMemoryRawProviderSink(),
+      runner: makeCampaignRunner(opts.candidateRunner),
+      onRunComplete: opts.outcomes
+        ? [traceAnalystOnRunComplete({ analyze: ..., save: ... })]
+        : [],
+      report: { comparator: variants[0]!.id },
+    })
+
+    // 2. RL-bridge analysis: preferences, verifiable rewards, sequential
+    //    interim verdict, reward-hacking diagnosis.
+    const analysis = await analyzeOptimizationResult({
+      result: pretendItsAPromptEvolution(campaign),
+      ctx: { experimentId: 'task', model: '...', commitSha: '...', promptHash: '...', configHash: '...' },
+      comparator: variants[0]!.id,
+      outcomes: opts.outcomes,
+    })
+
+    // 3. Periodic rubric recalibration via predictive validity.
+    if (researcher && iter > 0 && iter % 5 === 0) {
+      await researcher.runValidityCheck(campaign.runs)
+      // The researcher's `proposeChange` output can be folded into the
+      // mutator as a steering signal in the next iteration.
+    }
+
+    // 4. Pick champion + record this iteration.
+    const champion = pickChampion(campaign.runs)
+    reports.push({ iter, champion, score: champion.score, analysis })
+
+    // 5. Sequential stop: the anytime-valid e-value can decisively call
+    //    'promote_now' or 'reject_now' before iterations exhausted.
+    if (analysis.interimConfidence?.recommendation.decision === 'promote_now') {
+      break
+    }
+
+    // 6. Propose next variants via the mutator.
+    if (iter < opts.iterations - 1) {
+      variants = await opts.mutator(champion.variant, analysis)
+    }
+  }
+
+  return reports
+}
+```
+
+## Invariants every iteration must preserve
+
+1. **The campaign produces RunRecord[] with `scenarioId` populated.** Every
+   downstream primitive (preferences, sequential, predictive validity,
+   tournament) keys on this. `runEvalCampaign` populates it canonically;
+   if you adapt from `runPromptEvolution` use `trialsToRunRecords`.
+
+2. **Capture is wired by construction.** Don't pass `NoopRawProviderSink`
+   to `rawSinkFactory` unless the iteration is exploratory. Every
+   captured run is replayable, every replayable run is free judge-iteration
+   data for the next loop.
+
+3. **`commitSha` is real.** It's how downstream tooling (predictive
+   validity, contamination probe, tournament) ties iterations together.
+
+4. **The comparator is stable across iterations.** Either the original
+   `baseline` or whichever champion you froze. Shifting the comparator
+   between iterations corrupts the paired-delta semantics.
+
+5. **The mutator is deterministic given the analysis output.** Otherwise
+   the iteration isn't reproducible and the auto-research artifacts
+   become unfalsifiable. If you need stochastic mutation, seed the
+   mutator and emit the seed onto the run record.
+
+## When to run each primitive
+
+| Frequency | Primitive | Why |
+|---|---|---|
+| Every iteration | `runEvalCampaign` | core measurement |
+| Every iteration | `analyzeOptimizationResult` | preferences + verifiable rewards + reward-hacking |
+| Every iteration | `evaluateInterimReleaseConfidence` (via `analyzeOptimizationResult`) | anytime-valid stop signal |
+| Every 5–10 iterations | `rubricPredictiveValidity` | rubric weights drift; recalibrate |
+| Every release | `runContaminationProbe` | scenario set freshness |
+| Once per task | `runComputeCurve` | cost-quality frontier |
+| As-needed | `adversarialScenarioSearch` | discover failure modes the curated set missed |
+
+## When to drop into the smaller primitives
+
+Two cases:
+
+1. **Trajectory-shaped optimization with steering.** Use
+   `runMultiShotOptimization` directly — it already runs the inner
+   search-vs-holdout loop. Wrap with `analyzeOptimizationResult` after
+   for the RL bridge.
+
+2. **Prompt + code evolution with sandboxed code mutation.** Use
+   `runPromptEvolution` + `createCompositeMutator` directly. Same wrap
+   pattern.
+
+The auto-research loop above wraps these primitives in a higher-level
+loop that runs them across multiple campaigns. They're each one tick of
+the bigger loop.
+
+## What this does NOT do
+
+- It doesn't fine-tune model weights. That's the
+  [`fine-tune-with-prime-rl`](../examples/fine-tune-with-prime-rl/) example
+  — separate concern, separate trainer.
+- It doesn't drive a production deployment decision on its own. The
+  artifacts feed a launch-review process (humans, the `researchReport`
+  output, the `assertReleaseConfidence` gate). Loop ≠ promotion gate.
+- It doesn't substitute for a real preregistration trail. The
+  `preregistrationHash` field on the report exists so iterations can be
+  audited, but the auto-research loop *is* iterative and post-hoc by
+  definition. Use the standing `assertReleaseConfidence` gate at the
+  release boundary; use the auto-research loop everywhere upstream of it.
+
+## Reading order for the example
+
+1. [`examples/auto-research-with-agent-builder/README.md`](../examples/auto-research-with-agent-builder/README.md) — architectural picture.
+2. [`examples/auto-research-with-agent-builder/auto-research-with-agent-builder.ts`](../examples/auto-research-with-agent-builder/auto-research-with-agent-builder.ts) — runnable demo.
+3. Run it: `npx tsx examples/auto-research-with-agent-builder/auto-research-with-agent-builder.ts`.
+   It prints the iteration progression and the score climbing.