@tangle-network/agent-eval 0.24.0 → 0.27.0

package/dist/index.d.ts

@@ -6,12 +6,16 @@ import { A as AgentEvalError } from './errors-BZ9sTdz7.js';
 export { a as AgentEvalErrorCode, C as CaptureIntegrityError, b as ConfigError, J as JudgeError, N as NotFoundError, R as ReplayError, V as ValidationError, c as VerificationError } from './errors-BZ9sTdz7.js';
 import { b as FeedbackLabel, n as FeedbackTrajectoryStore, l as FeedbackTrajectory } from './feedback-trajectory-DfFdrraJ.js';
 export { F as FeedbackArtifactType, a as FeedbackAttempt, c as FeedbackLabelKind, d as FeedbackLabelSource, e as FeedbackOptimizerRow, f as FeedbackOutcome, g as FeedbackReplayAdapter, h as FeedbackReplayResult, i as FeedbackSeverity, j as FeedbackSplitPolicy, k as FeedbackTask, m as FeedbackTrajectoryFilter, o as FileSystemFeedbackTrajectoryStore, I as InMemoryFeedbackTrajectoryStore, P as PreferenceMemoryEntry, p as ProposedSideEffect, q as assignFeedbackSplit, r as controlRunToFeedbackTrajectory, s as createFeedbackTrajectory, t as feedbackTrajectoriesToDatasetScenarios, u as feedbackTrajectoriesToOptimizerRows, v as feedbackTrajectoryToDatasetScenario, w as feedbackTrajectoryToOptimizerRow, x as parseFeedbackTrajectoriesJsonl, y as renderPreferenceMemoryMarkdown, z as replayFeedbackTrajectories, A as replayFeedbackTrajectory, B as serializeFeedbackTrajectoriesJsonl, C as summarizePreferenceMemory, D as withAssignedFeedbackSplit } from './feedback-trajectory-DfFdrraJ.js';
-import { A as ActionableSideInfo, a4 as Objective, a5 as ParetoResult, E as EvolvableVariant, V as VariantAggregate, t as TrialResult, o as MutateAdapter, T as TrialCache } from './summary-report-BXGs_9V0.js';
-export { a as AsiSeverity, D as DEFAULT_MUTATION_PRIMITIVES, a6 as Direction, C as GainDistributionBin, F as GainDistributionFigureSpec, H as GainDistributionOptions, a3 as GateDecision, a7 as GateEvidence, G as GenerationReport, a8 as HeldOutGate, a9 as HeldOutGateConfig, aa as HeldOutGateRejectionCode, I as InMemoryTrialCache, M as MultiShotGateConfig, b as MultiShotGateResult, c as MultiShotMutateAdapter, d as MultiShotOptimizationConfig, e as MultiShotOptimizationResult, f as MultiShotRun, g as MultiShotRunInput, h as MultiShotRunner, i as MultiShotScore, j as MultiShotScorer, k as MultiShotSplit, l as MultiShotTrace, m as MultiShotTrialResult, n as MultiShotVariant, J as ParetoFigureSpec, K as ParetoPoint, P as PromptEvolutionConfig, p as PromptEvolutionEvent, q as PromptEvolutionResult, L as RESEARCH_REPORT_HARD_PAIR_FLOOR, R as ReflectionContext, r as ReflectionProposal, N as ResearchReport, O as ResearchReportCandidate, Q as ResearchReportDecision, U as ResearchReportMethodology, W as ResearchReportOptions, X as ResearchReportRecommendation, S as ScenarioAggregate, s as ScoreAdapter, Y as SummaryTable, Z as SummaryTableOptions, _ as SummaryTableRow, u as TrialTrace, v as buildReflectionPrompt, ab as crowdingDistance, w as defaultMultiShotObjectives, ac as dominates, $ as gainHistogram, a0 as paretoChart, ad as paretoFrontier, ae as paretoFrontierWithCrowding, x as parseReflectionResponse, a1 as researchReport, y as runMultiShotOptimization, z as runPromptEvolution, af as scalarScore, a2 as summaryTable, B as trialTraceFromMultiShotTrial } from './summary-report-BXGs_9V0.js';
+import { A as ActionableSideInfo, h as MultiShotRunner, j as MultiShotScorer, c as MultiShotMutateAdapter, a4 as HeldOutGateConfig, E as EvolvableVariant, m as MultiShotTrialResult, e as MultiShotOptimizationResult, a3 as GateDecision, a5 as Objective, a6 as ParetoResult, V as VariantAggregate, t as TrialResult, o as MutateAdapter, T as TrialCache } from './summary-report-DZVXOCK_.js';
+export { a as AsiSeverity, D as DEFAULT_MUTATION_PRIMITIVES, a7 as Direction, C as GainDistributionBin, F as GainDistributionFigureSpec, H as GainDistributionOptions, a8 as GateEvidence, G as GenerationReport, a9 as HeldOutGate, aa as HeldOutGateRejectionCode, I as InMemoryTrialCache, M as MultiShotGateConfig, b as MultiShotGateResult, d as MultiShotOptimizationConfig, f as MultiShotRun, g as MultiShotRunInput, i as MultiShotScore, k as MultiShotSplit, l as MultiShotTrace, n as MultiShotVariant, J as ParetoFigureSpec, K as ParetoPoint, P as PromptEvolutionConfig, p as PromptEvolutionEvent, q as PromptEvolutionResult, L as RESEARCH_REPORT_HARD_PAIR_FLOOR, R as ReflectionContext, r as ReflectionProposal, N as ResearchReport, O as ResearchReportCandidate, Q as ResearchReportDecision, U as ResearchReportMethodology, W as ResearchReportOptions, X as ResearchReportRecommendation, S as ScenarioAggregate, s as ScoreAdapter, Y as SummaryTable, Z as SummaryTableOptions, _ as SummaryTableRow, u as TrialTrace, v as buildReflectionPrompt, ab as crowdingDistance, w as defaultMultiShotObjectives, ac as dominates, $ as gainHistogram, a0 as paretoChart, ad as paretoFrontier, ae as paretoFrontierWithCrowding, x as parseReflectionResponse, a1 as researchReport, y as runMultiShotOptimization, z as runPromptEvolution, af as scalarScore, a2 as summaryTable, B as trialTraceFromMultiShotTrial } from './summary-report-DZVXOCK_.js';
 export { DataAcquisitionPlan, KnowledgeAcquisitionMode, KnowledgeBundle, KnowledgeFallbackPolicy, KnowledgeFreshness, KnowledgeImportance, KnowledgeReadinessReport, KnowledgeRecommendedAction, KnowledgeRequirement, KnowledgeRequirementCategory, KnowledgeResponsibleSurface, KnowledgeSensitivity, ScoreKnowledgeReadinessOptions, UserQuestion, acquisitionPlansForKnowledgeGaps, blockingKnowledgeEval, knowledgeReadinessTracePayload, scoreKnowledgeReadiness, userQuestionsForKnowledgeGaps } from './knowledge/index.js';
-import { i as ReleaseConfidenceThresholds, g as ReleaseConfidenceScorecard } from './release-report-TDPn1cxq.js';
-export { B as BootstrapOptions, a as BootstrapResult, J as JudgeReplayGateArgs, P as PairedBootstrapOptions, b as PairedBootstrapResult, R as ReleaseConfidenceAxis, c as ReleaseConfidenceAxisName, d as ReleaseConfidenceInput, e as ReleaseConfidenceIssue, f as ReleaseConfidenceMetrics, h as ReleaseConfidenceStatus, j as ReleaseTraceEvidence, k as RenderReleaseReportOptions, V as Verdict, l as assertReleaseConfidence, m as bhAdjust, n as bootstrapCi, o as evaluateReleaseConfidence, p as judgeReplayGate, q as pairedBootstrap, r as pairedWilcoxon, s as releaseTraceEvidenceFromMultiShotTrials, t as renderReleaseReport } from './release-report-TDPn1cxq.js';
-import { R as Run$1, S as Span, b as TraceEvent, A as Artifact$1, B as BudgetLedgerEntry, T as TraceStore, g as BudgetSpec, h as RunFilter, L as LlmSpan } from './store-Db2Bv8Cf.js';
+import { i as ReleaseConfidenceThresholds, g as ReleaseConfidenceScorecard } from './release-report-wfUySN5F.js';
+export { B as BootstrapOptions, a as BootstrapResult, J as JudgeReplayGateArgs, P as PairedBootstrapOptions, b as PairedBootstrapResult, R as ReleaseConfidenceAxis, c as ReleaseConfidenceAxisName, d as ReleaseConfidenceInput, e as ReleaseConfidenceIssue, f as ReleaseConfidenceMetrics, h as ReleaseConfidenceStatus, j as ReleaseTraceEvidence, k as RenderReleaseReportOptions, V as Verdict, l as assertReleaseConfidence, m as bhAdjust, n as bootstrapCi, o as evaluateReleaseConfidence, p as judgeReplayGate, q as pairedBootstrap, r as pairedWilcoxon, s as releaseTraceEvidenceFromMultiShotTrials, t as renderReleaseReport } from './release-report-wfUySN5F.js';
+import { a as FailureCluster } from './failure-cluster-C2EGSDiT.js';
+export { D as DEFAULT_FAILURE_RULES, b as FailureClassification, c as FailureContext, d as FailureRule, e as classifyFailure } from './failure-cluster-C2EGSDiT.js';
+import { a as RunSplitTag, R as RunRecord } from './run-record-CqzahIbx.js';
+export { c as RunJudgeMetadata, d as RunOutcome, e as RunRecordValidationError, b as RunTokenUsage, i as isRunRecord, p as parseRunRecordSafe, r as roundTripRunRecord, v as validateRunRecord } from './run-record-CqzahIbx.js';
+import { T as TraceStore, R as Run$1, S as Span, b as TraceEvent, A as Artifact$1, B as BudgetLedgerEntry, g as BudgetSpec, h as RunFilter, L as LlmSpan } from './store-Db2Bv8Cf.js';
 export { i as EventFilter, E as EventKind, j as FAILURE_CLASSES, F as FailureClass, k as FileSystemTraceStore, l as FileSystemTraceStoreOptions, G as GenericSpan, I as InMemoryTraceStore, J as JudgeSpan, M as Message, e as RetrievalSpan, m as RunLayer, n as RunStatus, f as SandboxSpan, o as SpanBase, p as SpanFilter, d as SpanKind, q as SpanStatus, r as TRACE_SCHEMA_VERSION, a as ToolSpan, s as isJudgeSpan, t as isLlmSpan, u as isRetrievalSpan, v as isSandboxSpan, w as isToolSpan } from './store-Db2Bv8Cf.js';
 import { S as SandboxDriver, H as HarnessConfig, a as SandboxHarnessResult } from './test-graded-scenario-B2kWEdh9.js';
 export { D as DockerSandboxDriver, c as SandboxHarness, d as SandboxResult, e as SubprocessSandboxDriver, f as SubprocessSandboxDriverOptions, g as TestGradedRunOptions, b as TestGradedRunResult, T as TestGradedScenario, h as TestOutputParser, i as composeParsers, j as jestTestParser, p as pytestTestParser, r as runTestGradedScenario, v as vitestTestParser } from './test-graded-scenario-B2kWEdh9.js';
@@ -20,25 +24,146 @@ export { R as RunCompleteHook, a as RunCompleteHookContext, S as SpanHandle, b a
 export { F as FileSystemRawProviderSink, d as FileSystemRawProviderSinkOptions, I as InMemoryRawProviderSink, e as InMemoryRawProviderSinkOptions, N as NoopRawProviderSink, P as ProviderRedactor, f as RawProviderDirection, c as RawProviderEvent, R as RawProviderSink, g as RawProviderSinkFilter, h as RunIntegrityError, a as RunIntegrityExpectations, i as RunIntegrityIssue, j as RunIntegrityIssueCode, b as RunIntegrityReport, k as assertRunCaptured, l as defaultProviderRedactor, p as providerFromBaseUrl, t as throwIfRunIncomplete } from './integrity-DK2EBVZC.js';
 export { D as DEFAULT_REDACTION_RULES, O as OTEL_AGENT_EVAL_SCOPE, a as OtlpExport, b as OtlpResourceSpans, c as OtlpSpan, R as REDACTION_VERSION, d as RedactionReport, e as RedactionRule, f as ReplayCache, g as ReplayCacheEntry, h as ReplayCacheMissError, i as ReplayCacheStats, j as ReplayFetchOptions, k as createReplayFetch, l as exportRunAsOtlp, m as iterateRawCalls, r as redactString, n as redactValue } from './replay-BL96gCEP.js';
 export { a as aggregateLlm, b as argHash, g as groupBy, j as judgeSpans, l as llmSpans, r as runFailureClass, c as runsForScenario, t as toolSpans } from './query-DODUYdPg.js';
-export { D as DEFAULT_FAILURE_RULES, b as FailureClassification, c as FailureContext, d as FailureRule, e as classifyFailure } from './failure-cluster-C2EGSDiT.js';
 import { a as BaselineReport } from './baseline-4R5deP0N.js';
 export { B as BaselineOptions, M as MetricSamples, b as MetricVerdict, T as ToolStats, d as ToolUseMetrics, e as ToolUseOptions, f as compareToBaseline, c as computeToolUseMetrics, i as iqr, w as welchsTTest } from './baseline-4R5deP0N.js';
 import { T as Trajectory, a as TrajectoryStep } from './trajectory-CnoBo-JY.js';
 export { b as buildTrajectory } from './trajectory-CnoBo-JY.js';
 import { a as DatasetScenario, c as Dataset } from './dataset-CiK_3LDr.js';
 export { d as DatasetDifficulty, b as DatasetManifest, e as DatasetProvenance, D as DatasetSplit, H as HoldoutLockedError, S as SliceOptions, h as hashScenarios } from './dataset-CiK_3LDr.js';
-export { C as CalibrationResult, a as CandidateScore, D as DEFAULT_RED_TEAM_CORPUS, E as EuRiskClass, G as GoldenItem, b as GovernanceContext, c as GovernanceFinding, d as GovernanceReport, P as PositionalBiasResult, R as RedTeamCase, e as RedTeamCategory, f as RedTeamFinding, g as RedTeamPayload, h as RedTeamReport, S as SelfPreferenceResult, U as UseCaseSignals, V as VerbosityBiasResult, i as calibrateJudge, j as classifyEuAiRisk, k as euAiActReport, n as nistAiRmfReport, p as positionalBias, r as redTeamDataset, l as redTeamReport, m as renderMarkdown, s as scoreRedTeamOutput, o as selfPreference, q as soc2Report, t as summarize, u as toolNamesForRun, v as verbosityBias } from './index-Oj9fAPPN.js';
+export { C as CalibrationResult, a as CandidateScore, b as ContinuousAgreement, c as ContinuousAgreementOptions, d as ContinuousCalibrationResult, D as DEFAULT_RED_TEAM_CORPUS, E as EuRiskClass, G as GoldenItem, e as GovernanceContext, f as GovernanceFinding, g as GovernanceReport, P as PositionalBiasResult, R as RedTeamCase, h as RedTeamCategory, i as RedTeamFinding, j as RedTeamPayload, k as RedTeamReport, S as SelfPreferenceResult, U as UseCaseSignals, V as VerbosityBiasResult, l as calibrateJudge, m as calibrateJudgeContinuous, n as classifyEuAiRisk, o as continuousAgreement, p as euAiActReport, q as nistAiRmfReport, r as positionalBias, s as redTeamDataset, t as redTeamReport, u as renderMarkdown, v as scoreRedTeamOutput, w as selfPreference, x as soc2Report, y as summarize, z as toolNamesForRun, A as verbosityBias } from './index-D3iBCjdF.js';
 import { a as PrmGrader } from './rubric-D5tjHNJQ.js';
 import { L as Layer, S as Severity, a as LayerResult, b as VerifyContext } from './multi-layer-verifier-LkP3LVKj.js';
 export { F as Finding, c as LayerStatus, M as MultiLayerVerifier, V as VerificationReport, d as VerifyOptions, g as gradeSemanticStatus } from './multi-layer-verifier-LkP3LVKj.js';
-import { L as LlmClientOptions } from './researcher-CUOiGcGv.js';
-export { C as CallbackResearcher, a as CallbackResearcherOptions, b as CampaignFactoryParams, c as CampaignIntegrityPolicy, d as CampaignRunContext, e as CampaignRunOutcome, f as CampaignRunner, g as CampaignScenario, h as CampaignVariant, E as EvalCampaignOptions, i as EvalCampaignResult, j as ExperimentPlan, k as ExperimentResult, F as FailedRun, l as FailureMode, m as LlmCallError, n as LlmCallRequest, o as LlmCallResult, p as LlmClient, q as LlmMessage, s as LlmRouteAssertionError, t as LlmRouteRequirements, u as LlmUsage, N as NoopResearcher, R as Researcher, S as SteeringChange, v as assertLlmRoute, w as callLlm, x as callLlmJson, y as probeLlm, r as runEvalCampaign, z as stripFencedJson } from './researcher-CUOiGcGv.js';
+import { L as LlmClientOptions } from './researcher-bGkI7vCl.js';
+export { C as CallbackResearcher, a as CallbackResearcherOptions, b as CampaignFactoryParams, c as CampaignIntegrityPolicy, d as CampaignRunContext, e as CampaignRunOutcome, f as CampaignRunner, g as CampaignScenario, h as CampaignVariant, E as EvalCampaignOptions, i as EvalCampaignResult, j as ExperimentPlan, k as ExperimentResult, F as FailedRun, l as FailureMode, m as LlmCallError, n as LlmCallRequest, o as LlmCallResult, p as LlmClient, q as LlmMessage, s as LlmRouteAssertionError, t as LlmRouteRequirements, u as LlmUsage, N as NoopResearcher, R as Researcher, S as SteeringChange, v as assertLlmRoute, w as callLlm, x as callLlmJson, y as probeLlm, r as runEvalCampaign, z as stripFencedJson } from './researcher-bGkI7vCl.js';
 export { B as BENCHMARK_SPLIT_SEED, a as BenchmarkAdapter, b as BenchmarkDatasetItem, c as BenchmarkEvaluation, d as benchmarkDeterministicSplit, i as benchmarks } from './index--fVrWDiR.js';
-import { R as RunRecord } from './run-record-CqzahIbx.js';
-export { c as RunJudgeMetadata, d as RunOutcome, e as RunRecordValidationError, a as RunSplitTag, b as RunTokenUsage, i as isRunRecord, p as parseRunRecordSafe, r as roundTripRunRecord, v as validateRunRecord } from './run-record-CqzahIbx.js';
 export { I as InterimReleaseConfidence, a as InterimReleaseConfidenceInput, P as PairedEvalueOptions, b as PairedEvalueSequence, c as PairedEvalueStep, S as SequentialDecision, e as evaluateInterimReleaseConfidence, p as pairedEvalueSequence } from './sequential-Dgz1n51-.js';
 import './outcome-store-D6KWmYvj.js';
 
+/**
+ * Automated pull request opener for the production loop.
+ *
+ * `runProductionLoop` produces a `promotedPrompt` string and a release
+ * scorecard. To close the eval → prod → eval cycle the framework needs
+ * to land that prompt as a reviewable code change. This module does
+ * exactly that:
+ *
+ *   1. Stage a branch off `baseBranch`.
+ *   2. Write each `fileChange` into the worktree.
+ *   3. Commit + push.
+ *   4. Open a PR via the GitHub API.
+ *
+ * Two transports ship in core:
+ *
+ *   - `ghCliClient(opts)` — shells out to the `gh` CLI. No extra deps,
+ *     re-uses the developer machine's `gh auth` state, works with both
+ *     github.com and GitHub Enterprise. This is the recommended default.
+ *   - `httpGithubClient(opts)` — direct `fetch` against `api.github.com`
+ *     with a bearer token. Useful in CI where `gh` may not be installed.
+ *
+ * Both implement the small `AutoPrClient` interface, so tests substitute
+ * a fake without spinning a process or network.
+ *
+ * @experimental — added in 0.25.0. Surface may evolve as consumers wire
+ * it into CI workflows.
+ */
+interface FileChange {
+    /** Repo-relative path. Forward slashes; no `..`. */
+    path: string;
+    /** New file contents. UTF-8. */
+    contents: string;
+    /** Optional explanatory comment shown in the commit body. */
+    rationale?: string;
+}
+interface RepoRef {
+    owner: string;
+    name: string;
+}
+interface ProposeAutomatedPullRequestInput {
+    repo: RepoRef;
+    /** Branch to base the PR on. Default `'main'`. */
+    baseBranch?: string;
+    /** New branch name. Use a prefix + a short stable id; no spaces. */
+    branchName: string;
+    fileChanges: FileChange[];
+    title: string;
+    body: string;
+    /** Optional GitHub usernames to request review from. */
+    reviewers?: string[];
+    /** Optional labels to apply. */
+    labels?: string[];
+    /** Commit author name. Default: derived from the GitHub client. */
+    authorName?: string;
+    /** Commit author email. Default: derived from the GitHub client. */
+    authorEmail?: string;
+    /** Dry-run — do not push or open a PR; just return the would-be plan. */
+    dryRun?: boolean;
+}
+interface ProposeAutomatedPullRequestResult {
+    prUrl: string;
+    branchName: string;
+    headSha: string;
+    dryRun: boolean;
+}
+/** Pluggable transport for the auto-PR pipeline. */
+interface AutoPrClient {
+    /**
+     * Create a branch from `baseBranch`, write file changes, commit, push,
+     * and open a PR. Returns the PR's HTML url and head SHA.
+     *
+     * Implementations must be idempotent on `branchName`: if the branch
+     * already exists with the same head SHA as the would-be commit, return
+     * the existing PR rather than failing. This makes the production loop
+     * safe to retry on transient errors.
+     */
+    proposeChange(input: ProposeAutomatedPullRequestInput): Promise<ProposeAutomatedPullRequestResult>;
+}
+declare function proposeAutomatedPullRequest(client: AutoPrClient, input: ProposeAutomatedPullRequestInput): Promise<ProposeAutomatedPullRequestResult>;
+interface HttpGithubClientOptions {
+    /** Personal access token, GitHub App token, or `GITHUB_TOKEN` from Actions. */
+    token: string;
+    /** Override for GitHub Enterprise. Default `'https://api.github.com'`. */
+    apiBase?: string;
+    /** Test seam — defaults to global `fetch`. */
+    fetchImpl?: typeof fetch;
+    /** Test seam — clock for commit timestamps. */
+    now?: () => Date;
+}
+/**
+ * Direct REST-API GitHub client. No external deps.
+ *
+ * Idempotency strategy: before creating refs/commits/PRs, check whether
+ * the branch already exists at the desired tree. If so, return the
+ * existing PR (or open one if missing). Errors from concurrent runs
+ * (`Reference already exists`) are caught and treated as success.
+ */
+declare function httpGithubClient(opts: HttpGithubClientOptions): AutoPrClient;
+interface GhCliClientOptions {
+    /** Override the CLI binary (`gh`). For testing. */
+    bin?: string;
+    /** Working directory containing a clone of `repo`. Default: process cwd. */
+    cwd?: string;
+    /** Test seam: process spawner. Default: node:child_process spawn. */
+    exec?: (bin: string, args: string[], opts: {
+        cwd: string;
+        stdin?: string;
+    }) => Promise<{
+        stdout: string;
+        stderr: string;
+        exitCode: number;
+    }>;
+}
+/**
+ * `gh` CLI transport. Requires:
+ *   - `gh` installed and authenticated (`gh auth status`).
+ *   - A local clone of the repo with a clean working tree.
+ *   - `git` on PATH.
+ *
+ * Uses `gh api` for repo metadata and `gh pr create` for the PR. The
+ * actual commit lands via `git`, which keeps `gh`'s footprint minimal.
+ */
+declare function ghCliClient(opts?: GhCliClientOptions): AutoPrClient;
+
 interface Scenario {
     id: string;
     persona: string;
@@ -595,6 +720,181 @@ declare class MetricsCollector {
     getConvergenceCurve(): number[];
 }
 
+/**
+ * ProductionLoop — the substrate that closes eval → prod → eval.
+ *
+ * Static prompts decay. Yesterday's regulation flips today; yesterday's
+ * tool quirk becomes today's incident. A production agent that ships a
+ * static prompt and never re-trains is on a clock.
+ *
+ * `runProductionLoop` is the orchestration layer over the eval substrate:
+ *
+ *   1. Ingest production traces + user feedback (via the wire HTTP
+ *      ingestion endpoints, or directly through any `TraceStore` and
+ *      `FeedbackTrajectoryStore` implementation).
+ *   2. Cluster the failures (`failureClusterView`) and prioritize by
+ *      size × severity.
+ *   3. If any cluster crosses the consumer's threshold, run a
+ *      `runMultiShotOptimization` round seeded by the current production
+ *      prompt against holdout-shape scenarios derived from the offending
+ *      cluster.
+ *   4. Gate the promoted prompt with `evaluateReleaseConfidence`. Fail
+ *      closed.
+ *   5. If the gate passes and an `AutoPrClient` is wired, open a PR with
+ *      the new prompt. Otherwise return the proposed change.
+ *
+ * One call = one cycle. Cron / GitHub Actions are the caller's job. The
+ * primitive is idempotent + replayable: re-running with the same
+ * `runId` will produce the same plan.
+ *
+ * @experimental — added in 0.25.0. Surface may evolve as the 5 product
+ * agents wire it in.
+ */
+
+interface FailureClusterConfig {
+    /** Minimum runs in a cluster before it triggers an evolve round. Default 5. */
+    minClusterSize?: number;
+    /**
+     * Severity threshold. A cluster is "actionable" when its size
+     * normalized by total runs exceeds this. Default 0.05 (5% of all runs).
+     */
+    minSeverityRatio?: number;
+    /**
+     * Maximum number of clusters to react to in one cycle. Acting on too
+     * many at once obscures attribution. Default 1 — the worst cluster.
+     */
+    maxClustersPerCycle?: number;
+}
+interface ProductionEvolveConfig<P = string> {
+    /** How to run a candidate prompt against a scenario. */
+    runner: MultiShotRunner<P>;
+    /** How to score the trajectory. Usually a calibrated judge. */
+    scorer: MultiShotScorer<P>;
+    /** How to mutate. Addendum-style mutators (append vs. rewrite) work best. */
+    mutator: MultiShotMutateAdapter<P>;
+    /** The current production prompt. Acts as the baseline + seed. */
+    baselinePrompt: P;
+    /** Stable id for the baseline variant. Default `'baseline'`. */
+    baselineId?: string;
+    /** Scenarios resembling production load. Used as the holdout split. */
+    holdoutScenarios: Scenario[];
+    /** Scenarios used during search. Default: derived from `holdoutScenarios` via deterministic split. */
+    searchScenarios?: Scenario[];
+    /** Gate config for the held-out promotion check. */
+    gate: HeldOutGateConfig;
+    /** Reps per (variant × scenario) cell. Default 3. */
+    reps?: number;
+    /** Number of mutation generations. Default 3. */
+    generations?: number;
+    /** Population size per generation. Default 4. */
+    populationSize?: number;
+    /** Concurrent score() calls. Default 1. */
+    scoreConcurrency?: number;
+    /**
+     * Optional bridge from a scored trial into a paper-grade RunRecord.
+     * If omitted, the loop synthesises a minimal record sufficient for
+     * `HeldOutGate` and `evaluateReleaseConfidence`.
+     */
+    toRunRecord?: (input: {
+        variant: EvolvableVariant<P>;
+        scenarioId: string;
+        rep: number;
+        split: RunSplitTag;
+        seed: number;
+        trial: MultiShotTrialResult;
+    }) => RunRecord;
+}
+interface ProductionShipConfig {
+    repo: RepoRef;
+    /** Branch name prefix. Final branch = `${branchPrefix}/${runId}`. */
+    branchPrefix: string;
+    /** Path (repo-relative) of the file holding the production prompt. */
+    promptFilePath: string;
+    /** Base branch for the PR. Default `'main'`. */
+    baseBranch?: string;
+    reviewers?: string[];
+    labels?: string[];
+    /** Required: the auto-PR transport. Use `ghCliClient()` or `httpGithubClient()`. */
+    client: AutoPrClient;
+    /** Skip the actual push + PR call — for sanity-checking the plan. Default false. */
+    dryRun?: boolean;
+    /** Render PR body from the loop's findings. Optional override. */
+    renderBody?: (ctx: ProductionLoopRenderContext) => string;
+    /** Render the file contents from the new prompt. Default: serialize as the file. */
+    renderPromptFile?: (newPrompt: string, oldFileContents: string | null) => string;
+    /** Read the current prompt file contents for diff context. Optional. */
+    readCurrentPromptFile?: () => Promise<string | null>;
+}
+interface ProductionLoopCronConfig {
+    cadence: 'weekly' | 'daily' | 'hourly';
+    /** Optional jitter (seconds) the consumer's scheduler should add. Surface-only. */
+    jitterSec?: number;
+}
+interface RunProductionLoopOptions<P = string> {
+    /** Stable id; deterministic outputs when reused. */
+    runId: string;
+    /** Human label — surfaces in PR titles and reports. */
+    target: string;
+    traceStore: TraceStore;
+    feedbackStore: FeedbackTrajectoryStore;
+    cluster: FailureClusterConfig;
+    evolve: ProductionEvolveConfig<P>;
+    /** When omitted, the loop returns the proposed prompt without opening a PR. */
+    ship?: ProductionShipConfig;
+    /** Surface-only — encodes scheduler expectations into the artifact. */
+    cron?: ProductionLoopCronConfig;
+    /** Release confidence thresholds. Default: library defaults. */
+    releaseThresholds?: ReleaseConfidenceThresholds;
+    /** Now() seam for reproducibility in tests. */
+    now?: () => Date;
+}
+type ProductionLoopDecision = 'no_actionable_failures' | 'evolve_yielded_no_improvement' | 'gate_failed' | 'proposed_change' | 'pr_opened';
+interface ProductionLoopRenderContext {
+    runId: string;
+    target: string;
+    decision: ProductionLoopDecision;
+    /** Clusters seen in production this cycle, sorted by severity. */
+    clusters: FailureCluster[];
+    /** The cluster the loop acted on (if any). */
+    actedOnCluster: FailureCluster | null;
+    /** Production runs observed this cycle. */
+    observedRunCount: number;
+    /** Feedback trajectories observed this cycle. */
+    observedFeedbackCount: number;
+    /** Evolve result (if evolve ran). */
+    evolution: MultiShotOptimizationResult<unknown> | null;
+    /** Release gate verdict (if evolve ran). */
+    release: ReleaseConfidenceScorecard | null;
+    /** Held-out gate decision (if a candidate was paired against the baseline). */
+    gate: GateDecision | null;
+    /** The baseline (current production) prompt as a string. */
+    baselinePromptString: string;
+    /** The proposed new prompt as a string. Empty if no change was proposed. */
+    promotedPromptString: string;
+}
+interface ProductionLoopResult {
+    runId: string;
+    target: string;
+    decision: ProductionLoopDecision;
+    startedAt: string;
+    finishedAt: string;
+    observedRunCount: number;
+    observedFeedbackCount: number;
+    clusters: FailureCluster[];
+    actedOnCluster: FailureCluster | null;
+    evolution: MultiShotOptimizationResult<unknown> | null;
+    release: ReleaseConfidenceScorecard | null;
+    gate: GateDecision | null;
+    /** Baseline prompt as it entered the cycle. */
+    baselinePrompt: unknown;
+    /** Promoted prompt — equals baseline when no change is proposed. */
+    promotedPrompt: unknown;
+    /** PR artifact when `ship` was wired and gate passed. */
+    pullRequest: ProposeAutomatedPullRequestResult | null;
+    cron: ProductionLoopCronConfig | null;
+}
+declare function runProductionLoop<P = string>(opts: RunProductionLoopOptions<P>): Promise<ProductionLoopResult>;
+
 /**
  * ScenarioRegistry — manages scenario discovery and filtering.
  *
@@ -4453,6 +4753,63 @@ declare class Mutex {
     get pending(): number;
 }
 
+/**
+ * Persona discovery — replaces every consumer's hardcoded TRAINING_PERSONA_FILES.
+ *
+ * Today's failure mode: each product agent (legal/gtm/tax/creative) defines
+ * a TRAINING_PERSONA_FILES const with 5 hardcoded filenames. When the 2yr
+ * rewrite added 10+ new personas, those personas existed on disk but the
+ * evolve runner never loaded them — the new rubric dims (audit_defendability,
+ * intake_discipline, etc) got no training signal. The personas were
+ * cosmetic, the rewrites partially uninformed.
+ *
+ * `discoverPersonas` walks a personas directory and returns every persona
+ * file matching the convention. Consumers can filter by include/exclude
+ * patterns. Default behavior — discover everything — eliminates the
+ * "forgot to add the new persona to the list" failure mode.
+ */
+interface DiscoverPersonasOptions {
+    /**
+     * Regex applied to filenames. Files that don't match are skipped.
+     * Default: `^[0-9]{2}-.+\.(yaml|yml|json|md)$` (the prevailing convention
+     * across legal/gtm/tax/creative: `NN-slug.yaml`).
+     */
+    pattern?: RegExp;
+    /**
+     * Filenames (or basenames) to skip. Use this to exclude WIP / archived
+     * personas without removing the file.
+     */
+    exclude?: readonly string[];
+    /**
+     * If set, return only personas whose basename contains one of these
+     * substrings (post-pattern filter). Used by the CLI's `--personas a,b,c`
+     * flag — consumers pass through.
+     */
+    include?: readonly string[];
+    /**
+     * Recurse into subdirectories. Default false (legal/gtm/tax/creative all
+     * store personas flat).
+     */
+    recursive?: boolean;
+}
+interface DiscoveredPersona {
+    /** Absolute file path. */
+    path: string;
+    /** Filename without directory. */
+    filename: string;
+    /** Filename without extension — the conventional persona id. */
+    id: string;
+}
+/**
+ * Walk `dir` and return every persona file matching the convention. Async
+ * because the consumer almost always wants this to be I/O-driven (so a new
+ * persona added on disk is picked up without a code change).
+ *
+ * Sorted by filename (which gives stable persona id order via the `NN-`
+ * numeric prefix convention) for reproducibility.
+ */
+declare function discoverPersonas(dir: string, opts?: DiscoverPersonasOptions): Promise<DiscoveredPersona[]>;
+
 /**
  * GoldenMatcher — fuzzy matcher for "did the agent produce the expected things?".
  *
@@ -4556,6 +4913,101 @@ declare class JsonlTrialCache implements TrialCache {
     setSync(key: string, value: TrialResult): void;
 }
 
+/**
+ * Judge-retry wrapper.
+ *
+ * Today's failure mode: a judge LLM call aborts mid-stream (connection
+ * dropped, model timed out, schema rejected) → consumer's try/catch swallows
+ * the error and returns `score: 0`. The eval composite then weights that
+ * zero into the mean, silently corrupting the score. Today's tax/gtm evals
+ * had `judge=0` across every trial — the prompt rewrites couldn't be
+ * evaluated honestly because the measurement instrument was broken.
+ *
+ * `withJudgeRetry` is the substrate fix. It wraps a single judge invocation
+ * with:
+ *
+ *   1. N retry attempts on transient failures (abort, timeout, network).
+ *   2. Optional fallback-model rotation — try the next model in the list
+ *      if the primary keeps aborting (a verbose new prompt may stream-abort
+ *      on claude-code/sonnet but succeed on kimi-code/k2p6).
+ *   3. Exponential backoff between attempts.
+ *   4. A typed outcome `{ succeeded, attempts, value, error }` that callers
+ *      MUST decide what to do with. No silent zero.
+ *
+ * The reporting contract: callers ship `TrialResult.judgeSucceeded = succeeded`
+ * and `TrialResult.judgeAttempts = attempts`. `aggregateTrials({mode: 'exclude-failed'})`
+ * then skips failed-judge trials when computing composites.
+ *
+ * The library does NOT decide what score to record on failure — that's the
+ * caller's product choice. Today's product agents (legal/gtm/tax/creative)
+ * should set `score: NaN` + `judgeSucceeded: false` + `error: ...` so the
+ * aggregator's exclude-failed mode drops the trial. Defaulting to 0 is what
+ * caused today's data corruption.
+ */
+/**
+ * Retry policy for judge LLM calls.
+ *
+ * Defaults are tuned for the verbose post-2yr-rewrite prompts that exceed
+ * the 60s `callLlm` default and abort on streaming. Pick a different timeout
+ * for cheap-and-quick judges (e.g., 30s) or longer for thinking models.
+ */
+interface JudgeRetryPolicy {
+    /** Max attempts per model. Default 3 (one initial + two retries). */
+    maxAttempts?: number;
+    /** Per-attempt timeout in ms. Default 90_000 (1.5×agent-eval's 60s default). */
+    timeoutMs?: number;
+    /**
+     * Models to try, in order. The first model is the primary; subsequent
+     * models are fallbacks invoked only when ALL retries on the previous
+     * model have been exhausted. Example: `['claude-code/sonnet', 'kimi-code/k2p6']`
+     * runs claude-code up to maxAttempts times, then falls back to kimi.
+     * If omitted, the caller's judge function controls model selection and
+     * the retries apply to that single model.
+     */
+    models?: readonly string[];
+    /** Exponential backoff function, default `attempt → min(500 * 2^attempt, 16_000)`. */
+    backoffMs?: (attempt: number) => number;
+    /**
+     * Predicate deciding whether an error should trigger a retry. Default
+     * retries on: AbortError, TimeoutError, `fetch failed`, `ECONNRESET`,
+     * `[This operation was aborted]`, and any LlmCallError with status in
+     * {429, 502, 503, 504}. JSON-parse errors are NOT retriable (the model
+     * needs prompt adjustment, not another shot).
+     */
+    isRetryable?: (err: unknown) => boolean;
+}
+/** Outcome of a wrapped judge invocation. */
+interface JudgeRetryOutcome<T> {
+    /** The judge's returned value when `succeeded === true`. */
+    value: T | null;
+    /** True iff one of the attempts completed without throwing. */
+    succeeded: boolean;
+    /** Total attempts made across all models. */
+    attempts: number;
+    /** Which model the successful attempt used (when succeeded). */
+    modelUsed?: string;
+    /** Last error captured when `succeeded === false`. */
+    error?: Error;
+    /** Per-attempt error log for forensics. */
+    attemptErrors: Array<{
+        attempt: number;
+        model: string;
+        error: string;
+    }>;
+}
+/**
+ * Wrap a judge call with retry + fallback-model + typed outcome semantics.
+ *
+ * The `judgeFn` signature is `(model: string, signal: AbortSignal) => Promise<T>`.
+ * The signal will be aborted at `timeoutMs`. Callers should pass the signal
+ * to their underlying fetch/SDK call so the abort actually fires.
+ *
+ * Returns a typed outcome — callers MUST inspect `succeeded` before using
+ * `value`. The library refuses to default to a silent zero score because that
+ * is exactly what caused today's eval data corruption.
+ */
+declare function withJudgeRetry<T>(judgeFn: (model: string, signal: AbortSignal) => Promise<T>, policy?: JudgeRetryPolicy): Promise<JudgeRetryOutcome<T>>;
+
 /**
  * LockedJsonlAppender — mutex-serialized JSONL append helper for arbitrary
  * payloads. The reference-replay store does the same thing for typed
@@ -4617,4 +5069,69 @@ interface ReferenceReplaySteeringRowsOptions<Input = unknown> {
 declare function referenceReplayRunsToSteeringRows<Input = unknown>(runs: ReferenceReplayRun<Input>[], options?: ReferenceReplaySteeringRowsOptions<Input>): SteeringOptimizationRow[];
 declare function referenceReplayScenarioToRunScore(scenarioScore: ReferenceReplayScenarioScore, durationMs?: number): RunScore;
 
-export { ActionableSideInfo, type ActiveLearningOptions, type AdapterRun, AgentDriver, type AgentDriverConfig, AgentEvalError, type AlignmentOp, type AntiSlopConfig, type AntiSlopIssue, type AntiSlopReport, Artifact$1 as Artifact, type ArtifactCheck, type Artifact as ArtifactCheckArtifact, type ArtifactResult, type ArtifactValidator, AxGepaSteeringOptimizer, type AxSteeringOptimizerConfig, BaselineReport, BehaviorAssertion, type BenchmarkReport, BenchmarkRunner, type BenchmarkRunnerConfig, type BisectOptions, type BisectResult, type BisectStep, BudgetBreachError, BudgetGuard, BudgetLedgerEntry, BudgetSpec, CallExpectation, type CanaryAlert, type CanaryKind, type CanaryLeak, type CanaryOptions, type CanaryReport, type CanarySeverity, type CandidateScenario, type CausalAttributionReport, type CheckResult, type CodeMutationOutcome, type CodeMutationRunner, type CollectedArtifacts, type CommandRunner, type CompletionCriterion, type CompositePolicy, type ConceptComplexity, type ConceptFinding, type ConceptSpec, type ConceptWeightStrategy, type ContinuityCheck, type ContinuityCheckResult, type ContinuityReport, type ContinuitySnapshotPair, type ContractMetric, type ContractReport, ControlEvalResult, ConvergenceTracker, type CostEntry, CostLedger, type CostLedgerGeneration, type CostLedgerSnapshot, type CostSummary, CostTracker, type CounterfactualContext, type CounterfactualMutation, type CounterfactualResult, type CounterfactualRunner, type CreateCompositeMutatorOpts, type CreateDefaultReviewerOptions, type CreateSandboxCodeMutatorOpts, type CreateSandboxPoolOpts, type CrossTraceDiff, type CrossTraceDiffOptions, D1ExperimentStore, type D1ExperimentStoreOptions, type D1Like, type D1PreparedStatementLike, DEFAULT_AGENT_SLOS, DEFAULT_COMPLEXITY_WEIGHTS, DEFAULT_FINDERS, DEFAULT_HARNESS_OBJECTIVES, DEFAULT_MUTATORS, DEFAULT_RUN_SCORE_WEIGHTS, DEFAULT_SEVERITY_WEIGHTS, Dataset, DatasetScenario, type DeployFamily, type DeployGateLayerInput, type DeployRunResult, type DeployRunner, type DirEntry, type DriverResult, type DriverState, DualAgentBench, type DualAgentBenchConfig, type DualAgentReport, type DualAgentRound, type DualAgentScenario, type DualAgentScenarioResult, ERROR_COUNT_PATTERNS, type ErrorCountPattern, type EvalResult, type EvolutionRound, EvolvableVariant, type ExecutorConfig, type Expectation, type Experiment, type Run as ExperimentRun, type ExperimentStore, ExperimentTracker, type ExportedRewardModel, type ExtractOptions, type ExtractResult, type FactorContribution, type FactorialCell, FeedbackLabel, type FeedbackPattern, FeedbackTrajectory, FeedbackTrajectoryStore, FileSystemExperimentStore, type FileSystemExperimentStoreOptions, type FlowAction, type FlowLayerEnv, type FlowLayerFactoryInput, type FlowRunner, type FlowRunnerStepResult, type FlowSpec, type FlowStep, type GoldenSeverity, type GoldenSpec, type HarnessAdapter, HarnessConfig, type HarnessExperimentConfig, type HarnessExperimentResult, type HarnessIntervention, type HarnessRunRequest, type HarnessRunResult, type HarnessScenario, type HarnessSelection, type HarnessVariant, type HarnessVariantReport, HoldoutAuditor, type HostedJudgeConfig, type HostedJudgeDimension, type HostedJudgeRequest, type HostedJudgeResponse, type HostedRunCriticConfig, type HostedRunScoreRequest, type HostedRunScoreResponse, type HypothesisManifest, type HypothesisResult, INTENT_MATCH_JUDGE_VERSION, type ImageData, InMemoryExperimentStore, InMemoryWorkspaceInspector, type InferenceScorer, type InspectorContext, type IntegrationGateSurface, type IntegrationInvokeFailureInput, type IntegrationManifestGateInput, type IntentMatchInput, type IntentMatchOptions, type IntentMatchResult, type InteractionContribution, JsonlTrialCache, type JudgeConfig, type JudgeFleetOptions, type JudgeFn, type JudgeInput, type JudgeReplayResult, type JudgeRubric, JudgeRunner, type JudgeScore, type KeywordConceptSpec, type KeywordCoverageFinding, type KeywordCoverageOptions, type KeywordCoverageResult, type LangfuseEnvelope, type LangfuseGeneration, type LangfuseScore, Layer, LayerResult, type LineageKind, type LineageKindResolver, type LineageNode, LineageRecorder, type LiveProofArtifact, type LiveProofConfig, type LiveProofContext, type LiveProofResult, LlmClientOptions, LlmSpan, LockedJsonlAppender, MODEL_PRICING, type MatchResult, type MatcherResult, type MeasurementPolicy, type MergeOptions, MetricsCollector, type MuffledFinder, type MuffledFinding, type MultiToolchainLayerConfig, MutateAdapter, type MutationAttempt, type MutationChannel, MutationTelemetry, type Mutator, Mutex, Objective, type Oracle, type OracleObservation, type OracleReport, type OracleResult, type OrthogonalityInput, type OrthogonalityResult, PairwiseSteeringOptimizer, type ParaphraseRobustnessScenarioInput, type ParaphraseRobustnessScenarioResult, ParetoResult, type PersonaConfig, type Playbook, type PlaybookEntry, type PoolSlot, ProductClient, type ProductClientConfig, type PromptHandle, PromptRegistry, TrialResult as PromptTrialResult, type ReferenceMatchResult, type ReferenceReplayAdapter, type ReferenceReplayAdapterFn, type ReferenceReplayAdapterLike, type ReferenceReplayAggregate, type ReferenceReplayCandidate, type ReferenceReplayCase, type ReferenceReplayCaseRun, type ReferenceReplayExecutionScenario, type ReferenceReplayItem, type ReferenceReplayMatch, type ReferenceReplayMatchStrategy, type ReferenceReplayMatcher, type ReferenceReplayPromotionDecision, type ReferenceReplayPromotionPolicy, type ReferenceReplayRun, type ReferenceReplayRunContext, type ReferenceReplayRunOptions, type ReferenceReplayRunStore, type ReferenceReplayScenario, type ReferenceReplayScenarioScore, type ReferenceReplayScore, type ReferenceReplayScoreOptions, type ReferenceReplaySplit, type ReferenceReplaySplitComparison, type ReferenceReplaySteeringRowsOptions, ReleaseConfidenceScorecard, ReleaseConfidenceThresholds, type ReviewerMemoryEntry, type ReviewerOutput, type ReviewerPromptInput, type ReviewerSoftFailDefaults, type ReviewerVerificationSummary, type RobustnessResult, type RouteMap, type RubricDimension, Run$1 as Run, type RunCommandInput, type RunCommandResult, type RunConfig, RunCritic, type RunCriticOptions, type RunDiff, RunFilter, RunRecord, type RunScore, type RunScoreWeights, type RunTrace, SEMANTIC_CONCEPT_JUDGE_VERSION, SandboxDriver, SandboxHarnessResult, type SandboxJudgeKind, type SandboxJudgeResult, type SandboxJudgeSpec, type SandboxPool, type ScanOptions, type Scenario, type ScenarioCost, type ScenarioFile, ScenarioRegistry, type ScenarioResult, type ScoredTarget, type SelfPlayOptions, type SelfPlayProposer, type SelfPlayScorer, type SemanticConceptJudgeInput, type SemanticConceptJudgeOptions, type SemanticConceptJudgeResult, type SeriesConvergenceOptions, type SeriesConvergenceResult, Severity, type SignedManifest, type SignedManifestAlgo, type Slo, type SloCheckResult, type SloComparator, type SloReport, type SloSeverity, type SlopCategory, type SlotFactory, Span, type SteeringBundle, type SteeringDelta, type SteeringOptimizationResult, type SteeringOptimizationRow, type SteeringOptimizationSelector, type SteeringOptimizerBackend, type SteeringOptimizerConfig, type SteeringRolePrompt, type StepAttribution, type SynthesisReason, type SynthesisTarget, type TestResult, type ThresholdContract, TokenCounter, type TokenSpec, TraceEmitter, TraceEvent, TraceStore, Trajectory, TrajectoryStep, type TrialAttempt, TrialCache, TrialTelemetry, type Turn, type TurnMetrics, type TurnResult, UNIVERSAL_FINDERS, type ValidationContext, type ValidationIssue, type ValidationResult, VariantAggregate, VerifyContext, type VisualDiffOptions, type VisualDiffResult, type ViteDeployRunnerInput, type WorkflowTopology, type WorkspaceAssertion, type WorkspaceAssertionResult, type WorkspaceInspector, type WorkspaceSnapshot, type WranglerDeployRunnerInput, adversarialJudge, aggregateRunScore, analyzeAntiSlop, analyzeSeries, attributeCounterfactuals, benjaminiHochberg, bisect, bonferroni, buildReviewerPrompt, byteLengthRange, canaryLeakView, canonicalize, causalAttribution, checkBehavioralCanary, checkCanaries, checkSlos, clamp01, codeExecutionJudge, cohensD, coherenceJudge, collectionPreserved, commitBisect, compareReferenceReplay, compilerJudge, composeValidators, confidenceInterval, containsAll, createAntiSlopJudge, createCompositeMutator, createCustomJudge, createDefaultReviewer, createDomainExpertJudge, createIntentMatchJudge, createSandboxCodeMutator, createSandboxPool, createSemanticConceptJudge, crossTraceDiff, decideReferenceReplayPromotion, decideReferenceReplayRunPromotion, defaultJudges, defaultReferenceReplayMatcher, deployGateLayer, distillPlaybook, estimateCost, estimateTokens, evaluateContract, evaluateHypothesis, evaluateOracles, executeScenario, expectAgent, exportRewardModel, extractAssetUrls, extractErrorCount, fileContains, fileExists, findAutoMatchNoExpectation, findConstructorCwdDropped, findFallbackToPass, findLiteralTruePass, findSkipCountsAsPass, flowLayer, formatBenchmarkReport, formatDriverReport, formatFindings, precision as goldenPrecision, hashContent, hashJson, htmlContainsElement, inMemoryReferenceReplayStore, integrationAsi, integrationGateEvals, integrationInvokeFailedPayload, integrationManifestResolvedPayload, integrationManifestValidatedPayload, interRaterReliability, jsonHasKeys, jsonShape, jsonlReferenceReplayStore, keyPreserved, linterJudge, loadScorerFromGrader, localCommandRunner, lowercaseMutator, mannWhitneyU, matchGoldens, mergeLayerResults, mergeSteeringBundle, multiToolchainLayer, normalizeScores, notBlocked, pairedTTest, paraphraseRobustness, paraphraseRobustnessScenarios, partialCredit, passOrthogonality, pixelDeltaRatio, politenessPrefixMutator, printDriverSummary, promptBisect, proposeSynthesisTargets, referenceReplayRunsToSteeringRows, referenceReplayScenarioToRunScore, regexMatch, regexMatches, renderMarkdownReport, renderPlaybookMarkdown, renderSteeringText, replayScorerOverCorpus, replayTraceThroughJudge, requiredSampleSize, resetLockedAppendersForTesting, rowCount, rowWhere, runAssertions, runBehavioralCanaries, runCanaries, runCounterfactual, runE2EWorkflow, runExpectations, runHarnessExperiment, runIntentMatchJudge, runJudgeFleet, runKeywordCoverageJudge, runKeywordCoverageJudgeUrl, runLiveProof, runReferenceReplay, runSelfPlay, runSemanticConceptJudge, scanForMuffledGates, scoreContinuity, scoreReferenceReplay, securityJudge, selectHarnessVariant, sentenceReorderMutator, signManifest, statusAdvanced, summarizeHarnessResults, testJudge, textInSnapshot, toLangfuseEnvelope, toPrometheusText, typoMutator, urlContains, verifyManifest, visualDiff, viteDeployRunner, weightedMean, weightedRecall, whitespaceCollapseMutator, wilcoxonSignedRank, wranglerDeployRunner };
+/**
+ * Trial-aggregator modes.
+ *
+ * The prompt-evolution loop's internal `aggregateTrials` defaulted to
+ * including every non-`error` trial in the mean — which corrupted the mean
+ * when a trial had `score: 0` because the judge silently aborted (the
+ * caller's try/catch swallowed the abort and returned zero). Today's
+ * tax/gtm evals show this: every trial scored judge=0 because the judge
+ * aborted, and the composite then reflected `structural * 0.3 + slop * 0.1`
+ * instead of the intended `judge * 0.6 + structural * 0.3 + slop * 0.1`.
+ *
+ * `aggregateTrialsByMode` is the substrate fix. Consumers can choose:
+ *
+ *   - `strict-fail` — any trial with `judgeSucceeded === false` fails the
+ *     whole aggregate. Right for production-gate runs where one corrupted
+ *     trial means "we don't know if the prompt is good, halt the gate."
+ *
+ *   - `exclude-failed` — drop trials with `judgeSucceeded === false` from
+ *     the mean; report `failedTrials` separately. Right for research /
+ *     comparison runs where you want to use the signal that DID land.
+ *     Default for new code.
+ *
+ *   - `zero-fill` — legacy behavior: failed trials count as score=0 in
+ *     the mean. Default ONLY for backwards-compat with adapters that
+ *     don't yet set `judgeSucceeded`. Migrate off this — it's the source
+ *     of today's data corruption.
+ */
+
+type AggregatorMode = 'strict-fail' | 'exclude-failed' | 'zero-fill';
+interface TrialAggregate {
+    /** Mean score over the trials counted by the chosen mode. */
+    meanScore: number;
+    /** Mean cost (legacy, kept for compatibility). */
+    meanCost: number;
+    /** Mean wall time (legacy). */
+    meanDurationMs: number;
+    /** ok-rate (legacy). */
+    okRate: number;
+    /** Trials counted in the mean (mode-dependent). */
+    countedTrials: number;
+    /** Trials excluded because `judgeSucceeded === false` (exclude-failed mode). */
+    excludedFailedTrials: number;
+    /** Total trials passed in. */
+    totalTrials: number;
+    /** Mean of every numeric metric across counted trials. */
+    metrics: Record<string, number>;
+    /**
+     * Set when mode is `strict-fail` AND at least one trial had
+     * `judgeSucceeded === false`. Caller should refuse to use this aggregate
+     * downstream — the eval is corrupt.
+     */
+    strictFailure?: {
+        failedCount: number;
+        firstError?: string;
+    };
+}
+/**
+ * Aggregate trials with explicit failed-judge handling. Returns counts for
+ * counted + excluded so callers can surface "the score is based on 7 of 10
+ * trials; 3 judges failed" instead of silently weighting zero.
+ */
+declare function aggregateTrialsByMode(trials: TrialResult[], opts: {
+    mode: AggregatorMode;
+}): TrialAggregate;
+
+export { ActionableSideInfo, type ActiveLearningOptions, type AdapterRun, AgentDriver, type AgentDriverConfig, AgentEvalError, type AggregatorMode, type AlignmentOp, type AntiSlopConfig, type AntiSlopIssue, type AntiSlopReport, Artifact$1 as Artifact, type ArtifactCheck, type Artifact as ArtifactCheckArtifact, type ArtifactResult, type ArtifactValidator, type AutoPrClient, AxGepaSteeringOptimizer, type AxSteeringOptimizerConfig, BaselineReport, BehaviorAssertion, type BenchmarkReport, BenchmarkRunner, type BenchmarkRunnerConfig, type BisectOptions, type BisectResult, type BisectStep, BudgetBreachError, BudgetGuard, BudgetLedgerEntry, BudgetSpec, CallExpectation, type CanaryAlert, type CanaryKind, type CanaryLeak, type CanaryOptions, type CanaryReport, type CanarySeverity, type CandidateScenario, type CausalAttributionReport, type CheckResult, type CodeMutationOutcome, type CodeMutationRunner, type CollectedArtifacts, type CommandRunner, type CompletionCriterion, type CompositePolicy, type ConceptComplexity, type ConceptFinding, type ConceptSpec, type ConceptWeightStrategy, type ContinuityCheck, type ContinuityCheckResult, type ContinuityReport, type ContinuitySnapshotPair, type ContractMetric, type ContractReport, ControlEvalResult, ConvergenceTracker, type CostEntry, CostLedger, type CostLedgerGeneration, type CostLedgerSnapshot, type CostSummary, CostTracker, type CounterfactualContext, type CounterfactualMutation, type CounterfactualResult, type CounterfactualRunner, type CreateCompositeMutatorOpts, type CreateDefaultReviewerOptions, type CreateSandboxCodeMutatorOpts, type CreateSandboxPoolOpts, type CrossTraceDiff, type CrossTraceDiffOptions, D1ExperimentStore, type D1ExperimentStoreOptions, type D1Like, type D1PreparedStatementLike, DEFAULT_AGENT_SLOS, DEFAULT_COMPLEXITY_WEIGHTS, DEFAULT_FINDERS, DEFAULT_HARNESS_OBJECTIVES, DEFAULT_MUTATORS, DEFAULT_RUN_SCORE_WEIGHTS, DEFAULT_SEVERITY_WEIGHTS, Dataset, DatasetScenario, type DeployFamily, type DeployGateLayerInput, type DeployRunResult, type DeployRunner, type DirEntry, type DiscoverPersonasOptions, type DiscoveredPersona, type DriverResult, type DriverState, DualAgentBench, type DualAgentBenchConfig, type DualAgentReport, type DualAgentRound, type DualAgentScenario, type DualAgentScenarioResult, ERROR_COUNT_PATTERNS, type ErrorCountPattern, type EvalResult, type EvolutionRound, EvolvableVariant, type ExecutorConfig, type Expectation, type Experiment, type Run as ExperimentRun, type ExperimentStore, ExperimentTracker, type ExportedRewardModel, type ExtractOptions, type ExtractResult, type FactorContribution, type FactorialCell, type FailureClusterConfig, FeedbackLabel, type FeedbackPattern, FeedbackTrajectory, FeedbackTrajectoryStore, type FileChange, FileSystemExperimentStore, type FileSystemExperimentStoreOptions, type FlowAction, type FlowLayerEnv, type FlowLayerFactoryInput, type FlowRunner, type FlowRunnerStepResult, type FlowSpec, type FlowStep, GateDecision, type GhCliClientOptions, type GoldenSeverity, type GoldenSpec, type HarnessAdapter, HarnessConfig, type HarnessExperimentConfig, type HarnessExperimentResult, type HarnessIntervention, type HarnessRunRequest, type HarnessRunResult, type HarnessScenario, type HarnessSelection, type HarnessVariant, type HarnessVariantReport, HeldOutGateConfig, HoldoutAuditor, type HostedJudgeConfig, type HostedJudgeDimension, type HostedJudgeRequest, type HostedJudgeResponse, type HostedRunCriticConfig, type HostedRunScoreRequest, type HostedRunScoreResponse, type HttpGithubClientOptions, type HypothesisManifest, type HypothesisResult, INTENT_MATCH_JUDGE_VERSION, type ImageData, InMemoryExperimentStore, InMemoryWorkspaceInspector, type InferenceScorer, type InspectorContext, type IntegrationGateSurface, type IntegrationInvokeFailureInput, type IntegrationManifestGateInput, type IntentMatchInput, type IntentMatchOptions, type IntentMatchResult, type InteractionContribution, JsonlTrialCache, type JudgeConfig, type JudgeFleetOptions, type JudgeFn, type JudgeInput, type JudgeReplayResult, type JudgeRetryOutcome, type JudgeRetryPolicy, type JudgeRubric, JudgeRunner, type JudgeScore, type KeywordConceptSpec, type KeywordCoverageFinding, type KeywordCoverageOptions, type KeywordCoverageResult, type LangfuseEnvelope, type LangfuseGeneration, type LangfuseScore, Layer, LayerResult, type LineageKind, type LineageKindResolver, type LineageNode, LineageRecorder, type LiveProofArtifact, type LiveProofConfig, type LiveProofContext, type LiveProofResult, LlmClientOptions, LlmSpan, LockedJsonlAppender, MODEL_PRICING, type MatchResult, type MatcherResult, type MeasurementPolicy, type MergeOptions, MetricsCollector, type MuffledFinder, type MuffledFinding, MultiShotMutateAdapter, MultiShotOptimizationResult, MultiShotRunner, MultiShotScorer, MultiShotTrialResult, type MultiToolchainLayerConfig, MutateAdapter, type MutationAttempt, type MutationChannel, MutationTelemetry, type Mutator, Mutex, Objective, type Oracle, type OracleObservation, type OracleReport, type OracleResult, type OrthogonalityInput, type OrthogonalityResult, PairwiseSteeringOptimizer, type ParaphraseRobustnessScenarioInput, type ParaphraseRobustnessScenarioResult, ParetoResult, type PersonaConfig, type Playbook, type PlaybookEntry, type PoolSlot, ProductClient, type ProductClientConfig, type ProductionEvolveConfig, type ProductionLoopCronConfig, type ProductionLoopDecision, type ProductionLoopRenderContext, type ProductionLoopResult, type ProductionShipConfig, type PromptHandle, PromptRegistry, TrialResult as PromptTrialResult, type ProposeAutomatedPullRequestInput, type ProposeAutomatedPullRequestResult, type ReferenceMatchResult, type ReferenceReplayAdapter, type ReferenceReplayAdapterFn, type ReferenceReplayAdapterLike, type ReferenceReplayAggregate, type ReferenceReplayCandidate, type ReferenceReplayCase, type ReferenceReplayCaseRun, type ReferenceReplayExecutionScenario, type ReferenceReplayItem, type ReferenceReplayMatch, type ReferenceReplayMatchStrategy, type ReferenceReplayMatcher, type ReferenceReplayPromotionDecision, type ReferenceReplayPromotionPolicy, type ReferenceReplayRun, type ReferenceReplayRunContext, type ReferenceReplayRunOptions, type ReferenceReplayRunStore, type ReferenceReplayScenario, type ReferenceReplayScenarioScore, type ReferenceReplayScore, type ReferenceReplayScoreOptions, type ReferenceReplaySplit, type ReferenceReplaySplitComparison, type ReferenceReplaySteeringRowsOptions, ReleaseConfidenceScorecard, ReleaseConfidenceThresholds, type RepoRef, type ReviewerMemoryEntry, type ReviewerOutput, type ReviewerPromptInput, type ReviewerSoftFailDefaults, type ReviewerVerificationSummary, type RobustnessResult, type RouteMap, type RubricDimension, Run$1 as Run, type RunCommandInput, type RunCommandResult, type RunConfig, RunCritic, type RunCriticOptions, type RunDiff, RunFilter, type RunProductionLoopOptions, RunRecord, type RunScore, type RunScoreWeights, RunSplitTag, type RunTrace, SEMANTIC_CONCEPT_JUDGE_VERSION, SandboxDriver, SandboxHarnessResult, type SandboxJudgeKind, type SandboxJudgeResult, type SandboxJudgeSpec, type SandboxPool, type ScanOptions, type Scenario, type ScenarioCost, type ScenarioFile, ScenarioRegistry, type ScenarioResult, type ScoredTarget, type SelfPlayOptions, type SelfPlayProposer, type SelfPlayScorer, type SemanticConceptJudgeInput, type SemanticConceptJudgeOptions, type SemanticConceptJudgeResult, type SeriesConvergenceOptions, type SeriesConvergenceResult, Severity, type SignedManifest, type SignedManifestAlgo, type Slo, type SloCheckResult, type SloComparator, type SloReport, type SloSeverity, type SlopCategory, type SlotFactory, Span, type SteeringBundle, type SteeringDelta, type SteeringOptimizationResult, type SteeringOptimizationRow, type SteeringOptimizationSelector, type SteeringOptimizerBackend, type SteeringOptimizerConfig, type SteeringRolePrompt, type StepAttribution, type SynthesisReason, type SynthesisTarget, type TestResult, type ThresholdContract, TokenCounter, type TokenSpec, TraceEmitter, TraceEvent, TraceStore, Trajectory, TrajectoryStep, type TrialAggregate, type TrialAttempt, TrialCache, TrialTelemetry, type Turn, type TurnMetrics, type TurnResult, UNIVERSAL_FINDERS, type ValidationContext, type ValidationIssue, type ValidationResult, VariantAggregate, VerifyContext, type VisualDiffOptions, type VisualDiffResult, type ViteDeployRunnerInput, type WorkflowTopology, type WorkspaceAssertion, type WorkspaceAssertionResult, type WorkspaceInspector, type WorkspaceSnapshot, type WranglerDeployRunnerInput, adversarialJudge, aggregateRunScore, aggregateTrialsByMode, analyzeAntiSlop, analyzeSeries, attributeCounterfactuals, benjaminiHochberg, bisect, bonferroni, buildReviewerPrompt, byteLengthRange, canaryLeakView, canonicalize, causalAttribution, checkBehavioralCanary, checkCanaries, checkSlos, clamp01, codeExecutionJudge, cohensD, coherenceJudge, collectionPreserved, commitBisect, compareReferenceReplay, compilerJudge, composeValidators, confidenceInterval, containsAll, createAntiSlopJudge, createCompositeMutator, createCustomJudge, createDefaultReviewer, createDomainExpertJudge, createIntentMatchJudge, createSandboxCodeMutator, createSandboxPool, createSemanticConceptJudge, crossTraceDiff, decideReferenceReplayPromotion, decideReferenceReplayRunPromotion, defaultJudges, defaultReferenceReplayMatcher, deployGateLayer, discoverPersonas, distillPlaybook, estimateCost, estimateTokens, evaluateContract, evaluateHypothesis, evaluateOracles, executeScenario, expectAgent, exportRewardModel, extractAssetUrls, extractErrorCount, fileContains, fileExists, findAutoMatchNoExpectation, findConstructorCwdDropped, findFallbackToPass, findLiteralTruePass, findSkipCountsAsPass, flowLayer, formatBenchmarkReport, formatDriverReport, formatFindings, ghCliClient, precision as goldenPrecision, hashContent, hashJson, htmlContainsElement, httpGithubClient, inMemoryReferenceReplayStore, integrationAsi, integrationGateEvals, integrationInvokeFailedPayload, integrationManifestResolvedPayload, integrationManifestValidatedPayload, interRaterReliability, jsonHasKeys, jsonShape, jsonlReferenceReplayStore, keyPreserved, linterJudge, loadScorerFromGrader, localCommandRunner, lowercaseMutator, mannWhitneyU, matchGoldens, mergeLayerResults, mergeSteeringBundle, multiToolchainLayer, normalizeScores, notBlocked, pairedTTest, paraphraseRobustness, paraphraseRobustnessScenarios, partialCredit, passOrthogonality, pixelDeltaRatio, politenessPrefixMutator, printDriverSummary, promptBisect, proposeAutomatedPullRequest, proposeSynthesisTargets, referenceReplayRunsToSteeringRows, referenceReplayScenarioToRunScore, regexMatch, regexMatches, renderMarkdownReport, renderPlaybookMarkdown, renderSteeringText, replayScorerOverCorpus, replayTraceThroughJudge, requiredSampleSize, resetLockedAppendersForTesting, rowCount, rowWhere, runAssertions, runBehavioralCanaries, runCanaries, runCounterfactual, runE2EWorkflow, runExpectations, runHarnessExperiment, runIntentMatchJudge, runJudgeFleet, runKeywordCoverageJudge, runKeywordCoverageJudgeUrl, runLiveProof, runProductionLoop, runReferenceReplay, runSelfPlay, runSemanticConceptJudge, scanForMuffledGates, scoreContinuity, scoreReferenceReplay, securityJudge, selectHarnessVariant, sentenceReorderMutator, signManifest, statusAdvanced, summarizeHarnessResults, testJudge, textInSnapshot, toLangfuseEnvelope, toPrometheusText, typoMutator, urlContains, verifyManifest, visualDiff, viteDeployRunner, weightedMean, weightedRecall, whitespaceCollapseMutator, wilcoxonSignedRank, withJudgeRetry, wranglerDeployRunner };