@tangle-network/agent-eval 0.11.1 → 0.13.0

package/dist/index.d.ts

@@ -944,6 +944,114 @@ interface RunDiff {
     }>;
 }
 
+/**
+ * FileSystemExperimentStore — NDJSON-backed `ExperimentStore` for local + CI.
+ *
+ * Mirrors the file layout of `FileSystemTraceStore`: two append-only NDJSON
+ * files (`experiments.ndjson` + `runs.ndjson`) under one directory, with size-
+ * based rollover. Writes are append-only so the file log doubles as an audit
+ * trail of every state transition the tracker ever wrote.
+ *
+ * Reads lazy-load every NDJSON file in the directory (including rolled-over
+ * archives), latest-write-wins per `id`. Subsequent writes update the
+ * in-memory index in place so reads after writes are O(1).
+ *
+ * Node-only — imports `node:fs/promises`. Don't import this from a Worker;
+ * use the in-memory store or the D1 store from `./experiment-tracker-d1`.
+ */
+
+interface FileSystemExperimentStoreOptions {
+    /** Directory the NDJSON files live in. Created on first write. */
+    dir: string;
+    /** Bytes after which a file is rolled over. Default 32 MB (matches FileSystemTraceStore). */
+    maxBytes?: number;
+}
+declare class FileSystemExperimentStore implements ExperimentStore {
+    private readonly dir;
+    private readonly maxBytes;
+    private index?;
+    private loaded;
+    constructor(options: FileSystemExperimentStoreOptions);
+    saveExperiment(exp: Experiment): Promise<void>;
+    getExperiment(id: string): Promise<Experiment | null>;
+    listExperiments(): Promise<Experiment[]>;
+    saveRun(run: Run$1): Promise<void>;
+    getRun(id: string): Promise<Run$1 | null>;
+    listRuns(experimentId: string): Promise<Run$1[]>;
+    private ensureDir;
+    private append;
+    private load;
+}
+
+/**
+ * D1ExperimentStore — Cloudflare D1-backed `ExperimentStore`.
+ *
+ * Workers-safe (uses only the `D1Database` binding the runtime injects). Two
+ * tables, no joins, no migrations beyond `ensureSchema()`. Schema designed so
+ * a Worker route can both write the row at run start and update it at run end
+ * without losing the original config — the row's lifecycle mirrors the
+ * `Run.status` field one-to-one.
+ *
+ * Why this lives next to `InMemoryExperimentStore`:
+ *   - bad-app, legal-agent, gtm-agent, film-agent all run as Workers
+ *   - Workers cannot use `node:fs`, so `FileSystemExperimentStore` doesn't apply
+ *   - Hand-rolling D1 SQL in every consumer is exactly the duplication this
+ *     module exists to prevent
+ *
+ * Schema versioning: the `meta` table records `schema_version` so a future
+ * column addition can be detected and migrated additively. Today's schema is
+ * v1; bump only on breaking shape changes.
+ */
+
+/**
+ * Minimal `D1Database` shape we depend on. Avoids pulling in
+ * `@cloudflare/workers-types` as a hard dep — consumers that already have
+ * those types installed can pass the binding directly.
+ */
+interface D1Like {
+    prepare(query: string): D1PreparedStatementLike;
+    batch?(statements: D1PreparedStatementLike[]): Promise<unknown[]>;
+    exec(query: string): Promise<unknown>;
+}
+interface D1PreparedStatementLike {
+    bind(...values: unknown[]): D1PreparedStatementLike;
+    first<T = Record<string, unknown>>(): Promise<T | null>;
+    all<T = Record<string, unknown>>(): Promise<{
+        results: T[];
+    }>;
+    run(): Promise<unknown>;
+}
+interface D1ExperimentStoreOptions {
+    /** D1 binding from `env`. */
+    db: D1Like;
+    /**
+     * Optional table-name prefix so multiple ExperimentStores can share a DB
+     * without colliding (e.g. `tax_eval_experiments` vs `legal_eval_experiments`).
+     * Default: `agent_eval_`.
+     */
+    tablePrefix?: string;
+}
+declare class D1ExperimentStore implements ExperimentStore {
+    private readonly db;
+    private readonly experimentsTable;
+    private readonly runsTable;
+    private readonly metaTable;
+    private schemaReady;
+    constructor(options: D1ExperimentStoreOptions);
+    /**
+     * Idempotent schema setup. Safe to call before every operation; the second
+     * call short-circuits via `schemaReady`. Most consumers will call it once
+     * during Worker bootstrap.
+     */
+    ensureSchema(): Promise<void>;
+    saveExperiment(exp: Experiment): Promise<void>;
+    getExperiment(id: string): Promise<Experiment | null>;
+    listExperiments(): Promise<Experiment[]>;
+    saveRun(run: Run$1): Promise<void>;
+    getRun(id: string): Promise<Run$1 | null>;
+    listRuns(experimentId: string): Promise<Run$1[]>;
+}
+
 /**
  * Prompt optimizer — A/B test prompt variants with statistical rigor.
  *
@@ -959,13 +1067,13 @@ interface RunDiff {
  * and returns a number per scenario. This lets the optimizer stay small +
  * testable.
  */
-interface PromptVariant {
+interface PromptVariant$1 {
     id: string;
     prompt: string;
     metadata?: Record<string, unknown>;
 }
 interface OptimizationConfig {
-    variants: PromptVariant[];
+    variants: PromptVariant$1[];
     /** How many trials per (variant, scenario) — controls CI tightness. Default 3. */
     trialsPerScenario?: number;
     /** Significance threshold for pairwise comparison (default 0.05). */
@@ -976,7 +1084,7 @@ interface OptimizationConfig {
      * monotonicity).
      */
     scoreVariant: (args: {
-        variant: PromptVariant;
+        variant: PromptVariant$1;
         scenarioId: string;
         trialIndex: number;
     }) => Promise<number>;
@@ -1750,6 +1858,46 @@ declare function dominates<T>(a: T, b: T, objectives: Objective<T>[]): boolean;
  * iff no other candidate dominates it.
  */
 declare function paretoFrontier<T>(candidates: T[], objectives: Objective<T>[]): ParetoResult<T>;
+/**
+ * Weighted-sum scalarisation. Use as a tie-break / single-winner selector
+ * when callers don't want to consume a frontier. Each objective contributes
+ * its normalised value (0..1 via min-max across the candidate pool) times
+ * its weight; missing weights default to 1/N.
+ *
+ * Direction is honoured automatically — `minimize` axes have their values
+ * inverted before scaling so "higher scalar = better" always holds.
+ */
+declare function scalarScore<T>(candidates: T[], objectives: Objective<T>[], options?: {
+    weights?: Partial<Record<string, number>>;
+}): Array<{
+    candidate: T;
+    score: number;
+}>;
+/**
+ * NSGA-II crowding distance — secondary sort for ties on the frontier.
+ *
+ * When the Pareto front collapses to a single point (or many candidates tie
+ * on dominance), naive selection picks arbitrarily and the population
+ * degenerates over generations. NSGA-II preserves diversity by preferring
+ * candidates with more empty space around them on the frontier.
+ *
+ * Returns an array of `{ candidate, distance }` in the SAME order as the
+ * input. Higher distance = more isolated = should be preferred when
+ * preserving diversity.
+ */
+declare function crowdingDistance<T>(candidates: T[], objectives: Objective<T>[]): Array<{
+    candidate: T;
+    distance: number;
+}>;
+/**
+ * Pareto frontier with tie-break by crowding distance — the canonical
+ * NSGA-II selection step. Returns the frontier sorted by descending crowding
+ * distance so callers can `.slice(0, k)` to pick K diverse winners.
+ */
+declare function paretoFrontierWithCrowding<T>(candidates: T[], objectives: Objective<T>[]): Array<{
+    candidate: T;
+    distance: number;
+}>;
 
 type HarnessIntervention = 'continue' | 'plan' | 'audit' | 'recover' | 'repair' | 'verify' | 'final_gate' | 'wait_for_measurement' | 'abort';
 interface WorkflowTopology {
@@ -6079,4 +6227,449 @@ interface ReferenceReplaySteeringRowsOptions<Input = unknown> {
 declare function referenceReplayRunsToSteeringRows<Input = unknown>(runs: ReferenceReplayRun<Input>[], options?: ReferenceReplaySteeringRowsOptions<Input>): SteeringOptimizationRow[];
 declare function referenceReplayScenarioToRunScore(scenarioScore: ReferenceReplayScenarioScore, durationMs?: number): RunScore;
 
-export { type ActiveLearningOptions, type AdapterRun, AgentDriver, type AgentDriverConfig, type AlignmentOp, type AntiSlopConfig, type AntiSlopIssue, type AntiSlopReport, type Artifact, type ArtifactCheck, type Artifact$1 as ArtifactCheckArtifact, type ArtifactResult, type ArtifactValidator, AxGepaSteeringOptimizer, type AxSteeringOptimizerConfig, type BaselineOptions, type BaselineReport, BehaviorAssertion, type BenchmarkReport, BenchmarkRunner, type BenchmarkRunnerConfig, type BestOfNResult, type BisectOptions, type BisectResult, type BisectStep, BudgetBreachError, type BudgetBreachFinding, type BudgetBreachReport, BudgetGuard, type BudgetLedgerEntry, type BudgetSpec, BuilderSession, type BuilderSessionInit, type CalibrationBin, type CalibrationOptions, type CalibrationReport, type CalibrationResult, CallExpectation, type CanaryLeak, type CandidateScenario, type CandidateScore, type CausalAttributionReport, type ChatSummary, type CheckResult, type CollectedArtifacts, type CommandRunner, type CompletionCriterion, type ConceptComplexity, type ConceptFinding, type ConceptSpec, type ConceptWeightStrategy, type ContinuityCheck, type ContinuityCheckResult, type ContinuityReport, type ContinuitySnapshotPair, type ContractMetric, type ContractReport, ConvergenceTracker, type CorrelationReport, type CorrelationResult, type CorrelationStudyOptions, type CorrelationStudyResult, type CostEntry, type CostSummary, CostTracker, type CounterfactualContext, type CounterfactualMutation, type CounterfactualResult, type CounterfactualRunner, type CreateDefaultReviewerOptions, type CrossTraceDiff, type CrossTraceDiffOptions, DEFAULT_AGENT_SLOS, DEFAULT_COMPLEXITY_WEIGHTS, DEFAULT_RULES as DEFAULT_FAILURE_RULES, DEFAULT_FINDERS, DEFAULT_HARNESS_OBJECTIVES, DEFAULT_MUTATORS, DEFAULT_REDACTION_RULES, DEFAULT_RED_TEAM_CORPUS, DEFAULT_RUN_SCORE_WEIGHTS, Dataset, type DatasetDifficulty, type DatasetManifest, type DatasetProvenance, type DatasetScenario, type DatasetSplit, type DeployFamily, type DeployGateLayerInput, type DeployRunResult, type DeployRunner, type DeploymentOutcome, type DirEntry, type Direction, type DivergenceOptions, type DivergenceReport, DockerSandboxDriver, type DriverResult, type DriverState, DualAgentBench, type DualAgentBenchConfig, type DualAgentReport, type DualAgentRound, type DualAgentScenario, type DualAgentScenarioResult, ERROR_COUNT_PATTERNS, type ErrorCountPattern, type EuRiskClass, type EvalMetricSpec, type EvalResult, type EventFilter, type EventKind, type EvolutionRound, type ExecutorConfig, type Expectation, type Experiment, type Run$1 as ExperimentRun, type ExperimentStore, ExperimentTracker, type ExportedRewardModel, type ExtractOptions, type ExtractResult, FAILURE_CLASSES, type FactorContribution, type FactorialCell, type FailureClass, type FailureClassification, type FailureCluster, type FailureClusterReport, type FailureContext, type FailureRule, type FeedbackPattern, FileSystemOutcomeStore, type FileSystemOutcomeStoreOptions, FileSystemTraceStore, type FileSystemTraceStoreOptions, type Finding, type FlowAction, type FlowLayerEnv, type FlowLayerFactoryInput, type FlowRunner, type FlowRunnerStepResult, type FlowSpec, type FlowStep, type GenericSpan, type GoldenItem, type GovernanceContext, type GovernanceFinding, type GovernanceReport, type GradedStep, type HarnessAdapter, type HarnessConfig, type HarnessExperimentConfig, type HarnessExperimentResult, type HarnessIntervention, type HarnessRunRequest, type HarnessRunResult, type HarnessScenario, type HarnessSelection, type HarnessVariant, type HarnessVariantReport, HoldoutAuditor, HoldoutLockedError, type HostedJudgeConfig, type HostedJudgeDimension, type HostedJudgeRequest, type HostedJudgeResponse, type HostedRunCriticConfig, type HostedRunScoreRequest, type HostedRunScoreResponse, type HypothesisManifest, type HypothesisResult, INTENT_MATCH_JUDGE_VERSION, type ImageData, InMemoryExperimentStore, InMemoryOutcomeStore, InMemoryTraceStore, InMemoryWorkspaceInspector, type InferenceScorer, type InspectorContext, type IntentMatchInput, type IntentMatchOptions, type IntentMatchResult, type InteractionContribution, type JudgeAgreementReport, type JudgeConfig, type JudgeFleetOptions, type JudgeFn, type JudgeInput, type JudgePair, type JudgeReplayResult, type JudgeRubric, JudgeRunner, type JudgeScore, type JudgeSpan, type KeywordConceptSpec, type KeywordCoverageFinding, type KeywordCoverageOptions, type KeywordCoverageResult, type LangfuseEnvelope, type LangfuseGeneration, type LangfuseScore, type Layer, type LayerCorrelation, type LayerResult, type LayerStatus, LlmCallError, type LlmCallRequest, type LlmCallResult, LlmClient, type LlmClientOptions, type LlmJsonCall, type LlmMessage, type LlmReviewerConfig, type LlmSpan, type LlmUsage, MODEL_PRICING, type MatcherResult, type MeasurementPolicy, type MergeOptions, type Message, type MetricSamples, type MetricVerdict, MetricsCollector, type MuffledFinder, type MuffledFinding, MultiLayerVerifier, type MultiToolchainLayerConfig, type Mutator, OTEL_AGENT_EVAL_SCOPE, type Objective, type OptimizationConfig, type OptimizationExample, OptimizationLoop, type OptimizationLoopConfig, type OptimizationLoopResult, type OptimizationResult, type Oracle, type OracleObservation, type OracleReport, type OracleResult, type OtlpExport, type OtlpResourceSpans, type OtlpSpan, type OutcomeFilter, type OutcomePair, type OutcomeStore, type PairwiseComparison, PairwiseSteeringOptimizer, type ParetoResult, type PersonaConfig, type Playbook, type PlaybookEntry, type PositionalBiasResult, type PrmGradedTrace, PrmGrader, type PrmTrainingSample, ProductClient, type ProductClientConfig, type ProjectKind, ProjectRegistry, type ProjectSummary, type ProjectTimelineEntry, type PromptHandle, PromptOptimizer, PromptRegistry, type PromptVariant, type ProposeFn, type ProposeInput, type ProposeOutput, type ProposeReviewConfig, type ProposeReviewReport, type ProposeReviewShot, REDACTION_VERSION, type RedTeamCase, type RedTeamCategory, type RedTeamFinding, type RedTeamPayload, type RedTeamReport, type RedactionReport, type RedactionRule, type ReferenceMatchResult, type ReferenceReplayAdapter, type ReferenceReplayAdapterFn, type ReferenceReplayAdapterLike, type ReferenceReplayAggregate, type ReferenceReplayCandidate, type ReferenceReplayCase, type ReferenceReplayCaseRun, type ReferenceReplayExecutionScenario, type ReferenceReplayItem, type ReferenceReplayMatch, type ReferenceReplayMatchStrategy, type ReferenceReplayMatcher, type ReferenceReplayPromotionDecision, type ReferenceReplayPromotionPolicy, type ReferenceReplayRun, type ReferenceReplayRunContext, type ReferenceReplayRunOptions, type ReferenceReplayRunStore, type ReferenceReplayScenario, type ReferenceReplayScenarioScore, type ReferenceReplayScore, type ReferenceReplayScoreOptions, type ReferenceReplaySplit, type ReferenceReplaySplitComparison, type ReferenceReplaySteeringRowsOptions, type RegressionOptions, type RegressionSpec, type RetrievalSpan, type Review, type ReviewFn, type ReviewInput, type ReviewMemoryEntry, type ReviewMemoryStore, type ReviewerMemoryEntry, type ReviewerOutput, type ReviewerPromptInput, type ReviewerSoftFailDefaults, type ReviewerVerificationSummary, type RobustnessResult, type RouteMap, type RubricDimension, type Run, type RunAppScenarioOptions, type RunCommandInput, type RunCommandResult, type RunConfig, RunCritic, type RunCriticOptions, type RunDiff, type RunFilter, type RunLayer, type RunOutcome, type RunScore, type RunScoreWeights, type RunStatus, type RunTrace, SEMANTIC_CONCEPT_JUDGE_VERSION, type SandboxDriver, SandboxHarness, type SandboxHarnessResult, type SandboxJudgeKind, type SandboxJudgeResult, type SandboxJudgeSpec, type SandboxResult, type SandboxSpan, type ScanOptions, type Scenario, type ScenarioCost, type ScenarioFile, ScenarioRegistry, type ScenarioResult, type ScoredTarget, type SelfPlayOptions, type SelfPlayProposer, type SelfPlayScorer, type SelfPreferenceResult, type SemanticConceptJudgeInput, type SemanticConceptJudgeOptions, type SemanticConceptJudgeResult, type SeriesConvergenceOptions, type SeriesConvergenceResult, type Severity, type ShipOptions, type SignedManifest, type SliceOptions, type Slo, type SloCheckResult, type SloComparator, type SloReport, type SloSeverity, type SlopCategory, type Span, type SpanBase, type SpanFilter, type SpanHandle, type SpanKind, type SpanStatus, type SteeringBundle, type SteeringDelta, type SteeringEvaluation, type SteeringOptimizationResult, type SteeringOptimizationRow, type SteeringOptimizationSelector, type SteeringOptimizerBackend, type SteeringOptimizerConfig, type SteeringRolePrompt, type SteeringVariantReport, type StepAttribution, type StepContext, type StepRubric, type StuckLoopFinding, type StuckLoopOptions, type StuckLoopReport, SubprocessSandboxDriver, type SubprocessSandboxDriverOptions, type SynthesisReason, type SynthesisTarget, TRACE_SCHEMA_VERSION, type TestGradedRunOptions, type TestGradedRunResult, type TestGradedScenario, type TestOutputParser, type TestResult, type ThreeLayerProjectReport, type ThresholdContract, TokenCounter, type TokenSpec, type ToolSpan, type ToolStats, type ToolUseMetrics, type ToolUseOptions, type ToolWasteFinding, type ToolWasteOptions, type ToolWasteReport, TraceEmitter, type TraceEmitterOptions, type TraceEvent, type TraceStore, type Trajectory, type TrajectoryStep, type Turn, type TurnMetrics, type TurnResult, UNIVERSAL_FINDERS, type UseCaseSignals, type ValidationContext, type ValidationIssue, type ValidationResult, type VariantScore, type VerbosityBiasResult, type Verification, type VerificationReport, type VerifyContext, type VerifyFn, type VerifyOptions, type VisualDiffOptions, type VisualDiffResult, type ViteDeployRunnerInput, type WorkflowTopology, type WorkspaceAssertion, type WorkspaceAssertionResult, type WorkspaceInspector, type WorkspaceSnapshot, adversarialJudge, aggregateLlm, aggregateRunScore, analyzeAntiSlop, analyzeSeries, argHash, attributeCounterfactuals, benjaminiHochberg, bisect, bonferroni, budgetBreachView, buildReviewerPrompt, buildTrajectory, byteLengthRange, calibrateJudge, calibrationCurve, callLlm, callLlmJson, canaryLeakView, causalAttribution, checkCanaries, checkSlos, clamp01, classifyEuAiRisk, classifyFailure, codeExecutionJudge, cohensD, coherenceJudge, collectionPreserved, commitBisect, compareReferenceReplay, compareToBaseline, compilerJudge, composeParsers, composeValidators, computeToolUseMetrics, confidenceInterval, containsAll, correlateLayers, correlationStudy, createAntiSlopJudge, createCustomJudge, createDefaultReviewer, createDomainExpertJudge, createIntentMatchJudge, createLlmReviewer, createSemanticConceptJudge, crossTraceDiff, decideReferenceReplayPromotion, decideReferenceReplayRunPromotion, defaultJudges, defaultReferenceReplayMatcher, deployGateLayer, distillPlaybook, dominates, estimateCost, estimateTokens, euAiActReport, evaluateContract, evaluateHypothesis, evaluateOracles, executeScenario, expectAgent, exportRewardModel, exportRunAsOtlp, exportTrainingData, extractAssetUrls, extractErrorCount, failureClusterView, fileContains, fileExists, findAutoMatchNoExpectation, findConstructorCwdDropped, findFallbackToPass, findLiteralTruePass, findSkipCountsAsPass, firstDivergenceView, flowLayer, formatBenchmarkReport, formatDriverReport, formatFindings, gradeSemanticStatus, groupBy, hashContent, hashScenarios, htmlContainsElement, inMemoryReferenceReplayStore, inMemoryReviewStore, interRaterReliability, iqr, isJudgeSpan, isLlmSpan, isPrmVerdict, isRetrievalSpan, isSandboxSpan, isToolSpan, jestTestParser, jsonHasKeys, jsonShape, jsonlReferenceReplayStore, jsonlReviewStore, judgeAgreementView, judgeSpans, keyPreserved, linterJudge, llmSpanFromProvider, llmSpans, loadScorerFromGrader, localCommandRunner, lowercaseMutator, mannWhitneyU, mergeLayerResults, mergeSteeringBundle, multiToolchainLayer, nistAiRmfReport, nonRefusalRubric, normalizeScores, notBlocked, outputLengthRubric, pairedTTest, paraphraseRobustness, paretoFrontier, partialCredit, pixelDeltaRatio, politenessPrefixMutator, positionalBias, printDriverSummary, prmBestOfN, prmEnsembleBestOfN, probeLlm, promptBisect, proposeSynthesisTargets, pytestTestParser, redTeamDataset, redTeamReport, redactString, redactValue, referenceReplayRunsToSteeringRows, referenceReplayScenarioToRunScore, regexMatch, regexMatches, regressionView, renderMarkdown, renderMarkdownReport, renderPlaybookMarkdown, renderSteeringText, replayScorerOverCorpus, replayTraceThroughJudge, requiredSampleSize, resumeBuilderSession, rowCount, rowWhere, runAssertions, runCounterfactual, runE2EWorkflow, runExpectations, runFailureClass, runHarnessExperiment, runIntentMatchJudge, runJudgeFleet, runKeywordCoverageJudge, runKeywordCoverageJudgeUrl, runProposeReview, runReferenceReplay, runSelfPlay, runSemanticConceptJudge, runTestGradedScenario, runsForScenario, scanForMuffledGates, scoreAllProjects, scoreContinuity, scoreProject, scoreRedTeamOutput, scoreReferenceReplay, securityJudge, selectHarnessVariant, selfPreference, sentenceReorderMutator, signManifest, soc2Report, statusAdvanced, stripFencedJson, stuckLoopView, summarize, summarizeHarnessResults, testJudge, textInSnapshot, toLangfuseEnvelope, toNdjson, toPrometheusText, toolIntentAlignmentRubric, toolNamesForRun, toolNonRedundantRubric, toolSpans, toolSuccessRubric, toolWasteView, typoMutator, urlContains, verbosityBias, verifyManifest, visualDiff, viteDeployRunner, vitestTestParser, weightedMean, welchsTTest, whitespaceCollapseMutator, wilcoxonSignedRank };
+/**
+ * PromptEvolutionLoop — population-based reflective-mutation evolution.
+ *
+ * Above the existing `AxGepaSteeringOptimizer` (which RANKS variants),
+ * this loop GENERATES variants. Each generation:
+ *   1. Score the population across (variant × scenario × rep).
+ *   2. Pick survivors from the Pareto frontier (with crowding-distance tie-break).
+ *   3. Ask the mutator for replacements until population size is restored.
+ *   4. Repeat for N generations OR until convergence.
+ *
+ * Domain-agnostic. Consumers supply:
+ *   - A seed population of `PromptVariant`s.
+ *   - A `ScoreAdapter` that runs (variant, scenario, rep) → `TrialResult`.
+ *   - A `MutateAdapter` that produces children given trace evidence.
+ *   - Pareto `Objective<TrialAggregate>[]` defining the multi-objective vector.
+ *
+ * The loop owns: population management, parallel scheduling (concurrency-
+ * limited), Pareto selection with crowding distance, generation reporting.
+ *
+ * It does NOT own: rendering trials to a model, executing prompts, choosing
+ * mutation primitives, persisting to disk. Those are the consumer's call.
+ */
+
+interface PromptVariant<P = unknown> {
+    /** Stable id for the variant — surfaces in reports and trial results. */
+    id: string;
+    /** Variant payload — interpretation is the consumer's responsibility. */
+    payload: P;
+    /** Generation index (0 = seed, then 1, 2, ...). */
+    generation: number;
+    /** Parent variant id when produced via mutation; absent for seeds. */
+    parentId?: string;
+    /** Human label for reports. */
+    label: string;
+    /** What the mutator was trying to fix. */
+    rationale?: string;
+}
+interface TrialResult {
+    variantId: string;
+    scenarioId: string;
+    rep: number;
+    ok: boolean;
+    /** Primary scalar score the consumer cares about (e.g., recall, accuracy). */
+    score: number;
+    /** Token cost (or any cost-like dimension). */
+    cost?: number;
+    /** Wall time in ms. */
+    durationMs?: number;
+    /** Free-form metric bag for objective accessors. */
+    metrics?: Record<string, number>;
+    error?: string;
+}
+/** Aggregated trial summary for one (variant, scenario) pair across reps. */
+interface ScenarioAggregate {
+    variantId: string;
+    scenarioId: string;
+    meanScore: number;
+    meanCost: number;
+    meanDurationMs: number;
+    okRate: number;
+    trials: number;
+    /** Mean of every numeric metric across reps. */
+    metrics: Record<string, number>;
+}
+/** Aggregated trial summary for one variant across all scenarios. */
+interface VariantAggregate {
+    variantId: string;
+    meanScore: number;
+    meanCost: number;
+    meanDurationMs: number;
+    okRate: number;
+    scenarios: ScenarioAggregate[];
+    /** Mean of every numeric metric, averaged across scenarios. */
+    metrics: Record<string, number>;
+}
+interface ScoreAdapter<P = unknown> {
+    score(args: {
+        variant: PromptVariant<P>;
+        scenarioId: string;
+        rep: number;
+    }): Promise<TrialResult>;
+}
+interface MutateAdapter<P = unknown> {
+    mutate(args: {
+        parent: PromptVariant<P>;
+        parentAggregate: VariantAggregate;
+        topTrials: TrialResult[];
+        bottomTrials: TrialResult[];
+        childCount: number;
+        generation: number;
+    }): Promise<PromptVariant<P>[]>;
+}
+interface PromptEvolutionConfig<P = unknown> {
+    runId: string;
+    /** What component is being mutated — surfaces in reports + reflection prompts. */
+    target: string;
+    seedVariants: PromptVariant<P>[];
+    scenarioIds: string[];
+    reps: number;
+    generations: number;
+    populationSize: number;
+    /** Maximum concurrent score() calls. */
+    scoreConcurrency: number;
+    scoreAdapter: ScoreAdapter<P>;
+    mutateAdapter: MutateAdapter<P>;
+    /** Pareto objectives over `VariantAggregate`. Ordered by importance. */
+    objectives: Objective<VariantAggregate>[];
+    /** Optional weights for the scalar tie-break selector (by objective name). */
+    scalarWeights?: Record<string, number>;
+    /** Stop early if a generation produces no Pareto improvement. Default true. */
+    earlyStopOnNoImprovement?: boolean;
+    onProgress?: (event: PromptEvolutionEvent) => void;
+    /**
+     * Optional cache key for memoising scored (variantId, scenarioId, rep)
+     * tuples. When provided AND a cache instance is passed, repeated trials
+     * skip re-scoring. Cache keys are stable across runs.
+     */
+    cache?: TrialCache;
+}
+interface TrialCache {
+    get(key: string): TrialResult | undefined;
+    set(key: string, value: TrialResult): void;
+}
+declare class InMemoryTrialCache implements TrialCache {
+    private store;
+    get(key: string): TrialResult | undefined;
+    set(key: string, value: TrialResult): void;
+    size(): number;
+    clear(): void;
+}
+type PromptEvolutionEvent = {
+    type: 'generation-start';
+    generation: number;
+    populationSize: number;
+} | {
+    type: 'trial-complete';
+    generation: number;
+    variantId: string;
+    scenarioId: string;
+    rep: number;
+    ok: boolean;
+    score: number;
+    cached: boolean;
+} | {
+    type: 'generation-complete';
+    report: GenerationReport<unknown>;
+} | {
+    type: 'converged';
+    generation: number;
+    reason: string;
+};
+interface GenerationReport<P = unknown> {
+    runId: string;
+    target: string;
+    generation: number;
+    variants: PromptVariant<P>[];
+    aggregates: VariantAggregate[];
+    /** Frontier candidates, sorted by descending crowding distance. */
+    paretoFrontIds: string[];
+    /** Scalar-best variant id — used for the single "winner" if callers want one. */
+    winnerId: string;
+    /** Trials that fed this generation (kept for downstream reporting). */
+    trials: TrialResult[];
+}
+interface PromptEvolutionResult<P = unknown> {
+    runId: string;
+    target: string;
+    generations: GenerationReport<P>[];
+    /** Best variant by scalar score in the final generation. */
+    bestVariant: PromptVariant<P>;
+    /** Best aggregate (matches bestVariant). */
+    bestAggregate: VariantAggregate;
+}
+declare function runPromptEvolution<P>(config: PromptEvolutionConfig<P>): Promise<PromptEvolutionResult<P>>;
+
+/**
+ * GoldenMatcher — fuzzy matcher for "did the agent produce the expected things?".
+ *
+ * Universal primitive across agent-eval consumers. Use it for:
+ *   - Test suites: did the run hit the expected assertions?
+ *   - Tool agents: did the agent emit the expected tool call sequence?
+ *   - Judges: did the verdict include the expected concepts?
+ *   - Design audits: did the auditor surface the planted defects?
+ *
+ * Match rule (per golden):
+ *   - Any phrase in `golden.any` (case-insensitive substring) appears in the
+ *     candidate's text fields, OR
+ *   - Any pattern in `golden.anyRegex` (case-insensitive) matches.
+ *
+ * Recall is severity-weighted by default: critical=3, major=2, minor=1.
+ * Missing one critical hurts more than missing three minors.
+ */
+type GoldenSeverity = 'critical' | 'major' | 'minor';
+interface GoldenSpec {
+    /** Stable identifier — survives across runs so consumers can grep by id. */
+    id: string;
+    /** Severity drives recall weighting. */
+    severity: GoldenSeverity;
+    /**
+     * Substring phrases (case-insensitive). A hit on ANY phrase counts as a
+     * match. Keep these SHORT (3-6 words) and SPECIFIC.
+     */
+    any: string[];
+    /** Optional regex patterns. ORed with `any`. */
+    anyRegex?: string[];
+    /** Free-form note — surfaces in reports for humans. */
+    hint?: string;
+    /** Optional category for grouping/filtering. */
+    category?: string;
+}
+interface MatchResult {
+    /** Same length as goldens; `true` when matched. */
+    matches: boolean[];
+    /** Convenience: count of hits. */
+    hits: number;
+    /** Convenience: total goldens. */
+    total: number;
+}
+/**
+ * Match each golden against `candidates`, where each candidate exposes one or
+ * more text fields the matcher should search. Defaults to searching all
+ * string-typed fields concatenated.
+ */
+declare function matchGoldens<T>(goldens: GoldenSpec[], candidates: T[], options?: {
+    /**
+     * Extract the searchable text for a candidate. Default: concatenate every
+     * top-level string field with a space.
+     */
+    text?: (candidate: T) => string;
+}): MatchResult;
+/** Severity weights — exposed so consumers can override (rare). */
+declare const DEFAULT_SEVERITY_WEIGHTS: Record<GoldenSeverity, number>;
+/** Severity-weighted recall over a MatchResult + the goldens that produced it. */
+declare function weightedRecall(goldens: GoldenSpec[], result: MatchResult, weights?: Record<GoldenSeverity, number>): number;
+/**
+ * Precision proxy: fraction of emitted candidates that match SOME golden.
+ *
+ * No human-labelled negatives means unmatched candidates are SOFT false
+ * positives — punishes verbose agents that pad with filler. Doesn't punish
+ * unknown-but-real findings; the way to tighten this is to grow the golden
+ * set, not to invent a stricter score.
+ */
+declare function precision<T>(goldens: GoldenSpec[], candidates: T[], options?: {
+    text?: (candidate: T) => string;
+}): number;
+
+/**
+ * Inter-critic / inter-pass orthogonality.
+ *
+ * Detects redundant ensembles. When you run N critics (or N audit passes,
+ * or N specialized agents) on the same input, you want them to disagree —
+ * each contributing distinct signal. If they all converge on the same set
+ * of findings, you're paying N× cost for ~1× signal.
+ *
+ * The metric is `1 − mean pairwise cosine similarity` over bags of words
+ * extracted from each pass's outputs. 1.0 = fully orthogonal,
+ * 0.0 = fully redundant.
+ *
+ * Universal primitive: pass anything that produces text (findings, tool
+ * calls rendered as JSON, verdict strings) and the matcher derives its own
+ * vocabulary.
+ */
+interface OrthogonalityInput<T> {
+    passes: Array<{
+        findings: T[];
+    }>;
+    /** Render one element to text. Default: defaultRender (concatenates string fields). */
+    text?: (item: T) => string;
+    /** Minimum token length kept in the bag. Default 4 (drops short fillers). */
+    minTokenLength?: number;
+}
+interface OrthogonalityResult {
+    /** 1 − mean pairwise cosine similarity across passes. 1=fully orthogonal, 0=fully redundant. */
+    orthogonality: number;
+    /** Number of passes considered. */
+    passCount: number;
+    /** Pairwise cosine similarities, in upper-triangular order (for debugging). */
+    similarities: number[];
+}
+declare function passOrthogonality<T>(input: OrthogonalityInput<T>): OrthogonalityResult;
+
+/**
+ * Bootstrap-CI promotion gate.
+ *
+ * In any iterative-improvement loop (GEPA, prompt evolution, dataset
+ * curation), the question is "did this generation actually improve, or are
+ * we celebrating noise?". With small N and noisy outcomes, point-estimate
+ * deltas lie. Bootstrap confidence intervals tell the operator whether the
+ * delta is real before code or prompts get promoted.
+ *
+ * This module is pure functions — no I/O, no model calls. Easy to unit-test
+ * and to compose into any verdict gate.
+ *
+ * Default gate:
+ *   - Bootstrap mean baseline vs candidate (1k resamples).
+ *   - Compute the delta distribution; pass if the lower CI bound > 0.
+ *   - Tunable confidence (default 95%) and resample count.
+ *
+ * Verdict semantics intentionally match the existing `experiments.jsonl`
+ * vocabulary:
+ *   - ADVANCE: candidate's CI lower bound > baseline mean (real win)
+ *   - KEEP:    overlap, but candidate point estimate >= baseline (neutral)
+ *   - REVERT:  candidate's CI upper bound < baseline mean (real regression)
+ *   - INCONCLUSIVE: not enough samples or CI straddles zero with no signal
+ */
+type Verdict = 'ADVANCE' | 'KEEP' | 'REVERT' | 'INCONCLUSIVE';
+interface BootstrapResult {
+    baselineMean: number;
+    candidateMean: number;
+    /** candidateMean - baselineMean, point estimate. */
+    delta: number;
+    /** Lower bound of the (1 - alpha) CI on the delta. */
+    ciLower: number;
+    /** Upper bound of the (1 - alpha) CI on the delta. */
+    ciUpper: number;
+    /** Number of bootstrap resamples used. */
+    iterations: number;
+    alpha: number;
+    verdict: Verdict;
+}
+interface BootstrapOptions {
+    /** Confidence level alpha (default 0.05 → 95% CI). */
+    alpha?: number;
+    /** Number of resamples (default 1000). */
+    iterations?: number;
+    /**
+     * Minimum total samples (baseline + candidate) below which we always
+     * return INCONCLUSIVE — bootstrap with too few samples is meaningless.
+     * Default 6 (combined).
+     */
+    minTotalSamples?: number;
+    /** RNG seed for reproducibility. Default: Math.random. */
+    seed?: number;
+}
+/**
+ * Compute the bootstrap CI on (candidateMean - baselineMean) and a verdict.
+ *
+ * Uses simple percentile bootstrap on the difference of resampled means.
+ * That's the standard non-parametric primitive — no distributional
+ * assumptions, robust to skew, easy to reason about.
+ */
+declare function bootstrapCi(baseline: number[], candidate: number[], options?: BootstrapOptions): BootstrapResult;
+/**
+ * Judge-replay promotion gate.
+ *
+ * The cheap inner-loop judge that drives an evolution run is by definition
+ * fast and noisy. When you're about to promote a winning variant to the
+ * canonical default, you want a STRONGER judge (a more expensive model, a
+ * human grader, a separately-trained reward model) to confirm the win
+ * generalises beyond the inner loop.
+ *
+ * This helper takes raw winner + baseline outputs, scores both through the
+ * stronger judge, and applies `bootstrapCi`. ADVANCE means the stronger
+ * judge agrees the winner is real with the configured confidence. Doesn't
+ * matter what shape your "output" is — pass a string, an object, anything
+ * the judge can read.
+ */
+interface JudgeReplayGateArgs<TOutput> {
+    baselineOutputs: TOutput[];
+    candidateOutputs: TOutput[];
+    /** Stronger judge — async to allow LLM calls. Return a 0..N scalar score. */
+    judge: (output: TOutput) => Promise<number> | number;
+    alpha?: number;
+    iterations?: number;
+    /** RNG seed for reproducibility. */
+    seed?: number;
+    /** Maximum concurrent judge calls. Default 4. */
+    judgeConcurrency?: number;
+}
+declare function judgeReplayGate<TOutput>(args: JudgeReplayGateArgs<TOutput>): Promise<BootstrapResult & {
+    baselineSamples: number;
+    candidateSamples: number;
+}>;
+
+/**
+ * Reflective mutation — primitives for trace-conditioned prompt rewriting.
+ *
+ * Used by `prompt-evolution.ts` (and any consumer running iterative
+ * improvement). Given a parent prompt + concrete trace evidence (top trials,
+ * bottom trials, missed expectations), produce an LLM-ready prompt that
+ * proposes targeted mutations — not blind rephrasings.
+ *
+ * Why this lives outside `prompt-evolution.ts`: any consumer that wants to
+ * run reflective rewriting WITHOUT the population/Pareto machinery can
+ * import these primitives directly.
+ *
+ * Quality bar (vs. naive "mutate this prompt"):
+ *   - Show parent ↔ children diff, not just one variant
+ *   - Quote specific missed goldens with their match phrases
+ *   - Surface the model's actual emitted output side-by-side with what was expected
+ *   - Quote concrete mutation primitives so the model has a vocabulary
+ */
+interface TrialTrace {
+    /** Stable id for the trial — surfaces in the prompt for grounding. */
+    id: string;
+    /** Score the trial received on its primary metric. */
+    score: number;
+    /** Candidate inputs the agent was given (e.g., the fixture or scenario). */
+    inputName?: string;
+    /**
+     * Goldens / expectations this trial was tested against, with whether each
+     * was matched. The reflection prompt quotes the missed ones specifically.
+     */
+    expectations?: Array<{
+        id: string;
+        phrase: string;
+        matched: boolean;
+    }>;
+    /** Free-form text — what the agent actually emitted (e.g., findings, plan). */
+    emitted?: string;
+    /** Optional structured metrics (recall, precision, cost, latency). */
+    metrics?: Record<string, number>;
+}
+interface ReflectionContext {
+    /** What is being mutated — appears in the system prompt for orientation. */
+    target: string;
+    /** Current variant's payload — JSON-serialised for the prompt. */
+    parentPayload: unknown;
+    /** Best-performing trials this generation. */
+    topTrials: TrialTrace[];
+    /** Worst-performing trials this generation — the missed-golden source. */
+    bottomTrials: TrialTrace[];
+    /** How many children the mutator should propose. */
+    childCount: number;
+    /** Optional: domain-specific mutation primitives the model can pick from. */
+    mutationPrimitives?: string[];
+}
+declare const DEFAULT_MUTATION_PRIMITIVES: string[];
+/**
+ * Build the LLM-ready reflection prompt. Output is plain text — pass it as
+ * the user message. The system message should be small and stable (e.g.
+ * "Output ONLY a JSON object matching the schema below.").
+ */
+declare function buildReflectionPrompt(ctx: ReflectionContext): string;
+interface ReflectionProposal {
+    label: string;
+    rationale: string;
+    payload: unknown;
+}
+/**
+ * Parse the model's JSON response back into proposals. Tolerates markdown
+ * fences and surrounding prose. Returns at most `maxProposals`.
+ */
+declare function parseReflectionResponse(raw: string, maxProposals?: number): ReflectionProposal[];
+
+export { type ActiveLearningOptions, type AdapterRun, AgentDriver, type AgentDriverConfig, type AlignmentOp, type AntiSlopConfig, type AntiSlopIssue, type AntiSlopReport, type Artifact, type ArtifactCheck, type Artifact$1 as ArtifactCheckArtifact, type ArtifactResult, type ArtifactValidator, AxGepaSteeringOptimizer, type AxSteeringOptimizerConfig, type BaselineOptions, type BaselineReport, BehaviorAssertion, type BenchmarkReport, BenchmarkRunner, type BenchmarkRunnerConfig, type BestOfNResult, type BisectOptions, type BisectResult, type BisectStep, type BootstrapOptions, type BootstrapResult, BudgetBreachError, type BudgetBreachFinding, type BudgetBreachReport, BudgetGuard, type BudgetLedgerEntry, type BudgetSpec, BuilderSession, type BuilderSessionInit, type CalibrationBin, type CalibrationOptions, type CalibrationReport, type CalibrationResult, CallExpectation, type CanaryLeak, type CandidateScenario, type CandidateScore, type CausalAttributionReport, type ChatSummary, type CheckResult, type CollectedArtifacts, type CommandRunner, type CompletionCriterion, type ConceptComplexity, type ConceptFinding, type ConceptSpec, type ConceptWeightStrategy, type ContinuityCheck, type ContinuityCheckResult, type ContinuityReport, type ContinuitySnapshotPair, type ContractMetric, type ContractReport, ConvergenceTracker, type CorrelationReport, type CorrelationResult, type CorrelationStudyOptions, type CorrelationStudyResult, type CostEntry, type CostSummary, CostTracker, type CounterfactualContext, type CounterfactualMutation, type CounterfactualResult, type CounterfactualRunner, type CreateDefaultReviewerOptions, type CrossTraceDiff, type CrossTraceDiffOptions, D1ExperimentStore, type D1ExperimentStoreOptions, type D1Like, type D1PreparedStatementLike, DEFAULT_AGENT_SLOS, DEFAULT_COMPLEXITY_WEIGHTS, DEFAULT_RULES as DEFAULT_FAILURE_RULES, DEFAULT_FINDERS, DEFAULT_HARNESS_OBJECTIVES, DEFAULT_MUTATION_PRIMITIVES, DEFAULT_MUTATORS, DEFAULT_REDACTION_RULES, DEFAULT_RED_TEAM_CORPUS, DEFAULT_RUN_SCORE_WEIGHTS, DEFAULT_SEVERITY_WEIGHTS, Dataset, type DatasetDifficulty, type DatasetManifest, type DatasetProvenance, type DatasetScenario, type DatasetSplit, type DeployFamily, type DeployGateLayerInput, type DeployRunResult, type DeployRunner, type DeploymentOutcome, type DirEntry, type Direction, type DivergenceOptions, type DivergenceReport, DockerSandboxDriver, type DriverResult, type DriverState, DualAgentBench, type DualAgentBenchConfig, type DualAgentReport, type DualAgentRound, type DualAgentScenario, type DualAgentScenarioResult, ERROR_COUNT_PATTERNS, type ErrorCountPattern, type EuRiskClass, type EvalMetricSpec, type EvalResult, type EventFilter, type EventKind, type EvolutionRound, type PromptVariant as EvolvableVariant, type ExecutorConfig, type Expectation, type Experiment, type Run$1 as ExperimentRun, type ExperimentStore, ExperimentTracker, type ExportedRewardModel, type ExtractOptions, type ExtractResult, FAILURE_CLASSES, type FactorContribution, type FactorialCell, type FailureClass, type FailureClassification, type FailureCluster, type FailureClusterReport, type FailureContext, type FailureRule, type FeedbackPattern, FileSystemExperimentStore, type FileSystemExperimentStoreOptions, FileSystemOutcomeStore, type FileSystemOutcomeStoreOptions, FileSystemTraceStore, type FileSystemTraceStoreOptions, type Finding, type FlowAction, type FlowLayerEnv, type FlowLayerFactoryInput, type FlowRunner, type FlowRunnerStepResult, type FlowSpec, type FlowStep, type GenerationReport, type GenericSpan, type GoldenItem, type GoldenSeverity, type GoldenSpec, type GovernanceContext, type GovernanceFinding, type GovernanceReport, type GradedStep, type HarnessAdapter, type HarnessConfig, type HarnessExperimentConfig, type HarnessExperimentResult, type HarnessIntervention, type HarnessRunRequest, type HarnessRunResult, type HarnessScenario, type HarnessSelection, type HarnessVariant, type HarnessVariantReport, HoldoutAuditor, HoldoutLockedError, type HostedJudgeConfig, type HostedJudgeDimension, type HostedJudgeRequest, type HostedJudgeResponse, type HostedRunCriticConfig, type HostedRunScoreRequest, type HostedRunScoreResponse, type HypothesisManifest, type HypothesisResult, INTENT_MATCH_JUDGE_VERSION, type ImageData, InMemoryExperimentStore, InMemoryOutcomeStore, InMemoryTraceStore, InMemoryTrialCache, InMemoryWorkspaceInspector, type InferenceScorer, type InspectorContext, type IntentMatchInput, type IntentMatchOptions, type IntentMatchResult, type InteractionContribution, type JudgeAgreementReport, type JudgeConfig, type JudgeFleetOptions, type JudgeFn, type JudgeInput, type JudgePair, type JudgeReplayGateArgs, type JudgeReplayResult, type JudgeRubric, JudgeRunner, type JudgeScore, type JudgeSpan, type KeywordConceptSpec, type KeywordCoverageFinding, type KeywordCoverageOptions, type KeywordCoverageResult, type LangfuseEnvelope, type LangfuseGeneration, type LangfuseScore, type Layer, type LayerCorrelation, type LayerResult, type LayerStatus, LlmCallError, type LlmCallRequest, type LlmCallResult, LlmClient, type LlmClientOptions, type LlmJsonCall, type LlmMessage, type LlmReviewerConfig, type LlmSpan, type LlmUsage, MODEL_PRICING, type MatchResult, type MatcherResult, type MeasurementPolicy, type MergeOptions, type Message, type MetricSamples, type MetricVerdict, MetricsCollector, type MuffledFinder, type MuffledFinding, MultiLayerVerifier, type MultiToolchainLayerConfig, type MutateAdapter, type Mutator, OTEL_AGENT_EVAL_SCOPE, type Objective, type OptimizationConfig, type OptimizationExample, OptimizationLoop, type OptimizationLoopConfig, type OptimizationLoopResult, type OptimizationResult, type Oracle, type OracleObservation, type OracleReport, type OracleResult, type OrthogonalityInput, type OrthogonalityResult, type OtlpExport, type OtlpResourceSpans, type OtlpSpan, type OutcomeFilter, type OutcomePair, type OutcomeStore, type PairwiseComparison, PairwiseSteeringOptimizer, type ParetoResult, type PersonaConfig, type Playbook, type PlaybookEntry, type PositionalBiasResult, type PrmGradedTrace, PrmGrader, type PrmTrainingSample, ProductClient, type ProductClientConfig, type ProjectKind, ProjectRegistry, type ProjectSummary, type ProjectTimelineEntry, type PromptEvolutionConfig, type PromptEvolutionEvent, type PromptEvolutionResult, type PromptHandle, PromptOptimizer, PromptRegistry, type TrialResult as PromptTrialResult, type PromptVariant$1 as PromptVariant, type ProposeFn, type ProposeInput, type ProposeOutput, type ProposeReviewConfig, type ProposeReviewReport, type ProposeReviewShot, REDACTION_VERSION, type RedTeamCase, type RedTeamCategory, type RedTeamFinding, type RedTeamPayload, type RedTeamReport, type RedactionReport, type RedactionRule, type ReferenceMatchResult, type ReferenceReplayAdapter, type ReferenceReplayAdapterFn, type ReferenceReplayAdapterLike, type ReferenceReplayAggregate, type ReferenceReplayCandidate, type ReferenceReplayCase, type ReferenceReplayCaseRun, type ReferenceReplayExecutionScenario, type ReferenceReplayItem, type ReferenceReplayMatch, type ReferenceReplayMatchStrategy, type ReferenceReplayMatcher, type ReferenceReplayPromotionDecision, type ReferenceReplayPromotionPolicy, type ReferenceReplayRun, type ReferenceReplayRunContext, type ReferenceReplayRunOptions, type ReferenceReplayRunStore, type ReferenceReplayScenario, type ReferenceReplayScenarioScore, type ReferenceReplayScore, type ReferenceReplayScoreOptions, type ReferenceReplaySplit, type ReferenceReplaySplitComparison, type ReferenceReplaySteeringRowsOptions, type ReflectionContext, type ReflectionProposal, type RegressionOptions, type RegressionSpec, type RetrievalSpan, type Review, type ReviewFn, type ReviewInput, type ReviewMemoryEntry, type ReviewMemoryStore, type ReviewerMemoryEntry, type ReviewerOutput, type ReviewerPromptInput, type ReviewerSoftFailDefaults, type ReviewerVerificationSummary, type RobustnessResult, type RouteMap, type RubricDimension, type Run, type RunAppScenarioOptions, type RunCommandInput, type RunCommandResult, type RunConfig, RunCritic, type RunCriticOptions, type RunDiff, type RunFilter, type RunLayer, type RunOutcome, type RunScore, type RunScoreWeights, type RunStatus, type RunTrace, SEMANTIC_CONCEPT_JUDGE_VERSION, type SandboxDriver, SandboxHarness, type SandboxHarnessResult, type SandboxJudgeKind, type SandboxJudgeResult, type SandboxJudgeSpec, type SandboxResult, type SandboxSpan, type ScanOptions, type Scenario, type ScenarioAggregate, type ScenarioCost, type ScenarioFile, ScenarioRegistry, type ScenarioResult, type ScoreAdapter, type ScoredTarget, type SelfPlayOptions, type SelfPlayProposer, type SelfPlayScorer, type SelfPreferenceResult, type SemanticConceptJudgeInput, type SemanticConceptJudgeOptions, type SemanticConceptJudgeResult, type SeriesConvergenceOptions, type SeriesConvergenceResult, type Severity, type ShipOptions, type SignedManifest, type SliceOptions, type Slo, type SloCheckResult, type SloComparator, type SloReport, type SloSeverity, type SlopCategory, type Span, type SpanBase, type SpanFilter, type SpanHandle, type SpanKind, type SpanStatus, type SteeringBundle, type SteeringDelta, type SteeringEvaluation, type SteeringOptimizationResult, type SteeringOptimizationRow, type SteeringOptimizationSelector, type SteeringOptimizerBackend, type SteeringOptimizerConfig, type SteeringRolePrompt, type SteeringVariantReport, type StepAttribution, type StepContext, type StepRubric, type StuckLoopFinding, type StuckLoopOptions, type StuckLoopReport, SubprocessSandboxDriver, type SubprocessSandboxDriverOptions, type SynthesisReason, type SynthesisTarget, TRACE_SCHEMA_VERSION, type TestGradedRunOptions, type TestGradedRunResult, type TestGradedScenario, type TestOutputParser, type TestResult, type ThreeLayerProjectReport, type ThresholdContract, TokenCounter, type TokenSpec, type ToolSpan, type ToolStats, type ToolUseMetrics, type ToolUseOptions, type ToolWasteFinding, type ToolWasteOptions, type ToolWasteReport, TraceEmitter, type TraceEmitterOptions, type TraceEvent, type TraceStore, type Trajectory, type TrajectoryStep, type TrialCache, type TrialTrace, type Turn, type TurnMetrics, type TurnResult, UNIVERSAL_FINDERS, type UseCaseSignals, type ValidationContext, type ValidationIssue, type ValidationResult, type VariantAggregate, type VariantScore, type VerbosityBiasResult, type Verdict, type Verification, type VerificationReport, type VerifyContext, type VerifyFn, type VerifyOptions, type VisualDiffOptions, type VisualDiffResult, type ViteDeployRunnerInput, type WorkflowTopology, type WorkspaceAssertion, type WorkspaceAssertionResult, type WorkspaceInspector, type WorkspaceSnapshot, adversarialJudge, aggregateLlm, aggregateRunScore, analyzeAntiSlop, analyzeSeries, argHash, attributeCounterfactuals, benjaminiHochberg, bisect, bonferroni, bootstrapCi, budgetBreachView, buildReflectionPrompt, buildReviewerPrompt, buildTrajectory, byteLengthRange, calibrateJudge, calibrationCurve, callLlm, callLlmJson, canaryLeakView, causalAttribution, checkCanaries, checkSlos, clamp01, classifyEuAiRisk, classifyFailure, codeExecutionJudge, cohensD, coherenceJudge, collectionPreserved, commitBisect, compareReferenceReplay, compareToBaseline, compilerJudge, composeParsers, composeValidators, computeToolUseMetrics, confidenceInterval, containsAll, correlateLayers, correlationStudy, createAntiSlopJudge, createCustomJudge, createDefaultReviewer, createDomainExpertJudge, createIntentMatchJudge, createLlmReviewer, createSemanticConceptJudge, crossTraceDiff, crowdingDistance, decideReferenceReplayPromotion, decideReferenceReplayRunPromotion, defaultJudges, defaultReferenceReplayMatcher, deployGateLayer, distillPlaybook, dominates, estimateCost, estimateTokens, euAiActReport, evaluateContract, evaluateHypothesis, evaluateOracles, executeScenario, expectAgent, exportRewardModel, exportRunAsOtlp, exportTrainingData, extractAssetUrls, extractErrorCount, failureClusterView, fileContains, fileExists, findAutoMatchNoExpectation, findConstructorCwdDropped, findFallbackToPass, findLiteralTruePass, findSkipCountsAsPass, firstDivergenceView, flowLayer, formatBenchmarkReport, formatDriverReport, formatFindings, precision as goldenPrecision, gradeSemanticStatus, groupBy, hashContent, hashScenarios, htmlContainsElement, inMemoryReferenceReplayStore, inMemoryReviewStore, interRaterReliability, iqr, isJudgeSpan, isLlmSpan, isPrmVerdict, isRetrievalSpan, isSandboxSpan, isToolSpan, jestTestParser, jsonHasKeys, jsonShape, jsonlReferenceReplayStore, jsonlReviewStore, judgeAgreementView, judgeReplayGate, judgeSpans, keyPreserved, linterJudge, llmSpanFromProvider, llmSpans, loadScorerFromGrader, localCommandRunner, lowercaseMutator, mannWhitneyU, matchGoldens, mergeLayerResults, mergeSteeringBundle, multiToolchainLayer, nistAiRmfReport, nonRefusalRubric, normalizeScores, notBlocked, outputLengthRubric, pairedTTest, paraphraseRobustness, paretoFrontier, paretoFrontierWithCrowding, parseReflectionResponse, partialCredit, passOrthogonality, pixelDeltaRatio, politenessPrefixMutator, positionalBias, printDriverSummary, prmBestOfN, prmEnsembleBestOfN, probeLlm, promptBisect, proposeSynthesisTargets, pytestTestParser, redTeamDataset, redTeamReport, redactString, redactValue, referenceReplayRunsToSteeringRows, referenceReplayScenarioToRunScore, regexMatch, regexMatches, regressionView, renderMarkdown, renderMarkdownReport, renderPlaybookMarkdown, renderSteeringText, replayScorerOverCorpus, replayTraceThroughJudge, requiredSampleSize, resumeBuilderSession, rowCount, rowWhere, runAssertions, runCounterfactual, runE2EWorkflow, runExpectations, runFailureClass, runHarnessExperiment, runIntentMatchJudge, runJudgeFleet, runKeywordCoverageJudge, runKeywordCoverageJudgeUrl, runPromptEvolution, runProposeReview, runReferenceReplay, runSelfPlay, runSemanticConceptJudge, runTestGradedScenario, runsForScenario, scalarScore, scanForMuffledGates, scoreAllProjects, scoreContinuity, scoreProject, scoreRedTeamOutput, scoreReferenceReplay, securityJudge, selectHarnessVariant, selfPreference, sentenceReorderMutator, signManifest, soc2Report, statusAdvanced, stripFencedJson, stuckLoopView, summarize, summarizeHarnessResults, testJudge, textInSnapshot, toLangfuseEnvelope, toNdjson, toPrometheusText, toolIntentAlignmentRubric, toolNamesForRun, toolNonRedundantRubric, toolSpans, toolSuccessRubric, toolWasteView, typoMutator, urlContains, verbosityBias, verifyManifest, visualDiff, viteDeployRunner, vitestTestParser, weightedMean, weightedRecall, welchsTTest, whitespaceCollapseMutator, wilcoxonSignedRank };