@tangle-network/agent-eval 0.85.0 → 0.86.0

package/dist/index.d.ts

@@ -1,6 +1,6 @@
 export { A as ActionExecutionPolicy, a as ActionPolicyDecision, C as ControlRunToRunRecordOptions, L as LlmJsonCall, b as LlmReviewerConfig, P as ProposeFn, c as ProposeInput, d as ProposeOutput, e as ProposeReviewConfig, f as ProposeReviewControlAction, g as ProposeReviewControlConfig, h as ProposeReviewControlResult, i as ProposeReviewControlState, j as ProposeReviewReport, k as ProposeReviewShot, R as Review, l as ReviewFn, m as ReviewInput, n as ReviewMemoryEntry, o as ReviewMemoryStore, p as RunEvidenceMetadata, V as Verification, q as VerifyFn, r as controlFailureClassFromVerification, s as controlRunToRunRecord, t as createLlmReviewer, u as evaluateActionPolicy, v as inMemoryReviewStore, w as jsonlReviewStore, x as runProposeReview, y as runProposeReviewAsControlLoop, z as scoreFromEvals } from './control-GeE8OhpN.js';
-import { R as RunRecord } from './run-record-De9VarXR.js';
-export { e as AGENT_PROFILE_KINDS, A as AgentProfileCell, d as AgentProfileCellInput, f as AgentProfileCellSchemaVersion, g as AgentProfileCellValidationError, h as AgentProfileDimensionValue, i as AgentProfileHarness, j as AgentProfileJson, k as AgentProfileKind, l as AgentProfileSource, m as AgentProfileSourceInput, J as JudgeScoresRecord, c as RunJudgeMetadata, n as RunOutcome, o as RunRecordValidationError, a as RunSplitTag, b as RunTokenUsage, S as SandboxAgentProfileLike, p as agentProfileCellHashMaterial, q as agentProfileCellKey, r as assertRunAgentProfileCell, s as buildAgentProfileCell, t as buildSandboxAgentProfileCell, u as groupRunsByAgentProfileCell, v as isRunRecord, w as parseRunRecordSafe, x as requireAgentProfileCell, y as roundTripRunRecord, z as toAgentProfileJson, B as validateAgentProfileCell, C as validateRunRecord, D as verifyAgentProfileCell } from './run-record-De9VarXR.js';
+import { R as RunRecord, a as RunSplitTag } from './run-record-De9VarXR.js';
+export { e as AGENT_PROFILE_KINDS, A as AgentProfileCell, d as AgentProfileCellInput, f as AgentProfileCellSchemaVersion, g as AgentProfileCellValidationError, h as AgentProfileDimensionValue, i as AgentProfileHarness, j as AgentProfileJson, k as AgentProfileKind, l as AgentProfileSource, m as AgentProfileSourceInput, J as JudgeScoresRecord, c as RunJudgeMetadata, n as RunOutcome, o as RunRecordValidationError, b as RunTokenUsage, S as SandboxAgentProfileLike, p as agentProfileCellHashMaterial, q as agentProfileCellKey, r as assertRunAgentProfileCell, s as buildAgentProfileCell, t as buildSandboxAgentProfileCell, u as groupRunsByAgentProfileCell, v as isRunRecord, w as parseRunRecordSafe, x as requireAgentProfileCell, y as roundTripRunRecord, z as toAgentProfileJson, B as validateAgentProfileCell, C as validateRunRecord, D as verifyAgentProfileCell } from './run-record-De9VarXR.js';
 export { B as BehavioralMetrics, z as ConceptComplexity, A as ConceptFinding, E as ConceptSpec, G as ConceptWeightStrategy, C as CreateAnalystAiConfig, H as DEFAULT_COMPLEXITY_WEIGHTS, D as DEFAULT_TRACE_ANALYST_KINDS, b as DefaultAnalystRegistryOptions, c as DiffPolicy, F as FAILURE_MODE_KIND_SPEC, f as FindingSubject, g as FindingSubjectKind, i as FindingsDiff, j as FindingsStore, I as IMPROVEMENT_KIND_SPEC, k as KNOWLEDGE_GAP_KIND_SPEC, l as KNOWLEDGE_POISONING_KIND_SPEC, P as PersistedFinding, J as SEMANTIC_CONCEPT_JUDGE_VERSION, m as SKILL_USAGE_ANALYST, a as SemanticConceptJudgeInput, S as SemanticConceptJudgeOptions, L as SemanticConceptJudgeResult, n as SkillUsageAnalyst, M as SuboptimalCode, N as SuboptimalSignal, r as buildDefaultAnalystRegistry, O as computeTraceMetrics, t as createAnalystAi, Q as createSemanticConceptJudge, u as defaultIsMaterial, v as diffFindings, R as runSemanticConceptJudge } from './semantic-concept-judge-DIEgr_6v.js';
 import { l as ChatRequest, p as CreateChatClientOpts } from './types-Cu3u_x59.js';
 export { A as Analyst, a as AnalystContext, g as AnalystCost, c as AnalystFinding, i as AnalystInputKind, j as AnalystRequirements, f as AnalystRunEvent, e as AnalystRunInputs, d as AnalystRunResult, b as AnalystRunSummary, h as AnalystSeverity, k as ChatCallOpts, C as ChatClient, m as ChatResponse, n as ChatTransport, o as CliBridgeTransportOpts, D as DirectProviderTransportOpts, E as EvidenceRef, M as MockTransportOpts, R as RouterTransportOpts, S as SandboxSdkTransportOpts, q as computeFindingId, r as createChatClient, s as makeFinding } from './types-Cu3u_x59.js';
@@ -23,7 +23,7 @@ export { C as CliffsMagnitude, c as CorpusAgreementOptions, d as CorpusAgreement
 import { a as AnalyzeTracesInput, A as AnalyzeTracesOptions, b as AnalyzeTracesResult } from './analyst-C8HHvfJp.js';
 export { c as AnalyzeTracesTurnSnapshot, d as analyzeTraces } from './analyst-C8HHvfJp.js';
 import { OtelExporter, OtelExportConfig } from './traces.js';
-export { CaptureFetchContext, CaptureFetchOptions, ExportableSpan, FlattenOtlpOptions, OTEL_AGENT_EVAL_SCOPE, OtlpExport, OtlpFileTraceStore, OtlpFileTraceStoreOptions, OtlpFlatLine, OtlpResourceSpans, OtlpSpan, OtlpToRunRecordsOptions, OtlpTraceRunRecord, ProjectedOtlpSpan, ReplayCache, ReplayCacheEntry, ReplayCacheMissError, ReplayCacheStats, ReplayFetchOptions, SpanNotFoundError, TRACE_ANALYST_ACTOR_DESCRIPTION, TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION, TRACE_ANALYST_SUBAGENT_DESCRIPTION, TraceAggregate, TraceAnalystHookOptions, TraceFileMissingError, TraceInsightContext, TraceInsightFinding, TraceInsightPanelRole, TraceInsightPromptInput, TraceInsightQualityGate, TraceInsightQuestion, TraceInsightReadiness, TraceInsightSuite, TraceInsightTask, TraceNotFoundError, TraceStoreSource, TraceStoreToOtlpOptions, TracesToOtlpResult, asNumber, asString, buildTraceAnalystTools, buildTraceInsightContext, buildTraceInsightPrompt, captureFetchToRawSink, convertTraceStoresToOtlp, createOtelExporter, createOtelTracingStore, createReplayFetch, defaultTraceInsightPanel, describeTraceInsightScope, domainEvidencePattern, exportRunAsOtlp, extractOtlpAttributes, firstNumberAttr, firstStringAttr, flattenOtlpExportToNdjson, inferDomainKeywords, inferOtlpKind, iterateRawCalls, otelRunCompleteHook, otlpToRunRecords, otlpToTraceRunRecords, planTraceInsightQuestions, projectOtlpFlatLine, readOtlpStatus, scoreTraceInsightReadiness, stringField, tokenizeDomainWords, traceAnalystFunctionGroup, traceAnalystOnRunComplete } from './traces.js';
+export { CaptureFetchContext, CaptureFetchOptions, ExportableSpan, ExtractedUsage, FlattenOtlpOptions, OTEL_AGENT_EVAL_SCOPE, OtlpExport, OtlpFileTraceStore, OtlpFileTraceStoreOptions, OtlpFlatLine, OtlpResourceSpans, OtlpSpan, OtlpToRunRecordsOptions, OtlpTraceRunRecord, ProjectedOtlpSpan, ReplayCache, ReplayCacheEntry, ReplayCacheMissError, ReplayCacheStats, ReplayFetchOptions, SpanNotFoundError, TRACE_ANALYST_ACTOR_DESCRIPTION, TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION, TRACE_ANALYST_SUBAGENT_DESCRIPTION, TraceAggregate, TraceAnalystHookOptions, TraceFileMissingError, TraceInsightContext, TraceInsightFinding, TraceInsightPanelRole, TraceInsightPromptInput, TraceInsightQualityGate, TraceInsightQuestion, TraceInsightReadiness, TraceInsightSuite, TraceInsightTask, TraceNotFoundError, TraceStoreSource, TraceStoreToOtlpOptions, TracesToOtlpResult, asNumber, asString, buildTraceAnalystTools, buildTraceInsightContext, buildTraceInsightPrompt, captureFetchToRawSink, convertTraceStoresToOtlp, createOtelExporter, createOtelTracingStore, createReplayFetch, defaultTraceInsightPanel, describeTraceInsightScope, domainEvidencePattern, exportRunAsOtlp, extractOtlpAttributes, extractUsage, extractUsageFromResponse, extractUsageFromSse, firstNumberAttr, firstStringAttr, flattenOtlpExportToNdjson, inferDomainKeywords, inferOtlpKind, iterateRawCalls, otelRunCompleteHook, otlpToRunRecords, otlpToTraceRunRecords, planTraceInsightQuestions, projectOtlpFlatLine, readOtlpStatus, scoreTraceInsightReadiness, stringField, tokenizeDomainWords, traceAnalystFunctionGroup, traceAnalystOnRunComplete } from './traces.js';
 export { D as DEFAULT_TRACE_ANALYST_BUDGETS, b as DatasetOverview, E as ErrorCluster, Q as QueryTracesPage, S as SearchSpanResult, c as SearchTraceResult, d as SpanMatchRecord, e as TRACE_ANALYST_TRUNCATION_MARKER_PREFIX, T as TraceAnalysisStore, f as TraceAnalystByteBudgets, g as TraceAnalystFilters, a as TraceAnalystSpan, h as TraceAnalystSpanKind, i as TraceAnalystSpanStatus, j as TraceAnalystTraceSummary, V as ViewSpansResult, k as ViewTraceOversized, l as ViewTraceResult } from './store-C1YxJDEK.js';
 import { S as SteeringBundle } from './harness-optimizer-EnEnQPsr.js';
 export { D as DEFAULT_HARNESS_OBJECTIVES, H as HarnessAdapter, a as HarnessExperimentConfig, b as HarnessExperimentResult, c as HarnessIntervention, d as HarnessRunRequest, e as HarnessRunResult, f as HarnessScenario, g as HarnessSelection, h as HarnessVariant, i as HarnessVariantReport, M as MeasurementPolicy, j as SteeringDelta, k as SteeringRolePrompt, W as WorkflowTopology, m as mergeSteeringBundle, r as renderSteeringText, l as runHarnessExperiment, s as selectHarnessVariant, n as summarizeHarnessResults } from './harness-optimizer-EnEnQPsr.js';
@@ -1275,6 +1275,118 @@ declare class BudgetGuard {
     get state(): Record<keyof BudgetSpec, number>;
 }
 
+/**
+ * CostLedger — per-run token + USD accounting with an explicit `costUnknown`
+ * axis, folded over the substrate's pricing resolver.
+ *
+ * `estimateCost` already resolves a model id to a price (exact table, then
+ * family regex) and warns-once on a miss, but it returns 0 for an unpriced
+ * model — indistinguishable downstream from a genuinely free run. Four
+ * consumers re-wrap it to surface that distinction (physim's `costForUsage` /
+ * `modelPriceKey` is the cleanest), and to bucket spend by "channel" (the
+ * logical role of the call: agent / judge / verifier / …) so a dashboard can
+ * answer "how much did judging cost vs the agent itself?".
+ *
+ * This is the canonical version. `modelPriceKey` exposes the resolver's verdict
+ * as a stable key (or null). `CostLedger` folds usage records into per-channel
+ * and total rollups, tracks `unpricedModels` so a $0 is never mistaken for a
+ * measured zero, and computes cost-per-completed-task.
+ */
+/** Logical role of an LLM call. Free-form union — consumers add their own
+ *  channels; the rollup keys on whatever string is supplied. */
+type CostChannel = 'agent' | 'judge' | 'verifier' | 'analyst' | 'driver' | (string & {});
+interface CostUsage {
+    inputTokens: number;
+    outputTokens: number;
+    cachedTokens?: number;
+}
+/**
+ * Resolve a model id to the stable pricing key the substrate's `MODEL_PRICING`
+ * / family resolver would use, or null when the id is unpriced. A non-null
+ * return means `estimateCost` will produce a real number for this id; null
+ * means any cost computed is `costUnknown` and the 0 must not aggregate as a
+ * measured cost.
+ */
+declare function modelPriceKey(model: string): string | null;
+interface CostResult {
+    costUsd: number;
+    /** True when `model` has no pricing — the 0 is "not priced", NOT "free". */
+    costUnknown: boolean;
+}
+/**
+ * Cost for one usage record. Resolves pricing via the substrate resolver and
+ * flags `costUnknown` when the model is unpriced so the 0 is observable rather
+ * than silently emitted as a measured cost. Cached tokens are billed at the
+ * model's input rate when present (no separate cache-discount table — callers
+ * that need provider-specific cache pricing supply `actualCostUsd` upstream).
+ */
+declare function costForUsage(model: string, usage: CostUsage): CostResult;
+interface CostLedgerEntry extends CostUsage {
+    model: string;
+    channel: CostChannel;
+    costUsd: number;
+    costUnknown: boolean;
+    /** Override the estimate with an observed provider cost. */
+    actualCostUsd?: number;
+    /** Free-form tags (scenario id, variant id, round, …). */
+    tags?: Record<string, string>;
+    timestamp: number;
+}
+interface ChannelRollup {
+    channel: CostChannel;
+    calls: number;
+    inputTokens: number;
+    outputTokens: number;
+    cachedTokens: number;
+    costUsd: number;
+    /** Calls whose model was unpriced (their costUsd is 0-but-unknown). */
+    unpricedCalls: number;
+}
+interface CostLedgerSummary {
+    totalCalls: number;
+    inputTokens: number;
+    outputTokens: number;
+    cachedTokens: number;
+    totalCostUsd: number;
+    /** Per-channel breakdown, sorted by channel name. */
+    byChannel: ChannelRollup[];
+    /** Distinct unpriced model ids seen — non-empty means totalCostUsd is a
+     *  lower bound (some calls priced to an unknown 0). */
+    unpricedModels: string[];
+    /** True when no unpriced model was charged — totalCostUsd is then exact. */
+    fullyPriced: boolean;
+}
+/**
+ * Append-only ledger of LLM spend for a single run. Record each call with its
+ * channel; read per-channel and total rollups plus the unpriced-model set.
+ * Pure accounting — no I/O. The `markCompleted` / `costPerCompletedTask` pair
+ * answers "dollars per finished task", the metric every optimizer's
+ * quality-vs-cost tradeoff needs.
+ */
+declare class CostLedger {
+    private readonly entries;
+    private completedTasks;
+    /**
+     * Record one LLM call. The cost is computed from pricing unless
+     * `actualCostUsd` is supplied (a finite observed cost from the provider
+     * response), in which case `costUnknown` is false regardless of pricing.
+     */
+    record(input: {
+        model: string;
+        channel: CostChannel;
+        usage: CostUsage;
+        actualCostUsd?: number;
+        tags?: Record<string, string>;
+        timestamp?: number;
+    }): CostLedgerEntry;
+    /** Increment the completed-task counter (used for cost-per-completed-task). */
+    markCompleted(count?: number): void;
+    list(): CostLedgerEntry[];
+    summary(): CostLedgerSummary;
+    /** Total spend divided by completed tasks; null when nothing completed. */
+    costPerCompletedTask(): number | null;
+}
+
 /**
  * Cost tracker — token + USD accounting per scenario and per run.
  *
@@ -1351,6 +1463,291 @@ interface CostSummary {
     costPerCompletedTaskUsd: number | null;
 }
 
+/**
+ * EvalTraceStore — JSONL save / query / compare over the analysis-time
+ * `RunRecord` row.
+ *
+ * `FileSystemTraceStore` (in `./trace/store`) persists the rich TraceSchema-v1
+ * span corpus — runs, spans, events, artifacts — the runtime emits live. That
+ * is NOT the thing three consumers hand-roll in `tests/eval/lib/trace-store.ts`.
+ * What they hand-roll is the *analysis* layer: append each finished run as one
+ * JSONL line keyed by scenario/candidate, then query it ("every run where score
+ * < 50"), pick the best run for a scenario (few-shot seeding), and compare two
+ * candidates on matched scenarios.
+ *
+ * The substrate already has the canonical analysis-time row — `RunRecord` — so
+ * this is that query/compare API expressed over `RunRecord[]`, with a
+ * JSONL-backed store. It does NOT fork `FileSystemTraceStore`; it sits beside it
+ * for the analysis projection.
+ */
+
+/** The score the query/compare layer ranks on: holdout when present (the
+ *  gated number), else search. Throws when a record carries neither — a
+ *  RunRecord is invalid without at least one, but a hand-built object might. */
+declare function runScore(record: RunRecord): number;
+interface RunRecordFilter {
+    experimentId?: string;
+    candidateId?: string;
+    scenarioId?: string;
+    model?: string;
+    splitTag?: RunSplitTag;
+    /** Inclusive lower bound on `runScore`. */
+    minScore?: number;
+    /** Inclusive upper bound on `runScore`. */
+    maxScore?: number;
+    /** Match a single tag in `outcome.raw` by exact numeric value. */
+    rawEquals?: {
+        key: string;
+        value: number;
+    };
+    /** Custom predicate, ANDed with the structured filters. */
+    where?: (record: RunRecord) => boolean;
+}
+interface CandidateComparison {
+    a: string;
+    b: string;
+    /** Scenario ids present for BOTH candidates — the paired comparison set. */
+    pairedScenarioIds: string[];
+    /** Mean `runScore` for candidate a over the paired scenarios. */
+    meanA: number;
+    /** Mean `runScore` for candidate b over the paired scenarios. */
+    meanB: number;
+    /** meanB − meanA. Positive ⇒ b scored higher on the matched scenarios. */
+    meanDelta: number;
+    /** Scenarios where b beat a (strictly), tied, and a beat b. */
+    bWins: number;
+    ties: number;
+    aWins: number;
+}
+/**
+ * Backing persistence for `EvalTraceStore`. The in-memory store is the default;
+ * the JSONL file store appends one validated `RunRecord` per line. Both keep an
+ * append order so `getBest` / `compareRuns` are reproducible.
+ */
+interface RunRecordBackend {
+    append(record: RunRecord): Promise<void>;
+    load(): Promise<RunRecord[]>;
+}
+declare function inMemoryRunRecordBackend(initial?: RunRecord[]): RunRecordBackend;
+/**
+ * JSONL-backed store at `path`, one `RunRecord` per line. Malformed lines fail
+ * loud on load (a corrupt corpus must not silently shrink the analysis set);
+ * pass `skipInvalid` only for forensics on a known-bad file.
+ */
+declare function jsonlRunRecordBackend(path: string, opts?: {
+    skipInvalid?: boolean;
+}): RunRecordBackend;
+/**
+ * Query / compare layer over a `RunRecord` corpus. Append finished runs, query
+ * with a structured filter, take the best run for a scenario, and compare two
+ * candidates on their matched scenarios. Persistence is injected via
+ * `RunRecordBackend` (in-memory by default, JSONL file via
+ * `jsonlRunRecordBackend`).
+ */
+declare class EvalTraceStore {
+    private readonly backend;
+    constructor(backend?: RunRecordBackend);
+    /** Validate and append one run. Throws on an invalid record — the corpus
+     *  stays paper-grade. */
+    append(record: RunRecord): Promise<void>;
+    all(): Promise<RunRecord[]>;
+    query(filter?: RunRecordFilter): Promise<RunRecord[]>;
+    /**
+     * Highest-scoring run for a scenario (optionally restricted to a candidate).
+     * Returns null when no run matches. Ties resolve to the earliest-appended run
+     * so the result is stable.
+     */
+    getBest(scenarioId: string, opts?: {
+        candidateId?: string;
+        splitTag?: RunSplitTag;
+    }): Promise<RunRecord | null>;
+    /**
+     * Compare two candidates on the scenarios they BOTH ran. When a candidate
+     * ran a scenario more than once, its best `runScore` for that scenario is
+     * used. Throws when there is no paired scenario — an unpaired "comparison" is
+     * not one.
+     */
+    compareRuns(candidateA: string, candidateB: string): Promise<CandidateComparison>;
+}
+
+/**
+ * Experiment tracker — git-provenanced experiment log with N-rep stats and a
+ * KEEP / REGRESSION / NOISE verdict against a parent.
+ *
+ * Every loop the fleet runs reduces to the same question: "I ran the candidate
+ * N times — is the median measurably better than the parent, or is the delta
+ * inside the noise band?" The hand-rolled copies bake a fixed score scale
+ * (percentage points), a fixed store path (`.evolve/experiments-v2.json`), and
+ * `execSync('git …')` straight into the module. This is the canonical version:
+ * provenance and persistence are injected, thresholds are configurable, and the
+ * stats + verdict are pure functions you can unit-test without a git repo or a
+ * filesystem.
+ *
+ * Stats per experiment: median / mean / min / max / iqr / stddev / passRate /
+ * n, plus a `stable` flag (`iqr < iqrUnstableAbove && stddev < stddevUnstableAbove`).
+ *
+ * Verdict against a parent (both must have `n >= minRepsForVerdict`):
+ *   - NOISE       — the candidate is too unstable to judge (`!stable`)
+ *   - KEEP        — `medianDelta >  keepThreshold`
+ *   - REGRESSION  — `medianDelta < -regressionThreshold`
+ *   - NOISE       — otherwise (delta inside the band)
+ * With no parent (or insufficient reps) the verdict is the neutral ITERATE.
+ */
+/** Verdict for one experiment relative to its parent. ITERATE is the neutral
+ *  "keep collecting reps / no parent to compare against" state. */
+type ExperimentVerdict = 'KEEP' | 'ITERATE' | 'NOISE' | 'REGRESSION';
+/** Git provenance for the working tree an experiment was run from. */
+interface ExperimentProvenance {
+    /** Commit sha (short or full — the tracker does not interpret it). */
+    commit: string;
+    /** First line of the commit message. */
+    message: string;
+    /** Files changed vs the parent commit, or a marker like 'uncommitted'. */
+    changedFiles: string[];
+}
+/** A single repetition of an experiment, carrying the score the verdict is
+ *  computed on plus any free-form per-rep metrics the consumer wants kept. */
+interface ExperimentRep {
+    /** 0-indexed repetition number within the experiment. */
+    rep: number;
+    /** The score this rep is judged on (same scale as the thresholds). */
+    score: number;
+    /** ISO timestamp the rep completed. */
+    timestamp: string;
+    /** Whether this rep passed the consumer's own gate — folded into `passRate`. */
+    passed?: boolean;
+    /** Free-form numeric metrics retained for later analysis. */
+    metrics?: Record<string, number>;
+}
+interface ExperimentStats {
+    median: number;
+    mean: number;
+    min: number;
+    max: number;
+    /** Inter-quartile range of the rep scores. */
+    iqr: number;
+    /** Population standard deviation of the rep scores. */
+    stddev: number;
+    /** Fraction of reps with `passed === true`, over reps that set `passed`.
+     *  null when no rep declared a pass/fail outcome. */
+    passRate: number | null;
+    /** Number of reps. */
+    n: number;
+    /** True when the sample is tight enough to trust for a verdict. */
+    stable: boolean;
+}
+interface Experiment {
+    /** Stable id for the experiment. */
+    id: string;
+    /** Free-form label / config descriptor. */
+    label: string;
+    /** Git provenance captured when the experiment was created. */
+    provenance: ExperimentProvenance;
+    /** Parent experiment id this candidate is compared against, if any. */
+    parentId?: string;
+    /** One-line summary of what changed from the parent. */
+    changeSummary: string;
+    reps: ExperimentRep[];
+    stats: ExperimentStats;
+    verdict: ExperimentVerdict;
+    /** ISO timestamp the experiment was created. */
+    createdAt: string;
+}
+interface ImprovementThresholds {
+    /** medianDelta strictly above this ⇒ KEEP. Default 5. */
+    keepThreshold?: number;
+    /** medianDelta strictly below the negative of this ⇒ REGRESSION. Default 5. */
+    regressionThreshold?: number;
+    /** iqr at or above this ⇒ unstable. Default 10. */
+    iqrUnstableAbove?: number;
+    /** stddev at or above this ⇒ unstable. Default Infinity (iqr-only stability). */
+    stddevUnstableAbove?: number;
+    /** Reps required on BOTH candidate and parent before a verdict is rendered.
+     *  Default 3. */
+    minRepsForVerdict?: number;
+}
+interface ImprovementVerdictResult {
+    verdict: ExperimentVerdict;
+    /** candidate.median − parent.median; null when no parent or insufficient reps. */
+    medianDelta: number | null;
+    /** Human-readable reason for the verdict — for dashboards and logs. */
+    reason: string;
+}
+/**
+ * Compute the N-rep statistics for a set of reps. Pure — no I/O. The `stable`
+ * flag is the trust gate the verdict depends on: a sample whose spread exceeds
+ * the configured bounds can't distinguish a real delta from run-to-run noise.
+ */
+declare function computeExperimentStats(reps: ExperimentRep[], thresholds?: ImprovementThresholds): ExperimentStats;
+/**
+ * Verdict for a candidate against its parent. Pure — operates on already-computed
+ * stats. KEEP/REGRESSION require both sides to have `>= minRepsForVerdict` reps
+ * AND the candidate to be `stable`; otherwise the result is NOISE (unstable) or
+ * ITERATE (not enough reps / no parent).
+ */
+declare function improvementVerdict(candidate: ExperimentStats, parent: ExperimentStats | null, thresholds?: ImprovementThresholds): ImprovementVerdictResult;
+/** Reads git provenance for the working tree. Inject a fake in tests; the
+ *  default implementation shells out to `git`. */
+type ProvenanceReader = () => ExperimentProvenance | Promise<ExperimentProvenance>;
+/** Persistence seam for the experiment log. Inject in-memory in tests; the
+ *  filesystem implementation is `fileExperimentStore`. */
+interface ExperimentStore {
+    load(): Promise<Experiment[]>;
+    save(experiments: Experiment[]): Promise<void>;
+}
+/**
+ * Default provenance reader: `git rev-parse HEAD`, the subject line, and the
+ * files changed vs `HEAD~1`. Fail-loud — a tracker that silently logs
+ * `commit: 'unknown'` corrupts the provenance the whole point of the log is to
+ * carry. When the working tree genuinely has no parent commit, pass an override.
+ */
+declare const gitProvenanceReader: ProvenanceReader;
+/** In-memory store — the default when no persistence is wanted (tests, ephemeral
+ *  runs). State lives on the instance. */
+declare function inMemoryExperimentStore(initial?: Experiment[]): ExperimentStore;
+/** Filesystem store — a single JSON array at `path`, created on first save. */
+declare function fileExperimentStore(path: string): ExperimentStore;
+interface ExperimentTrackerOptions {
+    store?: ExperimentStore;
+    provenanceReader?: ProvenanceReader;
+    thresholds?: ImprovementThresholds;
+    /** Clock seam for deterministic timestamps in tests. Default `Date.now`. */
+    now?: () => number;
+}
+interface CreateExperimentInput {
+    id: string;
+    label: string;
+    changeSummary: string;
+    parentId?: string;
+    /** Override provenance instead of reading from git (e.g. CI metadata). */
+    provenance?: ExperimentProvenance;
+}
+/**
+ * Stateful tracker over an `ExperimentStore`. Create an experiment (provenance
+ * is captured once), append reps as they complete (stats + verdict recompute on
+ * every append), and read the log back for a dashboard. All persistence and git
+ * access flow through the injected seams, so the tracker is fully testable
+ * without a repo or disk.
+ */
+declare class ExperimentTracker {
+    private readonly store;
+    private readonly provenanceReader;
+    private readonly thresholds;
+    private readonly now;
+    constructor(options?: ExperimentTrackerOptions);
+    create(input: CreateExperimentInput): Promise<Experiment>;
+    /** Append a rep (its `rep` index defaults to the current rep count) and
+     *  recompute stats + verdict. Returns the updated experiment. */
+    addRep(experimentId: string, rep: Omit<ExperimentRep, 'rep' | 'timestamp'> & {
+        rep?: number;
+        timestamp?: string;
+    }): Promise<Experiment>;
+    get(experimentId: string): Promise<Experiment | undefined>;
+    list(): Promise<Experiment[]>;
+    /** Full verdict (not just the enum) for an experiment vs its parent. */
+    verdictFor(experimentId: string): Promise<ImprovementVerdictResult>;
+}
+
 /**
  * muffled-gate-scanner — test helper that greps consumer source for
  * gate + measurement anti-patterns and fails with file:line locations.
@@ -1508,6 +1905,76 @@ interface OracleReport {
 /** Run all oracles against one observation and aggregate. */
 declare function evaluateOracles(obs: OracleObservation, oracles: Oracle[]): OracleReport;
 
+/**
+ * Deterministic held-out partitioning over stable ids.
+ *
+ * A promotion gate's paired-delta math needs a fixed partition: the same
+ * scenario id lands in the same bucket forever, regardless of when it ran or
+ * which loop emitted the record — otherwise the gate has nothing paired to
+ * compare against. Picking one global split per run is wrong: every run then
+ * stamps either ALL search or ALL holdout, and no id is ever observed on both
+ * sides. Per-id deterministic hashing fixes that.
+ *
+ * Two consumers hand-roll this (agent-builder's `deterministicSplit` and the
+ * frontier persona-splitter). The substrate already has a 3-way benchmark
+ * `deterministicSplit` in `./benchmarks`; this is the generic, validated batch
+ * partitioner: it takes a list of ids, splits them by a stable FNV-1a hash, and
+ * fails loud when the inputs can't support a trustworthy held-out comparison
+ * (duplicate ids, or a holdout set below the significance floor).
+ */
+/** 32-bit FNV-1a hash. Stable, allocation-free, deterministic across runtimes —
+ *  the same id+seed maps to the same bucket on every machine and process. */
+declare function fnv1a32(input: string): number;
+/** Map an id+seed to the half-open unit interval [0, 1) deterministically. */
+declare function hashToUnit(id: string, seed: string): number;
+interface PartitionHeldOutOptions {
+    /** Partition seed. Bumping it reshuffles every assignment — do that only when
+     *  the corpus/policy changes meaningfully so old and new records can't pair.
+     *  Default 'held-out-v1'. */
+    seed?: string;
+    /** Fraction routed to the held-out bucket, in (0, 1). Default 0.5. */
+    holdoutFraction?: number;
+    /**
+     * Minimum held-out ids required for the split to be admissible. Below this,
+     * a paired comparison has too few observations to be significant, so the
+     * partition throws rather than hand back an underpowered holdout. Default 1
+     * (only the empty-holdout degenerate case is rejected). Raise it to enforce
+     * real statistical power. */
+    minHoldout?: number;
+    /** Same floor for the search side. Default 1. */
+    minSearch?: number;
+}
+interface HeldOutPartition {
+    /** Ids the optimizer is allowed to read/optimize against. */
+    search: string[];
+    /** Ids reserved for gated, held-out evaluation. */
+    holdout: string[];
+    seed: string;
+    holdoutFraction: number;
+}
+/**
+ * Assign a single id to `'search'` or `'holdout'` deterministically. Same
+ * `(id, seed)` always returns the same tag — the invariant the gate relies on.
+ * Use this in a `splitTag:` field literal when stamping records one at a time;
+ * use `partitionHeldOut` when you have the whole id list up front and want the
+ * validated disjoint split.
+ */
+declare function assignHeldOutTag(id: string, options?: {
+    seed?: string;
+    holdoutFraction?: number;
+}): 'search' | 'holdout';
+/**
+ * Partition a list of stable ids into disjoint `search` / `holdout` sets by a
+ * deterministic hash. Fail-loud:
+ *   - empty input throws,
+ *   - duplicate ids throw (a dup would be observed on both sides, double-counting),
+ *   - a holdout (or search) set below its floor throws (underpowered comparison).
+ *
+ * Order within each bucket follows the input order, so the result is stable and
+ * reproducible.
+ */
+declare function partitionHeldOut(ids: string[], options?: PartitionHeldOutOptions): HeldOutPartition;
+
 /**
  * @stable
  *
@@ -4801,4 +5268,4 @@ declare namespace index {
   export { type index_AgentProfile as AgentProfile, type index_AgentProfileSection as AgentProfileSection, index_BASELINE_ROLES as BASELINE_ROLES, type index_BaselineRoleKey as BaselineRoleKey, type index_ProfileSkill as ProfileSkill, index_applyDomainPatch as applyDomainPatch, index_baselineProfile as baselineProfile, index_baselineProfileFromRole as baselineProfileFromRole, index_engineerRole as engineerRole, index_generalistRole as generalistRole, index_prodProfile as prodProfile, index_profileToSurface as profileToSurface, index_renderProfile as renderProfile, index_researcherRole as researcherRole, index_sectionHash as sectionHash };
 }
 
-export { type ActiveLearningOptions, type AdapterRun, AgentDriver, type AgentDriverConfig, AgentEvalError, AgentProfile$1 as AgentProfile, type AgreementResult, type AlignmentOp, AnalyzeTracesInput, AnalyzeTracesOptions, AnalyzeTracesResult, type AntiSlopConfig, type AntiSlopIssue, type AntiSlopReport, type AssertCrossFamilyOptions, type AssertSingleBackendOptions, type AutoPrClient, AxGepaSteeringOptimizer, type AxSteeringOptimizerConfig, type BackendDescriptor, BaselineReport, BehaviorAssertion, BenchmarkReport, BenchmarkRunner, BenchmarkRunnerConfig, type BisectOptions, type BisectResult, type BisectStep, BudgetBreachError, BudgetGuard, BudgetLedgerEntry, BudgetSpec, type BuildAgreementJudgeOptions, CallExpectation, type CanaryAlert, type CanaryKind, type CanaryLeak, type CanaryOptions, type CanaryReport, type CanarySeverity, type CandidateScenario, type CausalAttributionReport, type CellVerdict, ChatRequest, CheckResult, CollectedArtifacts, type CommandRunner, type CompareLabels, CompletionCriterion, type ContinuityCheck, type ContinuityCheckResult, type ContinuityReport, type ContinuitySnapshotPair, type ContractMetric, type ContractReport, ConvergenceTracker, type CostEntry, type CostSummary, CostTracker, type CounterfactualContext, type CounterfactualMutation, type CounterfactualResult, type CounterfactualRunner, CreateChatClientOpts, type CreateDefaultReviewerOptions, type CreateSandboxPoolOpts, CrossFamilyError, type CrossTraceDiff, type CrossTraceDiffOptions, DEFAULT_AGENT_SLOS, DEFAULT_FINDERS, DEFAULT_MUTATION_PRIMITIVES, DEFAULT_MUTATORS, DEFAULT_PR_REVIEW_SCORE_WEIGHTS, DEFAULT_SEVERITY_WEIGHTS, Dataset, DatasetScenario, type DecideNextUserTurnOpts, type DeployFamily, type DeployGateLayerInput, type DeployRunResult, type DeployRunner, type DescriptionLengthCandidate, type DescriptionLengthConfig, type DescriptionLengthDecision, type DescriptionLengthEvidence, DescriptionLengthGate, type DescriptionLengthRejectionCode, type DiffScorecardOptions, type DirEntry, type DiscoverPersonasOptions, type DiscoveredPersona, DriverResult, DriverState, DualAgentBench, type DualAgentBenchConfig, type DualAgentReport, type DualAgentRound, type DualAgentScenario, type DualAgentScenarioResult, ERROR_COUNT_PATTERNS, type EnsembleAggregate, type ErrorCountPattern, type EvolutionRound, type ExecutorConfig, type Expectation, type ExportedRewardModel, type ExtractOptions, type ExtractResult, type FactorContribution, type FactorialCell, FeedbackLabel, FeedbackTrajectory, FeedbackTrajectoryStore, type FieldAgreementSpec, type FileChange, type FlowAction, type FlowLayerEnv, type FlowLayerFactoryInput, type FlowRunner, type FlowRunnerStepResult, type FlowSpec, type FlowStep, type GhCliClientOptions, type GoldScenario, type GoldSplit, type GoldenSeverity, type GoldenSpec, HarnessConfig, HoldoutAuditor, type HttpGithubClientOptions, type HypothesisManifest, type HypothesisResult, INTENT_MATCH_JUDGE_VERSION, type ImageData, InMemoryWorkspaceInspector, type InferenceScorer, type InspectorContext, type IntentMatchInput, type IntentMatchOptions, type IntentMatchResult, type InteractionContribution, type JudgeFamily, type JudgeFleetOptions, JudgeFn, type JudgeReplayResult, type JudgeRetryOutcome, type JudgeRetryPolicy, JudgeRunner, type JudgeVerdict, type KeywordConceptSpec, type KeywordCoverageFinding, type KeywordCoverageOptions, type KeywordCoverageResult, type LangfuseEnvelope, type LangfuseGeneration, type LangfuseScore, Layer, LayerResult, type LiveProofArtifact, type LiveProofConfig, type LiveProofContext, type LiveProofResult, LlmClientOptions, LlmSpan, LockedJsonlAppender, MODEL_PRICING, type MatchResult, type MatcherResult, type MergeOptions, MetricsCollector, type ModelPreflight, ModelsUnreachableError, type MuffledFinder, type MuffledFinding, type MultiToolchainLayerConfig, type Mutator, Mutex, type Oracle, type OracleObservation, type OracleReport, type OracleResult, type OrthogonalityInput, type OrthogonalityResult, OtelExportConfig, OtelExporter, type OtelPipelineHandle, type OtelPipelineOptions, PairwiseSteeringOptimizer, type ParaphraseRobustnessScenarioInput, type ParaphraseRobustnessScenarioResult, type ParseStudentLabel, PersonaConfig, type Playbook, type PlaybookEntry, type PoolSlot, type PrReviewAuditCase, type PrReviewBenchmarkSummary, type PrReviewComment, type PrReviewMatchedFinding, type PrReviewOutcome, type PrReviewReferenceFinding, type PrReviewScore, type PrReviewScoreWeights, type PrReviewSeverity, type PrReviewSource, type PreflightModelsOptions, type PreflightOutcome, ProductClient, ProductClientConfig, type PromptHandle, PromptRegistry, type ProposeAutomatedPullRequestInput, type ProposeAutomatedPullRequestResult, type RecordRunsOptions, type ReferenceMatchResult, type ReferenceReplayAdapter, type ReferenceReplayAdapterFn, type ReferenceReplayAdapterLike, type ReferenceReplayAggregate, type ReferenceReplayCandidate, type ReferenceReplayCase, type ReferenceReplayCaseRun, type ReferenceReplayExecutionScenario, type ReferenceReplayItem, type ReferenceReplayMatch, type ReferenceReplayMatchStrategy, type ReferenceReplayMatcher, type ReferenceReplayPromotionDecision, type ReferenceReplayPromotionPolicy, type ReferenceReplayRun, type ReferenceReplayRunContext, type ReferenceReplayRunOptions, type ReferenceReplayRunStore, type ReferenceReplayScenario, type ReferenceReplayScenarioScore, type ReferenceReplayScore, type ReferenceReplayScoreOptions, type ReferenceReplaySplit, type ReferenceReplaySplitComparison, type ReferenceReplaySteeringRowsOptions, type ReflectionContext, type ReflectionProposal, ReleaseConfidenceScorecard, ReleaseConfidenceThresholds, type RenderStudentPrompt, type RepoRef, type ReviewerMemoryEntry, type ReviewerOutput, type ReviewerPromptInput, type ReviewerSoftFailDefaults, type ReviewerVerificationSummary, type RobustnessResult, Run, type RunCommandInput, type RunCommandResult, type RunDistillationOptions, type RunDistillationResult, RunFilter, RunRecord, RunScore, RunScoreWeights, SandboxDriver, SandboxHarnessResult, type SandboxJudgeKind, type SandboxJudgeResult, type SandboxJudgeSpec, type SandboxPool, type ScanOptions, Scenario, type ScenarioCost, ScenarioFile, ScenarioRegistry, ScenarioResult, type Scorecard, type ScorecardCell, type ScorecardCellDiff, type ScorecardDiff, type ScorecardEntry, type ScorecardLogLine, type ScoredTarget, type SelfPlayOptions, type SelfPlayProposer, type SelfPlayScorer, type SeriesConvergenceOptions, type SeriesConvergenceResult, Severity, type SignedManifest, type SignedManifestAlgo, type SingleBackendDivergence, SingleBackendError, type SingleBackendField, type SingleBackendReport, type Slo, type SloCheckResult, type SloComparator, type SloReport, type SloSeverity, type SlopCategory, type SlotFactory, type SplitGoldOptions, SteeringBundle, type SteeringOptimizationResult, type SteeringOptimizationRow, type SteeringOptimizationSelector, type SteeringOptimizerBackend, type SteeringOptimizerConfig, type StepAttribution, type SynthesisReason, type SynthesisTarget, TestResult, type ThresholdContract, TokenCounter, type TokenSpec, TraceEmitter, TraceStore, type TracedAnalystOptions, type TracedJudgeOptions, Trajectory, TrajectoryStep, type TrialTrace, TurnMetrics, UI_FINDING_SEVERITIES, UI_LENSES, UNIVERSAL_FINDERS, type UiFinding, type UiFindingScreenshot, type UiFindingSeverity, type UiLens, VerifyContext, type VisualDiffOptions, type VisualDiffResult, type ViteDeployRunnerInput, type WorkspaceAssertion, type WorkspaceAssertionResult, type WorkspaceInspector, type WorkspaceSnapshot, type WranglerDeployRunnerInput, adversarialJudge, aggregateJudgeVerdicts, aggregatePrReviewScore, analyzeAntiSlop, analyzeSeries, appendScorecard, assertCrossFamily, assertModelsServed, assertSingleBackend, attributeCounterfactuals, bisect, buildAgreementJudge, buildDriverSystemPrompt, buildReflectionPrompt, buildReviewerPrompt, canaryLeakView, canonicalize, causalAttribution, checkBehavioralCanary, checkCanaries, checkSlos, codeExecutionJudge, coherenceJudge, collectionPreserved, commentsForSource, commitBisect, compareReferenceReplay, compilerJudge, createAntiSlopJudge, createCustomJudge, createDefaultReviewer, createDomainExpertJudge, createIntentMatchJudge, createSandboxPool, crossTraceDiff, dataDescriptionBits, decideNextUserTurn, decideReferenceReplayPromotion, decideReferenceReplayRunPromotion, defaultJudges, defaultParseStudentLabel, defaultReferenceReplayMatcher, defaultRenderStudentPrompt, deployGateLayer, diffScorecard, discoverPersonas, distillPlaybook, estimateCost, estimateTokens, evaluateContract, evaluateHypothesis, evaluateOracles, executeScenario, expectAgent, exportRewardModel, extractAssetUrls, extractErrorCount, fieldAgreement, fileContains, fileExists, findAutoMatchNoExpectation, findConstructorCwdDropped, findFallbackToPass, findLiteralTruePass, findSkipCountsAsPass, flowLayer, formatBenchmarkReport, formatDriverReport, formatFindings, formatScorecardDiff, ghCliClient, precision as goldenPrecision, hashContent, hashJson, htmlContainsElement, httpGithubClient, inMemoryReferenceReplayStore, isModelPriced, isOtelConfigured, jsonShape, jsonlReferenceReplayStore, judgeFamily, keyPreserved, linterJudge, loadGoldScenarios, loadScorecard, loadScorerFromGrader, localCommandRunner, lowercaseMutator, matchGoldens, mergeLayerResults, modelDescriptionBits, multiToolchainLayer, notBlocked, paraphraseRobustness, paraphraseRobustnessScenarios, parseGoldJsonl, parseReflectionResponse, passOrthogonality, pixelDeltaRatio, politenessPrefixMutator, preflightModels, printDriverSummary, index as profile, promptBisect, proposeAutomatedPullRequest, proposeSynthesisTargets, recordRuns, recordRunsToScorecard, referenceReplayRunsToSteeringRows, referenceReplayScenarioToRunScore, regexMatches, renderMarkdownReport, renderPlaybookMarkdown, replayScorerOverCorpus, replayTraceThroughJudge, resetLockedAppendersForTesting, resolveModelPricing, rowCount, rowWhere, runAssertions, runBehavioralCanaries, runCanaries, runCounterfactual, runDistillation, runE2EWorkflow, runExpectations, runIntentMatchJudge, runJudgeFleet, runKeywordCoverageJudge, runKeywordCoverageJudgeUrl, runLiveProof, runReferenceReplay, runSelfPlay, scanForMuffledGates, scoreContinuity, scorePrReviewComments, scorePrReviewSource, scoreReferenceReplay, securityJudge, sentenceReorderMutator, signManifest, splitGold, statusAdvanced, summarizePrReviewBenchmark, testJudge, textInSnapshot, toLangfuseEnvelope, toPrometheusText, traceJudge, traceJudgeEnsemble, tracedAnalyzeTraces, typoMutator, urlContains, verifyManifest, visualDiff, viteDeployRunner, weightedRecall, whitespaceCollapseMutator, withJudgeRetry, withOtelPipeline, wranglerDeployRunner };
+export { type ActiveLearningOptions, type AdapterRun, AgentDriver, type AgentDriverConfig, AgentEvalError, AgentProfile$1 as AgentProfile, type AgreementResult, type AlignmentOp, AnalyzeTracesInput, AnalyzeTracesOptions, AnalyzeTracesResult, type AntiSlopConfig, type AntiSlopIssue, type AntiSlopReport, type AssertCrossFamilyOptions, type AssertSingleBackendOptions, type AutoPrClient, AxGepaSteeringOptimizer, type AxSteeringOptimizerConfig, type BackendDescriptor, BaselineReport, BehaviorAssertion, BenchmarkReport, BenchmarkRunner, BenchmarkRunnerConfig, type BisectOptions, type BisectResult, type BisectStep, BudgetBreachError, BudgetGuard, BudgetLedgerEntry, BudgetSpec, type BuildAgreementJudgeOptions, CallExpectation, type CanaryAlert, type CanaryKind, type CanaryLeak, type CanaryOptions, type CanaryReport, type CanarySeverity, type CandidateComparison, type CandidateScenario, type CausalAttributionReport, type CellVerdict, type ChannelRollup, ChatRequest, CheckResult, CollectedArtifacts, type CommandRunner, type CompareLabels, CompletionCriterion, type ContinuityCheck, type ContinuityCheckResult, type ContinuityReport, type ContinuitySnapshotPair, type ContractMetric, type ContractReport, ConvergenceTracker, type CostChannel, type CostEntry, CostLedger, type CostLedgerEntry, type CostLedgerSummary, type CostResult, type CostSummary, CostTracker, type CostUsage, type CounterfactualContext, type CounterfactualMutation, type CounterfactualResult, type CounterfactualRunner, CreateChatClientOpts, type CreateDefaultReviewerOptions, type CreateExperimentInput, type CreateSandboxPoolOpts, CrossFamilyError, type CrossTraceDiff, type CrossTraceDiffOptions, DEFAULT_AGENT_SLOS, DEFAULT_FINDERS, DEFAULT_MUTATION_PRIMITIVES, DEFAULT_MUTATORS, DEFAULT_PR_REVIEW_SCORE_WEIGHTS, DEFAULT_SEVERITY_WEIGHTS, Dataset, DatasetScenario, type DecideNextUserTurnOpts, type DeployFamily, type DeployGateLayerInput, type DeployRunResult, type DeployRunner, type DescriptionLengthCandidate, type DescriptionLengthConfig, type DescriptionLengthDecision, type DescriptionLengthEvidence, DescriptionLengthGate, type DescriptionLengthRejectionCode, type DiffScorecardOptions, type DirEntry, type DiscoverPersonasOptions, type DiscoveredPersona, DriverResult, DriverState, DualAgentBench, type DualAgentBenchConfig, type DualAgentReport, type DualAgentRound, type DualAgentScenario, type DualAgentScenarioResult, ERROR_COUNT_PATTERNS, type EnsembleAggregate, type ErrorCountPattern, EvalTraceStore, type EvolutionRound, type ExecutorConfig, type Expectation, type Experiment, type ExperimentProvenance, type ExperimentRep, type ExperimentStats, type ExperimentStore, ExperimentTracker, type ExperimentTrackerOptions, type ExperimentVerdict, type ExportedRewardModel, type ExtractOptions, type ExtractResult, type FactorContribution, type FactorialCell, FeedbackLabel, FeedbackTrajectory, FeedbackTrajectoryStore, type FieldAgreementSpec, type FileChange, type FlowAction, type FlowLayerEnv, type FlowLayerFactoryInput, type FlowRunner, type FlowRunnerStepResult, type FlowSpec, type FlowStep, type GhCliClientOptions, type GoldScenario, type GoldSplit, type GoldenSeverity, type GoldenSpec, HarnessConfig, type HeldOutPartition, HoldoutAuditor, type HttpGithubClientOptions, type HypothesisManifest, type HypothesisResult, INTENT_MATCH_JUDGE_VERSION, type ImageData, type ImprovementThresholds, type ImprovementVerdictResult, InMemoryWorkspaceInspector, type InferenceScorer, type InspectorContext, type IntentMatchInput, type IntentMatchOptions, type IntentMatchResult, type InteractionContribution, type JudgeFamily, type JudgeFleetOptions, JudgeFn, type JudgeReplayResult, type JudgeRetryOutcome, type JudgeRetryPolicy, JudgeRunner, type JudgeVerdict, type KeywordConceptSpec, type KeywordCoverageFinding, type KeywordCoverageOptions, type KeywordCoverageResult, type LangfuseEnvelope, type LangfuseGeneration, type LangfuseScore, Layer, LayerResult, type LiveProofArtifact, type LiveProofConfig, type LiveProofContext, type LiveProofResult, LlmClientOptions, LlmSpan, LockedJsonlAppender, MODEL_PRICING, type MatchResult, type MatcherResult, type MergeOptions, MetricsCollector, type ModelPreflight, ModelsUnreachableError, type MuffledFinder, type MuffledFinding, type MultiToolchainLayerConfig, type Mutator, Mutex, type Oracle, type OracleObservation, type OracleReport, type OracleResult, type OrthogonalityInput, type OrthogonalityResult, OtelExportConfig, OtelExporter, type OtelPipelineHandle, type OtelPipelineOptions, PairwiseSteeringOptimizer, type ParaphraseRobustnessScenarioInput, type ParaphraseRobustnessScenarioResult, type ParseStudentLabel, type PartitionHeldOutOptions, PersonaConfig, type Playbook, type PlaybookEntry, type PoolSlot, type PrReviewAuditCase, type PrReviewBenchmarkSummary, type PrReviewComment, type PrReviewMatchedFinding, type PrReviewOutcome, type PrReviewReferenceFinding, type PrReviewScore, type PrReviewScoreWeights, type PrReviewSeverity, type PrReviewSource, type PreflightModelsOptions, type PreflightOutcome, ProductClient, ProductClientConfig, type PromptHandle, PromptRegistry, type ProposeAutomatedPullRequestInput, type ProposeAutomatedPullRequestResult, type ProvenanceReader, type RecordRunsOptions, type ReferenceMatchResult, type ReferenceReplayAdapter, type ReferenceReplayAdapterFn, type ReferenceReplayAdapterLike, type ReferenceReplayAggregate, type ReferenceReplayCandidate, type ReferenceReplayCase, type ReferenceReplayCaseRun, type ReferenceReplayExecutionScenario, type ReferenceReplayItem, type ReferenceReplayMatch, type ReferenceReplayMatchStrategy, type ReferenceReplayMatcher, type ReferenceReplayPromotionDecision, type ReferenceReplayPromotionPolicy, type ReferenceReplayRun, type ReferenceReplayRunContext, type ReferenceReplayRunOptions, type ReferenceReplayRunStore, type ReferenceReplayScenario, type ReferenceReplayScenarioScore, type ReferenceReplayScore, type ReferenceReplayScoreOptions, type ReferenceReplaySplit, type ReferenceReplaySplitComparison, type ReferenceReplaySteeringRowsOptions, type ReflectionContext, type ReflectionProposal, ReleaseConfidenceScorecard, ReleaseConfidenceThresholds, type RenderStudentPrompt, type RepoRef, type ReviewerMemoryEntry, type ReviewerOutput, type ReviewerPromptInput, type ReviewerSoftFailDefaults, type ReviewerVerificationSummary, type RobustnessResult, Run, type RunCommandInput, type RunCommandResult, type RunDistillationOptions, type RunDistillationResult, RunFilter, RunRecord, type RunRecordBackend, type RunRecordFilter, RunScore, RunScoreWeights, RunSplitTag, SandboxDriver, SandboxHarnessResult, type SandboxJudgeKind, type SandboxJudgeResult, type SandboxJudgeSpec, type SandboxPool, type ScanOptions, Scenario, type ScenarioCost, ScenarioFile, ScenarioRegistry, ScenarioResult, type Scorecard, type ScorecardCell, type ScorecardCellDiff, type ScorecardDiff, type ScorecardEntry, type ScorecardLogLine, type ScoredTarget, type SelfPlayOptions, type SelfPlayProposer, type SelfPlayScorer, type SeriesConvergenceOptions, type SeriesConvergenceResult, Severity, type SignedManifest, type SignedManifestAlgo, type SingleBackendDivergence, SingleBackendError, type SingleBackendField, type SingleBackendReport, type Slo, type SloCheckResult, type SloComparator, type SloReport, type SloSeverity, type SlopCategory, type SlotFactory, type SplitGoldOptions, SteeringBundle, type SteeringOptimizationResult, type SteeringOptimizationRow, type SteeringOptimizationSelector, type SteeringOptimizerBackend, type SteeringOptimizerConfig, type StepAttribution, type SynthesisReason, type SynthesisTarget, TestResult, type ThresholdContract, TokenCounter, type TokenSpec, TraceEmitter, TraceStore, type TracedAnalystOptions, type TracedJudgeOptions, Trajectory, TrajectoryStep, type TrialTrace, TurnMetrics, UI_FINDING_SEVERITIES, UI_LENSES, UNIVERSAL_FINDERS, type UiFinding, type UiFindingScreenshot, type UiFindingSeverity, type UiLens, VerifyContext, type VisualDiffOptions, type VisualDiffResult, type ViteDeployRunnerInput, type WorkspaceAssertion, type WorkspaceAssertionResult, type WorkspaceInspector, type WorkspaceSnapshot, type WranglerDeployRunnerInput, adversarialJudge, aggregateJudgeVerdicts, aggregatePrReviewScore, analyzeAntiSlop, analyzeSeries, appendScorecard, assertCrossFamily, assertModelsServed, assertSingleBackend, assignHeldOutTag, attributeCounterfactuals, bisect, buildAgreementJudge, buildDriverSystemPrompt, buildReflectionPrompt, buildReviewerPrompt, canaryLeakView, canonicalize, causalAttribution, checkBehavioralCanary, checkCanaries, checkSlos, codeExecutionJudge, coherenceJudge, collectionPreserved, commentsForSource, commitBisect, compareReferenceReplay, compilerJudge, computeExperimentStats, costForUsage, createAntiSlopJudge, createCustomJudge, createDefaultReviewer, createDomainExpertJudge, createIntentMatchJudge, createSandboxPool, crossTraceDiff, dataDescriptionBits, decideNextUserTurn, decideReferenceReplayPromotion, decideReferenceReplayRunPromotion, defaultJudges, defaultParseStudentLabel, defaultReferenceReplayMatcher, defaultRenderStudentPrompt, deployGateLayer, diffScorecard, discoverPersonas, distillPlaybook, estimateCost, estimateTokens, evaluateContract, evaluateHypothesis, evaluateOracles, executeScenario, expectAgent, exportRewardModel, extractAssetUrls, extractErrorCount, fieldAgreement, fileContains, fileExists, fileExperimentStore, findAutoMatchNoExpectation, findConstructorCwdDropped, findFallbackToPass, findLiteralTruePass, findSkipCountsAsPass, flowLayer, fnv1a32, formatBenchmarkReport, formatDriverReport, formatFindings, formatScorecardDiff, ghCliClient, gitProvenanceReader, precision as goldenPrecision, hashContent, hashJson, hashToUnit, htmlContainsElement, httpGithubClient, improvementVerdict, inMemoryExperimentStore, inMemoryReferenceReplayStore, inMemoryRunRecordBackend, isModelPriced, isOtelConfigured, jsonShape, jsonlReferenceReplayStore, jsonlRunRecordBackend, judgeFamily, keyPreserved, linterJudge, loadGoldScenarios, loadScorecard, loadScorerFromGrader, localCommandRunner, lowercaseMutator, matchGoldens, mergeLayerResults, modelDescriptionBits, modelPriceKey, multiToolchainLayer, notBlocked, paraphraseRobustness, paraphraseRobustnessScenarios, parseGoldJsonl, parseReflectionResponse, partitionHeldOut, passOrthogonality, pixelDeltaRatio, politenessPrefixMutator, preflightModels, printDriverSummary, index as profile, promptBisect, proposeAutomatedPullRequest, proposeSynthesisTargets, recordRuns, recordRunsToScorecard, referenceReplayRunsToSteeringRows, referenceReplayScenarioToRunScore, regexMatches, renderMarkdownReport, renderPlaybookMarkdown, replayScorerOverCorpus, replayTraceThroughJudge, resetLockedAppendersForTesting, resolveModelPricing, rowCount, rowWhere, runAssertions, runBehavioralCanaries, runCanaries, runCounterfactual, runDistillation, runE2EWorkflow, runExpectations, runIntentMatchJudge, runJudgeFleet, runKeywordCoverageJudge, runKeywordCoverageJudgeUrl, runLiveProof, runReferenceReplay, runScore, runSelfPlay, scanForMuffledGates, scoreContinuity, scorePrReviewComments, scorePrReviewSource, scoreReferenceReplay, securityJudge, sentenceReorderMutator, signManifest, splitGold, statusAdvanced, summarizePrReviewBenchmark, testJudge, textInSnapshot, toLangfuseEnvelope, toPrometheusText, traceJudge, traceJudgeEnsemble, tracedAnalyzeTraces, typoMutator, urlContains, verifyManifest, visualDiff, viteDeployRunner, weightedRecall, whitespaceCollapseMutator, withJudgeRetry, withOtelPipeline, wranglerDeployRunner };