@tangle-network/agent-eval 0.13.0 → 0.14.1

package/README.md

@@ -78,8 +78,96 @@ The recipe for a code-generator eval is in [`SKILL.md` §Minimal working path](.
 | `clients/python/` | First-party Python client (`tangle-agent-eval` on PyPI). Version-locked to npm. | clients/python/README.md |
 | `BenchmarkRunner`, `executeScenario`, `ConvergenceTracker` | Multi-turn scenario execution + cross-run tracking. | SKILL.md |
 | `ExperimentTracker`, `PromptOptimizer`, `bisector` | A/B prompts, optimize steering, bisect regressions. | SKILL.md |
+| `runPromptEvolution`, `createCompositeMutator`, `createSandboxPool`, `createSandboxCodeMutator`, `MutationTelemetry`, `LineageRecorder`, `CostLedger`, `JsonlTrialCache` | Prompt + code evolution loops with bounded sandbox pools, durable JSONL telemetry, plateau-detecting composite mutators, crash-resumable trial cache. | §Evolution loop |
+| `reflective-mutation` (`buildReflectionPrompt`, `parseReflectionResponse`, `DEFAULT_MUTATION_PRIMITIVES`) | Trace-conditioned LLM mutator that reasons over top/bottom trials instead of blind rewrites. | inline JSDoc |
+| `correlationStudy`, `OutcomeStore`, `ProductRegistry` | Meta-eval: do our scores predict deployment outcomes (revenue, retention)? | inline JSDoc |
 | Telemetry (`telemetry/`, `telemetry/file`) | OTLP export, trace replay, file sinks. | inline JSDoc |
 
+## Evolution loop
+
+Closing the loop on a prompt or codebase is **two adapters + a config**. Compose `runPromptEvolution` with `createCompositeMutator` (plateau policy) and you get prompt-only optimization until improvement stalls, then automatic switch to code-channel mutations from a coding agent inside a `SandboxPool`.
+
+```ts
+import {
+  createSandboxPool,
+  createSandboxCodeMutator,
+  createCompositeMutator,
+  buildReflectionPrompt,
+  parseReflectionResponse,
+  runPromptEvolution,
+  MutationTelemetry,
+  LineageRecorder,
+  CostLedger,
+  JsonlTrialCache,
+} from '@tangle-network/agent-eval'
+
+// 1. Prompt mutator — reflective-mutation reasons over top/bottom trials
+const promptMutator = {
+  async mutate({ parent, topTrials, bottomTrials, childCount }) {
+    const ctx = { target: 'forge-prompt', parentPayload: parent.payload, topTrials, bottomTrials, childCount }
+    const reflection = buildReflectionPrompt(ctx)
+    const raw = await yourLlm(reflection)
+    return parseReflectionResponse(raw, childCount).map((p, i) => ({
+      id: `${parent.id}.g${parent.generation + 1}.prompt.${i}`,
+      payload: p.payload,
+      generation: parent.generation + 1,
+      parentId: parent.id,
+      label: p.label,
+      rationale: p.rationale,
+    }))
+  },
+}
+
+// 2. Code mutator — runs a coding agent in a sandbox slot, captures the diff
+const pool = createSandboxPool({
+  size: 4,
+  factory: {
+    async create(id) { return await yourSandboxClient.create({ name: id }) },
+    async reset(slot) { await slot.resource.exec('git reset --hard origin/main && git clean -fd') },
+    async destroy(slot) { await slot.resource.delete() },
+  },
+})
+const codeMutator = createSandboxCodeMutator({
+  pool,
+  runner: async ({ slot, parent, topTrials, bottomTrials }) => {
+    const result = await slot.resource.task(`Improve the prompt at /repo/forge-prompt.ts...`)
+    return [{ ok: true, latencyMs: result.durationMs, costUsd: result.costUsd, artifact: { diff: result.diff } }]
+  },
+  toVariantPayload: (outcome, parent) => ({ ...parent.payload, codeMutation: outcome.artifact }),
+})
+
+// 3. Compose — plateau policy auto-switches when prompt evolution stalls
+const composite = createCompositeMutator({
+  primary: promptMutator,
+  secondary: codeMutator,
+  policy: 'plateau',
+  plateauThreshold: 0.02,
+  plateauPatience: 2,
+})
+
+// 4. Run — durable telemetry to disk, crash-resumable
+const result = await runPromptEvolution({
+  runId: `forge_${Date.now()}`,
+  target: 'forge-prompt',
+  seedVariants: [{ id: 'v0', payload: { text: currentPrompt }, generation: 0, label: 'baseline' }],
+  scenarioIds: referenceCorpus.map(s => s.id),
+  reps: 3,
+  generations: 5,
+  populationSize: 4,
+  scoreAdapter: { /* runs your eval against (variant, scenario, rep) */ },
+  mutateAdapter: composite,
+  cache: new JsonlTrialCache('.evolve/cache.jsonl'),
+  objectives: [
+    { name: 'score', direction: 'maximize', value: a => a.meanScore },
+    { name: 'cost', direction: 'minimize', value: a => a.meanCost },
+  ],
+})
+```
+
+The `MutationTelemetry`, `LineageRecorder`, and `CostLedger` pass into the `code-mutator` (and any consumer that wants them) — they emit append-only JSONL of every attempt (success + failure with reason) and a snapshot lineage tree, so a finished run leaves a forensically complete trail under one directory.
+
+For the full primitive surface and rationale, read each module's JSDoc — `prompt-evolution.ts`, `composite-mutator.ts`, `sandbox-pool.ts`, `code-mutator.ts`, `reflective-mutation.ts`, `evolution-telemetry.ts`.
+
 ## Tech stack
 
 - TypeScript strict, no semicolons, single quotes, 2-space indent

package/dist/index.d.ts

@@ -6402,6 +6402,454 @@ interface PromptEvolutionResult<P = unknown> {
 }
 declare function runPromptEvolution<P>(config: PromptEvolutionConfig<P>): Promise<PromptEvolutionResult<P>>;
 
+/**
+ * concurrency — small primitives the evolution loop needs.
+ *
+ * `Mutex` is a zero-dep async lock with FIFO fairness. The evolution loop
+ * uses it to serialise checkout/build/commit sequences inside a single
+ * pool slot, and to gate concurrent JSONL writers (see
+ * `lockedJsonlReferenceReplayStore`).
+ *
+ * Deliberately minimal — no priority queue, no timeouts. If you need
+ * those, swap to `async-mutex` at the call site.
+ */
+declare class Mutex {
+    private locked;
+    private readonly waiters;
+    acquire(): Promise<() => void>;
+    private release;
+    runExclusive<T>(fn: () => Promise<T> | T): Promise<T>;
+    /** True iff someone holds the lock right now. Diagnostics only. */
+    get isLocked(): boolean;
+    /** Pending waiter count. Diagnostics only. */
+    get pending(): number;
+}
+
+/**
+ * JsonlTrialCache — `TrialCache` backed by a JSONL append-only file so a
+ * crashed `runPromptEvolution` can resume without re-running expensive
+ * trials. Last write wins on key collision; the file is forward-swept at
+ * construction.
+ *
+ * Tail corruption (partial line at the bottom from a hard kill) is
+ * tolerated — we skip unparseable lines and continue.
+ *
+ * The cache surface (`get` / `set`) is synchronous because `TrialCache`
+ * is. Writes are mutex-serialised through a `LockedJsonlAppender`
+ * (kicked off with `void`) so two in-process callers can't tear a long
+ * line that exceeds POSIX `PIPE_BUF`. Cross-process safety still
+ * requires fcntl/flock and is deliberately out of scope.
+ */
+
+declare class JsonlTrialCache implements TrialCache {
+    private readonly map;
+    private readonly path;
+    private readonly appender;
+    constructor(path: string);
+    get(key: string): TrialResult | undefined;
+    set(key: string, value: TrialResult): void;
+    size(): number;
+    /**
+     * Synchronous fallback path for tests / CLI tools that want to be sure
+     * the line is on disk before returning. Bypasses the mutex (single-
+     * threaded callers only).
+     */
+    setSync(key: string, value: TrialResult): void;
+}
+
+/**
+ * LockedJsonlAppender — mutex-serialized JSONL append helper for arbitrary
+ * payloads. The reference-replay store does the same thing for typed
+ * `ReferenceReplayRun` rows; this is the generic version used by
+ * `MutationTelemetry`, `TrialTelemetry`, and any other consumer that wants
+ * append-only durable telemetry without rolling its own lock.
+ *
+ * Locks are per absolute file path (process-local). Cross-process
+ * concurrency is NOT addressed — that's an fcntl/flock problem.
+ */
+declare class LockedJsonlAppender {
+    readonly path: string;
+    private readonly mutex;
+    constructor(path: string);
+    append(entry: unknown): Promise<void>;
+}
+/** Reset all internal mutex state — tests only. */
+declare function resetLockedAppendersForTesting(): void;
+
+/**
+ * evolution-telemetry — durable JSONL/JSON sinks for the evolution loop.
+ *
+ * `runPromptEvolution` exposes generation-level events but doesn't persist
+ * the per-mutation, per-trial, lineage, or cost breakdown. These four
+ * sinks fill that gap so a finished autoresearch run leaves a forensically
+ * complete trail under one directory:
+ *
+ *   - `mutations.jsonl` — every mutate attempt (success + failure) with
+ *     latency, agent steps, diff stats, cost.
+ *   - `trials.jsonl` — every TrialResult including cache hits, with
+ *     provenance (channel, runtime slot, generation).
+ *   - `lineage.json` — variant tree {id → {parent, generation, kind, …}},
+ *     incremental upsert.
+ *   - `cost-ledger.json` — running $ totals per source (mutator-prompt,
+ *     mutator-code, scorer-prompt, scorer-code) plus pool utilisation.
+ *
+ * All writes are mutex-serialised. The append-only sinks (mutations,
+ * trials) survive a hard kill; the snapshot sinks (lineage, cost-ledger)
+ * rewrite on every update so the latest state is always on disk.
+ *
+ * Generic over a payload P so any consumer of `runPromptEvolution<P>` can
+ * record lineage without leaking domain types.
+ */
+
+type MutationChannel = 'prompt' | 'code';
+interface MutationAttempt {
+    ts: number;
+    channel: MutationChannel;
+    generation: number;
+    parentId: string;
+    /** Successful child variant id, or null if the attempt failed. */
+    childId: string | null;
+    ok: boolean;
+    /**
+     * One of: 'parse_failure' | 'typecheck_failure' | 'no_changes' |
+     * 'agent_error' | 'commit_failure' | 'no_api_key' | 'no_valid_proposals'
+     * | 'reproduce_parent_failed' | 'branch_failed' | 'other'.
+     * Free-form to allow consumer-specific reasons.
+     */
+    failureReason?: string;
+    /** Free-form description of what the agent said it did. */
+    description?: string;
+    /** Latency of the LLM call (ms). */
+    latencyMs: number;
+    /** Bytes of generated diff (code channel only). */
+    diffBytes?: number;
+    /** Files touched (code channel only). */
+    filesTouched?: number;
+    /** Steps the agent ran (tool calls). */
+    agentSteps?: number;
+    /** Approx $ spent on this mutation (LLM tokens). */
+    costUsd?: number;
+    /** Runtime slot used (code channel only). */
+    runtimeSandboxId?: string;
+}
+declare class MutationTelemetry {
+    private readonly appender;
+    constructor(path: string);
+    record(attempt: MutationAttempt): Promise<void>;
+}
+interface TrialAttempt {
+    ts: number;
+    channel: MutationChannel;
+    generation: number;
+    variantId: string;
+    scenarioId: string;
+    rep: number;
+    ok: boolean;
+    score: number;
+    costUsd: number;
+    durationMs: number;
+    cached: boolean;
+    runtimeSandboxId?: string;
+    error?: string;
+    metrics?: Record<string, number>;
+}
+declare class TrialTelemetry {
+    private readonly appender;
+    constructor(path: string);
+    record(attempt: TrialAttempt): Promise<void>;
+}
+type LineageKind = 'seed' | 'prompt' | 'code';
+interface LineageNode {
+    id: string;
+    parentId: string | null;
+    generation: number;
+    kind: LineageKind;
+    rationale?: string;
+    /** Filled when scoring lands. */
+    meanScore?: number;
+    promotedToFrontier?: boolean;
+    /**
+     * The variant payload (e.g. evolved persona text, code mutation diff).
+     * Persisted so a winning variant can be reproduced after a run completes
+     * without re-running the optimizer. Optional — pass `omitPayload: true` to
+     * `upsertVariant` for cases where the payload is too large to log.
+     */
+    payload?: unknown;
+}
+/**
+ * `kindOf` decides whether a variant is a seed (no parent), code mutation,
+ * or prompt mutation. Default looks at `variant.payload.codeMutation` —
+ * that field is part of the audit-bench convention but cheap enough to
+ * accept any payload that mirrors it. Override by passing your own.
+ */
+type LineageKindResolver<P> = (variant: PromptVariant<P>) => LineageKind;
+/**
+ * Persistence shape:
+ *
+ *   `<path>`           — JSONL of upserts (event log). Each line is a
+ *                        partial node; replay folds them into the current
+ *                        state. Append-only, so cost is O(1) per upsert
+ *                        instead of the previous O(n²) full rewrite.
+ *   `<path>.snapshot`  — Optional consolidated snapshot, written on
+ *                        demand via `compact()` (e.g. at end of run).
+ *                        Read by external tools that don't want to
+ *                        replay the log.
+ *
+ * Loaded at construction time: if `<path>.snapshot` exists, parse it
+ * first; then replay any newer log lines on top. Falls back to log-only
+ * when no snapshot is present.
+ */
+declare class LineageRecorder<P = unknown> {
+    private readonly path;
+    private readonly snapshotPath;
+    private readonly mutex;
+    private readonly nodes;
+    private readonly kindOf;
+    constructor(path: string, kindOf?: LineageKindResolver<P>);
+    upsert(node: LineageNode): Promise<void>;
+    upsertVariant(variant: PromptVariant<P>, opts?: {
+        omitPayload?: boolean;
+    }): Promise<void>;
+    snapshot(): LineageNode[];
+    /**
+     * Write the current consolidated state to `<path>.snapshot` so external
+     * tools can read it without replaying the event log. Idempotent.
+     */
+    compact(): Promise<void>;
+}
+/** Per-generation cost rollup. Same shape as the totals, scoped to one gen. */
+interface CostLedgerGeneration {
+    generation: number;
+    mutatorPromptUsd: number;
+    mutatorCodeUsd: number;
+    scorerPromptUsd: number;
+    scorerCodeUsd: number;
+    trialsCounted: number;
+    cachedTrials: number;
+}
+interface CostLedgerSnapshot {
+    totalUsd: number;
+    mutatorPromptUsd: number;
+    mutatorCodeUsd: number;
+    scorerPromptUsd: number;
+    scorerCodeUsd: number;
+    trialsCounted: number;
+    cachedTrials: number;
+    poolBusyMs?: number;
+    poolUtilizationPct?: number;
+    /** Per-generation breakdown, sorted ascending. Empty when generations
+     *  weren't supplied to addMutation/addTrial. */
+    byGeneration: CostLedgerGeneration[];
+}
+declare class CostLedger {
+    private totals;
+    private readonly path;
+    private readonly mutex;
+    constructor(path: string);
+    private genBucket;
+    addMutation(channel: MutationChannel, usd: number, opts?: {
+        generation?: number;
+    }): Promise<void>;
+    addTrial(channel: MutationChannel, usd: number, cached: boolean, opts?: {
+        generation?: number;
+    }): Promise<void>;
+    setPoolUtilization(busyMs: number, totalMs: number): Promise<void>;
+    snapshot(): CostLedgerSnapshot;
+    private persist;
+}
+
+/**
+ * createCompositeMutator — combines two `MutateAdapter<P>`s under a policy.
+ *
+ *   prompt-only — every generation runs `primary` (typical: a reflective
+ *                 prompt mutator). The default.
+ *   secondary-only — every generation runs `secondary` (typical: a coding
+ *                    agent that edits the harness itself). Slow + expensive.
+ *   alternate    — even gens run `primary`, odd gens run `secondary`.
+ *   plateau      — start with `primary`; switch to a 50/50 split between
+ *                  `primary` and `secondary` after K gens with less than
+ *                  Δ improvement (auto-detect when prompt evolution has
+ *                  hit a structural ceiling).
+ *
+ * Naming is generic: the original audit-bench version called the channels
+ * "prompt" and "code" — those are the canonical use cases, but the
+ * primitive doesn't care what each mutator actually does.
+ */
+
+type CompositePolicy = 'primary-only' | 'secondary-only' | 'alternate' | 'plateau';
+interface CreateCompositeMutatorOpts<P> {
+    primary: MutateAdapter<P>;
+    secondary?: MutateAdapter<P>;
+    policy: CompositePolicy;
+    /** For 'plateau': minimum improvement (Δ meanScore) to count as progress. Default 0.02. */
+    plateauThreshold?: number;
+    /** For 'plateau': consecutive gens without progress that trigger split mode. Default 2. */
+    plateauPatience?: number;
+    /** Optional progress hook. */
+    onPolicyDecision?: (info: {
+        generation: number;
+        chose: 'primary' | 'secondary' | 'split';
+        reason: string;
+    }) => void;
+}
+declare function createCompositeMutator<P>(opts: CreateCompositeMutatorOpts<P>): MutateAdapter<P>;
+
+/**
+ * SandboxPool — bounded checkout/release pool for mutation slots.
+ *
+ * The composite-mutator's `code` channel needs an isolated workspace per
+ * mutation attempt: a git worktree, a sandbox container, a tmpdir clone —
+ * whatever the consumer's runtime is. Without a pool, every consumer
+ * re-implements the same machinery (mint N slots, check one out per
+ * mutation, reset before reuse, drain at the end, track utilisation for
+ * the cost ledger). This primitive ships that machinery so consumers
+ * supply only a `SlotFactory`.
+ *
+ * Generic over a slot resource `T` so the same pool serves git worktrees
+ * (T = path), Tangle sandboxes (T = SandboxBox), or anything else with
+ * the create/reset/destroy lifecycle.
+ *
+ * Concurrency: FIFO via the shared `Mutex` primitive. Each `checkout()`
+ * either takes an idle slot or queues until one is released. Lifecycle
+ * is single-process — multi-process pools need external coordination
+ * (file locks, etc.) and are deliberately out of scope.
+ */
+interface PoolSlot<T> {
+    /** Stable id assigned at slot creation. Use for telemetry / lineage. */
+    readonly id: string;
+    /** Consumer-defined resource. */
+    readonly resource: T;
+}
+interface SlotFactory<T> {
+    /** Build a new slot. Called lazily as the pool grows up to `size`. */
+    create(slotId: string): Promise<T>;
+    /**
+     * Reset a slot to a clean state before reuse. Called BEFORE every
+     * checkout returns it (including the first — so the factory's
+     * `create` can leave the slot dirty and let `reset` normalise).
+     * Optional; default is a no-op.
+     */
+    reset?(slot: PoolSlot<T>): Promise<void>;
+    /** Tear the slot down. Called by `drain()`. */
+    destroy(slot: PoolSlot<T>): Promise<void>;
+}
+interface SandboxPool<T> {
+    /**
+     * Take a slot. If all slots are busy, the promise resolves when one
+     * is released. Always pair with the returned `release` (or wrap with
+     * `withSlot`).
+     */
+    checkout(): Promise<{
+        slot: PoolSlot<T>;
+        release: () => void;
+    }>;
+    /**
+     * Run `fn` with a checked-out slot, releasing on completion or throw.
+     * The convenience wrapper most callers should use.
+     */
+    withSlot<R>(fn: (slot: PoolSlot<T>) => Promise<R>): Promise<R>;
+    /** Destroy every slot. Idempotent. */
+    drain(): Promise<void>;
+    /** How many slots have been minted (≤ `size`). */
+    poolSize(): number;
+    /** How many checkouts are currently outstanding. */
+    activeCheckouts(): number;
+    /** Snapshot of busy/total durations for the cost ledger. */
+    utilization(): {
+        busyMs: number;
+        totalMs: number;
+        checkouts: number;
+    };
+}
+interface CreateSandboxPoolOpts<T> {
+    /** Maximum concurrent slots. Slots are minted on first need, not eagerly. */
+    size: number;
+    factory: SlotFactory<T>;
+}
+declare function createSandboxPool<T>(opts: CreateSandboxPoolOpts<T>): SandboxPool<T>;
+
+/**
+ * createSandboxCodeMutator — `MutateAdapter<P>` that runs a coding agent
+ * inside a SandboxPool slot to produce code-channel variants.
+ *
+ * Composable shape (matches `reflective-mutation.ts`'s separation of
+ * "build the prompt" from "run the model"):
+ *
+ *   pool      → where mutations execute (any SlotFactory)
+ *   runner    → consumer-supplied: invokes the coding agent in a slot,
+ *               returns the diff/branch/whatever as `CodeMutationOutcome`s
+ *   toVariantPayload → maps outcome → P (consumer encodes the diff their
+ *                      way — patch string, branch ref, file map, etc)
+ *
+ * What this primitive owns (so consumers don't reinvent it every time):
+ *   - Pool checkout / release with reset between attempts
+ *   - Per-attempt mutex so a single slot can't be invoked concurrently
+ *   - Telemetry write-through (mutations.jsonl, lineage.json,
+ *     cost-ledger.json) when sinks are passed
+ *   - Stable child-id generation
+ *   - Failure capture (every attempt produces either a successful child
+ *     or a recorded failure with reason — never a silent drop)
+ *
+ * Consumers stay focused on the actual interesting parts: building the
+ * agent prompt, running the agent, capturing the diff.
+ */
+
+/**
+ * Result of one coding-agent invocation. The runner produces 1..N of
+ * these per `runner` call (a single agent session can sometimes
+ * produce multiple sibling diffs cheaply — runner decides).
+ */
+interface CodeMutationOutcome {
+    ok: boolean;
+    /** Stable id for the child variant if `ok`. The mutator falls back to
+     *  a generated id when omitted. */
+    childId?: string;
+    /** Free-form one-liner: "tightened tool descriptions in forge-tools.ts". */
+    description?: string;
+    /** What the runner was trying to fix (carried into PromptVariant.rationale). */
+    rationale?: string;
+    /** Caller-defined diff payload. Mapped into the variant's payload by
+     *  `toVariantPayload`; agent-eval treats it as opaque. */
+    artifact?: unknown;
+    /** When ok === false. Free-form: 'parse_failure' / 'agent_error' /
+     *  'no_changes' / 'commit_failed' / etc. */
+    failureReason?: string;
+    /** Telemetry stats. */
+    diffBytes?: number;
+    filesTouched?: number;
+    agentSteps?: number;
+    costUsd?: number;
+    latencyMs: number;
+}
+type CodeMutationRunner<T, P> = (args: {
+    slot: PoolSlot<T>;
+    parent: PromptVariant<P>;
+    parentAggregate: VariantAggregate;
+    topTrials: TrialResult[];
+    bottomTrials: TrialResult[];
+    childCount: number;
+    generation: number;
+}) => Promise<CodeMutationOutcome[]>;
+interface CreateSandboxCodeMutatorOpts<T, P> {
+    pool: SandboxPool<T>;
+    runner: CodeMutationRunner<T, P>;
+    /**
+     * Map an outcome into the variant payload `P`. Lets the consumer
+     * encode the diff however they want (file map, patch string, branch
+     * ref, snapshot id) without agent-eval taking a stance.
+     */
+    toVariantPayload(outcome: CodeMutationOutcome, parent: PromptVariant<P>): P;
+    /** Optional telemetry sinks. */
+    mutationTelemetry?: MutationTelemetry;
+    costLedger?: CostLedger;
+    lineage?: LineageRecorder<P>;
+    /** Override id generation. Default: `${parent.id}.g${generation}.code.${i}`. */
+    childIdFor?(parent: PromptVariant<P>, generation: number, index: number): string;
+    /** Default label for the variant (visible in reports). */
+    labelFor?(outcome: CodeMutationOutcome, parent: PromptVariant<P>, generation: number, index: number): string;
+}
+declare function createSandboxCodeMutator<T, P>(opts: CreateSandboxCodeMutatorOpts<T, P>): MutateAdapter<P>;
+
 /**
  * GoldenMatcher — fuzzy matcher for "did the agent produce the expected things?".
  *
@@ -6672,4 +7120,4 @@ interface ReflectionProposal {
  */
 declare function parseReflectionResponse(raw: string, maxProposals?: number): ReflectionProposal[];
 
-export { type ActiveLearningOptions, type AdapterRun, AgentDriver, type AgentDriverConfig, type AlignmentOp, type AntiSlopConfig, type AntiSlopIssue, type AntiSlopReport, type Artifact, type ArtifactCheck, type Artifact$1 as ArtifactCheckArtifact, type ArtifactResult, type ArtifactValidator, AxGepaSteeringOptimizer, type AxSteeringOptimizerConfig, type BaselineOptions, type BaselineReport, BehaviorAssertion, type BenchmarkReport, BenchmarkRunner, type BenchmarkRunnerConfig, type BestOfNResult, type BisectOptions, type BisectResult, type BisectStep, type BootstrapOptions, type BootstrapResult, BudgetBreachError, type BudgetBreachFinding, type BudgetBreachReport, BudgetGuard, type BudgetLedgerEntry, type BudgetSpec, BuilderSession, type BuilderSessionInit, type CalibrationBin, type CalibrationOptions, type CalibrationReport, type CalibrationResult, CallExpectation, type CanaryLeak, type CandidateScenario, type CandidateScore, type CausalAttributionReport, type ChatSummary, type CheckResult, type CollectedArtifacts, type CommandRunner, type CompletionCriterion, type ConceptComplexity, type ConceptFinding, type ConceptSpec, type ConceptWeightStrategy, type ContinuityCheck, type ContinuityCheckResult, type ContinuityReport, type ContinuitySnapshotPair, type ContractMetric, type ContractReport, ConvergenceTracker, type CorrelationReport, type CorrelationResult, type CorrelationStudyOptions, type CorrelationStudyResult, type CostEntry, type CostSummary, CostTracker, type CounterfactualContext, type CounterfactualMutation, type CounterfactualResult, type CounterfactualRunner, type CreateDefaultReviewerOptions, type CrossTraceDiff, type CrossTraceDiffOptions, D1ExperimentStore, type D1ExperimentStoreOptions, type D1Like, type D1PreparedStatementLike, DEFAULT_AGENT_SLOS, DEFAULT_COMPLEXITY_WEIGHTS, DEFAULT_RULES as DEFAULT_FAILURE_RULES, DEFAULT_FINDERS, DEFAULT_HARNESS_OBJECTIVES, DEFAULT_MUTATION_PRIMITIVES, DEFAULT_MUTATORS, DEFAULT_REDACTION_RULES, DEFAULT_RED_TEAM_CORPUS, DEFAULT_RUN_SCORE_WEIGHTS, DEFAULT_SEVERITY_WEIGHTS, Dataset, type DatasetDifficulty, type DatasetManifest, type DatasetProvenance, type DatasetScenario, type DatasetSplit, type DeployFamily, type DeployGateLayerInput, type DeployRunResult, type DeployRunner, type DeploymentOutcome, type DirEntry, type Direction, type DivergenceOptions, type DivergenceReport, DockerSandboxDriver, type DriverResult, type DriverState, DualAgentBench, type DualAgentBenchConfig, type DualAgentReport, type DualAgentRound, type DualAgentScenario, type DualAgentScenarioResult, ERROR_COUNT_PATTERNS, type ErrorCountPattern, type EuRiskClass, type EvalMetricSpec, type EvalResult, type EventFilter, type EventKind, type EvolutionRound, type PromptVariant as EvolvableVariant, type ExecutorConfig, type Expectation, type Experiment, type Run$1 as ExperimentRun, type ExperimentStore, ExperimentTracker, type ExportedRewardModel, type ExtractOptions, type ExtractResult, FAILURE_CLASSES, type FactorContribution, type FactorialCell, type FailureClass, type FailureClassification, type FailureCluster, type FailureClusterReport, type FailureContext, type FailureRule, type FeedbackPattern, FileSystemExperimentStore, type FileSystemExperimentStoreOptions, FileSystemOutcomeStore, type FileSystemOutcomeStoreOptions, FileSystemTraceStore, type FileSystemTraceStoreOptions, type Finding, type FlowAction, type FlowLayerEnv, type FlowLayerFactoryInput, type FlowRunner, type FlowRunnerStepResult, type FlowSpec, type FlowStep, type GenerationReport, type GenericSpan, type GoldenItem, type GoldenSeverity, type GoldenSpec, type GovernanceContext, type GovernanceFinding, type GovernanceReport, type GradedStep, type HarnessAdapter, type HarnessConfig, type HarnessExperimentConfig, type HarnessExperimentResult, type HarnessIntervention, type HarnessRunRequest, type HarnessRunResult, type HarnessScenario, type HarnessSelection, type HarnessVariant, type HarnessVariantReport, HoldoutAuditor, HoldoutLockedError, type HostedJudgeConfig, type HostedJudgeDimension, type HostedJudgeRequest, type HostedJudgeResponse, type HostedRunCriticConfig, type HostedRunScoreRequest, type HostedRunScoreResponse, type HypothesisManifest, type HypothesisResult, INTENT_MATCH_JUDGE_VERSION, type ImageData, InMemoryExperimentStore, InMemoryOutcomeStore, InMemoryTraceStore, InMemoryTrialCache, InMemoryWorkspaceInspector, type InferenceScorer, type InspectorContext, type IntentMatchInput, type IntentMatchOptions, type IntentMatchResult, type InteractionContribution, type JudgeAgreementReport, type JudgeConfig, type JudgeFleetOptions, type JudgeFn, type JudgeInput, type JudgePair, type JudgeReplayGateArgs, type JudgeReplayResult, type JudgeRubric, JudgeRunner, type JudgeScore, type JudgeSpan, type KeywordConceptSpec, type KeywordCoverageFinding, type KeywordCoverageOptions, type KeywordCoverageResult, type LangfuseEnvelope, type LangfuseGeneration, type LangfuseScore, type Layer, type LayerCorrelation, type LayerResult, type LayerStatus, LlmCallError, type LlmCallRequest, type LlmCallResult, LlmClient, type LlmClientOptions, type LlmJsonCall, type LlmMessage, type LlmReviewerConfig, type LlmSpan, type LlmUsage, MODEL_PRICING, type MatchResult, type MatcherResult, type MeasurementPolicy, type MergeOptions, type Message, type MetricSamples, type MetricVerdict, MetricsCollector, type MuffledFinder, type MuffledFinding, MultiLayerVerifier, type MultiToolchainLayerConfig, type MutateAdapter, type Mutator, OTEL_AGENT_EVAL_SCOPE, type Objective, type OptimizationConfig, type OptimizationExample, OptimizationLoop, type OptimizationLoopConfig, type OptimizationLoopResult, type OptimizationResult, type Oracle, type OracleObservation, type OracleReport, type OracleResult, type OrthogonalityInput, type OrthogonalityResult, type OtlpExport, type OtlpResourceSpans, type OtlpSpan, type OutcomeFilter, type OutcomePair, type OutcomeStore, type PairwiseComparison, PairwiseSteeringOptimizer, type ParetoResult, type PersonaConfig, type Playbook, type PlaybookEntry, type PositionalBiasResult, type PrmGradedTrace, PrmGrader, type PrmTrainingSample, ProductClient, type ProductClientConfig, type ProjectKind, ProjectRegistry, type ProjectSummary, type ProjectTimelineEntry, type PromptEvolutionConfig, type PromptEvolutionEvent, type PromptEvolutionResult, type PromptHandle, PromptOptimizer, PromptRegistry, type TrialResult as PromptTrialResult, type PromptVariant$1 as PromptVariant, type ProposeFn, type ProposeInput, type ProposeOutput, type ProposeReviewConfig, type ProposeReviewReport, type ProposeReviewShot, REDACTION_VERSION, type RedTeamCase, type RedTeamCategory, type RedTeamFinding, type RedTeamPayload, type RedTeamReport, type RedactionReport, type RedactionRule, type ReferenceMatchResult, type ReferenceReplayAdapter, type ReferenceReplayAdapterFn, type ReferenceReplayAdapterLike, type ReferenceReplayAggregate, type ReferenceReplayCandidate, type ReferenceReplayCase, type ReferenceReplayCaseRun, type ReferenceReplayExecutionScenario, type ReferenceReplayItem, type ReferenceReplayMatch, type ReferenceReplayMatchStrategy, type ReferenceReplayMatcher, type ReferenceReplayPromotionDecision, type ReferenceReplayPromotionPolicy, type ReferenceReplayRun, type ReferenceReplayRunContext, type ReferenceReplayRunOptions, type ReferenceReplayRunStore, type ReferenceReplayScenario, type ReferenceReplayScenarioScore, type ReferenceReplayScore, type ReferenceReplayScoreOptions, type ReferenceReplaySplit, type ReferenceReplaySplitComparison, type ReferenceReplaySteeringRowsOptions, type ReflectionContext, type ReflectionProposal, type RegressionOptions, type RegressionSpec, type RetrievalSpan, type Review, type ReviewFn, type ReviewInput, type ReviewMemoryEntry, type ReviewMemoryStore, type ReviewerMemoryEntry, type ReviewerOutput, type ReviewerPromptInput, type ReviewerSoftFailDefaults, type ReviewerVerificationSummary, type RobustnessResult, type RouteMap, type RubricDimension, type Run, type RunAppScenarioOptions, type RunCommandInput, type RunCommandResult, type RunConfig, RunCritic, type RunCriticOptions, type RunDiff, type RunFilter, type RunLayer, type RunOutcome, type RunScore, type RunScoreWeights, type RunStatus, type RunTrace, SEMANTIC_CONCEPT_JUDGE_VERSION, type SandboxDriver, SandboxHarness, type SandboxHarnessResult, type SandboxJudgeKind, type SandboxJudgeResult, type SandboxJudgeSpec, type SandboxResult, type SandboxSpan, type ScanOptions, type Scenario, type ScenarioAggregate, type ScenarioCost, type ScenarioFile, ScenarioRegistry, type ScenarioResult, type ScoreAdapter, type ScoredTarget, type SelfPlayOptions, type SelfPlayProposer, type SelfPlayScorer, type SelfPreferenceResult, type SemanticConceptJudgeInput, type SemanticConceptJudgeOptions, type SemanticConceptJudgeResult, type SeriesConvergenceOptions, type SeriesConvergenceResult, type Severity, type ShipOptions, type SignedManifest, type SliceOptions, type Slo, type SloCheckResult, type SloComparator, type SloReport, type SloSeverity, type SlopCategory, type Span, type SpanBase, type SpanFilter, type SpanHandle, type SpanKind, type SpanStatus, type SteeringBundle, type SteeringDelta, type SteeringEvaluation, type SteeringOptimizationResult, type SteeringOptimizationRow, type SteeringOptimizationSelector, type SteeringOptimizerBackend, type SteeringOptimizerConfig, type SteeringRolePrompt, type SteeringVariantReport, type StepAttribution, type StepContext, type StepRubric, type StuckLoopFinding, type StuckLoopOptions, type StuckLoopReport, SubprocessSandboxDriver, type SubprocessSandboxDriverOptions, type SynthesisReason, type SynthesisTarget, TRACE_SCHEMA_VERSION, type TestGradedRunOptions, type TestGradedRunResult, type TestGradedScenario, type TestOutputParser, type TestResult, type ThreeLayerProjectReport, type ThresholdContract, TokenCounter, type TokenSpec, type ToolSpan, type ToolStats, type ToolUseMetrics, type ToolUseOptions, type ToolWasteFinding, type ToolWasteOptions, type ToolWasteReport, TraceEmitter, type TraceEmitterOptions, type TraceEvent, type TraceStore, type Trajectory, type TrajectoryStep, type TrialCache, type TrialTrace, type Turn, type TurnMetrics, type TurnResult, UNIVERSAL_FINDERS, type UseCaseSignals, type ValidationContext, type ValidationIssue, type ValidationResult, type VariantAggregate, type VariantScore, type VerbosityBiasResult, type Verdict, type Verification, type VerificationReport, type VerifyContext, type VerifyFn, type VerifyOptions, type VisualDiffOptions, type VisualDiffResult, type ViteDeployRunnerInput, type WorkflowTopology, type WorkspaceAssertion, type WorkspaceAssertionResult, type WorkspaceInspector, type WorkspaceSnapshot, adversarialJudge, aggregateLlm, aggregateRunScore, analyzeAntiSlop, analyzeSeries, argHash, attributeCounterfactuals, benjaminiHochberg, bisect, bonferroni, bootstrapCi, budgetBreachView, buildReflectionPrompt, buildReviewerPrompt, buildTrajectory, byteLengthRange, calibrateJudge, calibrationCurve, callLlm, callLlmJson, canaryLeakView, causalAttribution, checkCanaries, checkSlos, clamp01, classifyEuAiRisk, classifyFailure, codeExecutionJudge, cohensD, coherenceJudge, collectionPreserved, commitBisect, compareReferenceReplay, compareToBaseline, compilerJudge, composeParsers, composeValidators, computeToolUseMetrics, confidenceInterval, containsAll, correlateLayers, correlationStudy, createAntiSlopJudge, createCustomJudge, createDefaultReviewer, createDomainExpertJudge, createIntentMatchJudge, createLlmReviewer, createSemanticConceptJudge, crossTraceDiff, crowdingDistance, decideReferenceReplayPromotion, decideReferenceReplayRunPromotion, defaultJudges, defaultReferenceReplayMatcher, deployGateLayer, distillPlaybook, dominates, estimateCost, estimateTokens, euAiActReport, evaluateContract, evaluateHypothesis, evaluateOracles, executeScenario, expectAgent, exportRewardModel, exportRunAsOtlp, exportTrainingData, extractAssetUrls, extractErrorCount, failureClusterView, fileContains, fileExists, findAutoMatchNoExpectation, findConstructorCwdDropped, findFallbackToPass, findLiteralTruePass, findSkipCountsAsPass, firstDivergenceView, flowLayer, formatBenchmarkReport, formatDriverReport, formatFindings, precision as goldenPrecision, gradeSemanticStatus, groupBy, hashContent, hashScenarios, htmlContainsElement, inMemoryReferenceReplayStore, inMemoryReviewStore, interRaterReliability, iqr, isJudgeSpan, isLlmSpan, isPrmVerdict, isRetrievalSpan, isSandboxSpan, isToolSpan, jestTestParser, jsonHasKeys, jsonShape, jsonlReferenceReplayStore, jsonlReviewStore, judgeAgreementView, judgeReplayGate, judgeSpans, keyPreserved, linterJudge, llmSpanFromProvider, llmSpans, loadScorerFromGrader, localCommandRunner, lowercaseMutator, mannWhitneyU, matchGoldens, mergeLayerResults, mergeSteeringBundle, multiToolchainLayer, nistAiRmfReport, nonRefusalRubric, normalizeScores, notBlocked, outputLengthRubric, pairedTTest, paraphraseRobustness, paretoFrontier, paretoFrontierWithCrowding, parseReflectionResponse, partialCredit, passOrthogonality, pixelDeltaRatio, politenessPrefixMutator, positionalBias, printDriverSummary, prmBestOfN, prmEnsembleBestOfN, probeLlm, promptBisect, proposeSynthesisTargets, pytestTestParser, redTeamDataset, redTeamReport, redactString, redactValue, referenceReplayRunsToSteeringRows, referenceReplayScenarioToRunScore, regexMatch, regexMatches, regressionView, renderMarkdown, renderMarkdownReport, renderPlaybookMarkdown, renderSteeringText, replayScorerOverCorpus, replayTraceThroughJudge, requiredSampleSize, resumeBuilderSession, rowCount, rowWhere, runAssertions, runCounterfactual, runE2EWorkflow, runExpectations, runFailureClass, runHarnessExperiment, runIntentMatchJudge, runJudgeFleet, runKeywordCoverageJudge, runKeywordCoverageJudgeUrl, runPromptEvolution, runProposeReview, runReferenceReplay, runSelfPlay, runSemanticConceptJudge, runTestGradedScenario, runsForScenario, scalarScore, scanForMuffledGates, scoreAllProjects, scoreContinuity, scoreProject, scoreRedTeamOutput, scoreReferenceReplay, securityJudge, selectHarnessVariant, selfPreference, sentenceReorderMutator, signManifest, soc2Report, statusAdvanced, stripFencedJson, stuckLoopView, summarize, summarizeHarnessResults, testJudge, textInSnapshot, toLangfuseEnvelope, toNdjson, toPrometheusText, toolIntentAlignmentRubric, toolNamesForRun, toolNonRedundantRubric, toolSpans, toolSuccessRubric, toolWasteView, typoMutator, urlContains, verbosityBias, verifyManifest, visualDiff, viteDeployRunner, vitestTestParser, weightedMean, weightedRecall, welchsTTest, whitespaceCollapseMutator, wilcoxonSignedRank };
+export { type ActiveLearningOptions, type AdapterRun, AgentDriver, type AgentDriverConfig, type AlignmentOp, type AntiSlopConfig, type AntiSlopIssue, type AntiSlopReport, type Artifact, type ArtifactCheck, type Artifact$1 as ArtifactCheckArtifact, type ArtifactResult, type ArtifactValidator, AxGepaSteeringOptimizer, type AxSteeringOptimizerConfig, type BaselineOptions, type BaselineReport, BehaviorAssertion, type BenchmarkReport, BenchmarkRunner, type BenchmarkRunnerConfig, type BestOfNResult, type BisectOptions, type BisectResult, type BisectStep, type BootstrapOptions, type BootstrapResult, BudgetBreachError, type BudgetBreachFinding, type BudgetBreachReport, BudgetGuard, type BudgetLedgerEntry, type BudgetSpec, BuilderSession, type BuilderSessionInit, type CalibrationBin, type CalibrationOptions, type CalibrationReport, type CalibrationResult, CallExpectation, type CanaryLeak, type CandidateScenario, type CandidateScore, type CausalAttributionReport, type ChatSummary, type CheckResult, type CodeMutationOutcome, type CodeMutationRunner, type CollectedArtifacts, type CommandRunner, type CompletionCriterion, type CompositePolicy, type ConceptComplexity, type ConceptFinding, type ConceptSpec, type ConceptWeightStrategy, type ContinuityCheck, type ContinuityCheckResult, type ContinuityReport, type ContinuitySnapshotPair, type ContractMetric, type ContractReport, ConvergenceTracker, type CorrelationReport, type CorrelationResult, type CorrelationStudyOptions, type CorrelationStudyResult, type CostEntry, CostLedger, type CostLedgerGeneration, type CostLedgerSnapshot, type CostSummary, CostTracker, type CounterfactualContext, type CounterfactualMutation, type CounterfactualResult, type CounterfactualRunner, type CreateCompositeMutatorOpts, type CreateDefaultReviewerOptions, type CreateSandboxCodeMutatorOpts, type CreateSandboxPoolOpts, type CrossTraceDiff, type CrossTraceDiffOptions, D1ExperimentStore, type D1ExperimentStoreOptions, type D1Like, type D1PreparedStatementLike, DEFAULT_AGENT_SLOS, DEFAULT_COMPLEXITY_WEIGHTS, DEFAULT_RULES as DEFAULT_FAILURE_RULES, DEFAULT_FINDERS, DEFAULT_HARNESS_OBJECTIVES, DEFAULT_MUTATION_PRIMITIVES, DEFAULT_MUTATORS, DEFAULT_REDACTION_RULES, DEFAULT_RED_TEAM_CORPUS, DEFAULT_RUN_SCORE_WEIGHTS, DEFAULT_SEVERITY_WEIGHTS, Dataset, type DatasetDifficulty, type DatasetManifest, type DatasetProvenance, type DatasetScenario, type DatasetSplit, type DeployFamily, type DeployGateLayerInput, type DeployRunResult, type DeployRunner, type DeploymentOutcome, type DirEntry, type Direction, type DivergenceOptions, type DivergenceReport, DockerSandboxDriver, type DriverResult, type DriverState, DualAgentBench, type DualAgentBenchConfig, type DualAgentReport, type DualAgentRound, type DualAgentScenario, type DualAgentScenarioResult, ERROR_COUNT_PATTERNS, type ErrorCountPattern, type EuRiskClass, type EvalMetricSpec, type EvalResult, type EventFilter, type EventKind, type EvolutionRound, type PromptVariant as EvolvableVariant, type ExecutorConfig, type Expectation, type Experiment, type Run$1 as ExperimentRun, type ExperimentStore, ExperimentTracker, type ExportedRewardModel, type ExtractOptions, type ExtractResult, FAILURE_CLASSES, type FactorContribution, type FactorialCell, type FailureClass, type FailureClassification, type FailureCluster, type FailureClusterReport, type FailureContext, type FailureRule, type FeedbackPattern, FileSystemExperimentStore, type FileSystemExperimentStoreOptions, FileSystemOutcomeStore, type FileSystemOutcomeStoreOptions, FileSystemTraceStore, type FileSystemTraceStoreOptions, type Finding, type FlowAction, type FlowLayerEnv, type FlowLayerFactoryInput, type FlowRunner, type FlowRunnerStepResult, type FlowSpec, type FlowStep, type GenerationReport, type GenericSpan, type GoldenItem, type GoldenSeverity, type GoldenSpec, type GovernanceContext, type GovernanceFinding, type GovernanceReport, type GradedStep, type HarnessAdapter, type HarnessConfig, type HarnessExperimentConfig, type HarnessExperimentResult, type HarnessIntervention, type HarnessRunRequest, type HarnessRunResult, type HarnessScenario, type HarnessSelection, type HarnessVariant, type HarnessVariantReport, HoldoutAuditor, HoldoutLockedError, type HostedJudgeConfig, type HostedJudgeDimension, type HostedJudgeRequest, type HostedJudgeResponse, type HostedRunCriticConfig, type HostedRunScoreRequest, type HostedRunScoreResponse, type HypothesisManifest, type HypothesisResult, INTENT_MATCH_JUDGE_VERSION, type ImageData, InMemoryExperimentStore, InMemoryOutcomeStore, InMemoryTraceStore, InMemoryTrialCache, InMemoryWorkspaceInspector, type InferenceScorer, type InspectorContext, type IntentMatchInput, type IntentMatchOptions, type IntentMatchResult, type InteractionContribution, JsonlTrialCache, type JudgeAgreementReport, type JudgeConfig, type JudgeFleetOptions, type JudgeFn, type JudgeInput, type JudgePair, type JudgeReplayGateArgs, type JudgeReplayResult, type JudgeRubric, JudgeRunner, type JudgeScore, type JudgeSpan, type KeywordConceptSpec, type KeywordCoverageFinding, type KeywordCoverageOptions, type KeywordCoverageResult, type LangfuseEnvelope, type LangfuseGeneration, type LangfuseScore, type Layer, type LayerCorrelation, type LayerResult, type LayerStatus, type LineageKind, type LineageKindResolver, type LineageNode, LineageRecorder, LlmCallError, type LlmCallRequest, type LlmCallResult, LlmClient, type LlmClientOptions, type LlmJsonCall, type LlmMessage, type LlmReviewerConfig, type LlmSpan, type LlmUsage, LockedJsonlAppender, MODEL_PRICING, type MatchResult, type MatcherResult, type MeasurementPolicy, type MergeOptions, type Message, type MetricSamples, type MetricVerdict, MetricsCollector, type MuffledFinder, type MuffledFinding, MultiLayerVerifier, type MultiToolchainLayerConfig, type MutateAdapter, type MutationAttempt, type MutationChannel, MutationTelemetry, type Mutator, Mutex, OTEL_AGENT_EVAL_SCOPE, type Objective, type OptimizationConfig, type OptimizationExample, OptimizationLoop, type OptimizationLoopConfig, type OptimizationLoopResult, type OptimizationResult, type Oracle, type OracleObservation, type OracleReport, type OracleResult, type OrthogonalityInput, type OrthogonalityResult, type OtlpExport, type OtlpResourceSpans, type OtlpSpan, type OutcomeFilter, type OutcomePair, type OutcomeStore, type PairwiseComparison, PairwiseSteeringOptimizer, type ParetoResult, type PersonaConfig, type Playbook, type PlaybookEntry, type PoolSlot, type PositionalBiasResult, type PrmGradedTrace, PrmGrader, type PrmTrainingSample, ProductClient, type ProductClientConfig, type ProjectKind, ProjectRegistry, type ProjectSummary, type ProjectTimelineEntry, type PromptEvolutionConfig, type PromptEvolutionEvent, type PromptEvolutionResult, type PromptHandle, PromptOptimizer, PromptRegistry, type TrialResult as PromptTrialResult, type PromptVariant$1 as PromptVariant, type ProposeFn, type ProposeInput, type ProposeOutput, type ProposeReviewConfig, type ProposeReviewReport, type ProposeReviewShot, REDACTION_VERSION, type RedTeamCase, type RedTeamCategory, type RedTeamFinding, type RedTeamPayload, type RedTeamReport, type RedactionReport, type RedactionRule, type ReferenceMatchResult, type ReferenceReplayAdapter, type ReferenceReplayAdapterFn, type ReferenceReplayAdapterLike, type ReferenceReplayAggregate, type ReferenceReplayCandidate, type ReferenceReplayCase, type ReferenceReplayCaseRun, type ReferenceReplayExecutionScenario, type ReferenceReplayItem, type ReferenceReplayMatch, type ReferenceReplayMatchStrategy, type ReferenceReplayMatcher, type ReferenceReplayPromotionDecision, type ReferenceReplayPromotionPolicy, type ReferenceReplayRun, type ReferenceReplayRunContext, type ReferenceReplayRunOptions, type ReferenceReplayRunStore, type ReferenceReplayScenario, type ReferenceReplayScenarioScore, type ReferenceReplayScore, type ReferenceReplayScoreOptions, type ReferenceReplaySplit, type ReferenceReplaySplitComparison, type ReferenceReplaySteeringRowsOptions, type ReflectionContext, type ReflectionProposal, type RegressionOptions, type RegressionSpec, type RetrievalSpan, type Review, type ReviewFn, type ReviewInput, type ReviewMemoryEntry, type ReviewMemoryStore, type ReviewerMemoryEntry, type ReviewerOutput, type ReviewerPromptInput, type ReviewerSoftFailDefaults, type ReviewerVerificationSummary, type RobustnessResult, type RouteMap, type RubricDimension, type Run, type RunAppScenarioOptions, type RunCommandInput, type RunCommandResult, type RunConfig, RunCritic, type RunCriticOptions, type RunDiff, type RunFilter, type RunLayer, type RunOutcome, type RunScore, type RunScoreWeights, type RunStatus, type RunTrace, SEMANTIC_CONCEPT_JUDGE_VERSION, type SandboxDriver, SandboxHarness, type SandboxHarnessResult, type SandboxJudgeKind, type SandboxJudgeResult, type SandboxJudgeSpec, type SandboxPool, type SandboxResult, type SandboxSpan, type ScanOptions, type Scenario, type ScenarioAggregate, type ScenarioCost, type ScenarioFile, ScenarioRegistry, type ScenarioResult, type ScoreAdapter, type ScoredTarget, type SelfPlayOptions, type SelfPlayProposer, type SelfPlayScorer, type SelfPreferenceResult, type SemanticConceptJudgeInput, type SemanticConceptJudgeOptions, type SemanticConceptJudgeResult, type SeriesConvergenceOptions, type SeriesConvergenceResult, type Severity, type ShipOptions, type SignedManifest, type SliceOptions, type Slo, type SloCheckResult, type SloComparator, type SloReport, type SloSeverity, type SlopCategory, type SlotFactory, type Span, type SpanBase, type SpanFilter, type SpanHandle, type SpanKind, type SpanStatus, type SteeringBundle, type SteeringDelta, type SteeringEvaluation, type SteeringOptimizationResult, type SteeringOptimizationRow, type SteeringOptimizationSelector, type SteeringOptimizerBackend, type SteeringOptimizerConfig, type SteeringRolePrompt, type SteeringVariantReport, type StepAttribution, type StepContext, type StepRubric, type StuckLoopFinding, type StuckLoopOptions, type StuckLoopReport, SubprocessSandboxDriver, type SubprocessSandboxDriverOptions, type SynthesisReason, type SynthesisTarget, TRACE_SCHEMA_VERSION, type TestGradedRunOptions, type TestGradedRunResult, type TestGradedScenario, type TestOutputParser, type TestResult, type ThreeLayerProjectReport, type ThresholdContract, TokenCounter, type TokenSpec, type ToolSpan, type ToolStats, type ToolUseMetrics, type ToolUseOptions, type ToolWasteFinding, type ToolWasteOptions, type ToolWasteReport, TraceEmitter, type TraceEmitterOptions, type TraceEvent, type TraceStore, type Trajectory, type TrajectoryStep, type TrialAttempt, type TrialCache, TrialTelemetry, type TrialTrace, type Turn, type TurnMetrics, type TurnResult, UNIVERSAL_FINDERS, type UseCaseSignals, type ValidationContext, type ValidationIssue, type ValidationResult, type VariantAggregate, type VariantScore, type VerbosityBiasResult, type Verdict, type Verification, type VerificationReport, type VerifyContext, type VerifyFn, type VerifyOptions, type VisualDiffOptions, type VisualDiffResult, type ViteDeployRunnerInput, type WorkflowTopology, type WorkspaceAssertion, type WorkspaceAssertionResult, type WorkspaceInspector, type WorkspaceSnapshot, adversarialJudge, aggregateLlm, aggregateRunScore, analyzeAntiSlop, analyzeSeries, argHash, attributeCounterfactuals, benjaminiHochberg, bisect, bonferroni, bootstrapCi, budgetBreachView, buildReflectionPrompt, buildReviewerPrompt, buildTrajectory, byteLengthRange, calibrateJudge, calibrationCurve, callLlm, callLlmJson, canaryLeakView, causalAttribution, checkCanaries, checkSlos, clamp01, classifyEuAiRisk, classifyFailure, codeExecutionJudge, cohensD, coherenceJudge, collectionPreserved, commitBisect, compareReferenceReplay, compareToBaseline, compilerJudge, composeParsers, composeValidators, computeToolUseMetrics, confidenceInterval, containsAll, correlateLayers, correlationStudy, createAntiSlopJudge, createCompositeMutator, createCustomJudge, createDefaultReviewer, createDomainExpertJudge, createIntentMatchJudge, createLlmReviewer, createSandboxCodeMutator, createSandboxPool, createSemanticConceptJudge, crossTraceDiff, crowdingDistance, decideReferenceReplayPromotion, decideReferenceReplayRunPromotion, defaultJudges, defaultReferenceReplayMatcher, deployGateLayer, distillPlaybook, dominates, estimateCost, estimateTokens, euAiActReport, evaluateContract, evaluateHypothesis, evaluateOracles, executeScenario, expectAgent, exportRewardModel, exportRunAsOtlp, exportTrainingData, extractAssetUrls, extractErrorCount, failureClusterView, fileContains, fileExists, findAutoMatchNoExpectation, findConstructorCwdDropped, findFallbackToPass, findLiteralTruePass, findSkipCountsAsPass, firstDivergenceView, flowLayer, formatBenchmarkReport, formatDriverReport, formatFindings, precision as goldenPrecision, gradeSemanticStatus, groupBy, hashContent, hashScenarios, htmlContainsElement, inMemoryReferenceReplayStore, inMemoryReviewStore, interRaterReliability, iqr, isJudgeSpan, isLlmSpan, isPrmVerdict, isRetrievalSpan, isSandboxSpan, isToolSpan, jestTestParser, jsonHasKeys, jsonShape, jsonlReferenceReplayStore, jsonlReviewStore, judgeAgreementView, judgeReplayGate, judgeSpans, keyPreserved, linterJudge, llmSpanFromProvider, llmSpans, loadScorerFromGrader, localCommandRunner, lowercaseMutator, mannWhitneyU, matchGoldens, mergeLayerResults, mergeSteeringBundle, multiToolchainLayer, nistAiRmfReport, nonRefusalRubric, normalizeScores, notBlocked, outputLengthRubric, pairedTTest, paraphraseRobustness, paretoFrontier, paretoFrontierWithCrowding, parseReflectionResponse, partialCredit, passOrthogonality, pixelDeltaRatio, politenessPrefixMutator, positionalBias, printDriverSummary, prmBestOfN, prmEnsembleBestOfN, probeLlm, promptBisect, proposeSynthesisTargets, pytestTestParser, redTeamDataset, redTeamReport, redactString, redactValue, referenceReplayRunsToSteeringRows, referenceReplayScenarioToRunScore, regexMatch, regexMatches, regressionView, renderMarkdown, renderMarkdownReport, renderPlaybookMarkdown, renderSteeringText, replayScorerOverCorpus, replayTraceThroughJudge, requiredSampleSize, resetLockedAppendersForTesting, resumeBuilderSession, rowCount, rowWhere, runAssertions, runCounterfactual, runE2EWorkflow, runExpectations, runFailureClass, runHarnessExperiment, runIntentMatchJudge, runJudgeFleet, runKeywordCoverageJudge, runKeywordCoverageJudgeUrl, runPromptEvolution, runProposeReview, runReferenceReplay, runSelfPlay, runSemanticConceptJudge, runTestGradedScenario, runsForScenario, scalarScore, scanForMuffledGates, scoreAllProjects, scoreContinuity, scoreProject, scoreRedTeamOutput, scoreReferenceReplay, securityJudge, selectHarnessVariant, selfPreference, sentenceReorderMutator, signManifest, soc2Report, statusAdvanced, stripFencedJson, stuckLoopView, summarize, summarizeHarnessResults, testJudge, textInSnapshot, toLangfuseEnvelope, toNdjson, toPrometheusText, toolIntentAlignmentRubric, toolNamesForRun, toolNonRedundantRubric, toolSpans, toolSuccessRubric, toolWasteView, typoMutator, urlContains, verbosityBias, verifyManifest, visualDiff, viteDeployRunner, vitestTestParser, weightedMean, weightedRecall, welchsTTest, whitespaceCollapseMutator, wilcoxonSignedRank };