@tangle-network/agent-eval 0.7.1 → 0.11.0

package/dist/index.d.ts

@@ -2828,6 +2828,26 @@ declare class CostTracker {
         timestamp?: number;
     }): CostEntry;
     markOutcome(scenarioId: string, completed: boolean): void;
+    /**
+     * Convenience: record + markOutcome in one call from a
+     * `{ usage, verdict }`-shaped response (starter-foundry's
+     * `invokeMetaJudge` returns this shape; consumers that wrap any
+     * judge/critic can follow the same convention).
+     *
+     * `usage.model` must be present in `MODEL_PRICING` for cost math to
+     * populate; otherwise totalCostUsd stays at 0 for the entry but
+     * tokens still aggregate.
+     */
+    recordVerdict(verdict: {
+        usage?: {
+            inputTokens: number;
+            outputTokens: number;
+            model: string;
+            cachedTokens?: number;
+            reasoningTokens?: number;
+        };
+        verdict?: 'pass' | 'fail' | 'borderline' | string;
+    }, scenarioId: string, tags?: Record<string, string>): CostEntry | null;
     get(scenarioId: string): ScenarioCost | undefined;
     list(): ScenarioCost[];
     summary(): CostSummary;
@@ -2843,6 +2863,108 @@ interface CostSummary {
     costPerCompletedTaskUsd: number | null;
 }
 
+/**
+ * muffled-gate-scanner — test helper that greps consumer source for
+ * gate + measurement anti-patterns and fails with file:line locations.
+ *
+ * Named pattern lives at starter-foundry's `.evolve/patterns/muffled-gate.md`;
+ * same shape applies to every consumer (a gate that should fail loud
+ * returns silent success; a metric that should emit a real number
+ * reports noise/empty).
+ *
+ * Usage (in a consumer project's test file):
+ *
+ *   import { scanForMuffledGates, DEFAULT_FINDERS } from '@tangle-network/agent-eval'
+ *
+ *   test('no muffled gates in eval surface', () => {
+ *     const findings = scanForMuffledGates({
+ *       repoRoot: process.cwd(),
+ *       scanFiles: ['src/eval/scaffold.ts', 'scripts/promote.mjs'],
+ *       finders: DEFAULT_FINDERS,
+ *     })
+ *     if (findings.length) assert.fail(formatFindings(findings))
+ *   })
+ *
+ * Customize by passing your own `finders` — each finder is
+ * `(file, text) => Finding[]` and runs per-file.
+ *
+ * Escape hatch: any line containing `muffle-ok:` is excluded from all
+ * finders, letting consumers opt a legitimate fallback out explicitly.
+ */
+interface MuffledFinding {
+    file: string;
+    line: number;
+    lineText: string;
+    pattern: string;
+}
+type MuffledFinder = (file: string, text: string) => MuffledFinding[];
+interface ScanOptions {
+    /** Absolute path to the repo root. */
+    repoRoot: string;
+    /** Explicit file list (paths relative to repoRoot) for context-specific finders. */
+    scanFiles: string[];
+    /**
+     * Auto-derived scan: walk these dirs for files matching importGlob + the
+     * string `importsContain` and run the universal finders on them. Pattern
+     * from starter-foundry H4 (research/decisions/001) — catches new files
+     * with agent-eval import that would otherwise escape context-specific
+     * scan lists.
+     */
+    autoDerive?: {
+        roots: string[];
+        extensions: RegExp;
+        importsContain: string;
+        universalFinders: MuffledFinder[];
+    };
+    /** Per-file finders (context-specific patterns). */
+    finders: MuffledFinder[];
+}
+/**
+ * Default finder: `command || true` in a testCommand/setupCommand/cmd/command
+ * string. Swallows exit codes.
+ */
+declare const findFallbackToPass: MuffledFinder;
+/**
+ * `testCommand: 'true'` literal silent-pass — an unknown-language dispatch
+ * arm that returns a no-op instead of throwing.
+ */
+declare const findLiteralTruePass: MuffledFinder;
+/**
+ * `new SubprocessSandboxDriver({ cwd: ... })` — constructor arg silently
+ * dropped in agent-eval <0.7.1. 0.7.1+ honors as fallback, but the form
+ * still invites confusion; prefer `new SubprocessSandboxDriver()` with
+ * cwd in the per-call HarnessConfig.
+ */
+declare const findConstructorCwdDropped: MuffledFinder;
+/**
+ * `if (!expected) return true` — matcher auto-passes when ground truth is
+ * absent. Inflates accuracy metrics for scenarios without expectations.
+ */
+declare const findAutoMatchNoExpectation: MuffledFinder;
+/**
+ * `if (p.skipped) return true` — skip-counts-as-pass in quality scorers.
+ * Use three-valued `true | false | 'skipped'` return + explicit partial
+ * credit instead.
+ */
+declare const findSkipCountsAsPass: MuffledFinder;
+/**
+ * The canonical default bundle. Callers can import these individually,
+ * replace them, or append custom finders for project-specific patterns.
+ */
+declare const DEFAULT_FINDERS: MuffledFinder[];
+/** Finders that should run on EVERY file with the target import, not just SCAN_FILES. */
+declare const UNIVERSAL_FINDERS: MuffledFinder[];
+/**
+ * Run all finders against the configured files. Returns a flat list of
+ * findings. Callers format + assert as they prefer.
+ */
+declare function scanForMuffledGates(opts: ScanOptions): MuffledFinding[];
+/**
+ * Format findings into a single assert.fail-ready message. Each finding
+ * carries file:line + pattern name + the offending line.
+ */
+declare function formatFindings(findings: MuffledFinding[]): string;
+
 /**
  * Series convergence — detects whether a sequence of scalar measurements
  * is stabilizing, drifting, or noisy.
@@ -4674,4 +4796,1278 @@ interface UseCaseSignals {
 declare function classifyEuAiRisk(signals: UseCaseSignals): EuRiskClass;
 declare function euAiActReport(ctx: GovernanceContext, signals: UseCaseSignals): Promise<GovernanceReport>;
 
-export { type ActiveLearningOptions, AgentDriver, type AgentDriverConfig, type AlignmentOp, type AntiSlopConfig, type AntiSlopIssue, type AntiSlopReport, type Artifact, type ArtifactCheck, type Artifact$1 as ArtifactCheckArtifact, type ArtifactResult, type ArtifactValidator, AxGepaSteeringOptimizer, type AxSteeringOptimizerConfig, type BaselineOptions, type BaselineReport, BehaviorAssertion, type BenchmarkReport, BenchmarkRunner, type BenchmarkRunnerConfig, type BestOfNResult, type BisectOptions, type BisectResult, type BisectStep, BudgetBreachError, type BudgetBreachFinding, type BudgetBreachReport, BudgetGuard, type BudgetLedgerEntry, type BudgetSpec, BuilderSession, type BuilderSessionInit, type CalibrationBin, type CalibrationOptions, type CalibrationReport, type CalibrationResult, CallExpectation, type CanaryLeak, type CandidateScenario, type CandidateScore, type CausalAttributionReport, type ChatSummary, type CheckResult, type CollectedArtifacts, type CompletionCriterion, type ContinuityCheck, type ContinuityCheckResult, type ContinuityReport, type ContinuitySnapshotPair, type ContractMetric, type ContractReport, ConvergenceTracker, type CorrelationReport, type CorrelationResult, type CorrelationStudyOptions, type CorrelationStudyResult, type CostEntry, type CostSummary, CostTracker, type CounterfactualContext, type CounterfactualMutation, type CounterfactualResult, type CounterfactualRunner, type CrossTraceDiff, type CrossTraceDiffOptions, DEFAULT_AGENT_SLOS, DEFAULT_RULES as DEFAULT_FAILURE_RULES, DEFAULT_HARNESS_OBJECTIVES, DEFAULT_MUTATORS, DEFAULT_REDACTION_RULES, DEFAULT_RED_TEAM_CORPUS, DEFAULT_RUN_SCORE_WEIGHTS, Dataset, type DatasetDifficulty, type DatasetManifest, type DatasetProvenance, type DatasetScenario, type DatasetSplit, type DeploymentOutcome, type Direction, type DivergenceOptions, type DivergenceReport, DockerSandboxDriver, type DriverResult, type DriverState, DualAgentBench, type DualAgentBenchConfig, type DualAgentReport, type DualAgentRound, type DualAgentScenario, type DualAgentScenarioResult, type EuRiskClass, type EvalMetricSpec, type EvalResult, type EventFilter, type EventKind, type EvolutionRound, type ExecutorConfig, type Expectation, type Experiment, type Run$1 as ExperimentRun, type ExperimentStore, ExperimentTracker, type ExportedRewardModel, FAILURE_CLASSES, type FactorContribution, type FactorialCell, type FailureClass, type FailureClassification, type FailureCluster, type FailureClusterReport, type FailureContext, type FailureRule, type FeedbackPattern, FileSystemOutcomeStore, type FileSystemOutcomeStoreOptions, FileSystemTraceStore, type FileSystemTraceStoreOptions, type GenericSpan, type GoldenItem, type GovernanceContext, type GovernanceFinding, type GovernanceReport, type GradedStep, type HarnessAdapter, type HarnessConfig, type HarnessExperimentConfig, type HarnessExperimentResult, type HarnessIntervention, type HarnessRunRequest, type HarnessRunResult, type HarnessScenario, type HarnessSelection, type HarnessVariant, type HarnessVariantReport, HoldoutAuditor, HoldoutLockedError, type HostedJudgeConfig, type HostedJudgeDimension, type HostedJudgeRequest, type HostedJudgeResponse, type HostedRunCriticConfig, type HostedRunScoreRequest, type HostedRunScoreResponse, type HypothesisManifest, type HypothesisResult, type ImageData, InMemoryExperimentStore, InMemoryOutcomeStore, InMemoryTraceStore, InMemoryWorkspaceInspector, type InferenceScorer, type InspectorContext, type InteractionContribution, type JudgeAgreementReport, type JudgeConfig, type JudgeFleetOptions, type JudgeFn, type JudgeInput, type JudgePair, type JudgeReplayResult, type JudgeRubric, JudgeRunner, type JudgeScore, type JudgeSpan, type LangfuseEnvelope, type LangfuseGeneration, type LangfuseScore, type LayerCorrelation, type LlmJsonCall, type LlmReviewerConfig, type LlmSpan, MODEL_PRICING, type MatcherResult, type MeasurementPolicy, type Message, type MetricSamples, type MetricVerdict, MetricsCollector, type Mutator, OTEL_AGENT_EVAL_SCOPE, type Objective, type OptimizationConfig, type OptimizationExample, OptimizationLoop, type OptimizationLoopConfig, type OptimizationLoopResult, type OptimizationResult, type Oracle, type OracleObservation, type OracleReport, type OracleResult, type OtlpExport, type OtlpResourceSpans, type OtlpSpan, type OutcomeFilter, type OutcomePair, type OutcomeStore, type PairwiseComparison, PairwiseSteeringOptimizer, type ParetoResult, type PersonaConfig, type Playbook, type PlaybookEntry, type PositionalBiasResult, type PrmGradedTrace, PrmGrader, type PrmTrainingSample, ProductClient, type ProductClientConfig, type ProjectKind, ProjectRegistry, type ProjectSummary, type ProjectTimelineEntry, type PromptHandle, PromptOptimizer, PromptRegistry, type PromptVariant, type ProposeFn, type ProposeInput, type ProposeOutput, type ProposeReviewConfig, type ProposeReviewReport, type ProposeReviewShot, REDACTION_VERSION, type RedTeamCase, type RedTeamCategory, type RedTeamFinding, type RedTeamPayload, type RedTeamReport, type RedactionReport, type RedactionRule, type RegressionOptions, type RegressionSpec, type RetrievalSpan, type Review, type ReviewFn, type ReviewInput, type ReviewMemoryEntry, type ReviewMemoryStore, type RobustnessResult, type RouteMap, type RubricDimension, type Run, type RunAppScenarioOptions, type RunConfig, RunCritic, type RunCriticOptions, type RunDiff, type RunFilter, type RunLayer, type RunOutcome, type RunScore, type RunScoreWeights, type RunStatus, type RunTrace, type SandboxDriver, SandboxHarness, type SandboxHarnessResult, type SandboxJudgeKind, type SandboxJudgeResult, type SandboxJudgeSpec, type SandboxResult, type SandboxSpan, type Scenario, type ScenarioCost, type ScenarioFile, ScenarioRegistry, type ScenarioResult, type ScoredTarget, type SelfPlayOptions, type SelfPlayProposer, type SelfPlayScorer, type SelfPreferenceResult, type SeriesConvergenceOptions, type SeriesConvergenceResult, type ShipOptions, type SignedManifest, type SliceOptions, type Slo, type SloCheckResult, type SloComparator, type SloReport, type SloSeverity, type SlopCategory, type Span, type SpanBase, type SpanFilter, type SpanHandle, type SpanKind, type SpanStatus, type SteeringBundle, type SteeringDelta, type SteeringEvaluation, type SteeringOptimizationResult, type SteeringOptimizationRow, type SteeringOptimizationSelector, type SteeringOptimizerBackend, type SteeringOptimizerConfig, type SteeringRolePrompt, type SteeringVariantReport, type StepAttribution, type StepContext, type StepRubric, type StuckLoopFinding, type StuckLoopOptions, type StuckLoopReport, SubprocessSandboxDriver, type SubprocessSandboxDriverOptions, type SynthesisReason, type SynthesisTarget, TRACE_SCHEMA_VERSION, type TestGradedRunOptions, type TestGradedRunResult, type TestGradedScenario, type TestOutputParser, type TestResult, type ThreeLayerProjectReport, type ThresholdContract, TokenCounter, type TokenSpec, type ToolSpan, type ToolStats, type ToolUseMetrics, type ToolUseOptions, type ToolWasteFinding, type ToolWasteOptions, type ToolWasteReport, TraceEmitter, type TraceEmitterOptions, type TraceEvent, type TraceStore, type Trajectory, type TrajectoryStep, type Turn, type TurnMetrics, type TurnResult, type UseCaseSignals, type ValidationContext, type ValidationIssue, type ValidationResult, type VariantScore, type VerbosityBiasResult, type Verification, type VerifyFn, type VisualDiffOptions, type VisualDiffResult, type WorkflowTopology, type WorkspaceAssertion, type WorkspaceAssertionResult, type WorkspaceInspector, type WorkspaceSnapshot, adversarialJudge, aggregateLlm, aggregateRunScore, analyzeAntiSlop, analyzeSeries, argHash, attributeCounterfactuals, benjaminiHochberg, bisect, bonferroni, budgetBreachView, buildTrajectory, byteLengthRange, calibrateJudge, calibrationCurve, canaryLeakView, causalAttribution, checkCanaries, checkSlos, clamp01, classifyEuAiRisk, classifyFailure, codeExecutionJudge, cohensD, coherenceJudge, collectionPreserved, commitBisect, compareToBaseline, compilerJudge, composeParsers, composeValidators, computeToolUseMetrics, confidenceInterval, containsAll, correlateLayers, correlationStudy, createAntiSlopJudge, createCustomJudge, createDomainExpertJudge, createLlmReviewer, crossTraceDiff, defaultJudges, distillPlaybook, dominates, estimateCost, estimateTokens, euAiActReport, evaluateContract, evaluateHypothesis, evaluateOracles, executeScenario, expectAgent, exportRewardModel, exportRunAsOtlp, exportTrainingData, failureClusterView, fileContains, fileExists, firstDivergenceView, formatBenchmarkReport, formatDriverReport, groupBy, hashContent, hashScenarios, inMemoryReviewStore, interRaterReliability, iqr, isJudgeSpan, isLlmSpan, isPrmVerdict, isRetrievalSpan, isSandboxSpan, isToolSpan, jestTestParser, jsonHasKeys, jsonShape, jsonlReviewStore, judgeAgreementView, judgeSpans, keyPreserved, linterJudge, llmSpanFromProvider, llmSpans, loadScorerFromGrader, lowercaseMutator, mannWhitneyU, mergeSteeringBundle, nistAiRmfReport, nonRefusalRubric, normalizeScores, notBlocked, outputLengthRubric, pairedTTest, paraphraseRobustness, paretoFrontier, partialCredit, pixelDeltaRatio, politenessPrefixMutator, positionalBias, printDriverSummary, prmBestOfN, prmEnsembleBestOfN, promptBisect, proposeSynthesisTargets, pytestTestParser, redTeamDataset, redTeamReport, redactString, redactValue, regexMatch, regexMatches, regressionView, renderMarkdown, renderMarkdownReport, renderPlaybookMarkdown, renderSteeringText, replayScorerOverCorpus, replayTraceThroughJudge, requiredSampleSize, resumeBuilderSession, rowCount, rowWhere, runAssertions, runCounterfactual, runE2EWorkflow, runExpectations, runFailureClass, runHarnessExperiment, runJudgeFleet, runProposeReview, runSelfPlay, runTestGradedScenario, runsForScenario, scoreAllProjects, scoreContinuity, scoreProject, scoreRedTeamOutput, securityJudge, selectHarnessVariant, selfPreference, sentenceReorderMutator, signManifest, soc2Report, statusAdvanced, stuckLoopView, summarize, summarizeHarnessResults, testJudge, textInSnapshot, toLangfuseEnvelope, toNdjson, toPrometheusText, toolIntentAlignmentRubric, toolNamesForRun, toolNonRedundantRubric, toolSpans, toolSuccessRubric, toolWasteView, typoMutator, urlContains, verbosityBias, verifyManifest, visualDiff, vitestTestParser, weightedMean, welchsTTest, whitespaceCollapseMutator, wilcoxonSignedRank };
+/**
+ * LLM client with graceful degrade.
+ *
+ * OpenAI-compatible `/v1/chat/completions` client with:
+ *   - Exponential-backoff retry on 429 + 5xx gateway errors (502/503/504).
+ *   - Retry on transient network errors (fetch failed, AbortError, ECONNRESET).
+ *   - Graceful json_schema → json_object degrade on 400 with schema-reject body.
+ *   - Fenced-JSON stripping (```json ... ```) for models that wrap structured output.
+ *   - Configurable base URL + api key / bearer, works with LiteLLM proxies, OpenAI
+ *     directly, cli-bridge subscriptions, and any router that speaks the spec.
+ *
+ * Usage:
+ *   const { value, result } = await callLlmJson<MyType>(
+ *     { model: 'gpt-4o', messages: [...], jsonSchema: { name: 'x', schema: {...} } },
+ *     { baseUrl: 'https://router.tangle.tools/v1', apiKey: process.env.KEY },
+ *   )
+ *
+ * This is THE llm-calling seam for agent-eval primitives that need structured
+ * output (semantic concept judge, reviewer directives, critic scores). Primitives
+ * that need free-form text use `callLlm` and parse output themselves.
+ */
+interface LlmMessage {
+    role: 'system' | 'user' | 'assistant';
+    /**
+     * Either a plain text content string OR a multimodal content array
+     * (text + image_url parts) for vision-capable models.
+     */
+    content: string | Array<{
+        type: 'text';
+        text: string;
+    } | {
+        type: 'image_url';
+        image_url: {
+            url: string;
+            detail?: 'auto' | 'low' | 'high';
+        };
+    }>;
+}
+interface LlmCallRequest {
+    model: string;
+    messages: LlmMessage[];
+    /** Optional JSON-mode response format (response_format: json_object). */
+    jsonMode?: boolean;
+    /** Optional structured output via JSON Schema. Falls back to json_object on 400. */
+    jsonSchema?: {
+        name: string;
+        schema: Record<string, unknown>;
+    };
+    temperature?: number;
+    maxTokens?: number;
+    /** Per-call timeout, default 60s. */
+    timeoutMs?: number;
+}
+interface LlmUsage {
+    promptTokens: number;
+    completionTokens: number;
+    totalTokens: number;
+    /** Proxies populate this when prompt caching is on. */
+    cachedPromptTokens?: number;
+}
+interface LlmCallResult {
+    /** The text content of the first choice. Empty string if none. */
+    content: string;
+    usage: LlmUsage;
+    /**
+     * Cost in USD. Pulled from proxy's `_response_cost` field when present;
+     * `null` when neither the proxy nor the caller can derive it.
+     */
+    costUsd: number | null;
+    /** Model name actually used (echoed from response). */
+    model: string;
+    /** Wall-clock duration of the HTTP call (last attempt, if retried). */
+    durationMs: number;
+    /** Raw response body. */
+    raw: Record<string, unknown>;
+}
+declare class LlmCallError extends Error {
+    readonly status: number;
+    readonly body: string;
+    readonly model: string;
+    constructor(message: string, status: number, body: string, model: string);
+}
+interface LlmClientOptions {
+    /** Base URL (without trailing slash). Must end at the `/v1` prefix. */
+    baseUrl?: string;
+    /** Bearer token — either `apiKey` or `bearer` populates `Authorization: Bearer ...`. */
+    apiKey?: string;
+    bearer?: string;
+    /** Override for the `Authorization` header (e.g. `X-Auth: ...`). Takes precedence over apiKey/bearer. */
+    authHeader?: {
+        name: string;
+        value: string;
+    };
+    /** Default timeout in ms. Per-call can override. */
+    defaultTimeoutMs?: number;
+    /** Max retry attempts on retriable errors. Default 3 (1 initial + 2 retries). */
+    maxRetries?: number;
+    /** Fetch implementation — defaults to global `fetch`. Override for custom transport (e.g. tests). */
+    fetch?: typeof fetch;
+}
+/**
+ * Strip a ```json / ``` code fence if the model emitted one.
+ * Idempotent for naked JSON. Some models (claude-code via router, certain
+ * deepseek models) wrap output even under json_object.
+ */
+declare function stripFencedJson(raw: string): string;
+/**
+ * Low-level call. Returns raw content + usage + cost. Retries on transient
+ * failures; does NOT degrade schema here — callers that want graceful
+ * degrade use `callLlmJson`.
+ */
+declare function callLlm(req: LlmCallRequest, opts?: LlmClientOptions): Promise<LlmCallResult>;
+/**
+ * Structured-output call. Returns parsed JSON plus the raw result envelope.
+ * Degrades `jsonSchema` → `jsonMode` on a 400 that names the schema param —
+ * critical for deepseek-v3/v4, kimi-k2.6, and other models that don't accept
+ * the `response_format.json_schema` shape but DO accept `json_object`.
+ */
+declare function callLlmJson<T = unknown>(req: LlmCallRequest, opts?: LlmClientOptions): Promise<{
+    value: T;
+    result: LlmCallResult;
+}>;
+/**
+ * Probe whether a model is reachable. Returns latency + null error on
+ * success; `ok=false` + error message on any failure (HTTP, timeout,
+ * network, parse). Designed for sweep preflights — fail loud at the
+ * boundary before burning a 30-leaf run on a misconfigured router.
+ *
+ * Sends a tiny `ping` message with `maxTokens=64`. Reasoning models
+ * (glm-5.1, deepseek-v4) can burn the entire budget on internal reasoning
+ * for short prompts, so don't tighten this further. We don't validate
+ * content; HTTP 200 means reachable.
+ */
+declare function probeLlm(model: string, opts?: LlmClientOptions & {
+    timeoutMs?: number;
+}): Promise<{
+    ok: boolean;
+    latencyMs: number;
+    error: string | null;
+}>;
+/**
+ * Stateful client — construct once with defaults, call many times.
+ * Thin wrapper around the free functions; exists for callers that want
+ * to inject a single configured instance into multiple primitives.
+ */
+declare class LlmClient {
+    private readonly opts;
+    constructor(opts?: LlmClientOptions);
+    call(req: LlmCallRequest, per?: LlmClientOptions): Promise<LlmCallResult>;
+    callJson<T = unknown>(req: LlmCallRequest, per?: LlmClientOptions): Promise<{
+        value: T;
+        result: LlmCallResult;
+    }>;
+}
+
+/**
+ * Multi-layer verifier — ordered pipeline of verification layers.
+ *
+ * Different contract from {@link JudgeRunner} (which runs parallel
+ * specs against a sandbox). MultiLayerVerifier is a DAG of layers
+ * (install → typecheck → build → lint → serve → semantic → …) with
+ * dependency-based skip, per-layer findings, soft-fail semantics, and
+ * an aggregated `blendedScore` across all passed layers.
+ *
+ * Use when you want:
+ *   - ordered stages where a failing upstream stage skips downstream ones
+ *   - each stage produces rich `findings` (severity + message + evidence)
+ *   - a single composite score across stages with per-stage weights
+ *   - soft-fail stages whose failure doesn't abort the pipeline
+ *
+ * Use {@link JudgeRunner} when you want:
+ *   - N independent judges running in parallel against the same artifact
+ *   - no inter-judge dependencies
+ *   - boolean `passed` per judge + overall
+ *
+ * Both primitives compose — JudgeRunner can be invoked as a single
+ * layer inside a MultiLayerVerifier if that suits the caller.
+ */
+type LayerStatus = 'pass' | 'fail' | 'skipped' | 'error' | 'timeout';
+type Severity = 'critical' | 'major' | 'minor' | 'info';
+interface Finding {
+    severity: Severity;
+    message: string;
+    evidence?: string;
+    /** Optional layer name the finding belongs to (set by the verifier if omitted). */
+    layer?: string;
+    /**
+     * Free-form structured payload — used by `multiToolchainLayer` to attach
+     * `{ adapter: 'pnpm' }`, by judges to attach evidence pointers, etc.
+     * Renderers MAY interrogate; agent-eval primitives never assume shape.
+     */
+    detail?: Record<string, unknown>;
+}
+interface LayerResult {
+    layer: string;
+    status: LayerStatus;
+    /** 0..1 score, optional — layers that don't produce a numeric score omit. */
+    score?: number;
+    durationMs: number;
+    findings: Finding[];
+    /** Short human-readable summary (one line). */
+    reason?: string;
+    /**
+     * Numeric layer-level diagnostics: error counts, warning counts,
+     * cyclomatic complexity, total adapter wall-time, etc. Keyed by
+     * diagnostic name; null = "diagnostic not applicable / not measured."
+     * Renderers that know the keys can display them; ones that don't,
+     * ignore. Free-form on purpose — consumers type the value shape in
+     * their own namespace. Added in 0.10.
+     */
+    diagnostics?: Record<string, number | null>;
+    /** Any rich per-layer detail — rendered as-is by consumers that know the layer. */
+    detail?: Record<string, unknown>;
+}
+interface VerifyContext<Env = unknown> {
+    /** Per-run opaque context the caller provides. Layers destructure what they need. */
+    env: Env;
+    /** Previously-computed results from layers that already ran. */
+    prior: Record<string, LayerResult>;
+    /** Signal — if aborted, layers MUST bail within reasonable wall. */
+    signal: AbortSignal;
+}
+interface Layer<Env = unknown> {
+    name: string;
+    /** Stages that must have `status: 'pass'` before this layer runs. */
+    dependsOn?: string[];
+    /**
+     * Weight in the composite `blendedScore`. Default 1.0. Layers with weight 0
+     * contribute findings but not score.
+     */
+    weight?: number;
+    /**
+     * If true, a `fail` status contributes to `blendedScore` (as 0) instead of
+     * being dropped — use for layers whose failure is a real signal. Default:
+     * fail drops from numerator + denominator, matching VB's existing semantics.
+     */
+    failContributesToScore?: boolean;
+    /** Optional per-layer wall-cap in ms. Honored by the verifier (AbortSignal). */
+    capMs?: number;
+    run: (ctx: VerifyContext<Env>) => Promise<LayerResult> | LayerResult;
+}
+interface VerifyOptions<Env = unknown> {
+    env: Env;
+    /**
+     * Overall wall cap. Default: sum of layer capMs, or Infinity if any layer
+     * omits a cap. The verifier short-circuits remaining layers on overall cap.
+     */
+    overallCapMs?: number;
+    /** Called with each layer result as it completes. */
+    onLayer?: (result: LayerResult) => void;
+}
+interface VerificationReport {
+    layers: LayerResult[];
+    passCount: number;
+    failCount: number;
+    skippedCount: number;
+    errorCount: number;
+    /** True iff at least one scored layer ran AND every scored layer passed. */
+    allPass: boolean;
+    /**
+     * Weighted mean of `score` across contributing layers. 0 when no layers
+     * contributed. See {@link Layer.failContributesToScore} for fail semantics.
+     */
+    blendedScore: number;
+    durationMs: number;
+    startedAt: string;
+    finishedAt: string;
+}
+/**
+ * Grade a semantic-concept-style judge result into a single layer status.
+ *
+ * Pass when overall score >= threshold AND no critical-severity concept gap.
+ * Fail otherwise. Use inside a `Layer.run` when wrapping a concept judge.
+ *
+ * Generalized from VerticalBench H3 fix: `failingConcepts.length === 0` was
+ * too strict — a single concept at 6/10 failed the entire layer despite
+ * overall score being >= 0.7. Now we trust the judge's own `severity` field:
+ * `critical` findings veto; `major`/`minor` reduce the score but don't veto.
+ */
+declare function gradeSemanticStatus(input: {
+    score: number;
+    findings: Array<{
+        severity: Severity;
+        present?: boolean;
+        score?: number;
+    }>;
+    available: boolean;
+    threshold?: number;
+}): LayerStatus;
+declare class MultiLayerVerifier<Env = unknown> {
+    private readonly layers;
+    constructor(layers: Layer<Env>[]);
+    run(opts: VerifyOptions<Env>): Promise<VerificationReport>;
+}
+
+/**
+ * CommandRunner — abstract subprocess execution surface.
+ *
+ * Layers in a {@link MultiLayerVerifier} that need to invoke external
+ * tools (compilers, test runners, package managers) call out via this
+ * interface rather than directly using `child_process`. Two reasons:
+ *
+ *   1. **Sandbox interchangeability.** A run that targets a sandbox box
+ *      (via SDK-specific Box.exec) and a run that targets the host both
+ *      satisfy this same contract. The harness doesn't care which.
+ *   2. **Testability.** Tests inject a fake runner and assert on calls
+ *      without spawning real subprocesses.
+ *
+ * agent-eval ships only the local implementation (host-process). Sandbox
+ * implementations live with their consumer because they depend on
+ * SDK-specific Box / Sandbox types that don't belong in this package.
+ */
+interface RunCommandInput {
+    /** Executable name, looked up via PATH unless absolute. */
+    cmd: string;
+    /** Argument vector, NOT shell-interpolated. Each element passed to argv. */
+    argv: string[];
+    /** Working directory. Defaults to runner's notion of cwd if omitted. */
+    cwd?: string;
+    /**
+     * Wall-clock cap in ms. The runner SHOULD return `timedOut: true` when
+     * exceeded; callers MAY treat status null + timedOut as "killed."
+     */
+    capMs?: number;
+    /** Env overrides merged on top of the runner's base environment. */
+    env?: Record<string, string>;
+    /** Optional stdin payload. */
+    stdin?: string;
+}
+interface RunCommandResult {
+    /** Exit code, or null when the process couldn't start / was killed. */
+    status: number | null;
+    stdout: string;
+    stderr: string;
+    durationMs: number;
+    timedOut: boolean;
+    /** Non-fatal runner-side error (binary missing, signal, etc.). */
+    runnerError?: string;
+}
+interface DirEntry {
+    name: string;
+    isDirectory: boolean;
+    isFile: boolean;
+    /** File size in bytes. `null` for directories (not stat'd). */
+    sizeBytes: number | null;
+}
+interface CommandRunner {
+    /** Identifier for telemetry + logs. Open-ended literal-union for new runners. */
+    readonly name: string;
+    /** Execute a command in the runner's environment. */
+    run(input: RunCommandInput): Promise<RunCommandResult>;
+    /** True iff `<name>` resolves on the runner's PATH. */
+    hasBin(name: string): Promise<boolean>;
+    /** True iff the given path exists in the runner's filesystem. */
+    fileExists(path: string): Promise<boolean>;
+    /** Read a file. Returns `null` if missing or unreadable. */
+    readFile(path: string): Promise<string | null>;
+    /** List a directory. Returns `[]` if unreadable / missing. */
+    readDir(path: string): Promise<DirEntry[]>;
+}
+/**
+ * Host-process runner. Uses node:child_process spawnSync (synchronous
+ * under the hood — wrapped in a Promise to satisfy the interface). For
+ * very long-running commands consider an async-spawn variant; this
+ * shape matches VB's existing behavior and is fine for build/test/lint
+ * subprocesses that finish in seconds-to-minutes.
+ */
+declare const localCommandRunner: CommandRunner;
+
+/**
+ * Multi-toolchain layer factory + merge helper.
+ *
+ * Some verification stages (install, typecheck, build, lint) run the
+ * SAME logical layer across multiple parallel adapters — pnpm AND npm
+ * AND cargo AND forge for a polyglot scaffold. The verifier presents
+ * one row per stage; the toolchain breakdown lives in `findings.detail`.
+ *
+ * This module provides the merge: take N independent `LayerResult`s
+ * (one per adapter) and reduce them to a single `LayerResult` whose
+ * status is the worst of the parts and whose findings cite the adapter
+ * that produced each one. Plus a {@link multiToolchainLayer} factory
+ * that runs the adapter calls in parallel + applies the reducer.
+ *
+ * Pure utility — composes with {@link MultiLayerVerifier}.{run}.
+ */
+
+interface AdapterRun {
+    /** Identifier for the adapter (e.g. 'pnpm', 'npm', 'cargo', 'forge'). */
+    adapter: string;
+    result: LayerResult;
+}
+interface MergeOptions {
+    /**
+     * How to combine per-adapter `durationMs`. Default `'max'` (parallel
+     * wall-clock). Set `'sum'` when reporting total work done across
+     * adapters rather than wall time.
+     */
+    mergeDuration?: 'max' | 'sum';
+    /**
+     * Prefix finding messages with a per-adapter tag (e.g. `[pnpm] typecheck failed`).
+     * Default: no prefix (renderers read `detail.adapter` instead).
+     */
+    messagePrefixer?: (adapter: string) => string;
+    /**
+     * How to reduce per-adapter `LayerResult.diagnostics` into the merged
+     * result's diagnostics. `'max'` (default) — for each key, merged =
+     * max across adapters where value is non-null (matches "if ANY adapter
+     * saw N errors, merged saw N"). `'sum'` — sum non-null values.
+     */
+    mergeDiagnostics?: 'max' | 'sum';
+}
+/**
+ * Reduce N adapter runs to a single `LayerResult` for a logical layer.
+ *
+ *   - status: worst of the parts (pass < skipped < fail < timeout < error)
+ *   - score: weighted mean of numeric scores (skip = no contribution)
+ *   - findings: union, each tagged with `detail.adapter`
+ *   - durationMs: `mergeDuration` option (default 'max' for parallel wall-clock)
+ *   - diagnostics: `mergeDiagnostics` option (default 'max' per key)
+ *   - reason: " · "-joined `name: status` per adapter
+ */
+declare function mergeLayerResults(name: string, perAdapter: AdapterRun[], options?: MergeOptions): LayerResult;
+interface MultiToolchainLayerConfig<Env, Adapter> {
+    name: string;
+    adapters: ReadonlyArray<Adapter>;
+    /** Adapter identifier — used in findings + reason. */
+    adapterName: (a: Adapter) => string;
+    /** Run a single adapter against the verify context. */
+    run: (a: Adapter, ctx: VerifyContext<Env>) => Promise<LayerResult> | LayerResult;
+    dependsOn?: string[];
+    weight?: number;
+    failContributesToScore?: boolean;
+    capMs?: number;
+    /**
+     * Per-adapter parallel cap. Defaults to 8 — defense in depth against a
+     * caller passing 50 adapters and fanning out 50 simultaneous subprocesses.
+     * Adapters that need higher concurrency raise this explicitly.
+     */
+    maxParallel?: number;
+}
+/**
+ * Build a {@link Layer} that fans the same logical stage across N adapters
+ * in parallel and merges via {@link mergeLayerResults}.
+ *
+ * Per-adapter throws are caught + converted to `status: 'error'` results
+ * so one bad adapter doesn't poison the whole layer.
+ */
+declare function multiToolchainLayer<Env, Adapter>(config: MultiToolchainLayerConfig<Env, Adapter>): Layer<Env>;
+
+/**
+ * Reviewer primitives — prompt builder + default ReviewFn factory.
+ *
+ * `buildReviewerPrompt` is the pure, LLM-agnostic piece: takes
+ * `ReviewerPromptInput` (user request, trace summary, verification
+ * summary, memory, optional extra context) and emits the system +
+ * user message pair. No LLM dependency — callers that want to drive
+ * their own transport get full control.
+ *
+ * `createDefaultReviewer` is the convenience factory: wires the prompt
+ * builder to `callLlmJson` with a default schema + soft-fail policy.
+ * Returns a function that maps `ReviewerPromptInput` to `ReviewerOutput`.
+ *
+ * Same pattern as `runSemanticConceptJudge` / `createSemanticConceptJudge`:
+ * low-level pure builder + high-level factory built on top.
+ */
+
+interface ReviewerMemoryEntry {
+    shot: number;
+    ts?: string;
+    observations?: string;
+    diagnosis?: string;
+    nextShotInstruction?: string;
+    shouldContinue?: boolean;
+    confidence?: number;
+}
+interface ReviewerVerificationSummary {
+    blendedScore: number;
+    allPass: boolean;
+    failCount: number;
+    failingLayers?: string[];
+}
+interface ReviewerPromptInput {
+    shot: number;
+    userRequest: string;
+    /**
+     * Compact trace summary — tool-call counts, errors, recent activity
+     * lines. Built by the caller from whatever trace format they have;
+     * agent-eval does not prescribe.
+     */
+    traceSummary: string;
+    verification: ReviewerVerificationSummary;
+    memory: ReviewerMemoryEntry[];
+    /**
+     * Optional extra context injected into the prompt between the trace
+     * and the verification blocks. Use for workdir file-tree snapshots,
+     * scaffold descriptions, or any environmental fact the reviewer
+     * needs to direct the next shot accurately.
+     */
+    extraContext?: string;
+    /**
+     * Optional extra section appended at the end of the prompt (e.g.
+     * leaf metadata, scenario id). Free-form — no agent-eval-shaped
+     * schema.
+     */
+    trailingContext?: string;
+}
+interface ReviewerOutput {
+    shot: number;
+    observations: string;
+    diagnosis: string;
+    nextShotInstruction: string;
+    shouldContinue: boolean;
+    /** 0..1 self-assessed confidence in the directive. */
+    confidence: number;
+    /** LLM cost in USD if the transport reports it, else null. */
+    costUsd: number | null;
+    durationMs: number;
+    /** False when the LLM errored or returned malformed JSON; caller soft-fails to defaults. */
+    available: boolean;
+    error?: string;
+}
+interface ReviewerSoftFailDefaults {
+    observations?: string;
+    diagnosis?: string;
+    nextShotInstruction?: string;
+    shouldContinue?: boolean;
+    confidence?: number;
+}
+interface CreateDefaultReviewerOptions {
+    /** Model id to call. */
+    model: string;
+    /** Per-call timeout. Default 180s. */
+    timeoutMs?: number;
+    /** LlmClient transport config (baseUrl, apiKey, authHeader, etc.). */
+    llm?: LlmClientOptions;
+    /**
+     * Override the prompt builder. Default: `buildReviewerPrompt`.
+     * Consumers with different reviewer voices pass their own.
+     */
+    promptBuilder?: (input: ReviewerPromptInput) => {
+        system: string;
+        user: string;
+    };
+    /**
+     * Soft-fail values when the LLM throws or returns unparseable JSON.
+     * Matches VerticalBench's shipped policy: continue with generic
+     * instruction at confidence 0.3 so the worker keeps trying.
+     */
+    softFailDefaults?: ReviewerSoftFailDefaults;
+}
+/**
+ * Build the reviewer's system + user messages. Pure function, no LLM
+ * call. Callers that want their own transport or a different structured
+ * output can use this and skip `createDefaultReviewer` entirely.
+ */
+declare function buildReviewerPrompt(input: ReviewerPromptInput): {
+    system: string;
+    user: string;
+};
+/**
+ * Factory: returns a function that invokes the default reviewer against
+ * an LLM and parses the structured output. Soft-fails to the provided
+ * defaults on LLM throw or JSON-parse error so the shot loop keeps
+ * moving rather than crashing.
+ */
+declare function createDefaultReviewer(options: CreateDefaultReviewerOptions): (input: ReviewerPromptInput) => Promise<ReviewerOutput>;
+
+/**
+ * Semantic concept judge — "does the built artifact actually implement
+ * the features the user asked for?"
+ *
+ * Distinct from the domain/code/coherence judges in `judges.ts`:
+ *   - those judges score free-form conversational agent outputs along
+ *     quality dimensions (accuracy, depth, etc.)
+ *   - this judge scores a *built artifact* (served HTML + source files)
+ *     against an explicit list of expected concepts, returning per-concept
+ *     {present, score 0-10, evidence, severity}.
+ *
+ * The judge is strict about distinguishing (a) a working implementation
+ * from (b) a keyword-present stub. "// TODO: mint button" is NOT present.
+ * Only real, functional, wired-up code counts.
+ *
+ * Use via {@link createSemanticConceptJudge} or directly via
+ * {@link runSemanticConceptJudge}. Soft-fails (available=false) on LLM
+ * or JSON-parse errors so the caller can treat that as "layer skipped"
+ * rather than "layer failed" in a multi-layer pipeline.
+ */
+
+/**
+ * Implementation complexity class for weighted scoring (added 0.11).
+ *
+ * - `render` (default): the concept is a UI surface that displays static
+ *   data — render a list, show a counter, lay out a button. Single-file
+ *   work, no external integration.
+ * - `integrate`: the concept requires wiring a real external system —
+ *   wallet connect (wagmi + RainbowKit + chain config), payment provider
+ *   (Stripe Elements + intent + webhook), an API client with auth.
+ *   Multi-file, library-knowledge, runtime correctness matters.
+ * - `compute`: the concept requires algorithmic work — solver, simulator,
+ *   constraint propagation, ML inference. Correctness > UI polish.
+ *
+ * Default weights (when applied via `weightConcepts: 'complexity'`):
+ *   render=1.0, integrate=2.0, compute=2.5
+ *
+ * Cross-vertical scoring without complexity weighting silently inflates
+ * the rate of UI-heavy verticals (healthcare, fintech dashboards) vs
+ * integration-heavy verticals (DeFi, wallets) — all concepts treated
+ * equally even though the agent does 2-3x the work for `integrate`.
+ */
+type ConceptComplexity = 'render' | 'integrate' | 'compute';
+interface ConceptSpec {
+    name: string;
+    /** Short hints that help the judge; not used for matching. */
+    keywords?: string[];
+    /** Optional explicit weight; default 1.0. Overrides complexity-derived weight. */
+    weight?: number;
+    /** Implementation complexity class. Default `render`. */
+    complexity?: ConceptComplexity;
+}
+interface ConceptFinding {
+    concept: string;
+    present: boolean;
+    /** 0..10. 10 = production-ready; 7 = functional thin; 4 = partial; 0 = absent. */
+    score: number;
+    evidence: string;
+    severity: Severity;
+}
+interface SemanticConceptJudgeInput {
+    /** Full natural-language prompt the agent was handed. */
+    userRequest: string;
+    /** Rendered HTML the preview returns (UI artifacts). Optional. */
+    servedHtml?: string;
+    /** Top-level source files from the agent's workdir. */
+    sourceFiles: Array<{
+        path: string;
+        content: string;
+    }>;
+    /** The expected concept list. */
+    expectedConcepts: ConceptSpec[];
+    /** Free-form metadata (id, difficulty) to inject into the prompt. */
+    artifactLabel?: string;
+    artifactDescription?: string;
+}
+interface SemanticConceptJudgeResult {
+    kind: 'semantic-concept';
+    version: string;
+    /** Normalized 0..1 score — mean of per-concept scores / 10. */
+    score: number;
+    presentCount: number;
+    totalCount: number;
+    findings: ConceptFinding[];
+    summary: string;
+    durationMs: number;
+    costUsd: number | null;
+    /** False on LLM/JSON error — treat as "skipped / unable to judge" in pipelines. */
+    available: boolean;
+    error?: string;
+}
+/**
+ * Score-aggregation strategy. Default `mean` (legacy behavior — 0.10
+ * and earlier always averaged 0-10 scores). `complexity` applies the
+ * default weight table (render=1, integrate=2, compute=2.5) unless a
+ * concept has an explicit `weight`. `explicit` honors only `weight`
+ * (defaulting to 1 for unspecified).
+ */
+type ConceptWeightStrategy = 'mean' | 'complexity' | 'explicit';
+declare const DEFAULT_COMPLEXITY_WEIGHTS: Record<ConceptComplexity, number>;
+interface SemanticConceptJudgeOptions {
+    /** Model id to call. Default 'claude-sonnet-4-6' via agent-eval defaults. */
+    model?: string;
+    /** Per-call timeout. Default 180s. */
+    timeoutMs?: number;
+    /** Pipeline budget for the prompt (source blob truncation). Default 45000. */
+    maxSourceChars?: number;
+    /** Per-file cap before inclusion. Default 20000. */
+    maxPerFileChars?: number;
+    /** HTML cap. Default 30000. */
+    maxHtmlChars?: number;
+    /** LlmClient config (baseUrl, apiKey, authHeader, …). */
+    llm?: LlmClientOptions;
+    /**
+     * Score aggregation strategy. Default `mean` for backward compatibility
+     * with 0.10 and earlier callers. Cross-vertical comparisons should use
+     * `complexity` to neutralize the integrate-vs-render asymmetry.
+     */
+    weightConcepts?: ConceptWeightStrategy;
+    /** Override the default complexity → weight table. */
+    complexityWeights?: Partial<Record<ConceptComplexity, number>>;
+}
+declare const SEMANTIC_CONCEPT_JUDGE_VERSION = "semantic-concept-judge-v1-2026-04-24";
+/**
+ * Run the semantic concept judge. Soft-fails to available=false on
+ * LLM/JSON errors — callers in a MultiLayerVerifier pipeline can treat
+ * that as "skip" rather than "fail."
+ */
+declare function runSemanticConceptJudge(input: SemanticConceptJudgeInput, options?: SemanticConceptJudgeOptions): Promise<SemanticConceptJudgeResult>;
+/**
+ * Factory: pin LLM options once, return a closure that accepts inputs.
+ * Convenient for pipelines that want to share a single LlmClient config.
+ */
+declare function createSemanticConceptJudge(options?: SemanticConceptJudgeOptions): (input: SemanticConceptJudgeInput) => Promise<SemanticConceptJudgeResult>;
+
+/**
+ * Intent-match judge — "did the agent build the right APP, ignoring
+ * whether every feature is wired up?"
+ *
+ * Distinct from {@link runSemanticConceptJudge} which scores per-concept
+ * presence. The semantic judge can return 0/4 concepts present even
+ * when the agent built a thoughtful, polished, on-brief app that just
+ * lacks one or two features. The semantic judge can also return 4/4
+ * present even when the agent shipped the wrong project (keyword-rich
+ * stub).
+ *
+ * Intent-match asks ONE question:
+ *   "Looking at the agent's work as a whole — independent of feature
+ *    coverage — is this an honest attempt at the user's request?"
+ *
+ * Returns a 0–1 score and a 1-sentence evidence string. Use as a sanity
+ * check on `completenessScore`-style metrics: if intent-match is high
+ * and concept count is low, the agent built the right thing but is
+ * missing features (ship and iterate). If intent-match is low, the
+ * agent built the wrong thing (reject regardless of concept count).
+ *
+ * Soft-fails on LLM/JSON error (`available: false`) so callers can
+ * treat failure as "judge skipped."
+ *
+ * Added in 0.11 to replace the lying `completenessScore: 1` field that
+ * VerticalBench shipped pre-Gen-48 — that field was keyword-driven and
+ * fired true on builds with zero spec concepts implemented.
+ */
+
+declare const INTENT_MATCH_JUDGE_VERSION = "intent-match-judge-v1-2026-04-24";
+interface IntentMatchInput {
+    /** The full natural-language prompt the agent was handed. */
+    userRequest: string;
+    /** Top-level source files from the agent's workdir. */
+    sourceFiles: Array<{
+        path: string;
+        content: string;
+    }>;
+    /** Rendered HTML the preview returned, when available. */
+    servedHtml?: string;
+    /** Optional metadata to inject (id, vertical, difficulty). */
+    artifactLabel?: string;
+    artifactDescription?: string;
+}
+interface IntentMatchResult {
+    kind: 'intent-match';
+    version: string;
+    /** 0..1 — 1 = unmistakably the right app, 0 = unrelated to the brief. */
+    score: number;
+    /** One-sentence rationale citing concrete evidence (file or HTML). */
+    evidence: string;
+    durationMs: number;
+    costUsd: number | null;
+    available: boolean;
+    error?: string;
+}
+interface IntentMatchOptions {
+    model?: string;
+    timeoutMs?: number;
+    maxSourceChars?: number;
+    maxPerFileChars?: number;
+    maxHtmlChars?: number;
+    llm?: LlmClientOptions;
+}
+/**
+ * Run the intent-match judge. Soft-fails to available=false on error.
+ */
+declare function runIntentMatchJudge(input: IntentMatchInput, options?: IntentMatchOptions): Promise<IntentMatchResult>;
+/**
+ * Factory: pin LLM options once, return a closure.
+ */
+declare function createIntentMatchJudge(options?: IntentMatchOptions): (input: IntentMatchInput) => Promise<IntentMatchResult>;
+
+/**
+ * Flow layer — drive a previewed app through a scripted user walk.
+ *
+ * The MultiLayerVerifier already had a `flow` slot wired in
+ * VerticalBench's verification-harness, but the layer module was
+ * always-skipped ("flow layer module not yet wired"). This adds the
+ * module: a Layer<Env> that takes a {@link FlowSpec} (URL + steps),
+ * boots a preview server via the supplied {@link FlowRunner}, executes
+ * each step, and returns a LayerResult whose `findings` enumerate
+ * which step failed.
+ *
+ * The runner is injected so this module can swap between:
+ *   - production: agent-browser CLI (a11y-tree based steps)
+ *   - test: in-memory mock that returns canned step outcomes
+ *   - future: Playwright, Puppeteer, custom scrapers
+ *
+ * Shipped in 0.11 alongside {@link runIntentMatchJudge} — together they
+ * close the "the agent shipped the wrong app and we didn't catch it"
+ * blind spot. Intent-match catches "wrong app entirely"; flow catches
+ * "right app but the buttons don't work."
+ */
+
+type FlowAction = 'navigate' | 'click' | 'fill' | 'expect-text' | 'expect-element' | 'expect-url' | 'wait';
+interface FlowStep {
+    /** What this step does. */
+    action: FlowAction;
+    /** Human-readable description for findings. */
+    describe?: string;
+    /**
+     * For navigate/expect-url: full URL. For click/fill/expect-element:
+     * accessible-name selector or CSS selector.
+     * For expect-text: substring expected on the page.
+     * For wait: ignored (use `value` for ms).
+     */
+    target?: string;
+    /** For fill: text to enter. For wait: ms. */
+    value?: string;
+    /** Severity of a failure. Default `major`. */
+    severity?: Severity;
+}
+interface FlowSpec {
+    /** Initial URL the runner should open. */
+    url: string;
+    /** Ordered steps. Stops at the first failure unless `continueOnFail: true`. */
+    steps: FlowStep[];
+    /** When true, execute every step even after a failure (collect all findings). */
+    continueOnFail?: boolean;
+    /** Per-step wall cap (ms). Default 15s. */
+    stepTimeoutMs?: number;
+}
+interface FlowRunnerStepResult {
+    ok: boolean;
+    /** Concrete observation: matched text snippet, captured URL, error message. */
+    evidence?: string;
+    /** Wall-clock duration of the step. */
+    durationMs?: number;
+}
+interface FlowRunner {
+    /** Open the target URL. Returns when the page is interactable. */
+    open(url: string): Promise<FlowRunnerStepResult>;
+    /** Execute one step. The runner owns interpretation of `target`. */
+    step(step: FlowStep): Promise<FlowRunnerStepResult>;
+    /** Tear down browser, free resources. Always called once per layer.run. */
+    close(): Promise<void>;
+}
+interface FlowLayerEnv {
+    /** Optional override per-call. Defaults supplied by the layer factory. */
+    flowSpec?: FlowSpec;
+}
+interface FlowLayerFactoryInput {
+    /** Static spec (used when env doesn't supply one). */
+    flowSpec?: FlowSpec;
+    /** Build the runner per call (lets the layer create + tear down per leaf). */
+    runner: () => FlowRunner | Promise<FlowRunner>;
+    /** Layer name. Default `flow`. */
+    name?: string;
+    /** Layer dependencies — default `['serve']` so a non-booting preview skips us. */
+    dependsOn?: string[];
+    /** Layer weight for blendedScore (0..1+). Default 1. */
+    weight?: number;
+    /** Cap for the entire flow run (ms). Default 60s. */
+    capMs?: number;
+}
+/**
+ * Build a flow layer that scripts a user walk via the supplied runner.
+ *
+ * Score: 1.0 when every step passed; otherwise 1 - (failedSteps / totalSteps).
+ * Status: `pass` iff every step passed; `fail` if any step failed; `error`
+ * on runner setup error; `skipped` when no flowSpec is available.
+ */
+declare function flowLayer<Env extends FlowLayerEnv = FlowLayerEnv>(input: FlowLayerFactoryInput): Layer<Env>;
+
+/**
+ * Deploy gate layer — would the agent's build actually publish?
+ *
+ * The product Blueprint Agent fronts promises "go from idea to live URL."
+ * Pre-Gen-48 the eval stopped at install/typecheck/build/serve — every
+ * one of which can pass while `vite build` (or `next build`, etc) fails
+ * on a production-only constraint (env-var requirement, dynamic import
+ * not statically resolvable, missing public asset).
+ *
+ * Deploy gate runs the production build via the supplied {@link DeployRunner}
+ * and asserts:
+ *   - command exited 0
+ *   - artifact dir contains an entry point (index.html for static SPAs,
+ *     equivalent per framework family)
+ *
+ * Shipped in 0.11 with the canonical `vite` runner. Future generations
+ * add wrangler-deploy --dry-run, next-build, etc — each as another
+ * runner factory.
+ */
+
+type DeployFamily = 'frontend-static' | 'nextjs' | 'remix' | 'fullstack-ts';
+interface DeployRunResult {
+    ok: boolean;
+    /** Stdout/stderr tail surfaced as evidence. Bounded in caller. */
+    output?: string;
+    /** Wall-clock duration of the build command. */
+    durationMs?: number;
+    /** Path to artifact directory the runner expects (dist/, .next/, build/, etc). */
+    artifactDir?: string;
+    /** True iff artifactDir contains the family's expected entry point. */
+    artifactValid?: boolean;
+}
+interface DeployRunner {
+    /** Run the production build. The runner owns command + cwd. */
+    run(): Promise<DeployRunResult>;
+}
+interface DeployGateLayerInput {
+    /** Build the runner per call. */
+    runner: () => DeployRunner | Promise<DeployRunner>;
+    /** Family hint — for logging, surfaced in diagnostics. */
+    family?: DeployFamily;
+    /** Layer name. Default `deploy`. */
+    name?: string;
+    /** Layer dependencies — default `['build']`. */
+    dependsOn?: string[];
+    /** Weight in blendedScore. Default 1. */
+    weight?: number;
+    /** Cap (ms). Default 120s — prod builds are slower than dev. */
+    capMs?: number;
+    /** When true, treat artifactValid=false as a fail (default true). */
+    requireArtifact?: boolean;
+}
+/**
+ * Build a deploy gate layer that runs the production build and verifies
+ * the artifact. Pass: ok && artifactValid. Score: 1.0 (pass) or 0 (fail).
+ *
+ * For families where artifact-validation isn't applicable (e.g. a
+ * server-rendered build that prints a manifest), set `requireArtifact:
+ * false` and rely on the runner's own ok signal.
+ */
+declare function deployGateLayer<Env = unknown>(input: DeployGateLayerInput): Layer<Env>;
+interface ViteDeployRunnerInput {
+    /** Workdir to build. The runner cd's here. */
+    workdir: string;
+    /**
+     * Function to run a shell command in `workdir`. Same shape as
+     * agent-eval's CommandRunner.run for compositional reuse.
+     */
+    exec: (cmd: string, opts?: {
+        cwd?: string;
+        timeoutMs?: number;
+    }) => Promise<{
+        stdout: string;
+        stderr: string;
+        exitCode: number;
+    }>;
+    /**
+     * Function to test whether a path exists in the workdir. Inject
+     * `(p) => existsSync(join(workdir, p))` for host runs.
+     */
+    exists: (relativePath: string) => boolean | Promise<boolean>;
+    /** Build command. Default `npm run build`. */
+    buildCommand?: string;
+    /** Artifact directory to validate. Default `dist`. */
+    artifactDir?: string;
+    /** Entry-point file under artifactDir. Default `index.html`. */
+    artifactEntry?: string;
+    /** Per-build cap (ms). Default 90s. */
+    timeoutMs?: number;
+}
+/**
+ * Canonical runner for `frontend-static` family — runs the build script,
+ * validates `<artifactDir>/<artifactEntry>` exists. Use as the `runner:`
+ * factory for {@link deployGateLayer}.
+ */
+declare function viteDeployRunner(input: ViteDeployRunnerInput): DeployRunner;
+
+/**
+ * Keyword-coverage judge — baseline complement to the semantic concept
+ * judge.
+ *
+ * Where {@link runSemanticConceptJudge} uses an LLM to read source code
+ * and decide whether a concept is REALLY implemented (not just
+ * keyword-mentioned), this judge does the cheap, deterministic version:
+ * fetch the served preview, concatenate every linked CSS/JS asset, and
+ * substring-match each expected concept's keywords against the
+ * concatenated haystack. Optional `requiredElement` selector adds a
+ * structural gate so "supply counter" can require an actual `<input>` or
+ * `<table>`, not just a comment containing the word.
+ *
+ * Use both judges. Keyword coverage is a fast 0-cost gate — a stub page
+ * with the right keywords passes here, fails the semantic judge. Score
+ * divergence between the two is itself a signal: high keyword coverage
+ * + low semantic = "the agent slapped the right words on the right
+ * scaffold but didn't wire any of it up."
+ *
+ * Pure functions, soft-fail on fetch error, no LLM dependency.
+ */
+interface KeywordConceptSpec {
+    name: string;
+    keywords: string[];
+    /**
+     * Optional CSS selector that must match in the HTML for the concept
+     * to count as present. Tiny subset:
+     *   - `tag`               (e.g. `form`)
+     *   - `tag[attr="value"]` (e.g. `input[type="number"]`)
+     *   - `tag[attr]`         (presence only)
+     * Anything more complex is rejected with `null` (treated as
+     * "unenforced", not "failed").
+     */
+    requiredElement?: string;
+}
+interface KeywordCoverageFinding {
+    concept: string;
+    found: boolean;
+    matchedKeywords: string[];
+    /** True iff the optional requiredElement selector matched; null when no selector. */
+    requiredElementPresent: boolean | null;
+}
+interface KeywordCoverageResult {
+    /** 0..1 share of concepts satisfied. */
+    score: number;
+    presentCount: number;
+    totalCount: number;
+    findings: KeywordCoverageFinding[];
+    durationMs: number;
+    /** Total bytes assembled across html + linked assets. */
+    totalAssembledBytes: number;
+    /** Soft-failure reason if the audit couldn't run. */
+    error?: string;
+}
+interface KeywordCoverageOptions {
+    /** Override fetch implementation — for tests. */
+    fetch?: typeof fetch;
+    /** Per-asset fetch timeout (default 3s). */
+    assetTimeoutMs?: number;
+    /** Initial-HTML fetch timeout (default 5s). */
+    htmlTimeoutMs?: number;
+}
+/**
+ * Element-presence check using a tiny CSS-selector subset. Returns
+ * null when the selector isn't supported — caller treats that as
+ * "unenforced" rather than "failed."
+ */
+declare function htmlContainsElement(html: string, selector: string): boolean | null;
+/**
+ * Pull every `<link rel=stylesheet href>` and `<script src>` from a
+ * raw HTML body. Returns absolute URLs resolved against `baseUrl`.
+ * Permissive regex — agent-authored markup doesn't always quote
+ * attributes the same way.
+ */
+declare function extractAssetUrls(html: string, baseUrl: string): string[];
+/**
+ * Score expected concepts against an already-fetched HTML payload + any
+ * pre-fetched CSS/JS assets. Use when the runner has the bytes in hand
+ * and doesn't want a fresh HTTP round-trip — e.g. sandbox runtime where
+ * the preview content was fetched via curl from inside the container.
+ */
+declare function runKeywordCoverageJudge(html: string, expectedConcepts: ReadonlyArray<KeywordConceptSpec>, assets?: ReadonlyArray<string>): KeywordCoverageResult;
+/**
+ * URL-fetch flavor — GET the preview, parallel-fetch every linked
+ * stylesheet + script (with bounded timeouts, soft-fail individually),
+ * then score via {@link runKeywordCoverageJudge}.
+ */
+declare function runKeywordCoverageJudgeUrl(previewUrl: string, expectedConcepts: ReadonlyArray<KeywordConceptSpec>, options?: KeywordCoverageOptions): Promise<KeywordCoverageResult>;
+
+/**
+ * Toolchain error-count extractor.
+ *
+ * Given stderr/stdout from a compiler or test runner, count the number
+ * of reported errors/failures. Patterns are deliberately narrow —
+ * unknown stderr returns `null` rather than zero so callers can
+ * distinguish "no errors" from "different toolchain, couldn't parse".
+ *
+ * All patterns are anchored to the start of a line and use bounded
+ * character classes to avoid catastrophic backtracking on pathological
+ * inputs.
+ *
+ * Add new toolchains by appending to {@link ERROR_COUNT_PATTERNS};
+ * order matters only in the sense that the first matching pattern wins.
+ */
+interface ErrorCountPattern {
+    /** Stable identifier for logging + tests. */
+    name: string;
+    /** Must be global (`g` flag) — the extractor counts matches. */
+    regex: RegExp;
+    /** Optional post-processing to extract a count from a single captured match. */
+    transform?: (match: RegExpMatchArray) => number;
+}
+declare const ERROR_COUNT_PATTERNS: ErrorCountPattern[];
+interface ExtractOptions {
+    /** Restrict to named patterns — default: all patterns. */
+    only?: string[];
+    /** Additional patterns to consider BEFORE the built-in list. */
+    extra?: ErrorCountPattern[];
+}
+interface ExtractResult {
+    /** Total count of matched errors, or null when no pattern matched. */
+    count: number | null;
+    /** Name of the pattern that matched, or null. */
+    matched: string | null;
+    /** Original matches for callers that want to surface specifics. */
+    samples: string[];
+}
+/**
+ * Try each pattern in order; return the first with matches.
+ *
+ * Returning `null` (instead of zero) on no-match is deliberate — a
+ * callsite that greps for "typescript errors" on cargo output should
+ * NOT treat that as "zero TS errors" because the toolchain is wrong.
+ */
+declare function extractErrorCount(text: string, opts?: ExtractOptions): ExtractResult;
+
+/**
+ * Reference replay — score an agent against withheld historical outcomes.
+ *
+ * This is the generic version of the public-audit replay pattern:
+ * run a candidate system on an old task, keep the reference answers hidden
+ * until after execution, then score recall/precision and gate promotion
+ * across train/dev/test/holdout splits.
+ */
+type ReferenceReplaySplit = 'train' | 'dev' | 'test' | 'holdout';
+interface ReferenceReplayItem {
+    id: string;
+    title: string;
+    description?: string;
+    severity?: string;
+    tags?: string[];
+    weight?: number;
+}
+interface ReferenceReplayCandidate {
+    id: string;
+    title: string;
+    description?: string;
+    severity?: string;
+    tags?: string[];
+    metadata?: Record<string, unknown>;
+}
+interface ReferenceReplayScenario {
+    id: string;
+    split?: ReferenceReplaySplit;
+    references: ReferenceReplayItem[];
+    candidates: ReferenceReplayCandidate[];
+    metadata?: Record<string, unknown>;
+}
+interface ReferenceReplayCase<Input = unknown> {
+    id: string;
+    split?: ReferenceReplaySplit;
+    input: Input;
+    references: ReferenceReplayItem[];
+    metadata?: Record<string, unknown>;
+}
+interface ReferenceReplayExecutionScenario<Input = unknown> {
+    id: string;
+    split: ReferenceReplaySplit;
+    input: Input;
+    metadata?: Record<string, unknown>;
+}
+interface ReferenceReplayRunContext {
+    runId: string;
+    caseIndex: number;
+    abortSignal?: AbortSignal;
+}
+interface ReferenceReplayAdapter<Input = unknown> {
+    run(scenario: ReferenceReplayExecutionScenario<Input>, context: ReferenceReplayRunContext): Promise<ReferenceReplayCandidate[]>;
+}
+type ReferenceReplayAdapterFn<Input = unknown> = (scenario: ReferenceReplayExecutionScenario<Input>, context: ReferenceReplayRunContext) => Promise<ReferenceReplayCandidate[]>;
+type ReferenceReplayAdapterLike<Input = unknown> = ReferenceReplayAdapter<Input> | ReferenceReplayAdapterFn<Input>;
+interface ReferenceReplayMatch {
+    scenarioId: string;
+    referenceId: string;
+    candidateId: string | null;
+    score: number;
+    matched: boolean;
+    weight: number;
+    reason: string;
+}
+interface ReferenceReplayScenarioScore {
+    scenarioId: string;
+    split: ReferenceReplaySplit;
+    matched: number;
+    total: number;
+    falsePositives: number;
+    matchedWeight: number;
+    totalWeight: number;
+    precision: number;
+    recall: number;
+    f1: number;
+    matches: ReferenceReplayMatch[];
+}
+interface ReferenceReplayAggregate {
+    matched: number;
+    total: number;
+    falsePositives: number;
+    matchedWeight: number;
+    totalWeight: number;
+    precision: number;
+    recall: number;
+    f1: number;
+    weightedRecall: number;
+}
+interface ReferenceReplayScore {
+    scenarios: ReferenceReplayScenarioScore[];
+    aggregate: ReferenceReplayAggregate;
+    bySplit: Partial<Record<ReferenceReplaySplit, ReferenceReplayAggregate>>;
+}
+interface ReferenceMatchResult {
+    score: number;
+    reason?: string;
+}
+type ReferenceReplayMatcher = (reference: ReferenceReplayItem, candidate: ReferenceReplayCandidate, scenario: ReferenceReplayScenario) => ReferenceMatchResult;
+interface ReferenceReplayScoreOptions {
+    matcher?: ReferenceReplayMatcher;
+    matchThreshold?: number;
+    includeHoldout?: boolean;
+    splits?: ReferenceReplaySplit[];
+}
+interface ReferenceReplayPromotionPolicy {
+    /** Splits that must improve or stay flat. Default: ['dev', 'test']. */
+    requiredSplits?: ReferenceReplaySplit[];
+    /** Minimum aggregate F1 lift required on required splits. Default 0. */
+    minF1Delta?: number;
+    /** Maximum F1 drop allowed on any compared split. Default 0. */
+    maxRegression?: number;
+    /** If true, holdout must be present and must not regress. Default true. */
+    requireHoldoutNonRegression?: boolean;
+}
+interface ReferenceReplaySplitComparison {
+    split: ReferenceReplaySplit;
+    baselineF1: number;
+    candidateF1: number;
+    f1Delta: number;
+    baselineRecall: number;
+    candidateRecall: number;
+    recallDelta: number;
+}
+interface ReferenceReplayPromotionDecision {
+    promote: boolean;
+    reason: string;
+    aggregateDelta: number;
+    comparisons: ReferenceReplaySplitComparison[];
+    regressions: ReferenceReplaySplitComparison[];
+}
+interface ReferenceReplayCaseRun<Input = unknown> {
+    caseId: string;
+    split: ReferenceReplaySplit;
+    input: Input;
+    metadata?: Record<string, unknown>;
+    references: ReferenceReplayItem[];
+    candidates: ReferenceReplayCandidate[];
+    score: ReferenceReplayScenarioScore;
+    durationMs: number;
+    error?: string;
+}
+interface ReferenceReplayRun<Input = unknown> {
+    id: string;
+    variantId?: string;
+    startedAt: number;
+    completedAt: number;
+    durationMs: number;
+    cases: ReferenceReplayCaseRun<Input>[];
+    score: ReferenceReplayScore;
+    metadata?: Record<string, unknown>;
+}
+interface ReferenceReplayRunOptions<Input = unknown> extends ReferenceReplayScoreOptions {
+    adapter: ReferenceReplayAdapterLike<Input>;
+    runId?: string;
+    variantId?: string;
+    metadata?: Record<string, unknown>;
+    store?: ReferenceReplayRunStore<Input>;
+    abortSignal?: AbortSignal;
+    continueOnError?: boolean;
+    now?: () => number;
+}
+interface ReferenceReplayRunStore<Input = unknown> {
+    save(run: ReferenceReplayRun<Input>): Promise<void>;
+    list(): Promise<ReferenceReplayRun<Input>[]>;
+}
+declare function runReferenceReplay<Input = unknown>(cases: ReferenceReplayCase<Input>[], options: ReferenceReplayRunOptions<Input>): Promise<ReferenceReplayRun<Input>>;
+declare function decideReferenceReplayRunPromotion(baseline: ReferenceReplayRun, candidate: ReferenceReplayRun, policy?: ReferenceReplayPromotionPolicy): ReferenceReplayPromotionDecision;
+declare function inMemoryReferenceReplayStore<Input = unknown>(initial?: ReferenceReplayRun<Input>[]): ReferenceReplayRunStore<Input>;
+declare function jsonlReferenceReplayStore<Input = unknown>(path: string): ReferenceReplayRunStore<Input>;
+declare function scoreReferenceReplay(scenarios: ReferenceReplayScenario[], options?: ReferenceReplayScoreOptions): ReferenceReplayScore;
+declare function compareReferenceReplay(baseline: ReferenceReplayScore, candidate: ReferenceReplayScore): ReferenceReplaySplitComparison[];
+declare function decideReferenceReplayPromotion(baseline: ReferenceReplayScore, candidate: ReferenceReplayScore, policy?: ReferenceReplayPromotionPolicy): ReferenceReplayPromotionDecision;
+declare function defaultReferenceReplayMatcher(reference: ReferenceReplayItem, candidate: ReferenceReplayCandidate): ReferenceMatchResult;
+
+export { type ActiveLearningOptions, type AdapterRun, AgentDriver, type AgentDriverConfig, type AlignmentOp, type AntiSlopConfig, type AntiSlopIssue, type AntiSlopReport, type Artifact, type ArtifactCheck, type Artifact$1 as ArtifactCheckArtifact, type ArtifactResult, type ArtifactValidator, AxGepaSteeringOptimizer, type AxSteeringOptimizerConfig, type BaselineOptions, type BaselineReport, BehaviorAssertion, type BenchmarkReport, BenchmarkRunner, type BenchmarkRunnerConfig, type BestOfNResult, type BisectOptions, type BisectResult, type BisectStep, BudgetBreachError, type BudgetBreachFinding, type BudgetBreachReport, BudgetGuard, type BudgetLedgerEntry, type BudgetSpec, BuilderSession, type BuilderSessionInit, type CalibrationBin, type CalibrationOptions, type CalibrationReport, type CalibrationResult, CallExpectation, type CanaryLeak, type CandidateScenario, type CandidateScore, type CausalAttributionReport, type ChatSummary, type CheckResult, type CollectedArtifacts, type CommandRunner, type CompletionCriterion, type ConceptComplexity, type ConceptFinding, type ConceptSpec, type ConceptWeightStrategy, type ContinuityCheck, type ContinuityCheckResult, type ContinuityReport, type ContinuitySnapshotPair, type ContractMetric, type ContractReport, ConvergenceTracker, type CorrelationReport, type CorrelationResult, type CorrelationStudyOptions, type CorrelationStudyResult, type CostEntry, type CostSummary, CostTracker, type CounterfactualContext, type CounterfactualMutation, type CounterfactualResult, type CounterfactualRunner, type CreateDefaultReviewerOptions, type CrossTraceDiff, type CrossTraceDiffOptions, DEFAULT_AGENT_SLOS, DEFAULT_COMPLEXITY_WEIGHTS, DEFAULT_RULES as DEFAULT_FAILURE_RULES, DEFAULT_FINDERS, DEFAULT_HARNESS_OBJECTIVES, DEFAULT_MUTATORS, DEFAULT_REDACTION_RULES, DEFAULT_RED_TEAM_CORPUS, DEFAULT_RUN_SCORE_WEIGHTS, Dataset, type DatasetDifficulty, type DatasetManifest, type DatasetProvenance, type DatasetScenario, type DatasetSplit, type DeployFamily, type DeployGateLayerInput, type DeployRunResult, type DeployRunner, type DeploymentOutcome, type DirEntry, type Direction, type DivergenceOptions, type DivergenceReport, DockerSandboxDriver, type DriverResult, type DriverState, DualAgentBench, type DualAgentBenchConfig, type DualAgentReport, type DualAgentRound, type DualAgentScenario, type DualAgentScenarioResult, ERROR_COUNT_PATTERNS, type ErrorCountPattern, type EuRiskClass, type EvalMetricSpec, type EvalResult, type EventFilter, type EventKind, type EvolutionRound, type ExecutorConfig, type Expectation, type Experiment, type Run$1 as ExperimentRun, type ExperimentStore, ExperimentTracker, type ExportedRewardModel, type ExtractOptions, type ExtractResult, FAILURE_CLASSES, type FactorContribution, type FactorialCell, type FailureClass, type FailureClassification, type FailureCluster, type FailureClusterReport, type FailureContext, type FailureRule, type FeedbackPattern, FileSystemOutcomeStore, type FileSystemOutcomeStoreOptions, FileSystemTraceStore, type FileSystemTraceStoreOptions, type Finding, type FlowAction, type FlowLayerEnv, type FlowLayerFactoryInput, type FlowRunner, type FlowRunnerStepResult, type FlowSpec, type FlowStep, type GenericSpan, type GoldenItem, type GovernanceContext, type GovernanceFinding, type GovernanceReport, type GradedStep, type HarnessAdapter, type HarnessConfig, type HarnessExperimentConfig, type HarnessExperimentResult, type HarnessIntervention, type HarnessRunRequest, type HarnessRunResult, type HarnessScenario, type HarnessSelection, type HarnessVariant, type HarnessVariantReport, HoldoutAuditor, HoldoutLockedError, type HostedJudgeConfig, type HostedJudgeDimension, type HostedJudgeRequest, type HostedJudgeResponse, type HostedRunCriticConfig, type HostedRunScoreRequest, type HostedRunScoreResponse, type HypothesisManifest, type HypothesisResult, INTENT_MATCH_JUDGE_VERSION, type ImageData, InMemoryExperimentStore, InMemoryOutcomeStore, InMemoryTraceStore, InMemoryWorkspaceInspector, type InferenceScorer, type InspectorContext, type IntentMatchInput, type IntentMatchOptions, type IntentMatchResult, type InteractionContribution, type JudgeAgreementReport, type JudgeConfig, type JudgeFleetOptions, type JudgeFn, type JudgeInput, type JudgePair, type JudgeReplayResult, type JudgeRubric, JudgeRunner, type JudgeScore, type JudgeSpan, type KeywordConceptSpec, type KeywordCoverageFinding, type KeywordCoverageOptions, type KeywordCoverageResult, type LangfuseEnvelope, type LangfuseGeneration, type LangfuseScore, type Layer, type LayerCorrelation, type LayerResult, type LayerStatus, LlmCallError, type LlmCallRequest, type LlmCallResult, LlmClient, type LlmClientOptions, type LlmJsonCall, type LlmMessage, type LlmReviewerConfig, type LlmSpan, type LlmUsage, MODEL_PRICING, type MatcherResult, type MeasurementPolicy, type MergeOptions, type Message, type MetricSamples, type MetricVerdict, MetricsCollector, type MuffledFinder, type MuffledFinding, MultiLayerVerifier, type MultiToolchainLayerConfig, type Mutator, OTEL_AGENT_EVAL_SCOPE, type Objective, type OptimizationConfig, type OptimizationExample, OptimizationLoop, type OptimizationLoopConfig, type OptimizationLoopResult, type OptimizationResult, type Oracle, type OracleObservation, type OracleReport, type OracleResult, type OtlpExport, type OtlpResourceSpans, type OtlpSpan, type OutcomeFilter, type OutcomePair, type OutcomeStore, type PairwiseComparison, PairwiseSteeringOptimizer, type ParetoResult, type PersonaConfig, type Playbook, type PlaybookEntry, type PositionalBiasResult, type PrmGradedTrace, PrmGrader, type PrmTrainingSample, ProductClient, type ProductClientConfig, type ProjectKind, ProjectRegistry, type ProjectSummary, type ProjectTimelineEntry, type PromptHandle, PromptOptimizer, PromptRegistry, type PromptVariant, type ProposeFn, type ProposeInput, type ProposeOutput, type ProposeReviewConfig, type ProposeReviewReport, type ProposeReviewShot, REDACTION_VERSION, type RedTeamCase, type RedTeamCategory, type RedTeamFinding, type RedTeamPayload, type RedTeamReport, type RedactionReport, type RedactionRule, type ReferenceMatchResult, type ReferenceReplayAdapter, type ReferenceReplayAdapterFn, type ReferenceReplayAdapterLike, type ReferenceReplayAggregate, type ReferenceReplayCandidate, type ReferenceReplayCase, type ReferenceReplayCaseRun, type ReferenceReplayExecutionScenario, type ReferenceReplayItem, type ReferenceReplayMatch, type ReferenceReplayMatcher, type ReferenceReplayPromotionDecision, type ReferenceReplayPromotionPolicy, type ReferenceReplayRun, type ReferenceReplayRunContext, type ReferenceReplayRunOptions, type ReferenceReplayRunStore, type ReferenceReplayScenario, type ReferenceReplayScenarioScore, type ReferenceReplayScore, type ReferenceReplayScoreOptions, type ReferenceReplaySplit, type ReferenceReplaySplitComparison, type RegressionOptions, type RegressionSpec, type RetrievalSpan, type Review, type ReviewFn, type ReviewInput, type ReviewMemoryEntry, type ReviewMemoryStore, type ReviewerMemoryEntry, type ReviewerOutput, type ReviewerPromptInput, type ReviewerSoftFailDefaults, type ReviewerVerificationSummary, type RobustnessResult, type RouteMap, type RubricDimension, type Run, type RunAppScenarioOptions, type RunCommandInput, type RunCommandResult, type RunConfig, RunCritic, type RunCriticOptions, type RunDiff, type RunFilter, type RunLayer, type RunOutcome, type RunScore, type RunScoreWeights, type RunStatus, type RunTrace, SEMANTIC_CONCEPT_JUDGE_VERSION, type SandboxDriver, SandboxHarness, type SandboxHarnessResult, type SandboxJudgeKind, type SandboxJudgeResult, type SandboxJudgeSpec, type SandboxResult, type SandboxSpan, type ScanOptions, type Scenario, type ScenarioCost, type ScenarioFile, ScenarioRegistry, type ScenarioResult, type ScoredTarget, type SelfPlayOptions, type SelfPlayProposer, type SelfPlayScorer, type SelfPreferenceResult, type SemanticConceptJudgeInput, type SemanticConceptJudgeOptions, type SemanticConceptJudgeResult, type SeriesConvergenceOptions, type SeriesConvergenceResult, type Severity, type ShipOptions, type SignedManifest, type SliceOptions, type Slo, type SloCheckResult, type SloComparator, type SloReport, type SloSeverity, type SlopCategory, type Span, type SpanBase, type SpanFilter, type SpanHandle, type SpanKind, type SpanStatus, type SteeringBundle, type SteeringDelta, type SteeringEvaluation, type SteeringOptimizationResult, type SteeringOptimizationRow, type SteeringOptimizationSelector, type SteeringOptimizerBackend, type SteeringOptimizerConfig, type SteeringRolePrompt, type SteeringVariantReport, type StepAttribution, type StepContext, type StepRubric, type StuckLoopFinding, type StuckLoopOptions, type StuckLoopReport, SubprocessSandboxDriver, type SubprocessSandboxDriverOptions, type SynthesisReason, type SynthesisTarget, TRACE_SCHEMA_VERSION, type TestGradedRunOptions, type TestGradedRunResult, type TestGradedScenario, type TestOutputParser, type TestResult, type ThreeLayerProjectReport, type ThresholdContract, TokenCounter, type TokenSpec, type ToolSpan, type ToolStats, type ToolUseMetrics, type ToolUseOptions, type ToolWasteFinding, type ToolWasteOptions, type ToolWasteReport, TraceEmitter, type TraceEmitterOptions, type TraceEvent, type TraceStore, type Trajectory, type TrajectoryStep, type Turn, type TurnMetrics, type TurnResult, UNIVERSAL_FINDERS, type UseCaseSignals, type ValidationContext, type ValidationIssue, type ValidationResult, type VariantScore, type VerbosityBiasResult, type Verification, type VerificationReport, type VerifyContext, type VerifyFn, type VerifyOptions, type VisualDiffOptions, type VisualDiffResult, type ViteDeployRunnerInput, type WorkflowTopology, type WorkspaceAssertion, type WorkspaceAssertionResult, type WorkspaceInspector, type WorkspaceSnapshot, adversarialJudge, aggregateLlm, aggregateRunScore, analyzeAntiSlop, analyzeSeries, argHash, attributeCounterfactuals, benjaminiHochberg, bisect, bonferroni, budgetBreachView, buildReviewerPrompt, buildTrajectory, byteLengthRange, calibrateJudge, calibrationCurve, callLlm, callLlmJson, canaryLeakView, causalAttribution, checkCanaries, checkSlos, clamp01, classifyEuAiRisk, classifyFailure, codeExecutionJudge, cohensD, coherenceJudge, collectionPreserved, commitBisect, compareReferenceReplay, compareToBaseline, compilerJudge, composeParsers, composeValidators, computeToolUseMetrics, confidenceInterval, containsAll, correlateLayers, correlationStudy, createAntiSlopJudge, createCustomJudge, createDefaultReviewer, createDomainExpertJudge, createIntentMatchJudge, createLlmReviewer, createSemanticConceptJudge, crossTraceDiff, decideReferenceReplayPromotion, decideReferenceReplayRunPromotion, defaultJudges, defaultReferenceReplayMatcher, deployGateLayer, distillPlaybook, dominates, estimateCost, estimateTokens, euAiActReport, evaluateContract, evaluateHypothesis, evaluateOracles, executeScenario, expectAgent, exportRewardModel, exportRunAsOtlp, exportTrainingData, extractAssetUrls, extractErrorCount, failureClusterView, fileContains, fileExists, findAutoMatchNoExpectation, findConstructorCwdDropped, findFallbackToPass, findLiteralTruePass, findSkipCountsAsPass, firstDivergenceView, flowLayer, formatBenchmarkReport, formatDriverReport, formatFindings, gradeSemanticStatus, groupBy, hashContent, hashScenarios, htmlContainsElement, inMemoryReferenceReplayStore, inMemoryReviewStore, interRaterReliability, iqr, isJudgeSpan, isLlmSpan, isPrmVerdict, isRetrievalSpan, isSandboxSpan, isToolSpan, jestTestParser, jsonHasKeys, jsonShape, jsonlReferenceReplayStore, jsonlReviewStore, judgeAgreementView, judgeSpans, keyPreserved, linterJudge, llmSpanFromProvider, llmSpans, loadScorerFromGrader, localCommandRunner, lowercaseMutator, mannWhitneyU, mergeLayerResults, mergeSteeringBundle, multiToolchainLayer, nistAiRmfReport, nonRefusalRubric, normalizeScores, notBlocked, outputLengthRubric, pairedTTest, paraphraseRobustness, paretoFrontier, partialCredit, pixelDeltaRatio, politenessPrefixMutator, positionalBias, printDriverSummary, prmBestOfN, prmEnsembleBestOfN, probeLlm, promptBisect, proposeSynthesisTargets, pytestTestParser, redTeamDataset, redTeamReport, redactString, redactValue, regexMatch, regexMatches, regressionView, renderMarkdown, renderMarkdownReport, renderPlaybookMarkdown, renderSteeringText, replayScorerOverCorpus, replayTraceThroughJudge, requiredSampleSize, resumeBuilderSession, rowCount, rowWhere, runAssertions, runCounterfactual, runE2EWorkflow, runExpectations, runFailureClass, runHarnessExperiment, runIntentMatchJudge, runJudgeFleet, runKeywordCoverageJudge, runKeywordCoverageJudgeUrl, runProposeReview, runReferenceReplay, runSelfPlay, runSemanticConceptJudge, runTestGradedScenario, runsForScenario, scanForMuffledGates, scoreAllProjects, scoreContinuity, scoreProject, scoreRedTeamOutput, scoreReferenceReplay, securityJudge, selectHarnessVariant, selfPreference, sentenceReorderMutator, signManifest, soc2Report, statusAdvanced, stripFencedJson, stuckLoopView, summarize, summarizeHarnessResults, testJudge, textInSnapshot, toLangfuseEnvelope, toNdjson, toPrometheusText, toolIntentAlignmentRubric, toolNamesForRun, toolNonRedundantRubric, toolSpans, toolSuccessRubric, toolWasteView, typoMutator, urlContains, verbosityBias, verifyManifest, visualDiff, viteDeployRunner, vitestTestParser, weightedMean, welchsTTest, whitespaceCollapseMutator, wilcoxonSignedRank };