@tangle-network/agent-eval 0.117.0 → 0.118.0

package/dist/benchmarks/index.d.ts

@@ -1,14 +1,958 @@
-export { B as BENCHMARK_SPLIT_SEED, a as BenchmarkAdapter, b as BenchmarkDatasetItem, k as BenchmarkDistribution, c as BenchmarkEvaluation, d as BenchmarkFamily, l as BenchmarkMetricCalibrationOptions, m as BenchmarkMetricCalibrationResult, n as BenchmarkReport, e as BenchmarkResponder, o as BenchmarkRunOptions, p as BenchmarkRunResult, f as BenchmarkScenario, q as BenchmarkSliceSummary, g as BenchmarkSource, h as BenchmarkTaskKind, r as BuildStandardRetrievalItemsOptions, R as RetrievalIdAdapterOptions, S as StandardRetrievalArtifact, s as StandardRetrievalDocument, t as StandardRetrievalEvaluationOptions, u as StandardRetrievalPayload, v as StandardRetrievalQrel, w as StandardRetrievalQuery, x as StandardRetrievalResult, y as buildStandardRetrievalItems, z as calibrateBenchmarkMetric, A as createRetrievalIdBenchmarkAdapter, i as deterministicSplit, C as evaluateStandardRetrieval, D as normalizeRetrievedDocumentIds, E as parseBeirCorpusJsonl, F as parseBeirQueriesJsonl, G as parseJsonlRows, H as parseQrels, I as parseTsvRows, J as renderBenchmarkReportMarkdown, K as retrievalMetricsAtCutoff, L as routing, M as runBenchmarkAdapter, N as summarizeBenchmarkCampaign } from '../index-PdX4VnPA.js';
-import '../types-BSw1rOUB.js';
-import '../policy-edit-wG9uFEFm.js';
-import '../run-record-BDH49H2E.js';
-import '@tangle-network/agent-interface';
-import '../errors-oeQrLqXC.js';
-import '../schema-B3Q3l9Z_.js';
-import '../store-C1YxJDEK.js';
-import '../types-BkfcQnxV.js';
-import '../cost-ledger-DWy3XdJc.js';
-import '@tangle-network/tcloud';
-import '../llm-client-qoDd18Qz.js';
-import '../raw-provider-sink-C46HDghv.js';
-import '../storage-DrX3v_5B.js';
+type AgentProfileCellSchemaVersion = 'agent-profile-cell/v1';
+type AgentProfileJson = string | number | boolean | null | AgentProfileJson[] | {
+    [key: string]: AgentProfileJson;
+};
+type AgentProfileDimensionValue = string | number | boolean | null;
+interface AgentProfileSource {
+    /** Runtime/profile contract being fingerprinted, e.g. `agent-interface-profile`. */
+    kind: string;
+    /** sha256 over the canonical source profile object. */
+    hash: string;
+}
+interface AgentProfileHarness {
+    id: string;
+    version?: string;
+    hash?: string;
+}
+interface AgentProfileCell {
+    schemaVersion: AgentProfileCellSchemaVersion;
+    cellId: string;
+    profileId: string;
+    sourceProfile: AgentProfileSource;
+    harness?: AgentProfileHarness;
+    model?: string;
+    promptHash?: string;
+    dimensions?: Record<string, AgentProfileDimensionValue>;
+}
+
+/**
+ * Paper-grade RunRecord schema + runtime validator.
+ *
+ * Every run that participates in a promotion gate, paper table, or
+ * researcher loop SHOULD be recorded as a `RunRecord`. The mandatory
+ * fields are exactly those the paper "Two Loops, Three Roles" requires
+ * for reproducibility: who/what/when/cost/seed/hash, plus the search vs
+ * holdout split tag and either a `searchScore` or a `holdoutScore`.
+ *
+ * This is intentionally NOT a replacement for the rich `Run` /
+ * `ProposeReviewReport` / `ScenarioResult` types already in the
+ * package. Those are runtime structures with full provenance. A
+ * `RunRecord` is the analysis-time projection — the JSON-friendly
+ * row you'd put in a parquet file or paste into a notebook.
+ *
+ * Validate at the boundary:
+ *
+ *   const rec = validateRunRecord(rawJson)         // throws on missing
+ *   const ok  = isRunRecord(rawJson)               // boolean check
+ *   const rec = parseRunRecordSafe(rawJson)        // { ok, value | error }
+ *
+ * The validator runs in pure TS — zod is intentionally NOT a
+ * dependency. Round-trip tested in `tests/run-record.test.ts`.
+ */
+
+/** Search/dev/holdout split tag. 'search' is the paper-grade alias for the
+ *  combined train+test pool that the optimizer is allowed to read. */
+type RunSplitTag = 'search' | 'dev' | 'holdout';
+interface RunTokenUsage {
+    input: number;
+    /** All generated tokens charged as output, including reasoning tokens. */
+    output: number;
+    /** Reasoning-token subset of `output`, when the provider reports it. */
+    reasoning?: number;
+    /** Prompt tokens served from a provider cache. */
+    cached?: number;
+    /** Prompt tokens written into a provider cache. */
+    cacheWrite?: number;
+}
+
+type CostChannel = 'agent' | 'judge' | 'verifier' | 'analyst' | 'driver' | (string & {});
+interface CostUsage {
+    inputTokens: number;
+    outputTokens: number;
+    cachedTokens?: number;
+}
+interface CostCallBase {
+    callId: string;
+    channel: CostChannel;
+    phase: string;
+    actor: string;
+    model: string;
+    maximumCostUsd?: number;
+    tags?: Record<string, string>;
+    timestamp: number;
+}
+interface CostReceipt extends CostCallBase, CostUsage {
+    status: 'settled';
+    costUsd: number;
+    costUnknown: boolean;
+    usageUnknown?: boolean;
+    pricing?: {
+        inputUsdPerThousand: number;
+        outputUsdPerThousand: number;
+    };
+    actualCostUsd?: number;
+    error?: string;
+}
+interface CostReceiptInput extends CostUsage {
+    model: string;
+    actualCostUsd?: number;
+    costUnknown?: boolean;
+    usageUnknown?: boolean;
+}
+type MaximumCharge = {
+    externallyEnforcedMaximumUsd: number;
+} | ({
+    model: string;
+} & CostUsage);
+interface RunPaidCallInput<T> {
+    callId?: string;
+    channel: CostChannel;
+    phase: string;
+    actor: string;
+    /** Used before a provider receipt exists and on failures without one. */
+    model?: string;
+    tags?: Record<string, string>;
+    signal?: AbortSignal;
+    /** Provider-enforced dollar maximum, or maximum priced token usage. Required when capped. */
+    maximumCharge?: MaximumCharge;
+    /** `callId` can be forwarded as the provider's idempotency key. */
+    execute(signal: AbortSignal, callId: string): Promise<T>;
+    receipt(value: T): CostReceiptInput;
+    receiptFromError?(error: Error): CostReceiptInput | undefined;
+}
+type PaidCallResult<T> = {
+    succeeded: true;
+    callId: string;
+    value: T;
+    receipt: CostReceipt;
+} | {
+    succeeded: false;
+    callId?: string;
+    error: Error;
+    receipt?: CostReceipt;
+};
+interface ChannelRollup {
+    channel: CostChannel;
+    calls: number;
+    inputTokens: number;
+    outputTokens: number;
+    cachedTokens: number;
+    costUsd: number;
+    unpricedCalls: number;
+    unknownUsageCalls: number;
+}
+interface CostLedgerSummary {
+    totalCalls: number;
+    pendingCalls: number;
+    unresolvedCalls: number;
+    reservedCostUsd: number;
+    inputTokens: number;
+    outputTokens: number;
+    cachedTokens: number;
+    totalCostUsd: number;
+    byChannel: ChannelRollup[];
+    unpricedModels: string[];
+    fullyPriced: boolean;
+    usageComplete: boolean;
+    accountingComplete: boolean;
+    incompleteReasons: string[];
+}
+
+/**
+ * LLM client with graceful degrade.
+ *
+ * OpenAI-compatible `/v1/chat/completions` client with:
+ *   - Exponential-backoff retry on 429 + 5xx gateway errors (502/503/504).
+ *   - Retry on transient network errors (fetch failed, AbortError, ECONNRESET).
+ *   - Graceful json_schema → json_object degrade on 400 with schema-reject body.
+ *   - Fenced-JSON stripping (```json ... ```) for models that wrap structured output.
+ *   - Configurable base URL + api key / bearer, works with LiteLLM proxies, OpenAI
+ *     directly, cli-bridge subscriptions, and any router that speaks the spec.
+ *
+ * Usage:
+ *   const { value, result } = await callLlmJson<MyType>(
+ *     { model: 'gpt-4o', messages: [...], jsonSchema: { name: 'x', schema: {...} } },
+ *     { baseUrl: 'https://router.tangle.tools/v1', apiKey: process.env.KEY },
+ *   )
+ *
+ * This is THE llm-calling seam for agent-eval primitives that need structured
+ * output (semantic concept judge, reviewer directives, critic scores). Primitives
+ * that need free-form text use `callLlm` and parse output themselves.
+ */
+
+interface LlmUsage {
+    promptTokens: number;
+    completionTokens: number;
+    totalTokens: number;
+    /** False when the provider omitted or malformed prompt/completion usage. */
+    captured?: boolean;
+    /** Proxies populate this when prompt caching is on. */
+    cachedPromptTokens?: number;
+}
+interface LlmCallResult {
+    /** The text content of the first choice. Empty string if none. */
+    content: string;
+    usage: LlmUsage;
+    /**
+     * Cost in USD. Pulled from proxy's `_response_cost` field when present;
+     * `null` when neither the proxy nor the caller can derive it.
+     */
+    costUsd: number | null;
+    /** Model name actually used (echoed from response). */
+    model: string;
+    /** Wall-clock duration of the HTTP call (last attempt, if retried). */
+    durationMs: number;
+    /**
+     * `finish_reason` echoed from the first choice (`stop`, `length`,
+     * `content_filter`, `tool_calls`, ...). `null` when the provider omits it.
+     * Exposed so a free-form `callLlm` caller CAN detect a truncated answer
+     * (`length`) instead of treating a cut-off completion as complete. Note:
+     * `callLlm` does not itself reject on it — acting on this signal is the
+     * caller's responsibility (in-repo free-form drivers do not yet enforce it).
+     */
+    finishReason?: string | null;
+    /**
+     * True when `content.trim()` is empty. An empty completion is a silent zero
+     * for free-form `callLlm` callers; this flag is the signal a caller can
+     * inspect to fail loud rather than proceed on an empty string. `callLlm`
+     * surfaces it but does not throw on it.
+     */
+    contentEmpty?: boolean;
+    /** Raw response body. */
+    raw: Record<string, unknown>;
+}
+type LlmCallMetadata = Pick<LlmCallResult, 'usage' | 'costUsd' | 'model' | 'durationMs'>;
+
+/**
+ * Analyst contract — the missing orchestration layer over agent-eval's
+ * existing analyzers (analyzeTraces, MultiLayerVerifier, RunCritic,
+ * SemanticConceptJudge, JudgeFn, ...).
+ *
+ * Each existing primitive returns its own output shape. The Analyst
+ * contract is the single envelope every primitive lifts into, so a
+ * registry can run N analysts against a run and a single renderer can
+ * compose findings without knowing which analyzer produced them.
+ *
+ * The contract is intentionally domain-agnostic: nothing here knows
+ * about code, voice, RAG, or any particular agent stack. Analysts
+ * declare what INPUT KIND they need (a trace store, an artifact dir,
+ * a RunRecord, a JudgeInput, or `custom`), and the registry routes
+ * the matching input from `AnalystRunInputs`.
+ */
+
+interface EvidenceRef {
+    /**
+     * Where the evidence lives. `span` and `event` refer to OTLP trace
+     * elements; `artifact` to a file inside the run's artifact tree;
+     * `finding` to another AnalystFinding (cross-analyst chaining);
+     * `metric` to a named scalar reading the renderer knows how to read.
+     */
+    kind: 'span' | 'event' | 'artifact' | 'finding' | 'metric';
+    uri: string;
+    excerpt?: string;
+}
+
+type PolicyEditSchemaVersion = 'policy-edit/v1';
+declare const POLICY_EDIT_AXES: readonly ["carrier", "representation", "budget", "sampling", "output_contract", "tool_contract", "routing", "memory", "agent_profile", "deployment_target"];
+type PolicyEditAxis = (typeof POLICY_EDIT_AXES)[number];
+declare const POLICY_EDIT_TARGET_SURFACES: readonly ["prompt", "tool-contract", "runtime-config", "memory", "agent-profile", "code", "deployment"];
+type PolicyEditTargetSurface = (typeof POLICY_EDIT_TARGET_SURFACES)[number];
+type PolicyEditRisk = 'low' | 'medium' | 'high' | 'unknown';
+type PolicyEditGainDirection = 'increase' | 'decrease';
+type PolicyEditGainUnit = 'absolute' | 'relative' | 'percent' | 'score';
+interface PolicyEditTarget {
+    surface: PolicyEditTargetSurface;
+    /** Stable path inside the target surface, for example `system-prompt:tools`
+     * or `budget.maxTurns`. */
+    path?: string;
+    /** Optional canonical deployment identity. Store the existing cell, not a
+     * local profile shape. */
+    agentProfileCell?: AgentProfileCell;
+    /** Human label when the path is not enough for a readable audit trail. */
+    label?: string;
+}
+type PolicyEditChange = {
+    kind: 'text';
+    mode: 'append' | 'prepend' | 'replace';
+    value: string;
+    /** Required when `mode === 'replace'`; exact match only. */
+    find?: string;
+} | {
+    kind: 'json';
+    mode: 'set' | 'merge' | 'remove';
+    path: string;
+    value?: AgentProfileJson;
+};
+interface PolicyEditExpectedGain {
+    /** Metric this edit is expected to move, e.g. `holdout.composite`. */
+    metric: string;
+    direction: PolicyEditGainDirection;
+    /** Positive magnitude in the metric's native units. */
+    amount: number;
+    unit?: PolicyEditGainUnit;
+    rationale?: string;
+}
+interface PolicyEditSource {
+    findingIds: string[];
+    analystIds: string[];
+    evidenceRefs: EvidenceRef[];
+    /** Mirrors `AnalystFinding.derived_from_judge`; admission rejects it. */
+    derivedFromJudge?: boolean;
+}
+interface PolicyEdit {
+    schemaVersion: PolicyEditSchemaVersion;
+    editId: string;
+    axis: PolicyEditAxis;
+    target: PolicyEditTarget;
+    change: PolicyEditChange;
+    claim: string;
+    expectedGain: PolicyEditExpectedGain;
+    confidence: number;
+    risk: PolicyEditRisk;
+    source: PolicyEditSource;
+    rationale?: string;
+    validationPlan?: string;
+    metadata?: Record<string, unknown>;
+}
+declare const POLICY_EDIT_CANDIDATE_RECORD_SCHEMA: "tangle.policy-edit-candidate.v1";
+/** JSON-safe attribution carried with a measured candidate and its scores. */
+interface PolicyEditCandidateRecord {
+    schema: typeof POLICY_EDIT_CANDIDATE_RECORD_SCHEMA;
+    policyEdit: PolicyEdit;
+}
+
+/**
+ * Pass A substrate types — `runCampaign` is the one primitive every
+ * eval flow composes from. Three contracts in this file:
+ *
+ *   - `Scenario`            input set
+ *   - `DispatchFn`          how to run one scenario → artifact
+ *   - `CampaignResult`      defined output schema (the contract downstream tools depend on)
+ *
+ * Three more lifted from earlier substrate work (re-exported):
+ *
+ *   - `JudgeConfig`         pluggable dimensional scorer (0.38)
+ *   - `Mutator`             optimization-loop surface mutator
+ *   - `Gate`                promotion gate (`HeldOutGate` and friends adapt to this)
+ *
+ * No new architecture vs 0.38 — Pass A formalizes the shapes so consumers
+ * can build dashboards / CI gates / regression diffs against a stable schema.
+ */
+
+/** Stable identifier + kind tag for any scenario. Consumers
+ *  extend with their per-domain payload (persona, task, requirement, ...). */
+interface Scenario {
+    id: string;
+    kind: string;
+    tags?: string[];
+}
+/** Context handed to every dispatch invocation. Scoped — every
+ *  trace/span carries the cellId, every artifact write lands under the cell's
+ *  artifact root, the cost meter accumulates per cell. */
+interface DispatchContext {
+    cellId: string;
+    rep: number;
+    generation?: number;
+    seed: number;
+    signal: AbortSignal;
+    trace: CampaignTraceWriter;
+    artifacts: CampaignArtifactWriter;
+    cost: CampaignCostMeter;
+    /** Populated when this run is part of a multi-cycle improvement loop. */
+    cycleId?: string;
+    /** Populated when the substrate resumed from a prior cache hit. */
+    resumedFrom?: string;
+    /**
+     * Opaque placement key supplied by `RunCampaignOptions.cellPlacement`.
+     * The substrate forwards it through unchanged; placement-aware Dispatch
+     * implementations (e.g. `httpDispatch` from `/adapters/http`) read it to
+     * route the cell to the right worker / region / sandbox. `undefined`
+     * when no placement strategy is configured.
+     */
+    placement?: string;
+}
+/** The canonical judge verdict shape — one declaration, shared by campaign
+ *  judges and the multishot judge runner (which re-exports this type).
+ *
+ *  Scale is PRODUCER-DEFINED: campaign convention is [0,1]; the legacy
+ *  multishot runner emits 0-10. Cross-scale comparison must go through
+ *  `detectScale` (src/campaign/gates/statistical-heldout.ts, used by
+ *  promotion-policy) — never renormalize a producer's values in place, as
+ *  downstream thresholds (`composite >= 5` in multishot/matrix.ts, live-soak
+ *  `>= 7` gates) key on the producer's native scale. */
+interface JudgeScore {
+    dimensions: Record<string, number>;
+    composite: number;
+    notes: string;
+    /** Provider metadata for display and diagnostics; accounting uses CostLedger receipts. */
+    llmCall?: LlmCallMetadata;
+    /** Set when the judge itself failed (call error, unparseable output).
+     *  `composite`/`dimensions` carry no signal — aggregators MUST exclude
+     *  failed scores from means instead of folding them into zeros. */
+    failed?: true;
+    /** Ensemble extras (populated by `ensembleJudge`): max per-dimension
+     *  spread across surviving judges — the inter-rater signal. */
+    maxDisagreement?: number;
+    /** Ensemble extras: judge identities whose verdict failed. */
+    failedJudges?: string[];
+    /** Ensemble extras: each surviving judge's per-dimension scores. */
+    perJudge?: Record<string, Record<string, number>>;
+}
+/** Five-valued verdict taxonomy (MOSS-paper alignment). */
+type GateDecision = 'ship' | 'hold' | 'need_more_work' | 'model_ceiling' | 'arch_ceiling';
+interface GateResult {
+    decision: GateDecision;
+    reasons: string[];
+    contributingGates: Array<{
+        name: string;
+        passed: boolean;
+        detail: unknown;
+    }>;
+    delta?: number;
+}
+/** Scoped trace writer handed to each dispatch — every span
+ *  auto-tagged with the cellId so traces filter cleanly. */
+interface CampaignTraceWriter {
+    span(name: string, attributes?: Record<string, unknown>): TraceSpan;
+    flush(): Promise<void>;
+}
+interface TraceSpan {
+    end(attributes?: Record<string, unknown>): void;
+    setAttribute(key: string, value: unknown): void;
+}
+/** Scoped artifact writer — `write(path, content)` lands under
+ *  `<runDir>/<cellId>/<path>`. */
+interface CampaignArtifactWriter {
+    write(path: string, content: string | Uint8Array): Promise<string>;
+    writeJson(path: string, value: unknown): Promise<string>;
+}
+/** Token usage accumulated for a cell. Aliased to the canonical `RunTokenUsage`
+ *  (run-record.ts, same package) so a cell maps onto a `RunRecord` for the
+ *  backend-integrity guard with ONE source of truth — a field added to
+ *  `RunTokenUsage` is a compile error here, not a silent drift. */
+type CampaignTokenUsage = RunTokenUsage;
+/** Cell-scoped paid-call entry point. The dispatch places every paid operation
+ *  inside `runPaidCall`; the returned provider result supplies one receipt with
+ *  cost, tokens, and resolved model. Calls made outside this method are not
+ *  admitted or captured. */
+interface CampaignCostMeter {
+    /** The only paid-call path. Returns a typed result; callers must inspect it. */
+    runPaidCall<T>(input: Omit<RunPaidCallInput<T>, 'channel' | 'phase' | 'tags'> & {
+        channel?: CostChannel;
+    }): Promise<PaidCallResult<T>>;
+}
+interface CampaignCellResult<TArtifact> {
+    /** Manifest that produced this cell. Resumability refuses to reuse a cell
+     *  whose manifest differs from the current run. */
+    manifestHash?: string;
+    cellId: string;
+    scenarioId: string;
+    rep: number;
+    generation?: number;
+    artifact: TArtifact;
+    judgeScores: Record<string, JudgeScore>;
+    costUsd: number;
+    /** True when at least one priced receipt used the model table instead of a provider bill. */
+    costEstimated?: boolean;
+    /** Exact durable receipts required to reuse this cached result. */
+    costCallIds?: string[];
+    /** Agent-call token usage committed by `ctx.cost.runPaidCall`.
+     *  `{ input: 0, output: 0 }` when no paid agent call was recorded. */
+    tokenUsage: CampaignTokenUsage;
+    /** Concrete model from the latest committed agent receipt. Consumed by
+     *  `buildRunRecord` to pin the model when the declared profile uses a
+     *  runtime-resolved sentinel. */
+    resolvedModel?: string;
+    durationMs: number;
+    seed: number;
+    cached: boolean;
+    error?: string;
+}
+interface JudgeAggregate {
+    mean: number;
+    stdev: number;
+    ci95: [number, number];
+    n: number;
+}
+interface ScenarioAggregate {
+    meanComposite: number;
+    ci95: [number, number];
+    n: number;
+}
+interface GenerationRecord {
+    generationIndex: number;
+    candidates: GenerationCandidate[];
+    promoted: string[];
+}
+/** One scored candidate surface in a generation. `dimensions` + `scenarios`
+ *  let a reflective proposer ground its next proposal on WHICH
+ *  dimensions the candidate is weakest on and WHICH scenarios it best/worst
+ *  handled — the evidence a blind `Mutator` cannot see. */
+interface GenerationCandidate {
+    surfaceHash: string;
+    composite: number;
+    ci95: [number, number];
+    /** Exact surface this candidate mutated. */
+    parentSurfaceHash?: string;
+    /** Measured search-split composite of the exact parent surface. */
+    parentComposite?: number;
+    /** Candidate composite minus its parent's composite. Present only when the
+     *  candidate completed the designed denominator. */
+    observedDeltaFromParent?: number;
+    /** Whether this candidate had a scorable result for every designed campaign
+     *  cell and was therefore eligible for ranking, promotion, and Pareto
+     *  selection. Older externally-authored records may omit this field; loop
+     *  records always populate it. */
+    eligibleForPromotion?: boolean;
+    /** Exact denominator receipt for selection eligibility. Scores stay
+     *  descriptive: an incomplete candidate is retained with its observed score
+     *  and errors instead of receiving an invented penalty. */
+    coverage?: {
+        expectedCells: number;
+        scorableCells: number;
+        unscorableCells: Array<{
+            cellId: string;
+            reason: string;
+        }>;
+    };
+    /** Mean score per judge dimension across all cells (scenarios × reps ×
+     *  judges that reported the dimension). */
+    dimensions: Record<string, number>;
+    /** Per-scenario composite (mean over reps + judges), plus the judge's
+     *  free-form `notes` for that scenario — the "why it scored low" evidence a
+     *  reflective proposer grounds its next edit on. Keep `notes` GENERALIZABLE
+     *  (which checks/lines/dimensions failed and how), NOT case-specific ground
+     *  truth: leaking expected answers into the prompt is memorization, and the
+     *  held-out gate would reject it anyway. */
+    scenarios: Array<{
+        scenarioId: string;
+        composite: number;
+        notes?: string;
+    }>;
+    /** Proposer-supplied short label for the change. Present when the proposer
+     *  returned a `ProposedCandidate`; absent for bare-surface mutators. */
+    label?: string;
+    /** Proposer-supplied rationale — WHY this candidate was proposed. The
+     *  "because rationale Z" the audit requires to survive to the result.
+     *  Present when the proposer returned a `ProposedCandidate`. */
+    rationale?: string;
+    /** Exact structured cause threaded from the proposer, when available. */
+    candidateRecord?: PolicyEditCandidateRecord;
+}
+interface CampaignAggregates {
+    byJudge: Record<string, JudgeAggregate>;
+    byScenario: Record<string, ScenarioAggregate>;
+    /** Canonical campaign accounting, including worker and judge calls. */
+    cost: CostLedgerSummary;
+    /** Compatibility alias of `cost.totalCostUsd`. */
+    totalCostUsd: number;
+    cellsExecuted: number;
+    cellsSkipped: number;
+    cellsCached: number;
+    cellsFailed: number;
+}
+interface CampaignResult<TArtifact = unknown, TScenario extends Scenario = Scenario> {
+    /** sha256(scenarios, judges, dispatch source ref, optimizer config, seed). Stable identity for reruns. */
+    manifestHash: string;
+    seed: number;
+    startedAt: string;
+    endedAt: string;
+    durationMs: number;
+    cells: Array<CampaignCellResult<TArtifact>>;
+    aggregates: CampaignAggregates;
+    optimization?: {
+        generations: GenerationRecord[];
+        winnerSurfaceHash?: string;
+    };
+    gate?: GateResult;
+    prUrl?: string;
+    runDir: string;
+    artifactsByPath: Record<string, string>;
+    /** Substrate strips the input scenarios to id+kind for the result manifest;
+     *  consumers needing full payload look it up via the original input. The
+     *  type parameter `TScenario` is propagated for downstream consumers that
+     *  want narrowed types when extending `CampaignResult`. */
+    scenarios: Array<Pick<TScenario, 'id' | 'kind'>>;
+}
+
+/**
+ * Shared types for the reference benchmark wrappers under
+ * `src/benchmarks/`. Each wrapper exports the three functions in
+ * `BenchmarkAdapter` plus its own typed `DatasetItem` shape.
+ */
+
+type BenchmarkTaskKind = 'retrieval' | 'rag-answer' | 'hallucination' | 'kb-improvement' | 'routing' | 'custom';
+type BenchmarkFamily = 'beir' | 'mteb-retrieval' | 'msmarco' | 'trec-dl' | 'miracl' | 'lotte' | 'bright' | 'crag' | 'hotpotqa' | 'kilt' | 'ragtruth' | 'faithbench' | 'first-party' | 'custom';
+interface BenchmarkDatasetItem<TPayload = unknown> {
+    /** Stable dataset-local item id (used for split assignment + paper
+     *  references). Unique within a benchmark. */
+    id: string;
+    /** Free-form payload. Each benchmark defines its own shape. */
+    payload: TPayload;
+    /** Optional precomputed split. When absent, adapters use assignSplit(id). */
+    split?: RunSplitTag;
+    /** Benchmark family this row came from, e.g. `beir` or `crag`. */
+    family?: BenchmarkFamily | string;
+    /** Benchmark-local task kind, e.g. retrieval vs answer quality. */
+    taskKind?: BenchmarkTaskKind | string;
+    /** Slice labels such as language, domain, freshness, multihop, long-tail. */
+    tags?: string[];
+    /** Dataset provenance, version, URL, or license notes. */
+    source?: BenchmarkSource;
+    metadata?: Record<string, unknown>;
+}
+interface BenchmarkEvaluation {
+    /** [0, 1] score for the response on this item. Exact-match
+     *  benchmarks use 0/1; partial-credit benchmarks may return
+     *  fractional values. */
+    score: number;
+    /** Optional pass/fail projection. Defaults to `score > 0` when absent. */
+    passed?: boolean;
+    /** Numeric sub-metrics. These become report dimensions. */
+    dimensions?: Record<string, number>;
+    /** Optional bag of raw scoring signals — e.g. parsed numeric
+     *  answer, regex match, judge sub-scores. */
+    raw: Record<string, unknown>;
+    notes?: string;
+}
+interface BenchmarkSource {
+    name?: string;
+    url?: string;
+    version?: string;
+    license?: string;
+    citation?: string;
+}
+/** Common signature implemented by every adapter under `src/benchmarks/*`. */
+interface BenchmarkAdapter<_TItem = unknown, TPayload = unknown, TArtifact = string> {
+    /** Stable benchmark id such as `beir/nfcorpus` or `crag/smoke`. */
+    id?: string;
+    family?: BenchmarkFamily | string;
+    taskKind?: BenchmarkTaskKind | string;
+    description?: string;
+    source?: BenchmarkSource;
+    defaultMetric?: string;
+    /** Load the dataset for the given split. May hit the network on
+     *  first call but should be cache-friendly. Adapters that don't
+     *  ship the dataset itself MUST throw a clearly-marked error
+     *  pointing the caller at the loader script. */
+    loadDataset(split: RunSplitTag): Promise<BenchmarkDatasetItem<TPayload>[]>;
+    /** Score a single response. Pure with respect to the inputs. */
+    evaluate(item: BenchmarkDatasetItem<TPayload>, artifact: TArtifact): Promise<BenchmarkEvaluation>;
+    /** Deterministic split assignment via item id hashing. The
+     *  fraction of items in each split is implementation-defined but
+     *  MUST be stable across processes and platforms. */
+    assignSplit(itemId: string): RunSplitTag;
+}
+interface BenchmarkScenario<TPayload = unknown> extends Scenario {
+    kind: 'benchmark';
+    benchmarkId: string;
+    family: BenchmarkFamily | string;
+    taskKind: BenchmarkTaskKind | string;
+    splitTag: RunSplitTag;
+    item: BenchmarkDatasetItem<TPayload>;
+}
+type BenchmarkResponder<TPayload = unknown, TArtifact = string> = (input: {
+    scenario: BenchmarkScenario<TPayload>;
+    item: BenchmarkDatasetItem<TPayload>;
+    context: DispatchContext;
+}) => Promise<TArtifact> | TArtifact;
+/** Split-assignment seed shared across all benchmarks. Bumping this
+ *  value reshuffles every split — do NOT do that lightly. */
+declare const BENCHMARK_SPLIT_SEED = "agent-eval-v1";
+/**
+ * Assign an item id to one of `'search' | 'dev' | 'holdout'` using a
+ * stable 32-bit hash of `${seed}::${id}`. Default proportions:
+ *
+ *   search: 60%   (optimization-readable)
+ *   dev:    20%   (held-out for tuning, leak-on-purpose during dev)
+ *   holdout:20%   (paper-grade held-out, gated reads)
+ */
+declare function deterministicSplit(itemId: string, seed?: string): RunSplitTag;
+
+interface BenchmarkMetricCalibrationOptions<TPayload = unknown, TArtifact = string> {
+    adapter: BenchmarkAdapter<BenchmarkDatasetItem<TPayload>, TPayload, TArtifact>;
+    item: BenchmarkDatasetItem<TPayload>;
+    weakArtifact: TArtifact;
+    strongArtifact: TArtifact;
+    maxWeakScore?: number;
+    minStrongScore?: number;
+    minGap?: number;
+}
+interface BenchmarkMetricCalibrationResult {
+    passed: boolean;
+    weak: BenchmarkEvaluation;
+    strong: BenchmarkEvaluation;
+    weakScore: number;
+    strongScore: number;
+    gap: number;
+    reasons: string[];
+}
+declare function calibrateBenchmarkMetric<TPayload = unknown, TArtifact = string>(options: BenchmarkMetricCalibrationOptions<TPayload, TArtifact>): Promise<BenchmarkMetricCalibrationResult>;
+
+/**
+ * Synthetic routing dataset. 16 tasks across 4 categories. Used as a
+ * deterministic, dependency-free benchmark for any router that maps a
+ * natural-language request to one of a fixed set of route labels.
+ *
+ * Format (see `routing/README.md` for prose):
+ *
+ *   {
+ *     id:           stable per-task ID (matches across processes).
+ *     category:     one of the four route labels.
+ *     prompt:       the user-facing request the router must classify.
+ *     route:        the ground-truth route the router should pick.
+ *     synonyms:     other strings that count as a correct answer.
+ *     hardNegatives:close-but-wrong route labels — used to detect the
+ *                   "always picks the popular route" failure mode.
+ *   }
+ *
+ * The four categories are intentionally cross-domain (file ops,
+ * math, search, conversation) so a router that collapses to one
+ * category is easy to spot.
+ */
+interface RoutingItem {
+    id: string;
+    category: 'file' | 'math' | 'search' | 'chat';
+    prompt: string;
+    /** Canonical correct route label. */
+    route: string;
+    /** Alternate route labels that also count as correct. */
+    synonyms: string[];
+    /** Wrong-but-tempting route labels (for analysis, not grading). */
+    hardNegatives: string[];
+}
+declare const ROUTING_DATASET: RoutingItem[];
+
+/**
+ * Routing benchmark — synthetic, dependency-free, ships in the
+ * package. 16 cross-category items in `dataset.ts`. See
+ * `routing/README.md` for the format.
+ *
+ * `evaluate` does case-insensitive exact match against the canonical
+ * route plus declared synonyms. The first valid route token in the
+ * response wins; everything else is ignored. Wrong answers also
+ * report whether they hit a hard negative — useful when triaging
+ * "always picks the popular route" failure modes.
+ */
+
+type RoutingPayload = RoutingItem;
+type RoutingDatasetItem = BenchmarkDatasetItem<RoutingPayload>;
+declare class RoutingAdapter implements BenchmarkAdapter<RoutingDatasetItem, RoutingPayload> {
+    readonly id = "first-party/routing";
+    readonly family = "first-party";
+    readonly taskKind = "routing";
+    readonly description = "Synthetic fixed-route classification smoke benchmark";
+    readonly defaultMetric = "route_exact_match";
+    loadDataset(split: RunSplitTag): Promise<RoutingDatasetItem[]>;
+    evaluate(item: RoutingDatasetItem, response: string): Promise<BenchmarkEvaluation>;
+    assignSplit(itemId: string): RunSplitTag;
+}
+/**
+ * Pull route-shaped tokens out of a model response. Routes look like
+ * `category.action` (`fs.write`, `chat.reply`). Bare alphanumerics
+ * are not routes, but `category.action` patterns are robust to most
+ * model wrappers (JSON output, prose explanations, code fences).
+ */
+declare function extractRouteTokens(response: string): string[];
+declare const loadDataset: (split: RunSplitTag) => Promise<RoutingDatasetItem[]>;
+declare const evaluate: (item: RoutingDatasetItem, response: string) => Promise<BenchmarkEvaluation>;
+declare const assignSplit: (itemId: string) => RunSplitTag;
+
+declare const index_ROUTING_DATASET: typeof ROUTING_DATASET;
+type index_RoutingAdapter = RoutingAdapter;
+declare const index_RoutingAdapter: typeof RoutingAdapter;
+type index_RoutingDatasetItem = RoutingDatasetItem;
+type index_RoutingItem = RoutingItem;
+type index_RoutingPayload = RoutingPayload;
+declare const index_assignSplit: typeof assignSplit;
+declare const index_evaluate: typeof evaluate;
+declare const index_extractRouteTokens: typeof extractRouteTokens;
+declare const index_loadDataset: typeof loadDataset;
+declare namespace index {
+  export { index_ROUTING_DATASET as ROUTING_DATASET, index_RoutingAdapter as RoutingAdapter, type index_RoutingDatasetItem as RoutingDatasetItem, type index_RoutingItem as RoutingItem, type index_RoutingPayload as RoutingPayload, index_assignSplit as assignSplit, index_evaluate as evaluate, index_extractRouteTokens as extractRouteTokens, index_loadDataset as loadDataset };
+}
+
+/**
+ * `CampaignStorage` — the filesystem seam `runCampaign` writes through
+ * (run/cell dirs, the resumability cache, per-cell artifacts, trace spans).
+ *
+ * The default (`fsCampaignStorage`) is the Node filesystem — identical
+ * behavior to the inline `node:fs` calls it replaces, so existing CLI
+ * consumers are unaffected. `inMemoryCampaignStorage` keeps everything in a
+ * `Map`, so the substrate runs in environments WITHOUT a filesystem
+ * (Cloudflare Workers, Deno Deploy, other edge runtimes) — the campaign
+ * still produces its `CampaignResult` (cells + aggregates) in memory;
+ * artifacts/traces simply aren't persisted to disk.
+ *
+ * Paths are opaque keys to the in-memory adapter — it does not parse them,
+ * so the same `join(...)`-built paths work unchanged across both adapters.
+ */
+interface CampaignStorage {
+    /** Ensure a directory exists (recursive). No-op for in-memory. */
+    ensureDir(dir: string): void;
+    /** Does this path exist (as a written file or an ensured dir)? */
+    exists(path: string): boolean;
+    /** Read a UTF-8 file; `undefined` when missing or unreadable. */
+    read(path: string): string | undefined;
+    /** Write a file (string or bytes). Parent dir is assumed ensured. */
+    write(path: string, content: string | Uint8Array): void;
+    /** Append only when the current UTF-8 byte length matches `expectedBytes`.
+     * Returns the new length, or undefined when another writer won. */
+    append?(path: string, content: string, expectedBytes: number): number | undefined;
+}
+
+interface BenchmarkRunOptions<TPayload = unknown, TArtifact = string> {
+    adapter: BenchmarkAdapter<BenchmarkDatasetItem<TPayload>, TPayload, TArtifact>;
+    respond: BenchmarkResponder<TPayload, TArtifact>;
+    splits?: readonly RunSplitTag[];
+    runDir: string;
+    repo?: string;
+    seed?: number;
+    reps?: number;
+    resumable?: boolean;
+    costCeiling?: number;
+    maxConcurrency?: number;
+    dispatchTimeoutMs?: number;
+    expectUsage?: 'assert' | 'warn' | 'off';
+    storage?: CampaignStorage;
+    now?: () => Date;
+}
+interface BenchmarkReport {
+    benchmarkId: string;
+    family: BenchmarkFamily | string;
+    taskKind: BenchmarkTaskKind | string;
+    source?: BenchmarkAdapter['source'];
+    runDir: string;
+    manifestHash: string;
+    seed: number;
+    startedAt: string;
+    endedAt: string;
+    durationMs: number;
+    totalItems: number;
+    totalCells: number;
+    cellsFailed: number;
+    cellsCached: number;
+    totalCostUsd: number;
+    splits: Record<string, BenchmarkSliceSummary>;
+    tags: Record<string, BenchmarkSliceSummary>;
+    dimensions: Record<string, BenchmarkDistribution>;
+    score: BenchmarkDistribution;
+    costUsd: BenchmarkDistribution;
+    latencyMs: BenchmarkDistribution;
+}
+interface BenchmarkSliceSummary {
+    n: number;
+    meanScore: number;
+    passRate: number;
+    score: BenchmarkDistribution;
+    costUsd: BenchmarkDistribution;
+    latencyMs: BenchmarkDistribution;
+}
+interface BenchmarkDistribution {
+    n: number;
+    min: number;
+    mean: number;
+    median: number;
+    p90: number;
+    max: number;
+}
+interface BenchmarkRunResult<TPayload = unknown, TArtifact = string> {
+    scenarios: Array<BenchmarkScenario<TPayload>>;
+    campaign: CampaignResult<TArtifact, BenchmarkScenario<TPayload>>;
+    report: BenchmarkReport;
+    reportJsonPath: string;
+    reportMarkdownPath: string;
+}
+declare function runBenchmarkAdapter<TPayload = unknown, TArtifact = string>(options: BenchmarkRunOptions<TPayload, TArtifact>): Promise<BenchmarkRunResult<TPayload, TArtifact>>;
+declare function summarizeBenchmarkCampaign<TPayload, TArtifact>(input: {
+    adapter: BenchmarkAdapter<BenchmarkDatasetItem<TPayload>, TPayload, TArtifact>;
+    scenarios: Array<BenchmarkScenario<TPayload>>;
+    campaign: CampaignResult<TArtifact, BenchmarkScenario<TPayload>>;
+}): BenchmarkReport;
+declare function renderBenchmarkReportMarkdown(report: BenchmarkReport): string;
+
+interface StandardRetrievalDocument {
+    id: string;
+    title?: string;
+    text: string;
+    metadata?: Record<string, unknown>;
+}
+interface StandardRetrievalQuery {
+    id: string;
+    text: string;
+    metadata?: Record<string, unknown>;
+}
+interface StandardRetrievalQrel {
+    queryId: string;
+    documentId: string;
+    score: number;
+}
+interface StandardRetrievalPayload {
+    queryId: string;
+    query: string;
+    expectedDocumentIds: string[];
+    expectedScores: Record<string, number>;
+    corpus?: Record<string, StandardRetrievalDocument>;
+    metadata?: Record<string, unknown>;
+}
+interface BuildStandardRetrievalItemsOptions {
+    benchmarkId: string;
+    family: BenchmarkFamily | string;
+    queries: readonly StandardRetrievalQuery[];
+    qrels: readonly StandardRetrievalQrel[];
+    corpus?: readonly StandardRetrievalDocument[];
+    includeCorpusInPayload?: boolean;
+    source?: BenchmarkSource;
+    tags?: readonly string[];
+    splitOf?: (queryId: string) => RunSplitTag;
+}
+interface RetrievalIdAdapterOptions extends BuildStandardRetrievalItemsOptions {
+    responseIdPattern?: RegExp;
+    cutoffs?: readonly number[];
+    primaryMetric?: string;
+    passMetric?: string;
+    passThreshold?: number;
+}
+interface StandardRetrievalResult {
+    id?: string;
+    documentId?: string;
+    docId?: string;
+    score?: number;
+}
+type StandardRetrievalArtifact = string | readonly string[] | readonly StandardRetrievalResult[] | {
+    ids?: readonly string[];
+    documentIds?: readonly string[];
+    results?: readonly StandardRetrievalResult[];
+};
+interface StandardRetrievalEvaluationOptions {
+    responseIdPattern?: RegExp;
+    cutoffs?: readonly number[];
+    primaryMetric?: string;
+    passMetric?: string;
+    passThreshold?: number;
+}
+declare function parseJsonlRows<T = unknown>(text: string): T[];
+declare function parseTsvRows(text: string): string[][];
+declare function parseQrels(text: string): StandardRetrievalQrel[];
+declare function parseBeirCorpusJsonl(text: string): StandardRetrievalDocument[];
+declare function parseBeirQueriesJsonl(text: string): StandardRetrievalQuery[];
+declare function buildStandardRetrievalItems(options: BuildStandardRetrievalItemsOptions): Array<BenchmarkDatasetItem<StandardRetrievalPayload>>;
+declare function createRetrievalIdBenchmarkAdapter(options: RetrievalIdAdapterOptions): BenchmarkAdapter<BenchmarkDatasetItem<StandardRetrievalPayload>, StandardRetrievalPayload, StandardRetrievalArtifact>;
+declare function evaluateStandardRetrieval(payload: StandardRetrievalPayload, artifact: StandardRetrievalArtifact, options?: StandardRetrievalEvaluationOptions): {
+    score: number;
+    passed: boolean;
+    dimensions: Record<string, number>;
+    raw: {
+        rankedDocumentIds: string[];
+        expectedDocumentIds: string[];
+        expectedScores: Record<string, number>;
+    };
+};
+declare function normalizeRetrievedDocumentIds(artifact: StandardRetrievalArtifact, responseIdPattern?: RegExp): string[];
+declare function retrievalMetricsAtCutoff(input: {
+    rankedDocumentIds: readonly string[];
+    expectedScores: Record<string, number>;
+    cutoff: number;
+}): Record<string, number>;
+
+export { BENCHMARK_SPLIT_SEED, type BenchmarkAdapter, type BenchmarkDatasetItem, type BenchmarkDistribution, type BenchmarkEvaluation, type BenchmarkFamily, type BenchmarkMetricCalibrationOptions, type BenchmarkMetricCalibrationResult, type BenchmarkReport, type BenchmarkResponder, type BenchmarkRunOptions, type BenchmarkRunResult, type BenchmarkScenario, type BenchmarkSliceSummary, type BenchmarkSource, type BenchmarkTaskKind, type BuildStandardRetrievalItemsOptions, type RetrievalIdAdapterOptions, type StandardRetrievalArtifact, type StandardRetrievalDocument, type StandardRetrievalEvaluationOptions, type StandardRetrievalPayload, type StandardRetrievalQrel, type StandardRetrievalQuery, type StandardRetrievalResult, buildStandardRetrievalItems, calibrateBenchmarkMetric, createRetrievalIdBenchmarkAdapter, deterministicSplit, evaluateStandardRetrieval, normalizeRetrievedDocumentIds, parseBeirCorpusJsonl, parseBeirQueriesJsonl, parseJsonlRows, parseQrels, parseTsvRows, renderBenchmarkReportMarkdown, retrievalMetricsAtCutoff, index as routing, runBenchmarkAdapter, summarizeBenchmarkCampaign };