@tangle-network/agent-eval 0.21.0 → 0.23.0

package/dist/eval-campaign-Ds5QljIh.d.ts

@@ -0,0 +1,573 @@
+import { C as GateDecision, F as ResearchReportOptions, H as ResearchReport } from './summary-report-Ce1r4EYo.js';
+import { R as RunRecord, a as RunSplitTag, b as RunTokenUsage, c as RunJudgeMetadata } from './run-record-DNiOMBrZ.js';
+import { R as RawProviderSink, P as ProviderRedactor, a as RunIntegrityExpectations, b as RunIntegrityReport } from './integrity-Cr5YodSY.js';
+import { T as TraceEmitter, R as RunCompleteHook } from './emitter-B2XqDKFU.js';
+import { T as TraceStore } from './store-u47QaJ9G.js';
+
+/**
+ * LLM client with graceful degrade.
+ *
+ * OpenAI-compatible `/v1/chat/completions` client with:
+ *   - Exponential-backoff retry on 429 + 5xx gateway errors (502/503/504).
+ *   - Retry on transient network errors (fetch failed, AbortError, ECONNRESET).
+ *   - Graceful json_schema → json_object degrade on 400 with schema-reject body.
+ *   - Fenced-JSON stripping (```json ... ```) for models that wrap structured output.
+ *   - Configurable base URL + api key / bearer, works with LiteLLM proxies, OpenAI
+ *     directly, cli-bridge subscriptions, and any router that speaks the spec.
+ *
+ * Usage:
+ *   const { value, result } = await callLlmJson<MyType>(
+ *     { model: 'gpt-4o', messages: [...], jsonSchema: { name: 'x', schema: {...} } },
+ *     { baseUrl: 'https://router.tangle.tools/v1', apiKey: process.env.KEY },
+ *   )
+ *
+ * This is THE llm-calling seam for agent-eval primitives that need structured
+ * output (semantic concept judge, reviewer directives, critic scores). Primitives
+ * that need free-form text use `callLlm` and parse output themselves.
+ */
+
+interface LlmMessage {
+    role: 'system' | 'user' | 'assistant';
+    /**
+     * Either a plain text content string OR a multimodal content array
+     * (text + image_url parts) for vision-capable models.
+     */
+    content: string | Array<{
+        type: 'text';
+        text: string;
+    } | {
+        type: 'image_url';
+        image_url: {
+            url: string;
+            detail?: 'auto' | 'low' | 'high';
+        };
+    }>;
+}
+interface LlmCallRequest {
+    model: string;
+    messages: LlmMessage[];
+    /** Optional JSON-mode response format (response_format: json_object). */
+    jsonMode?: boolean;
+    /** Optional structured output via JSON Schema. Falls back to json_object on 400. */
+    jsonSchema?: {
+        name: string;
+        schema: Record<string, unknown>;
+    };
+    temperature?: number;
+    maxTokens?: number;
+    /** Per-call timeout, default 60s. */
+    timeoutMs?: number;
+}
+interface LlmUsage {
+    promptTokens: number;
+    completionTokens: number;
+    totalTokens: number;
+    /** Proxies populate this when prompt caching is on. */
+    cachedPromptTokens?: number;
+}
+interface LlmCallResult {
+    /** The text content of the first choice. Empty string if none. */
+    content: string;
+    usage: LlmUsage;
+    /**
+     * Cost in USD. Pulled from proxy's `_response_cost` field when present;
+     * `null` when neither the proxy nor the caller can derive it.
+     */
+    costUsd: number | null;
+    /** Model name actually used (echoed from response). */
+    model: string;
+    /** Wall-clock duration of the HTTP call (last attempt, if retried). */
+    durationMs: number;
+    /** Raw response body. */
+    raw: Record<string, unknown>;
+}
+declare class LlmCallError extends Error {
+    readonly status: number;
+    readonly body: string;
+    readonly model: string;
+    constructor(message: string, status: number, body: string, model: string);
+}
+interface LlmClientOptions {
+    /** Base URL (without trailing slash). Must end at the `/v1` prefix. */
+    baseUrl?: string;
+    /** Bearer token — either `apiKey` or `bearer` populates `Authorization: Bearer ...`. */
+    apiKey?: string;
+    bearer?: string;
+    /** Override for the `Authorization` header (e.g. `X-Auth: ...`). Takes precedence over apiKey/bearer. */
+    authHeader?: {
+        name: string;
+        value: string;
+    };
+    /** Default timeout in ms. Per-call can override. */
+    defaultTimeoutMs?: number;
+    /** Max retry attempts on retriable errors. Default 3 (1 initial + 2 retries). */
+    maxRetries?: number;
+    /** Fetch implementation — defaults to global `fetch`. Override for custom transport (e.g. tests). */
+    fetch?: typeof fetch;
+    /**
+     * Optional raw HTTP capture sink. When provided, every request, response,
+     * and error (across all retry attempts) is recorded to the sink, with auth
+     * headers and credential-shaped body fields redacted by default. This is
+     * the layer-1 forensics primitive: structured `LlmSpan`s record intent,
+     * raw events record what actually crossed the wire.
+     */
+    rawSink?: RawProviderSink;
+    /**
+     * Logical provider id attached to raw events. When omitted, derived from
+     * `baseUrl` via `providerFromBaseUrl`.
+     */
+    provider?: string;
+    /** Trace context attached to raw events; populated by emitter-aware callers. */
+    traceContext?: {
+        runId?: string;
+        spanId?: string;
+    };
+    /** Override the redaction strategy for this call. Defaults to `defaultProviderRedactor`. */
+    redactor?: ProviderRedactor;
+}
+/**
+ * Strip a ```json / ``` code fence if the model emitted one.
+ * Idempotent for naked JSON. Some models (claude-code via router, certain
+ * deepseek models) wrap output even under json_object.
+ */
+declare function stripFencedJson(raw: string): string;
+/**
+ * Low-level call. Returns raw content + usage + cost. Retries on transient
+ * failures; does NOT degrade schema here — callers that want graceful
+ * degrade use `callLlmJson`.
+ */
+declare function callLlm(req: LlmCallRequest, opts?: LlmClientOptions): Promise<LlmCallResult>;
+/**
+ * Structured-output call. Returns parsed JSON plus the raw result envelope.
+ * Degrades `jsonSchema` → `jsonMode` on a 400 that names the schema param —
+ * critical for deepseek-v3/v4, kimi-k2.6, and other models that don't accept
+ * the `response_format.json_schema` shape but DO accept `json_object`.
+ */
+declare function callLlmJson<T = unknown>(req: LlmCallRequest, opts?: LlmClientOptions): Promise<{
+    value: T;
+    result: LlmCallResult;
+}>;
+declare class LlmRouteAssertionError extends Error {
+    readonly code: 'no_explicit_base_url' | 'base_url_blocked' | 'base_url_not_allowed' | 'no_auth' | 'wrong_provider';
+    readonly baseUrl: string;
+    constructor(message: string, code: 'no_explicit_base_url' | 'base_url_blocked' | 'base_url_not_allowed' | 'no_auth' | 'wrong_provider', baseUrl: string);
+}
+interface LlmRouteRequirements {
+    /**
+     * Throw if `opts.baseUrl` is undefined, i.e. the call would fall back to
+     * `DEFAULT_BASE_URL`. Set this for evaluation runs where silently using
+     * the public/free-tier router is a defect — the launch reviewer needs to
+     * know exactly which provider answered.
+     */
+    requireExplicitBaseUrl?: boolean;
+    /**
+     * Allowlist of acceptable base URLs. Strings match by prefix
+     * (case-insensitive); RegExps test against the full base URL.
+     */
+    allowedBaseUrls?: Array<string | RegExp>;
+    /** Blocklist that takes precedence over `allowedBaseUrls`. */
+    blockedBaseUrls?: Array<string | RegExp>;
+    /** Throw if no auth header / api key is configured. */
+    requireAuth?: boolean;
+    /**
+     * Logical provider id the configured `baseUrl` is expected to match (via
+     * `providerFromBaseUrl`). Mainly useful when paired with `requireExplicitBaseUrl`.
+     */
+    expectedProvider?: string;
+}
+/**
+ * Fail-loud assertion that the configured LLM client points at the route
+ * the caller intends. Designed for the matrix-runner preflight: invoke
+ * once before any LLM call to catch misconfiguration before a sweep burns
+ * dollars on the wrong provider.
+ *
+ * Throws `LlmRouteAssertionError`. Pure — no I/O — so it's safe to call
+ * from constructors and CI gates.
+ */
+declare function assertLlmRoute(opts: LlmClientOptions, req?: LlmRouteRequirements): void;
+/**
+ * Probe whether a model is reachable. Returns latency + null error on
+ * success; `ok=false` + error message on any failure (HTTP, timeout,
+ * network, parse). Designed for sweep preflights — fail loud at the
+ * boundary before burning a 30-leaf run on a misconfigured router.
+ *
+ * Sends a tiny `ping` message with `maxTokens=64`. Reasoning models
+ * (glm-5.1, deepseek-v4) can burn the entire budget on internal reasoning
+ * for short prompts, so don't tighten this further. We don't validate
+ * content; HTTP 200 means reachable.
+ */
+declare function probeLlm(model: string, opts?: LlmClientOptions & {
+    timeoutMs?: number;
+}): Promise<{
+    ok: boolean;
+    latencyMs: number;
+    error: string | null;
+}>;
+/**
+ * Stateful client — construct once with defaults, call many times.
+ * Thin wrapper around the free functions; exists for callers that want
+ * to inject a single configured instance into multiple primitives.
+ */
+declare class LlmClient {
+    private readonly opts;
+    constructor(opts?: LlmClientOptions);
+    call(req: LlmCallRequest, per?: LlmClientOptions): Promise<LlmCallResult>;
+    callJson<T = unknown>(req: LlmCallRequest, per?: LlmClientOptions): Promise<{
+        value: T;
+        result: LlmCallResult;
+    }>;
+}
+
+/**
+ * Researcher interface — stable hook for an external autonomous-research
+ * agent to drive the meta-loop.
+ *
+ * Implementations live downstream (typically in a private repo that
+ * runs the actual LLM). This package ships only the contract + a
+ * `NoopResearcher` so consumers can wire the surface without being
+ * forced to implement every method up front.
+ *
+ * The four methods mirror the four stages of the paper "Two Loops,
+ * Three Roles":
+ *
+ *   inspectFailures   — given the observed runs, what failure modes
+ *                       are present? (data → diagnosis)
+ *   proposeChange     — given diagnosed failure modes, what
+ *                       structural changes should we try?
+ *                       (diagnosis → plan delta)
+ *   applyChange       — fold the proposed deltas into a concrete
+ *                       experiment plan against an existing baseline.
+ *                       (plan delta → executable plan)
+ *   evaluateChange    — run the plan, return runs + the gate verdict.
+ *                       (executable plan → verdict)
+ *
+ * Composition is the discipline: a Researcher implementation MUST
+ * keep these four steps separate and inspectable. Conflating
+ * "diagnose + propose + run" into a single LLM call defeats the
+ * point of the framework — you can't audit which step lied.
+ *
+ * THIS INTERFACE IS STABLE. Breaking changes require a new module
+ * (e.g. `Researcher2`) so existing implementations keep working.
+ */
+
+/** A diagnosed failure mode with the run-IDs that exhibit it. */
+interface FailureMode {
+    /** Short machine-readable code. Must be stable across runs of the
+     *  same researcher to enable longitudinal tracking. */
+    code: string;
+    /** Human-readable description for the paper / dashboard. */
+    description: string;
+    evidence: {
+        /** Run IDs (from `RunRecord.runId`) where this failure mode was
+         *  observed. */
+        runIds: string[];
+        /** Number of run samples that informed the diagnosis. */
+        samples: number;
+    };
+}
+/** A single steering change the researcher wants to try. */
+interface SteeringChange {
+    kind: 'reviewer_prompt' | 'skill_add' | 'skill_remove' | 'threshold' | 'budget';
+    /** Implementation-specific payload. Researcher implementations
+     *  define the schema — keep this `unknown` here to avoid coupling
+     *  the public interface to any one researcher's internal model. */
+    payload: unknown;
+    /** Why the researcher proposed this change. Goes into the audit
+     *  trail next to the failure-mode evidence. */
+    rationale: string;
+    /** Optional self-reported expected delta on the headline metric. */
+    expectedDelta?: number;
+}
+/** A single experiment plan, mapped onto the search/holdout splits. */
+interface ExperimentPlan {
+    baselineCandidateId: string;
+    proposedCandidateId: string;
+    changes: SteeringChange[];
+    /** USD ceiling for the entire experiment. The runner must stop
+     *  before exceeding this and report a partial result. */
+    evaluationBudgetUsd: number;
+    /** Item IDs (your dataset keys) for the search vs holdout splits. */
+    splits: {
+        search: string[];
+        holdout: string[];
+    };
+}
+/** Result of running a plan: every run, plus the gate verdict. */
+interface ExperimentResult {
+    plan: ExperimentPlan;
+    runs: RunRecord[];
+    gateDecision: GateDecision;
+}
+/**
+ * The researcher loop. Stable, four-step, inspectable.
+ *
+ *   ┌──────────┐  inspectFailures  ┌──────────┐  proposeChange ┌──────────┐
+ *   │   runs   │ ─────────────────▶│ failures │ ──────────────▶│ changes  │
+ *   └──────────┘                   └──────────┘                └────┬─────┘
+ *                                                                   │
+ *                                                                   ▼
+ *                              ┌────────────────┐  applyChange ┌────────┐
+ *                              │ ExperimentPlan │ ◀────────────│  base  │
+ *                              └────────┬───────┘              └────────┘
+ *                                       │
+ *                       evaluateChange  ▼
+ *                              ┌────────────────┐
+ *                              │ ExperimentResult│
+ *                              └────────────────┘
+ */
+interface Researcher {
+    inspectFailures(runs: RunRecord[]): Promise<FailureMode[]>;
+    proposeChange(failures: FailureMode[]): Promise<SteeringChange[]>;
+    applyChange(changes: SteeringChange[], baseline: ExperimentPlan): Promise<ExperimentPlan>;
+    evaluateChange(plan: ExperimentPlan): Promise<ExperimentResult>;
+}
+interface CallbackResearcherOptions {
+    inspectFailures: Researcher['inspectFailures'];
+    proposeChange: Researcher['proposeChange'];
+    applyChange: Researcher['applyChange'];
+    evaluateChange: Researcher['evaluateChange'];
+}
+/**
+ * Minimal concrete researcher for tests, scripts, and small integrations.
+ * Larger autonomous researchers can still implement `Researcher` directly.
+ */
+declare class CallbackResearcher implements Researcher {
+    private readonly callbacks;
+    constructor(callbacks: CallbackResearcherOptions);
+    inspectFailures(runs: RunRecord[]): Promise<FailureMode[]>;
+    proposeChange(failures: FailureMode[]): Promise<SteeringChange[]>;
+    applyChange(changes: SteeringChange[], baseline: ExperimentPlan): Promise<ExperimentPlan>;
+    evaluateChange(plan: ExperimentPlan): Promise<ExperimentResult>;
+}
+/**
+ * No-op researcher — fails loud on every method. Use as a placeholder
+ * in code paths that wire the interface but don't have an implementation
+ * yet. Importantly, this does NOT silently succeed: a no-op researcher
+ * that returned empty arrays would muffle the loop's signal that
+ * nobody implemented the brain.
+ */
+declare class NoopResearcher implements Researcher {
+    private readonly hint;
+    constructor(hint?: string);
+    inspectFailures(_runs: RunRecord[]): Promise<FailureMode[]>;
+    proposeChange(_failures: FailureMode[]): Promise<SteeringChange[]>;
+    applyChange(_changes: SteeringChange[], _baseline: ExperimentPlan): Promise<ExperimentPlan>;
+    evaluateChange(_plan: ExperimentPlan): Promise<ExperimentResult>;
+}
+
+/**
+ * EvalCampaign — opinionated matrix runner that wires the four
+ * capture-integrity directives by construction.
+ *
+ * Every consumer that ran a launch-grade benchmark before 0.22 reinvented
+ * the same shape: matrix runner → for each (variant, scenario, seed) →
+ * start a TraceEmitter → call LLMs → end the run → maybe analyze.
+ * The bug class blueprint-agent reported (raw events not captured, route
+ * silently wrong, integrity not asserted, analyst never ran) lives at the
+ * integration boundary — not the agent-eval API surface. The four
+ * directives in `SKILL.md § Capture integrity` are mitigations.
+ *
+ * `EvalCampaign` is the structural fix. Consumers don't wire the integrity
+ * surface anymore; the campaign owns it. Specifically, the campaign:
+ *
+ *   - calls `assertLlmRoute` once at preflight before any work runs
+ *   - constructs a per-run `TraceStore` and `RawProviderSink` via factories
+ *   - constructs the `TraceEmitter` with `onRunComplete: [analyst hook]`
+ *   - hands the runner an `LlmClientOptions` pre-wired with the sink and
+ *     trace context — the runner can't accidentally call an LLM without
+ *     capturing the raw HTTP envelope
+ *   - calls `assertRunCaptured` after every `endRun` and routes failures
+ *     through a configurable policy (`throw` / `mark_failed` / `log`)
+ *   - assembles per-run `RunRecord`s and runs `researchReport` at the end
+ *     so the campaign artifact is launch-decision-grade by default
+ *   - embeds the campaign fingerprint (a SHA-256 over the canonicalised
+ *     run set) and optional `preregistrationHash` in the report
+ *
+ * The runner contract is intentionally narrow: produce a `CampaignRunOutcome`
+ * given a fully-wired `CampaignRunContext`. Everything orchestration-shaped
+ * lives in the campaign. This is the inversion-of-control point — consumers
+ * stop writing matrix runners and start writing scenario-runners.
+ *
+ * Out of scope for v1 (tracked in `docs/research-report-methodology.md`):
+ *
+ *   - Distributed/cluster execution (concurrency is local async)
+ *   - Adaptive sampling / sequential interim looks
+ *   - Resume from partial state across crashes
+ *   - LLM-call retry beyond what `LlmClient` already does
+ */
+
+interface CampaignVariant<V> {
+    id: string;
+    payload: V;
+}
+interface CampaignScenario {
+    scenarioId: string;
+    /** Free-form metadata propagated to runs and reports. */
+    tags?: Record<string, string>;
+}
+interface CampaignRunContext<V> {
+    /** Stable run id. The campaign generates this; the runner does not. */
+    runId: string;
+    /** Logical experiment id (campaignId by default; overridable per-run via opts). */
+    experimentId: string;
+    variant: V;
+    variantId: string;
+    scenarioId: string;
+    scenarioTags: Record<string, string>;
+    seed: number;
+    splitTag: RunSplitTag;
+    /**
+     * The TraceEmitter for this run, with `onRunComplete` hooks pre-wired
+     * (analyst auto-execution if configured, plus integrity check). The
+     * runner MUST call `emitter.startRun` before doing any work and either
+     * `emitter.endRun` or `emitter.abortRun` before returning.
+     */
+    emitter: TraceEmitter;
+    store: TraceStore;
+    rawSink: RawProviderSink;
+    /**
+     * Pre-wired LLM client options — `rawSink` and `traceContext` are populated
+     * so any `callLlm(req, ctx.llmOpts)` automatically captures raw HTTP. The
+     * runner can spread additional fields if needed.
+     */
+    llmOpts: LlmClientOptions;
+}
+interface CampaignRunOutcome {
+    /** Did the run pass? Mirrors `RunOutcome.pass` semantics. */
+    pass: boolean;
+    /** Score for the run on its split. Maps to `searchScore` or `holdoutScore`. */
+    score: number;
+    /** Mandatory cost in USD. Use 0 + raw.cost_unknown=1 only if truly unknown. */
+    costUsd: number;
+    tokenUsage: RunTokenUsage;
+    /** Snapshot model id (e.g. `claude-sonnet-4-6@2025-04-15`). */
+    model: string;
+    /** sha256 of the effective prompt sent to the model. */
+    promptHash: string;
+    /** sha256 of the effective config (model, temperature, tools, judges, splits). */
+    configHash: string;
+    /** Optional extra numeric metrics to land in `outcome.raw`. */
+    raw?: Record<string, number>;
+    /** Optional failure-taxonomy tag if the run failed. */
+    failureMode?: string;
+    /** Optional judge metadata when a judge was used. */
+    judgeMetadata?: RunJudgeMetadata;
+}
+type CampaignRunner<V> = (ctx: CampaignRunContext<V>) => Promise<CampaignRunOutcome>;
+type CampaignIntegrityPolicy = 'throw' | 'mark_failed' | 'log';
+interface EvalCampaignOptions<V> {
+    /**
+     * Stable id for the campaign. Used as the default `experimentId` on
+     * every run, and folded into the campaign fingerprint.
+     */
+    campaignId: string;
+    variants: CampaignVariant<V>[];
+    scenarios: CampaignScenario[];
+    /** Default `[0, 1, 2]`. */
+    seeds?: number[];
+    /** Default `'holdout'` — the split that anchors a launch decision. */
+    splitTag?: RunSplitTag;
+    /** Git SHA the campaign is run against. Mandatory; `RunRecord` rejects unset. */
+    commitSha: string;
+    /**
+     * LLM client config. Augmented per-run with `rawSink` and `traceContext`
+     * before being passed to the runner. The campaign asserts this config
+     * matches `routeRequirements` once at preflight.
+     */
+    llmOpts: LlmClientOptions;
+    /**
+     * Default `{ requireExplicitBaseUrl: true, requireAuth: true }` — fail
+     * loud if the campaign would silently fall back to the public router or
+     * run unauthenticated. Override with an empty object to disable.
+     */
+    routeRequirements?: LlmRouteRequirements;
+    /**
+     * Per-run TraceStore factory. Common shape: a fresh store per run keyed
+     * on `runId`. Implementations that share a store across the campaign
+     * are valid — the campaign only writes through `emitter`.
+     */
+    storeFactory: (params: CampaignFactoryParams) => TraceStore;
+    /**
+     * Per-run RawProviderSink factory. Defaults to `FileSystemRawProviderSink`
+     * rooted at `${workDir}/raw-events/${runId}` if `workDir` is supplied;
+     * otherwise required. Forensic capture is non-negotiable in a campaign
+     * run — pass `NoopRawProviderSink` explicitly if you want to opt out.
+     */
+    rawSinkFactory?: (params: CampaignFactoryParams) => RawProviderSink;
+    /**
+     * Filesystem root for default `rawSinkFactory`. Ignored if
+     * `rawSinkFactory` is supplied.
+     */
+    workDir?: string;
+    /**
+     * Extra `onRunComplete` hooks the campaign appends (after its own
+     * integrity-check hook). Pass `traceAnalystOnRunComplete(...)` here.
+     */
+    onRunComplete?: RunCompleteHook[];
+    /**
+     * Per-run integrity expectations. Defaults to:
+     *   `{ llmSpansMin: 1, requireRawCoverageOfLlmSpans: true, requireOutcome: true }`.
+     * Override (e.g. `{ llmSpansMin: 0 }`) for runs that don't call LLMs.
+     */
+    integrity?: RunIntegrityExpectations;
+    /** Behaviour when integrity fails. Default `'mark_failed'`. */
+    onIntegrityFailure?: CampaignIntegrityPolicy;
+    /**
+     * Per-run runner. Receives a fully-wired context; produces an outcome
+     * the campaign converts into a `RunRecord`.
+     */
+    runner: CampaignRunner<V>;
+    /**
+     * If set, the campaign computes `researchReport` at the end. `comparator`
+     * is a `variantId`. Other fields are forwarded verbatim.
+     */
+    report?: {
+        comparator?: string;
+    } & Omit<ResearchReportOptions, 'comparator' | 'preregistrationHash' | 'generatedAt'>;
+    /**
+     * Hash of a signed `HypothesisManifest` (see `pre-registration.ts`).
+     * Embedded in the campaign fingerprint and the research report.
+     */
+    preregistrationHash?: string;
+    /** Local concurrency. Default `1` (sequential). */
+    concurrency?: number;
+    /**
+     * Override the time source. Tests pass a mock to make wallMs deterministic.
+     */
+    now?: () => number;
+    /** Override the runId generator. Tests pin this. */
+    runId?: (params: CampaignFactoryParams) => string;
+}
+interface CampaignFactoryParams {
+    campaignId: string;
+    runId: string;
+    variantId: string;
+    scenarioId: string;
+    seed: number;
+}
+interface FailedRun {
+    runId: string;
+    variantId: string;
+    scenarioId: string;
+    seed: number;
+    reason: string;
+    error?: string;
+}
+interface EvalCampaignResult {
+    campaignId: string;
+    /** SHA-256 over canonicalised `(variantIds, scenarioIds, seeds, comparator, splitTag, baseUrl, provider, preregistrationHash)`. */
+    campaignFingerprint: string;
+    preregistrationHash: string | null;
+    /** Successful runs only. Failed runs land in `failedRuns`. */
+    runs: RunRecord[];
+    /** Integrity reports for every successful run. */
+    integrityReports: RunIntegrityReport[];
+    failedRuns: FailedRun[];
+    /** Computed when `report` is set on options. */
+    report?: ResearchReport;
+    startedAt: string;
+    endedAt: string;
+}
+declare function runEvalCampaign<V>(opts: EvalCampaignOptions<V>): Promise<EvalCampaignResult>;
+
+export { CallbackResearcher as C, type EvalCampaignOptions as E, type FailedRun as F, type LlmClientOptions as L, NoopResearcher as N, type Researcher as R, type SteeringChange as S, type CallbackResearcherOptions as a, type CampaignFactoryParams as b, type CampaignIntegrityPolicy as c, type CampaignRunContext as d, type CampaignRunOutcome as e, type CampaignRunner as f, type CampaignScenario as g, type CampaignVariant as h, type EvalCampaignResult as i, type ExperimentPlan as j, type ExperimentResult as k, type FailureMode as l, LlmCallError as m, type LlmCallRequest as n, type LlmCallResult as o, LlmClient as p, type LlmMessage as q, runEvalCampaign as r, LlmRouteAssertionError as s, type LlmRouteRequirements as t, type LlmUsage as u, assertLlmRoute as v, callLlm as w, callLlmJson as x, probeLlm as y, stripFencedJson as z };

package/dist/{feedback-trajectory-CB0A32o3.d.ts → feedback-trajectory-c43WGtTX.d.ts}

@@ -343,4 +343,4 @@ declare function controlRunToFeedbackTrajectory<TState, TAction, TActionResult>(
     createdAt?: string;
 }): FeedbackTrajectory;
 
-export { type ProposedSideEffect as A, allCriticalPassed as B, type ControlSeverity as C, assignFeedbackSplit as D, controlRunToFeedbackTrajectory as E, type FeedbackLabel as F, createFeedbackTrajectory as G, feedbackTrajectoriesToDatasetScenarios as H, InMemoryFeedbackTrajectoryStore as I, feedbackTrajectoriesToOptimizerRows as J, feedbackTrajectoryToDatasetScenario as K, feedbackTrajectoryToOptimizerRow as L, objectiveEval as M, parseFeedbackTrajectoriesJsonl as N, renderPreferenceMemoryMarkdown as O, type PreferenceMemoryEntry as P, replayFeedbackTrajectories as Q, replayFeedbackTrajectory as R, type StopDecision as S, runAgentControlLoop as T, serializeFeedbackTrajectoriesJsonl as U, stopOnNoProgress as V, stopOnRepeatedAction as W, subjectiveEval as X, summarizePreferenceMemory as Y, withAssignedFeedbackSplit as Z, type FeedbackTrajectoryStore as a, type FeedbackTrajectory as b, type ControlEvalResult as c, type ControlActionFailureMode as d, type ControlActionOutcome as e, type ControlBudget as f, type ControlContext as g, type ControlDecision as h, type ControlRunResult as i, type ControlRuntimeConfig as j, type ControlRuntimeError as k, type ControlStep as l, type ControlStopPolicies as m, type FeedbackArtifactType as n, type FeedbackAttempt as o, type FeedbackLabelKind as p, type FeedbackLabelSource as q, type FeedbackOptimizerRow as r, type FeedbackOutcome as s, type FeedbackReplayAdapter as t, type FeedbackReplayResult as u, type FeedbackSeverity as v, type FeedbackSplitPolicy as w, type FeedbackTask as x, type FeedbackTrajectoryFilter as y, FileSystemFeedbackTrajectoryStore as z };
+export { replayFeedbackTrajectory as A, serializeFeedbackTrajectoriesJsonl as B, summarizePreferenceMemory as C, withAssignedFeedbackSplit as D, type ControlSeverity as E, type FeedbackArtifactType as F, type ControlEvalResult as G, type ControlActionFailureMode as H, InMemoryFeedbackTrajectoryStore as I, type ControlActionOutcome as J, type ControlBudget as K, type ControlContext as L, type ControlDecision as M, type ControlRunResult as N, type ControlRuntimeConfig as O, type PreferenceMemoryEntry as P, type ControlRuntimeError as Q, type ControlStep as R, type ControlStopPolicies as S, type StopDecision as T, allCriticalPassed as U, objectiveEval as V, runAgentControlLoop as W, stopOnNoProgress as X, stopOnRepeatedAction as Y, subjectiveEval as Z, type FeedbackAttempt as a, type FeedbackLabel as b, type FeedbackLabelKind as c, type FeedbackLabelSource as d, type FeedbackOptimizerRow as e, type FeedbackOutcome as f, type FeedbackReplayAdapter as g, type FeedbackReplayResult as h, type FeedbackSeverity as i, type FeedbackSplitPolicy as j, type FeedbackTask as k, type FeedbackTrajectory as l, type FeedbackTrajectoryFilter as m, type FeedbackTrajectoryStore as n, FileSystemFeedbackTrajectoryStore as o, type ProposedSideEffect as p, assignFeedbackSplit as q, controlRunToFeedbackTrajectory as r, createFeedbackTrajectory as s, feedbackTrajectoriesToDatasetScenarios as t, feedbackTrajectoriesToOptimizerRows as u, feedbackTrajectoryToDatasetScenario as v, feedbackTrajectoryToOptimizerRow as w, parseFeedbackTrajectoriesJsonl as x, renderPreferenceMemoryMarkdown as y, replayFeedbackTrajectories as z };

package/dist/{index-c5saLbKD.d.ts → index-DDTlbHEK.d.ts}

@@ -1,4 +1,4 @@
-import { R as RunSplitTag } from './run-record-CX_jcAyr.js';
+import { a as RunSplitTag } from './run-record-DNiOMBrZ.js';
 
 /**
  * Shared types for the reference benchmark wrappers under