@tangle-network/agent-eval 0.117.1 → 0.118.0

package/dist/pre-registration-BWQhJ3vz.d.ts

@@ -1,761 +0,0 @@
-import { A as AgentEvalError } from './errors-oeQrLqXC.js';
-import { R as RunRecord } from './run-record-BDH49H2E.js';
-import { P as PairedBootstrapOptions, M as McNemarResult, R as RiskDifferenceResult, a as PairedBootstrapResult } from './statistics-KUnG73jH.js';
-import { z } from 'zod';
-import { C as ChatClient } from './policy-edit-wG9uFEFm.js';
-import { a as JudgeDimension, S as Scenario, b as JudgeConfig } from './types-BSw1rOUB.js';
-import { C as CostLedger, c as CostReceiptInput } from './cost-ledger-DWy3XdJc.js';
-import { TCloud } from '@tangle-network/tcloud';
-import { R as RawProviderSink } from './raw-provider-sink-C46HDghv.js';
-import { D as DefaultVerdict } from './verdict-C9MlYujm.js';
-
-/**
- * Backend-integrity guard: distinguish "agent failed" from "eval ran against
- * a stub / unconfigured backend." Without this guard a canonical eval can
- * silently report `0/N passed` and look like an agent-quality problem when
- * the LLM was never actually called — the failure mode we just hit running
- * the 4-vertical parallel eval (legal-sandbox-stub returned hard-coded 33-104
- * char strings; gtm/creative defaulted to a cli-bridge that wasn't running).
- *
- * The shape:
- *
- *   const report = summarizeBackendIntegrity(records)
- *   assertRealBackend(records)   // throws BackendIntegrityError if 100% stub
- *
- * A record is "stub-mode" if its `tokenUsage.input === 0 && tokenUsage.output === 0`.
- * (`costUsd` alone is unreliable — some backends successfully call LLMs but
- *  don't propagate pricing, producing real tokens with $0 cost.)
- *
- * Verdicts:
- *   - `real`   — at least one record has nonzero token usage
- *   - `stub`   — every record is stub-mode (eval ran blind)
- *   - `mixed`  — some records real, some stub (partial backend failure;
- *                often the 429-cascade or auth-half-failed case)
- */
-
-interface BackendIntegrityReport {
-    /** Total records inspected. */
-    totalRecords: number;
-    /** Records with input=0 AND output=0 (a stub fingerprint). */
-    stubRecords: number;
-    /** Records with nonzero token usage (real LLM activity). */
-    realRecords: number;
-    /** Records where output>0 but costUsd=0 (real LLM, broken cost ledger). */
-    uncostedRecords: number;
-    /** Sum of input tokens across all records. */
-    totalInputTokens: number;
-    /** Sum of output tokens across all records. */
-    totalOutputTokens: number;
-    /** Sum of costUsd across all records. */
-    totalCostUsd: number;
-    /** Worst-case integrity verdict. */
-    verdict: 'real' | 'mixed' | 'stub';
-    /** Human-readable diagnosis suitable for terminal output. */
-    diagnosis: string;
-}
-/**
- * Error thrown when an integrity assertion fails. Caller can pattern-match
- * by `code === 'AGENT_EVAL_BACKEND_STUB'` to differentiate from other
- * errors.
- */
-declare class BackendIntegrityError extends AgentEvalError {
-    readonly report: BackendIntegrityReport;
-    constructor(message: string, report: BackendIntegrityReport);
-}
-/**
- * Inspect a batch of RunRecords and return an integrity report. Pure
- * function — no I/O, no logging. The caller decides what to do with the
- * verdict (print warning, throw, gate CI, etc.).
- */
-declare function summarizeBackendIntegrity(records: ReadonlyArray<RunRecord>): BackendIntegrityReport;
-/**
- * Throw BackendIntegrityError if the verdict is 'stub' — i.e. every record
- * shows zero LLM activity. Non-strict callers can pass `{ allowMixed: false }`
- * to also reject mixed verdicts (recommended for CI gates).
- *
- * Real backends pass through silently.
- */
-declare function assertRealBackend(records: ReadonlyArray<RunRecord>, opts?: {
-    allowMixed?: boolean;
-}): BackendIntegrityReport;
-
-/**
- * Matched-pair arm comparison — "did the treatment arm beat the baseline arm
- * on the SAME work items?"
- *
- * An arm A/B over run records is only trustworthy when it is PAIRED: the same
- * task/scenario/seed evaluated under both arms, compared item-by-item, so
- * inter-item difficulty variance cancels instead of masquerading as an arm
- * effect. This module owns the two error-prone steps every consumer otherwise
- * hand-rolls:
- *
- *   1. Pairing — matching rows across arms by `pairKey` (and by `repKey`
- *      within multi-rep items), with leftovers REPORTED rather than silently
- *      dropped (a silently unbalanced pairing biases every paired statistic
- *      downstream). Pairing never keys on outcome content: matching reps by
- *      their outcomes deflates discordant-pair counts and makes McNemar
- *      anti-conservative, so reps pair only by (`pairKey`, `repKey`) identity.
- *   2. Composition — feeding the matched pairs to the correct paired
- *      estimators that already live in `statistics`: `mcnemar` +
- *      `pairedRiskDifference` for pass/fail, `pairedBootstrap` +
- *      `wilcoxonSignedRank` for continuous metrics. No statistic is
- *      re-implemented here.
- *
- * The row shape is deliberately structural — callers project a `RunRecord`
- * (or any record) into `{ pairKey, arm, pass?, metrics? }`. Arm names are
- * caller-supplied parameters; the module ships no domain literal.
- */
-
-/** One arm observation of one work item. Structural on purpose: callers
- *  project their own record type (e.g. a `RunRecord`) into this shape. */
-interface PairedArmRow {
-    /** Matching key — rows sharing a `pairKey` across both arms form pairs
-     *  (typically the task/scenario/seed identity). */
-    pairKey: string;
-    /** Rep identity within a `pairKey` (e.g. a seed or rep number). Required on
-     *  every row of a `pairKey` that has more than one rep in either arm; reps
-     *  then pair only on exact (`pairKey`, `repKey`) match, never on outcome
-     *  content. Optional when each arm has at most one rep of the item. */
-    repKey?: string;
-    /** Arm label this row was produced under. */
-    arm: string;
-    /** Binary outcome; omit when the comparison has no pass/fail notion. */
-    pass?: boolean;
-    /** Named numeric measurements (score, cost, latency, …). */
-    metrics?: Record<string, number>;
-}
-interface PairArmsOptions {
-    /** Arm treated as the control side of every pair. */
-    baselineArm: string;
-    /** Arm treated as the treatment side of every pair. */
-    treatmentArm: string;
-}
-/** One matched (baseline, treatment) observation of the same work item. */
-interface MatchedPair {
-    pairKey: string;
-    /** 0-based position of this pair within its `pairKey`, ordered by sorted
-     *  `repKey` (always 0 for a single-rep item). The rep identity itself is on
-     *  the rows (`baseline.repKey` / `treatment.repKey`). */
-    repIndex: number;
-    baseline: PairedArmRow;
-    treatment: PairedArmRow;
-}
-interface PairArmsResult {
-    /** Matched pairs, ordered by (`pairKey`, `repIndex`). */
-    pairs: MatchedPair[];
-    /** Baseline rows left without a treatment counterpart — reported, never
-     *  silently dropped. */
-    unpairedBaseline: PairedArmRow[];
-    /** Treatment rows left without a baseline counterpart. */
-    unpairedTreatment: PairedArmRow[];
-}
-/**
- * Match rows across two arms into (baseline, treatment) pairs by `pairKey`.
- *
- * A `pairKey` with at most one row per arm pairs directly, no `repKey`
- * needed. A `pairKey` with multiple reps in either arm requires `repKey` on
- * every one of its rows, and reps pair only on exact (`pairKey`, `repKey`)
- * match — pairing is keyed purely on row identity, never on outcome content
- * (outcome-keyed matching deflates discordant counts and biases McNemar), and
- * is therefore independent of input order. Reps whose `repKey` has no
- * counterpart in the other arm, and items present in only one arm, land in
- * the unpaired lists — reported, never truncated.
- *
- * Fail-loud: throws when either named arm has zero rows (an unknown arm
- * name would otherwise read as "everything unpaired"), when the two arm
- * names are equal, when a multi-rep `pairKey` has a row without `repKey`, or
- * when a (`pairKey`, arm) group repeats a `repKey` (the match would be
- * ambiguous).
- */
-declare function pairArms(rows: readonly PairedArmRow[], opts: PairArmsOptions): PairArmsResult;
-/** Paired pass/fail comparison over the pairs where BOTH sides carry `pass`. */
-interface PairedCorrectness {
-    /** Discordant pairs where the treatment passed and the baseline failed. */
-    b10: number;
-    /** Discordant pairs where the baseline passed and the treatment failed. */
-    b01: number;
-    /** Exact McNemar significance over the paired outcomes (`b === b10`, `c === b01`). */
-    mcnemar: McNemarResult;
-    /** Paired effect size: p(treatment) − p(baseline) with a paired-variance CI. */
-    riskDifference: RiskDifferenceResult;
-}
-/** Paired delta summary for one named metric (delta = treatment − baseline). */
-interface PairedMetricDelta {
-    name: string;
-    /** Pairs where BOTH sides carry a finite value for this metric. */
-    n: number;
-    /** Pairs where at least one side does not carry the metric. */
-    nMissing: number;
-    /** Median paired delta; NaN when `n === 0` (no data ≠ measured zero). */
-    medianDelta: number;
-    /** Mean paired delta; NaN when `n === 0`. */
-    meanDelta: number;
-    /** Bootstrap CI on the paired delta (`pairedBootstrap`); null when
-     *  `n === 0` — a zero-width [0, 0] interval on no data would read as a
-     *  measured tight null. */
-    bootstrapCi: PairedBootstrapResult | null;
-    /** Wilcoxon signed-rank test on the paired deltas; null when `n === 0`. */
-    wilcoxon: {
-        w: number;
-        p: number;
-    } | null;
-}
-interface ComparePairedArmsOptions extends PairArmsOptions {
-    /** Metrics to compare. Default: every metric name observed on any matched
-     *  pair, sorted. A name that appears on no pair is still reported (with
-     *  `n = 0`) so a misspelled metric is visible instead of vanishing. */
-    metricNames?: string[];
-    /** Passed through to `pairedBootstrap` — set `seed` for reproducible CIs. */
-    bootstrap?: PairedBootstrapOptions;
-}
-interface PairedArmsComparison {
-    nPairs: number;
-    nUnpairedBaseline: number;
-    nUnpairedTreatment: number;
-    /** null when no matched pair carries `pass` on both sides — a pass/fail
-     *  verdict over rows that never measured pass/fail would be fabricated. */
-    correctness: PairedCorrectness | null;
-    metricDeltas: PairedMetricDelta[];
-}
-/**
- * Full matched-pair arm comparison: pair via {@link pairArms}, then compose
- * the paired estimators from `statistics` over the matched pairs.
- *
- * Correctness uses only the pairs where both sides carry `pass` (`mcnemar.n`
- * is that subset's size); each metric uses only the pairs where both sides
- * carry a finite value for it, with the remainder counted in `nMissing`.
- * Deltas are treatment − baseline throughout.
- *
- * Fail-loud: inherits {@link pairArms}'s unknown-arm throw, and throws on a
- * non-finite metric value — silently treating corrupt telemetry as "metric
- * absent" would misreport it as missing coverage.
- */
-declare function comparePairedArms(rows: readonly PairedArmRow[], opts: ComparePairedArmsOptions): PairedArmsComparison;
-
-/**
- * Artifact validators.
- *
- * Generic "score a produced artifact" primitive. Tax uses it for PDF form
- * correctness, research for sourced briefs, browser for task assertions, coding
- * for social posts. One interface, many validators; all plug into
- * `BenchmarkRunner` the same way.
- *
- * A validator receives an `Artifact` (file on disk, JSON blob, text, binary)
- * plus a `ValidationContext` (scenario id, the turns that produced it) and
- * returns a `ValidationResult` with pass/fail + 0..1 score + structured
- * issues.
- */
-interface Artifact {
-    /** Logical kind — validators type-guard on this */
-    kind: 'file' | 'json' | 'text' | 'binary' | string;
-    /** Filesystem-style path, optional */
-    path?: string;
-    /** String content for text/json/file kinds */
-    content?: string;
-    /** Binary content (if kind === 'binary') */
-    bytes?: Uint8Array;
-    /** Caller-supplied metadata (mimeType, sha256, size, etc.) */
-    metadata?: Record<string, unknown>;
-}
-interface ValidationContext {
-    scenarioId: string;
-    turnIndex?: number;
-    /** Prior artifacts for multi-artifact scenarios */
-    priorArtifacts?: Artifact[];
-    /** Free-form hints the validator uses for domain-specific checks */
-    hints?: Record<string, unknown>;
-}
-interface ValidationIssue {
-    severity: 'error' | 'warning' | 'info';
-    message: string;
-    /** Optional path into the artifact (e.g. JSON path or byte offset) */
-    locus?: string;
-}
-interface ValidationResult {
-    pass: boolean;
-    /** 0–1 normalized score. Validators should be monotonic in pass-ness. */
-    score: number;
-    issues: ValidationIssue[];
-    /** Diagnostic payload for reporters */
-    evidence?: Record<string, unknown>;
-}
-interface ArtifactValidator {
-    /** Stable identifier for the validator; appears in reports. */
-    name: string;
-    /** Optional description for human-facing reports. */
-    description?: string;
-    /** Called once per artifact; validators are expected to be pure + idempotent. */
-    validate(artifact: Artifact, context: ValidationContext): Promise<ValidationResult>;
-}
-/**
- * Run every validator on the same artifact; aggregate pass as AND, score as
- * (weighted) mean, issues concatenated. Weights default to 1 each.
- */
-declare function composeValidators(validators: ArtifactValidator[], options?: {
-    name?: string;
-    weights?: number[];
-}): ArtifactValidator;
-/** Pass if the artifact body matches a provided regex. */
-declare function regexMatch(name: string, pattern: RegExp): ArtifactValidator;
-/** Pass if JSON parses and every required key is present. */
-declare function jsonHasKeys(name: string, requiredPaths: string[]): ArtifactValidator;
-/** Pass if min ≤ byte length ≤ max. */
-declare function byteLengthRange(name: string, min: number, max: number): ArtifactValidator;
-/** Pass if the artifact contains every required substring (case-insensitive by default). */
-declare function containsAll(name: string, required: string[], options?: {
-    caseSensitive?: boolean;
-}): ArtifactValidator;
-
-/**
- * Completion verifier — the task-completion oracle.
- *
- * Answers the only eval question that is not a proxy: did the agent actually
- * COMPLETE the task — produce every required deliverable, persisted and
- * correct — rather than describe what should be done. A fluent transcript
- * that never produces the artifact scores zero here.
- *
- * Per requirement, a two-stage check:
- *   1. Structural — a produced item (vault artifact / approved proposal /
- *      tool call) of the right kind is matched against the requirement and
- *      carries non-empty content. Deterministic; no LLM.
- *   2. Correctness — only if structurally present AND the matched item
- *      carries content, one targeted check decides whether that item
- *      actually fulfils the requirement. A hallucinated artifact fails here;
- *      an absent one already failed stage 1.
- *
- * `completionRate` is satisfied / MEASURABLE requirements (unmeasured rows —
- * checker failures — are excluded from the denominator, never scored as
- * zeros). Quality dimensions are meaningless on an incomplete task — callers
- * gate on `fullyComplete` / `completionRate` before scoring quality.
- */
-
-/** What kind of produced state can satisfy a requirement structurally. */
-type SatisfiedBy = 'artifact' | 'proposal' | 'tool-call' | 'any';
-interface CompletionRequirement {
-    /** Stable id from the task gold (e.g. a persona's `expected_requirements[].req_id`). */
-    reqId: string;
-    /** Human-readable description of the required deliverable. */
-    title: string;
-    /** Optional kind/category hint, matched against a produced item's kind. */
-    category?: string;
-    /** What produced state satisfies this requirement. Defaults to 'any'. */
-    satisfiedBy?: SatisfiedBy;
-}
-interface TaskGold {
-    taskId: string;
-    requirements: CompletionRequirement[];
-}
-interface ProducedProposal {
-    id: string;
-    title: string;
-    status: 'pending' | 'approved' | 'rejected';
-    /** Optional persisted body — when present, enables a correctness check. */
-    content?: string;
-}
-/** Everything observable about what a run actually produced. */
-interface ProducedState {
-    /** Persisted vault artifacts. Reuses the shared `Artifact` shape. */
-    artifacts: Artifact[];
-    /** Proposals / filings the agent created. */
-    proposals: ProducedProposal[];
-    /** Names of tools the agent invoked. */
-    toolCalls: string[];
-}
-interface RequirementCheck {
-    reqId: string;
-    title: string;
-    /** A produced item of the right kind matched the requirement, non-empty. */
-    structurallyPresent: boolean;
-    /**
-     * Whether the matched item actually fulfils the requirement. `null` when
-     * not structurally present, when the matched item carries no content
-     * to assess, or when the correctness check itself failed (`unmeasured`).
-     */
-    correct: boolean | null;
-    /** structurallyPresent && !unmeasured && correct !== false. */
-    satisfied: boolean;
-    /**
-     * Set when the correctness check itself errored (LLM call failure or an
-     * unparseable response after retry). The requirement's fulfilment is
-     * UNKNOWN — `correct` stays null, `satisfied` is false, and
-     * `completionVerdict` excludes the row from `completionRate`'s
-     * denominator. Never folded into a zero: a synthetic zero is
-     * indistinguishable from a real failure (see `JudgeParseError`).
-     */
-    unmeasured?: true;
-    /** Why the correctness check could not be measured (present iff `unmeasured`). */
-    unmeasuredReason?: string;
-    /** Human-readable evidence for the verdict. */
-    evidence: string[];
-}
-/** Extends the substrate verdict spine: `valid` = `fullyComplete` and
- *  `score` = `completionRate` — derived in `completionVerdict()`, the one
- *  place those equalities hold by construction. */
-interface CompletionVerdict extends DefaultVerdict {
-    taskId: string;
-    requirements: RequirementCheck[];
-    /** satisfied / MEASURABLE requirements (unmeasured rows leave the denominator). */
-    completionRate: number;
-    /** Every measurable requirement satisfied (false when anything is unmeasured). */
-    fullyComplete: boolean;
-    /** Requirements whose correctness check errored — reported, never scored as zero. */
-    unmeasuredCount: number;
-}
-/**
- * Construct a `CompletionVerdict` from the per-requirement checks, deriving
- * `completionRate` / `fullyComplete` and the spine fields (`valid` =
- * `fullyComplete`, `score` = `completionRate`) in one place. Throws on zero
- * requirements — a verdict over nothing is a misconfiguration, mirroring
- * `verifyCompletion`'s gold-spec guard.
- */
-declare function completionVerdict(input: {
-    taskId: string;
-    requirements: RequirementCheck[];
-}): CompletionVerdict;
-/**
- * Decides whether a produced item's content actually fulfils a requirement.
- * Injected so the structural verifier stays pure and unit-testable; the
- * production implementation is `createLlmCorrectnessChecker`.
- */
-type CorrectnessChecker = (requirement: CompletionRequirement, content: string) => Promise<{
-    correct: boolean;
-    reason: string;
-}>;
-/**
- * Verify whether a run completed the task. `checkCorrectness` is injected —
- * `createLlmCorrectnessChecker` for production, a deterministic stub in tests.
- *
- * Throws on a gold spec with no requirements: an eval task that requires
- * nothing is a misconfiguration, not a vacuously-complete task.
- */
-declare function verifyCompletion(gold: TaskGold, state: ProducedState, checkCorrectness: CorrectnessChecker): Promise<CompletionVerdict>;
-interface LlmCorrectnessCheckerOpts {
-    model?: string;
-    /** Optional ledger for direct use. */
-    costLedger?: CostLedger;
-    costPhase?: string;
-    costTags?: Record<string, string>;
-    signal?: AbortSignal;
-    /** Exact maximum provider attempts configured on the supplied TCloud client. */
-    tcloudMaximumAttempts?: number;
-    /** Usage/cost retained by a failed provider response; enables a safe retry. */
-    receiptFromError?: (error: Error, attempt: number) => CostReceiptInput | undefined;
-    /** Max chars of artifact content sent to the checker. */
-    maxContentChars?: number;
-    /**
-     * Checker LLM calls per requirement before giving up (parse failures and
-     * call errors both consume attempts). The failure then surfaces as an
-     * `unmeasured` requirement, never a zero.
-     */
-    maxAttempts?: number;
-    /**
-     * Forensic capture of every checker request/response/error — without it a
-     * checker failure is unauditable (the agent-turn raws never contain the
-     * checker's own calls). Same sink contract as `LlmClient`.
-     */
-    rawSink?: RawProviderSink;
-}
-/**
- * Parse the correctness checker's model response. Tolerates a response
- * truncated mid-JSON (max_tokens cap) by auto-closing the prefix — the
- * verdict boolean usually lands in the first few tokens, so a recovered
- * prefix with a boolean `correct` is a real measurement, not a guess.
- * Fails loud (JudgeParseError) when no boolean verdict is recoverable.
- */
-declare function parseCorrectnessResponse(raw: string): {
-    correct: boolean;
-    reason: string;
-};
-/**
- * Production `CorrectnessChecker` — one LLM call per matched artifact,
- * deterministic (temperature 0), structured JSON out. Judges fulfilment
- * only: a plan, a gesture, or a description of what should be done does not
- * fulfil a requirement — the artifact must BE the deliverable.
- */
-declare function createLlmCorrectnessChecker(tc: TCloud, opts?: LlmCorrectnessCheckerOpts): CorrectnessChecker;
-/**
- * Deterministic `CorrectnessChecker` — the no-LLM counterpart to
- * `createLlmCorrectnessChecker`. A produced item fulfils a requirement when its
- * content is substantive (≥ `minContentLength` chars) AND recalls ≥ `minRecall`
- * of the requirement title's significant tokens. No network.
- *
- * Polarity-blind: token recall credits a negation that contains the
- * requirement's tokens ("I will NOT produce the comparison" recalls every token
- * of "produce the comparison"). The structural match stage is ALSO lexical, so
- * pairing the two collapses to a single gameable gate. Use this only as an
- * opt-in structural pre-filter or for tasks whose requirements have no polarity
- * to invert; for produced-state grading the correctness checker MUST be semantic
- * (`createLlmCorrectnessChecker`). See the anti-game fixtures in the test suite.
- */
-declare function createTokenRecallChecker(opts?: {
-    minRecall?: number;
-    minContentLength?: number;
-}): CorrectnessChecker;
-
-/**
- * `llmJudge` — the single-LLM-call bridge that turns a rubric prompt into a
- * canonical campaign `JudgeConfig`.
- *
- * The `JudgeConfig` contract (src/campaign/types.ts) is deliberately a
- * function, not a fixed LLM-prompt shape: real consumers judge with
- * ensembles, deterministic checks, or one LLM call. `ensembleJudge`
- * (src/judge-panel.ts) covers the multi-model case; `buildAgreementJudge`
- * (src/campaign/distillation) covers the pure-comparator case. `llmJudge`
- * covers the common single-call case the `JudgeConfig` doc-comment names:
- * one model call against `prompt`, parsed into the canonical `JudgeScore`
- * (`{ dimensions, composite, notes }`) on the campaign [0,1] scale.
- *
- * Transport is injected as a `ChatClient` (src/analyst/chat-client.ts) — the
- * substrate's transport-agnostic LLM seam — so the judge stays decoupled from
- * router-vs-sandbox-vs-cli-bridge and is unit-testable with the `mock`
- * transport. The composite is computed by `weightedComposite` (the same
- * sum-normalized weighting `ensembleJudge` uses), so a lift is attributable to
- * the dimension scores, not to a bespoke reducer.
- *
- * Fail-loud throughout: an unparseable model response throws `JudgeParseError`;
- * a response missing a declared dimension throws; an out-of-range score throws.
- * A thrown judge is recorded by the campaign engine as a failed cell, never
- * folded into a silent zero.
- */
-
-/** A rubric dimension as a bare key or the full `{ key, description }` shape. A
- *  bare string uses the key as its own description. */
-type LlmJudgeDimension = string | JudgeDimension;
-interface LlmJudgeOptions<TArtifact, TScenario extends Scenario = Scenario> {
-    /** The injected LLM transport. One `chat()` call per `score()`. Required —
-     *  there is no default route, so a misconfigured judge fails at construction,
-     *  never silently against the free-tier router. */
-    chat: ChatClient;
-    /** Rubric dimensions the model scores. Each becomes a `[0,1]` field of the
-     *  returned `JudgeScore.dimensions`. Defaults to a single `quality` dimension. */
-    dimensions?: LlmJudgeDimension[];
-    /** Model id. Falls back to `chat.defaultModel`; one of the two MUST resolve. */
-    model?: string;
-    /** Explicit scoring revision for opaque transport or renderer changes. */
-    judgeVersion?: string;
-    temperature?: number;
-    maxTokens?: number;
-    /** Composite weights forwarded to `weightedComposite`: a partial map selects
-     *  AND weights exactly the named dimensions. Omit for a uniform mean. */
-    weights?: Record<string, number>;
-    /** Scale the model is prompted to score on, normalized into `[0,1]`:
-     *   - `'unit'`  (default): the model returns `[0,1]` directly.
-     *   - `'ten'`:  the model returns `[0,10]`; divided by 10 here.
-     *  The prompt is annotated with the expected range either way. */
-    scale?: 'unit' | 'ten';
-    /** Run this judge only on matching scenarios (mirrors `JudgeConfig.appliesTo`). */
-    appliesTo?: (scenario: TScenario) => boolean;
-    /** Render the artifact + scenario into the user message. Default:
-     *  pretty-printed JSON of `{ scenario, artifact }`. */
-    renderUser?: (input: {
-        artifact: TArtifact;
-        scenario: TScenario;
-    }) => string;
-    /** Strict runtime contract; its JSON Schema is sent to the provider. */
-    costLedger?: CostLedger;
-    responseSchema?: {
-        name: string;
-        schema: z.ZodObject;
-    };
-}
-/**
- * Build a campaign-shaped `JudgeConfig` whose `score()` makes ONE LLM call
- * against `prompt` and reduces the model's per-dimension scores to a canonical
- * `JudgeScore` in `[0,1]`.
- *
- * The model is instructed to return JSON `{ "dimensions": { <key>: <number>, … },
- * "notes": "…" }`; the helper strips fenced JSON, validates every declared
- * dimension is present and in range, normalizes by `scale`, and composites via
- * `weightedComposite`.
- */
-declare function llmJudge<TArtifact = unknown, TScenario extends Scenario = Scenario>(name: string, prompt: string, opts: LlmJudgeOptions<TArtifact, TScenario>): JudgeConfig<TArtifact, TScenario>;
-
-/**
- * Produced-state extraction — normalize a run's runtime event stream into the
- * typed `ProducedState` the completion oracle consumes.
- *
- * `ProducedState` answers "what did the agent actually produce" — vault
- * artifacts, proposals, tool calls. The runtime emits these as a stream of
- * events; this module is the single normalization point from that stream to
- * the shape `verifyCompletion` expects.
- *
- * Input is structurally typed (`RuntimeEventLike`) so this module does not
- * depend on agent-runtime — agent-runtime's `RuntimeStreamEvent` satisfies it
- * structurally. The `content` on `ArtifactEventLike` and the whole
- * `proposal_created` variant are the runtime-side enrichments this contract
- * requires; the runtime emits them, this module consumes them.
- */
-
-/** A tool the agent invoked. */
-interface ToolCallEventLike {
-    type: 'tool_call';
-    toolName: string;
-}
-/**
- * An artifact the agent produced. `content` is the enriched field — the
- * runtime's base `artifact` event carries only metadata; the completion
- * oracle needs the body to verify the deliverable, so the runtime emits it.
- */
-interface ArtifactEventLike {
-    type: 'artifact';
-    artifactId: string;
-    name?: string;
-    mimeType?: string;
-    uri?: string;
-    content?: string;
-}
-/** A proposal / filing the agent created. */
-interface ProposalEventLike {
-    type: 'proposal_created';
-    proposalId: string;
-    title: string;
-    status?: 'pending' | 'approved' | 'rejected';
-    content?: string;
-}
-/**
- * The subset of runtime stream events `extractProducedState` consumes.
- * agent-runtime's full `RuntimeStreamEvent` union satisfies this structurally;
- * the `{ type: string }` catch-all keeps the input permissive so callers can
- * pass the whole unfiltered telemetry stream — unrecognized events are skipped.
- */
-type RuntimeEventLike = ToolCallEventLike | ArtifactEventLike | ProposalEventLike | {
-    type: string;
-};
-/**
- * Normalize a run's runtime event stream into `ProducedState`.
- *
- * Pure and total — unrecognized event types are skipped. `toolCalls` is
- * deduplicated by name in first-seen order (completion cares about a tool's
- * presence, not its call count). An artifact with neither a name nor a uri
- * still yields an entry keyed by its `artifactId` so it is never silently
- * dropped; an artifact with no `content` yields empty content, which the
- * completion oracle's structural check then rejects on its own.
- */
-declare function extractProducedState(events: readonly RuntimeEventLike[]): ProducedState;
-
-/**
- * Pre-registered hypotheses — declare what you're testing BEFORE the
- * run, check it AFTER. Prevents p-hacking, optional stopping, and the
- * "we ran until it looked good" failure mode.
- *
- * Manifest is a plain JSON-friendly object. Sign it with a content hash
- * + timestamp; the registered record becomes immutable. Post-run,
- * evaluate the manifest against observed results — the library refuses
- * to let you re-interpret a different metric as the declared one.
- */
-interface HypothesisManifest {
-    id: string;
-    /** Human prose — goes into the audit trail. */
-    hypothesis: string;
-    /** Metric the hypothesis claims to move. */
-    metric: string;
-    /** 'increase' = candidate should score higher than baseline; 'decrease' = lower. */
-    direction: 'increase' | 'decrease';
-    /** Minimum effect size to count (same units as the metric). */
-    minEffect: number;
-    /** Alpha threshold. */
-    alpha: number;
-    /** Target statistical power at which sample size was pre-computed. */
-    power: number;
-    /** Declared N per arm before running. */
-    preRegisteredN: number;
-    /** ISO8601 timestamp the manifest was registered. */
-    registeredAt: string;
-    /** Optional identifiers to tie into the trace corpus. */
-    baselineLabel?: string;
-    candidateLabel?: string;
-}
-/**
- * Identifier for the hashing scheme used to produce `contentHash`.
- *
- * `'sha256-content'` — sha256 hex over the canonicalized manifest with
- * the `contentHash` and `algo` fields stripped. Held as a string union
- * so future schemes can be added without breaking parsers; SignedManifest
- * values without `algo` deserialize cleanly because the field is optional.
- */
-type SignedManifestAlgo = 'sha256-content';
-interface SignedManifest extends HypothesisManifest {
-    /** sha256 hex of canonicalized manifest (everything except contentHash and algo). */
-    contentHash: string;
-    /**
-     * Algorithm string describing how `contentHash` was produced.
-     *
-     * Optional on the type so serialized manifests without it still parse,
-     * but ALWAYS populated by {@link signManifest}. Consumers that want to
-     * enforce a known algorithm should reject manifests where this field
-     * is missing or unrecognized.
-     */
-    algo?: SignedManifestAlgo;
-}
-interface HypothesisResult {
-    manifest: SignedManifest;
-    observedN: number;
-    observedEffect: number;
-    observedPValue: number;
-    /** True iff the observed effect hits the pre-declared direction with
-     *  magnitude ≥ minEffect AND p < alpha. */
-    confirmed: boolean;
-    /** Enumerated reasons the hypothesis was rejected (each a machine-tag). */
-    rejectionReasons: Array<'wrong_direction' | 'effect_too_small' | 'not_significant' | 'undersampled'>;
-    notes?: string;
-}
-/**
- * Deterministic JSON canonicalization — sort object keys recursively.
- *
- * Two semantically-equal objects produce byte-identical canonicalized output;
- * this is what makes a content-hash stable across encoders, key insertion
- * orders, and runtime versions. Exported for any consumer that needs the same
- * canonicalization guarantee outside the manifest-signing path (e.g., signing
- * an artifact bundle, hashing a dataset version, etc.).
- */
-declare function canonicalize(v: unknown): unknown;
-/**
- * SHA-256 hex (full 64 chars) over the canonicalized JSON encoding of `obj`.
- *
- * The same primitive `signManifest` and `verifyManifest` are built on, exposed
- * directly so consumers signing arbitrary structured content (artifact bundles,
- * production packets, dataset manifests, etc.) don't have to re-derive
- * canonicalize+sha256 from scratch.
- *
- * Stable across:
- *   - object key insertion order (canonicalization sorts keys recursively)
- *   - encoder choice (UTF-8 via TextEncoder, fixed)
- *   - runtime (uses the Web Crypto subtle digest, present in Node ≥18 and browsers)
- *
- * Named `hashJson` to disambiguate from `prompt-registry.ts`'s `hashContent`,
- * which takes a string input and returns a truncated 12-char prompt id.
- * Use `hashJson` when you mean "canonicalize then hash."
- *
- * @example
- *   const hash = await hashJson({ id: '1', kind: 'spec' })
- *   // 'a3f1...' (64 hex chars)
- */
-declare function hashJson<T>(obj: T): Promise<string>;
-/**
- * Sign a manifest with a SHA-256 content hash.
- *
- * The hash covers the canonicalized manifest with the `contentHash`
- * and `algo` fields stripped; this lets verifiers re-sign the rest and
- * compare. Returned manifest always carries `algo: 'sha256-content'`
- * so downstream consumers can identify the scheme; manifests without
- * `algo` still verify because it is stripped before hashing on both sides.
- */
-declare function signManifest(m: HypothesisManifest): Promise<SignedManifest>;
-/**
- * Verify that a signed manifest has not been tampered with.
- *
- * Strips `contentHash` and `algo` before re-signing so manifests without
- * `algo` verify identically to ones that carry it.
- */
-declare function verifyManifest(m: SignedManifest): Promise<boolean>;
-/**
- * Evaluate a pre-registered hypothesis against observed results.
- * Mechanical — no re-interpretation permitted.
- */
-declare function evaluateHypothesis(manifest: SignedManifest, observed: {
-    n: number;
-    effect: number;
-    pValue: number;
-}): Promise<HypothesisResult>;
-
-export { verifyCompletion as $, type Artifact as A, type BackendIntegrityReport as B, type CompletionRequirement as C, canonicalize as D, comparePairedArms as E, completionVerdict as F, composeValidators as G, type HypothesisManifest as H, containsAll as I, createLlmCorrectnessChecker as J, createTokenRecallChecker as K, type LlmJudgeDimension as L, type MatchedPair as M, evaluateHypothesis as N, extractProducedState as O, type PairedArmsComparison as P, hashJson as Q, type RuntimeEventLike as R, type SignedManifest as S, type TaskGold as T, jsonHasKeys as U, type ValidationContext as V, pairArms as W, parseCorrectnessResponse as X, regexMatch as Y, signManifest as Z, summarizeBackendIntegrity as _, type CompletionVerdict as a, verifyManifest as a0, type ProducedState as b, type CorrectnessChecker as c, type LlmJudgeOptions as d, type ArtifactEventLike as e, type ArtifactValidator as f, BackendIntegrityError as g, type ComparePairedArmsOptions as h, type HypothesisResult as i, type LlmCorrectnessCheckerOpts as j, type PairArmsOptions as k, llmJudge as l, type PairArmsResult as m, type PairedArmRow as n, type PairedCorrectness as o, type PairedMetricDelta as p, type ProducedProposal as q, type ProposalEventLike as r, type RequirementCheck as s, type SatisfiedBy as t, type SignedManifestAlgo as u, type ToolCallEventLike as v, type ValidationIssue as w, type ValidationResult as x, assertRealBackend as y, byteLengthRange as z };