@tangle-network/agent-eval 0.120.2 → 0.120.3

package/dist/meta-eval/index.d.ts

@@ -0,0 +1,1030 @@
+type RunStatus = 'running' | 'completed' | 'failed' | 'aborted';
+interface BudgetSpec {
+    tokens?: number;
+    wallMs?: number;
+    calls?: number;
+    usd?: number;
+}
+interface RunOutcome$1 {
+    score?: number;
+    pass?: boolean;
+    failureClass?: FailureClass;
+    notes?: string;
+}
+/**
+ * Layer — optional classification in a nested build workflow.
+ * `builder`: the meta-agent editing a project (e.g. agent-builder Forge chat).
+ * `app-build`: sandbox harness that compiled + tested the generated scaffold.
+ * `app-runtime`: a run of the generated agent against a domain scenario.
+ * `meta`: any meta-eval (judge replay, correlation analysis).
+ */
+type RunLayer = 'builder' | 'app-build' | 'app-runtime' | 'meta' | 'custom';
+interface Run {
+    runId: string;
+    /**
+     * Stable identifier of the scenario being executed.
+     *
+     * Always populated on the persisted Run — but `TraceEmitter.startRun` accepts
+     * input WITHOUT this field, substituting a sensible default
+     * (`run.layer ?? run.tags?.['kind'] ?? 'runtime'`) when the caller has no
+     * curated scenario to anchor to (runtime / operator / meta-eval runs). This
+     * keeps the persisted shape unambiguous for downstream filters + aggregations
+     * while removing the boilerplate of inventing placeholder ids at the call site.
+     */
+    scenarioId: string;
+    variantId?: string;
+    datasetVersion?: string;
+    /** Git SHA of agent code at run time. */
+    codeSha?: string;
+    /** Hash of the prompt template + any system prompt. */
+    promptSha?: string;
+    /** Model id + date + system-prompt hash, concatenated. */
+    modelFingerprint?: string;
+    seed?: number;
+    /** Arbitrary environment markers (shell, docker version, tz). */
+    envFingerprint?: Record<string, string>;
+    /** Version of the redaction rules applied to this run. */
+    redactionVersion?: string;
+    /** Parent run in a nested build workflow. A builder run's children are
+     *  app-build runs; those children are app-runtime runs. */
+    parentRunId?: string;
+    /** Stable project identifier — groups runs across chats + sessions. */
+    projectId?: string;
+    /** Chat/conversation identifier within a project. */
+    chatId?: string;
+    /** Layer classification — hint for aggregation; not enforced. */
+    layer?: RunLayer;
+    startedAt: number;
+    endedAt?: number;
+    status: RunStatus;
+    outcome?: RunOutcome$1;
+    budget?: BudgetSpec;
+    /** Free-form labels for downstream grouping. */
+    tags?: Record<string, string>;
+}
+type SpanKind = 'agent' | 'llm' | 'tool' | 'retrieval' | 'judge' | 'sandbox' | 'custom';
+type SpanStatus = 'ok' | 'error';
+interface SpanBase {
+    spanId: string;
+    parentSpanId?: string;
+    runId: string;
+    kind: SpanKind;
+    name: string;
+    startedAt: number;
+    endedAt?: number;
+    status?: SpanStatus;
+    error?: string;
+    /** Anything not covered by typed fields. Kept deliberately free-form. */
+    attributes?: Record<string, unknown>;
+}
+interface Message {
+    role: 'system' | 'user' | 'assistant' | 'tool';
+    content: string;
+    tokens?: number;
+    /** Multi-modal content descriptors; blobs themselves live in Artifacts. */
+    images?: Array<{
+        artifactId?: string;
+        url?: string;
+        mime?: string;
+    }>;
+}
+interface LlmSpan extends SpanBase {
+    kind: 'llm';
+    model: string;
+    messages: Message[];
+    output?: string;
+    inputTokens?: number;
+    /** All generated tokens, including the reasoning subset when present. */
+    outputTokens?: number;
+    cachedTokens?: number;
+    cacheWriteTokens?: number;
+    /** Reasoning-token subset of `outputTokens`. */
+    reasoningTokens?: number;
+    costUsd?: number;
+    finishReason?: string;
+}
+interface ToolSpan extends SpanBase {
+    kind: 'tool';
+    toolName: string;
+    args: unknown;
+    /** False when the source observed the call but did not capture its arguments. */
+    argsCaptured?: boolean;
+    result?: unknown;
+    latencyMs?: number;
+}
+interface RetrievalSpan extends SpanBase {
+    kind: 'retrieval';
+    query: string;
+    hits: Array<{
+        docId: string;
+        score: number;
+        content?: string;
+    }>;
+}
+interface JudgeSpan extends SpanBase {
+    kind: 'judge';
+    judgeId: string;
+    /** Span this judgment applies to. */
+    targetSpanId: string;
+    dimension: string;
+    /** Numeric score (free-range; interpretation up to the judge). */
+    score: number;
+    rationale?: string;
+    evidence?: string;
+}
+interface SandboxSpan extends SpanBase {
+    kind: 'sandbox';
+    image?: string;
+    command?: string;
+    exitCode?: number;
+    testsTotal?: number;
+    testsPassed?: number;
+    stdoutHash?: string;
+    stderrHash?: string;
+    /** Duration in ms; the harness fills this explicitly (endedAt - startedAt may miss setup). */
+    wallMs?: number;
+}
+interface GenericSpan extends SpanBase {
+    kind: 'agent' | 'custom';
+}
+type Span = LlmSpan | ToolSpan | RetrievalSpan | JudgeSpan | SandboxSpan | GenericSpan;
+type EventKind = 'log' | 'error' | 'budget_decrement' | 'budget_breach' | 'state_mutation' | 'policy_violation' | 'redaction_applied' | 'custom';
+interface TraceEvent {
+    eventId: string;
+    runId: string;
+    spanId?: string;
+    kind: EventKind;
+    timestamp: number;
+    payload: Record<string, unknown>;
+}
+interface BudgetLedgerEntry {
+    runId: string;
+    dimension: keyof BudgetSpec;
+    limit: number;
+    consumed: number;
+    remaining: number;
+    timestamp: number;
+    breached: boolean;
+    /** Span that triggered this entry, if any. */
+    spanId?: string;
+}
+interface Artifact {
+    artifactId: string;
+    runId: string;
+    spanId?: string;
+    contentType: string;
+    sizeBytes: number;
+    /** sha256 in hex. */
+    hash: string;
+    /** External storage URL (R2, S3, filesystem path). */
+    storageUrl?: string;
+    /** Inline content for small blobs — keep under ~64KB. */
+    inlineContent?: string;
+}
+type FailureClass = 'success' | 'reasoning_error' | 'tool_selection_error' | 'tool_argument_error' | 'tool_recovery_failure' | 'hallucination' | 'instruction_following' | 'safety_refusal_miss' | 'policy_violation' | 'budget_exceeded' | 'format_drift' | 'permission_escalation' | 'pii_leak' | 'cost_overrun' | 'timeout' | 'sandbox_failure' | 'missing_user_data' | 'missing_domain_data' | 'missing_codebase_context' | 'missing_runtime_context' | 'missing_credentials' | 'missing_integration_connection' | 'missing_integration_scope' | 'integration_approval_required' | 'integration_auth_expired' | 'integration_provider_failure' | 'bad_integration_manifest' | 'unsafe_integration_write_denied' | 'stale_external_data' | 'bad_retrieval' | 'insufficient_evidence' | 'contradictory_evidence' | 'ambiguous_user_intent' | 'knowledge_readiness_blocked' | 'unknown';
+
+interface RunFilter {
+    scenarioId?: string;
+    variantId?: string;
+    status?: RunStatus;
+    since?: number;
+    until?: number;
+    tag?: {
+        key: string;
+        value: string;
+    };
+    parentRunId?: string;
+    projectId?: string;
+    chatId?: string;
+    layer?: RunLayer;
+}
+interface SpanFilter {
+    runId?: string;
+    parentSpanId?: string;
+    kind?: SpanKind;
+    name?: string;
+    toolName?: string;
+    judgeId?: string;
+    since?: number;
+    until?: number;
+}
+interface EventFilter {
+    runId?: string;
+    spanId?: string;
+    kind?: EventKind;
+    since?: number;
+    until?: number;
+}
+interface TraceStore {
+    appendRun(run: Run): Promise<void>;
+    updateRun(runId: string, patch: Partial<Run>): Promise<void>;
+    appendSpan(span: Span): Promise<void>;
+    updateSpan(spanId: string, patch: Partial<Span>): Promise<void>;
+    appendEvent(event: TraceEvent): Promise<void>;
+    appendArtifact(artifact: Artifact): Promise<void>;
+    appendBudgetEntry(entry: BudgetLedgerEntry): Promise<void>;
+    getRun(runId: string): Promise<Run | undefined>;
+    listRuns(filter?: RunFilter): Promise<Run[]>;
+    spans(filter?: SpanFilter): Promise<Span[]>;
+    events(filter?: EventFilter): Promise<TraceEvent[]>;
+    budget(runId: string): Promise<BudgetLedgerEntry[]>;
+    artifacts(runId: string): Promise<Artifact[]>;
+}
+
+/**
+ * OutcomeStore — deployment outcomes attached to Run IDs.
+ *
+ * Outcomes arrive asynchronously from production telemetry after the
+ * eval run completed: user ratings, retention flags, conversion events,
+ * revenue, support-ticket rate, anything a product team can measure.
+ * The store is a peer to TraceStore — separate lifecycle, same runId
+ * foreign key.
+ *
+ * The whole point of this module is to make the meta-eval correlation
+ * question computable: `correlate(evalMetric, outcomeMetric) → r, ρ, n, CI`.
+ */
+interface DeploymentOutcome {
+    runId: string;
+    capturedAt: number;
+    /** Numeric outcomes keyed by name — retention_7d, csat, revenue_usd, etc. */
+    metrics: Record<string, number>;
+    /** Dimensions for stratified analysis — cohort, region, user_segment. */
+    labels?: Record<string, string>;
+    /** Free-form provenance (source system, pipeline version). */
+    source?: string;
+}
+interface OutcomeFilter {
+    runIds?: string[];
+    since?: number;
+    until?: number;
+    label?: {
+        key: string;
+        value: string;
+    };
+    source?: string;
+}
+interface OutcomeStore {
+    append(outcome: DeploymentOutcome): Promise<void>;
+    /** All outcomes attached to this run (a single run can have many — multiple
+     *  capture windows over deployment time). */
+    forRun(runId: string): Promise<DeploymentOutcome[]>;
+    list(filter?: OutcomeFilter): Promise<DeploymentOutcome[]>;
+}
+declare class InMemoryOutcomeStore implements OutcomeStore {
+    private items;
+    append(outcome: DeploymentOutcome): Promise<void>;
+    forRun(runId: string): Promise<DeploymentOutcome[]>;
+    list(filter?: OutcomeFilter): Promise<DeploymentOutcome[]>;
+}
+interface FileSystemOutcomeStoreOptions {
+    dir: string;
+    maxBytes?: number;
+}
+declare class FileSystemOutcomeStore implements OutcomeStore {
+    private dir;
+    private maxBytes;
+    private memo?;
+    private loaded;
+    constructor(options: FileSystemOutcomeStoreOptions);
+    private ensureDir;
+    append(outcome: DeploymentOutcome): Promise<void>;
+    private load;
+    forRun(runId: string): Promise<DeploymentOutcome[]>;
+    list(filter?: OutcomeFilter): Promise<DeploymentOutcome[]>;
+}
+
+/**
+ * Correlation study — "does our eval score predict real-world outcomes?"
+ *
+ * This is the load-bearing signal. Takes a TraceStore + OutcomeStore,
+ * joins on runId, computes Pearson + Spearman + bootstrap CI for every
+ * (evalMetric, outcomeMetric) pair the caller declares.
+ *
+ * Without this number the framework is ornamental. With it and r > 0.6
+ * the framework is a moat — no other agent-eval tool publishes one.
+ */
+
+interface EvalMetricSpec {
+    id: string;
+    /** Extract a scalar from a run (defaults cover score/pass/durationMs/costUsd/tokens). */
+    extract?: (run: Run, store: TraceStore) => Promise<number | null>;
+}
+interface OutcomePair {
+    evalMetric: string;
+    outcomeMetric: string;
+}
+interface CorrelationResult {
+    evalMetric: string;
+    outcomeMetric: string;
+    n: number;
+    pearson: number;
+    spearman: number;
+    /** 95% bootstrap CI for Pearson. */
+    pearsonCi95: {
+        lower: number;
+        upper: number;
+    };
+    /** Rough verdict: 'strong' ≥ 0.7, 'moderate' ≥ 0.4, else 'weak'. */
+    verdict: 'strong' | 'moderate' | 'weak';
+}
+interface CorrelationStudyResult {
+    pairs: CorrelationResult[];
+    joinedSamples: number;
+    skippedRuns: number;
+}
+interface CorrelationStudyOptions {
+    /** Only join outcomes captured within this window after run.startedAt. */
+    maxCaptureLagMs?: number;
+    /** Restrict to a subset of outcomes (cohort, region, source). */
+    outcomeFilter?: OutcomeFilter;
+    /** Which outcome per run to use when multiple exist. Default 'latest'. */
+    reduction?: 'latest' | 'mean' | 'max';
+    /** Bootstrap iterations for the CI. Default 500. */
+    bootstrapIterations?: number;
+}
+declare function correlationStudy(traceStore: TraceStore, outcomeStore: OutcomeStore, evalMetrics: EvalMetricSpec[], outcomeMetricNames: string[], options?: CorrelationStudyOptions): Promise<CorrelationStudyResult>;
+
+/**
+ * Calibration curve — binned "if eval says X, what does reality show?"
+ *
+ * Companion to correlationStudy. Raw correlation is a single number;
+ * the calibration curve shows *where* the eval is well-calibrated vs
+ * overconfident / underconfident. Buckets the eval metric, computes
+ * mean outcome per bucket, reports expected-calibration-error (ECE).
+ */
+
+interface CalibrationBin {
+    lower: number;
+    upper: number;
+    n: number;
+    evalMean: number;
+    outcomeMean: number;
+    /** |outcomeMean − evalMean|; contributes to ECE weighted by n/total. */
+    gap: number;
+}
+interface CalibrationReport {
+    evalMetric: string;
+    outcomeMetric: string;
+    n: number;
+    bins: CalibrationBin[];
+    /** Expected Calibration Error — Σ (n_i/N) × |outcomeMean_i − evalMean_i|. */
+    ece: number;
+    /** Max bin gap — upper bound on miscalibration. */
+    maxGap: number;
+}
+interface CalibrationOptions {
+    bins?: number;
+    /** Equal-width (fixed bin edges) or equal-frequency (quantile bins). */
+    binning?: 'equal-width' | 'equal-frequency';
+    /** Clip eval values to [lo, hi] before binning. */
+    range?: {
+        lo: number;
+        hi: number;
+    };
+}
+interface CalibrationPair {
+    evalScore: number;
+    outcome: number;
+}
+declare function calibrationCurve(traceStore: TraceStore, outcomeStore: OutcomeStore, evalMetric: EvalMetricSpec, outcomeMetric: string, options?: CalibrationOptions): Promise<CalibrationReport | null>;
+declare function calibrationFromPairs(inputPairs: CalibrationPair[], evalMetric: string, outcomeMetric: string, options?: CalibrationOptions): CalibrationReport | null;
+
+type AgentProfileCellSchemaVersion = 'agent-profile-cell/v1';
+type AgentProfileDimensionValue = string | number | boolean | null;
+interface AgentProfileSource {
+    /** Runtime/profile contract being fingerprinted, e.g. `agent-interface-profile`. */
+    kind: string;
+    /** sha256 over the canonical source profile object. */
+    hash: string;
+}
+interface AgentProfileHarness {
+    id: string;
+    version?: string;
+    hash?: string;
+}
+interface AgentProfileCell {
+    schemaVersion: AgentProfileCellSchemaVersion;
+    cellId: string;
+    profileId: string;
+    sourceProfile: AgentProfileSource;
+    harness?: AgentProfileHarness;
+    model?: string;
+    promptHash?: string;
+    dimensions?: Record<string, AgentProfileDimensionValue>;
+}
+
+/**
+ * Paper-grade RunRecord schema + runtime validator.
+ *
+ * Every run that participates in a promotion gate, paper table, or
+ * researcher loop SHOULD be recorded as a `RunRecord`. The mandatory
+ * fields are exactly those the paper "Two Loops, Three Roles" requires
+ * for reproducibility: who/what/when/cost/seed/hash, plus the search vs
+ * holdout split tag and either a `searchScore` or a `holdoutScore`.
+ *
+ * This is intentionally NOT a replacement for the rich `Run` /
+ * `ProposeReviewReport` / `ScenarioResult` types already in the
+ * package. Those are runtime structures with full provenance. A
+ * `RunRecord` is the analysis-time projection — the JSON-friendly
+ * row you'd put in a parquet file or paste into a notebook.
+ *
+ * Validate at the boundary:
+ *
+ *   const rec = validateRunRecord(rawJson)         // throws on missing
+ *   const ok  = isRunRecord(rawJson)               // boolean check
+ *   const rec = parseRunRecordSafe(rawJson)        // { ok, value | error }
+ *
+ * The validator runs in pure TS — zod is intentionally NOT a
+ * dependency. Round-trip tested in `tests/run-record.test.ts`.
+ */
+
+/** Search/dev/holdout split tag. 'search' is the paper-grade alias for the
+ *  combined train+test pool that the optimizer is allowed to read. */
+type RunSplitTag = 'search' | 'dev' | 'holdout';
+interface RunTokenUsage {
+    input: number;
+    /** All generated tokens charged as output, including reasoning tokens. */
+    output: number;
+    /** Reasoning-token subset of `output`, when the provider reports it. */
+    reasoning?: number;
+    /** Prompt tokens served from a provider cache. */
+    cached?: number;
+    /** Prompt tokens written into a provider cache. */
+    cacheWrite?: number;
+}
+/**
+ * How a run's USD amount was obtained.
+ *
+ * `costUsd` remains mandatory for wire compatibility. New producers should
+ * always populate this discriminated union so a missing bill is never
+ * mistaken for an observed zero-dollar run. For `uncaptured`, `costUsd` uses
+ * the legacy `0` sentinel while this field carries the truthful null.
+ */
+type RunCostProvenance = {
+    kind: 'observed';
+    usd: number;
+} | {
+    kind: 'estimated';
+    usd: number;
+} | {
+    kind: 'uncaptured';
+    usd: null;
+};
+interface RunJudgeMetadata {
+    model: string;
+    promptVersion: string;
+    /** [0,1] confidence the judge declared. Constant judge confidence
+     *  across many runs is a fallback signal (see `canary.ts`). */
+    confidence: number;
+    /** True if the judge degraded to a fallback path (rules-only,
+     *  prior-call cache, etc.). The canary uses this to alert. */
+    fallback: boolean;
+}
+/**
+ * Per-judge / per-dimension breakdown for runs scored by an ensemble of
+ * judges over a multi-dimensional rubric.
+ *
+ * The collapsed `outcome.searchScore` / `holdoutScore` carries the
+ * composite the gate uses. The full breakdown belongs here so consumers
+ * can answer "which judge disagreed?", "which dimension dragged the
+ * composite down?", and "did half the panel fail?" without re-running.
+ *
+ * `perJudge[judgeId][dim]` is the canonical source; `perDimMean` and
+ * `composite` are convenience projections — derivable but precomputed so
+ * downstream IRR primitives (`interRaterReliability`,
+ * `corpusInterRaterAgreement`) and reporters don't pay the same
+ * aggregation twice.
+ *
+ * Fail-loud discipline: judges that errored out land in `failedJudges`
+ * by id. A missing key in `perJudge` is ambiguous (silent zero vs not
+ * run); the explicit list makes a partial-failure recorded as such.
+ */
+interface JudgeScoresRecord {
+    /** Per-judge per-dimension scores. `{ "kimi-k2.6": { helpfulness: 0.8, clarity: 0.7 }, ... }`. */
+    perJudge: Record<string, Record<string, number>>;
+    /** Per-dim mean across judges. Convenience — derivable from `perJudge`. */
+    perDimMean: Record<string, number>;
+    /** Composite mean across all dims and judges. Mirrors the score
+     *  the gate sees on `outcome.searchScore` / `holdoutScore`. */
+    composite: number;
+    /** Judges that errored or returned an unparseable verdict. Recorded
+     *  by id (e.g. `['glm-5.1']`) so a partial-failure case is explicit,
+     *  not inferred from missing keys in `perJudge`. */
+    failedJudges?: string[];
+    /** Free-form notes the judges emitted (joined across judges or
+     *  first-judge only — consumer's choice). */
+    notes?: string;
+}
+interface RunOutcome {
+    /** Score on the search/optimization split. Optional because a
+     *  holdout-only evaluation only fills `holdoutScore`. */
+    searchScore?: number;
+    /** Score on the held-out split. Optional because a search-only run
+     *  only fills `searchScore`. At least one must be present. */
+    holdoutScore?: number;
+    /** Bag of any other metric the run produced — judge dimensions,
+     *  pass/fail counters, latency stats, etc. Numeric only — keeps
+     *  reporters honest. */
+    raw: Record<string, number>;
+    /** Per-judge / per-dim breakdown. Consumers writing ensemble
+     *  judgements populate this; substrate primitives like
+     *  `interRaterReliability` and `corpusInterRaterAgreement` accept
+     *  these records as input. Optional — single-judge or scalar-only
+     *  runs leave it unset. */
+    judgeScores?: JudgeScoresRecord;
+    /** Authenticity / realness verdict — did the run build the REAL thing on the
+     *  intended infra, or fake it (see `./authenticity`)? Optional: only domains
+     *  with an authenticity config populate it. Carried in the corpus so the
+     *  flywheel / off-policy learning can optimize for real completion, not gamed
+     *  pass-rate. `score` is 0-1; `gated` is the anti-Goodhart flag — a gated run
+     *  must not count as a real success regardless of `score`. */
+    realness?: {
+        score: number;
+        gated: boolean;
+        reason?: string;
+    };
+}
+/**
+ * Mandatory paper-grade fields for a single evaluation run. Optional
+ * fields are extension points; mandatory fields throw if missing.
+ *
+ * Hash discipline:
+ *   - `promptHash` is the sha256 of the EFFECTIVE prompt sent to the
+ *     model (after any steering bundle merge).
+ *   - `configHash` is the sha256 of the effective run config (model,
+ *     temperature, tools, judges, splits). The pair (promptHash,
+ *     configHash) uniquely identifies an experiment cell.
+ *
+ * Model snapshot discipline:
+ *   - `model` MUST encode a snapshot version. Bare aliases like
+ *     `claude-sonnet-4` or `gpt-4o` are banned — they remap silently.
+ *     Use `claude-sonnet-4-6@2025-04-15` or `gpt-4o-2024-11-20`.
+ */
+interface RunRecord {
+    /** UUID for the run. */
+    runId: string;
+    /** Logical experiment grouping (a treatment vs a baseline within
+     *  the same sweep should share `experimentId`). */
+    experimentId: string;
+    /** Stable identifier for the candidate (variant) being run. The
+     *  promotion gate compares two `candidateId`s on matched items. */
+    candidateId: string;
+    /** RNG seed for the run. Always recorded — silent re-seeding is
+     *  the most common cause of non-reproducible numbers. */
+    seed: number;
+    /** Model identifier WITH snapshot version. */
+    model: string;
+    /** sha256 of the effective prompt (post-steering). */
+    promptHash: string;
+    /** sha256 of the effective config. */
+    configHash: string;
+    /** Git SHA the harness was run from. */
+    commitSha: string;
+    /** End-to-end wall-clock duration in milliseconds. */
+    wallMs: number;
+    /** Time spent queued before execution started, if known. */
+    queueMs?: number;
+    /** Total USD cost. Mandatory — runs without a cost number are
+     *  unbounded by definition and must not be admitted into the gate.
+     *  `0` is retained as the compatibility sentinel for an uncaptured amount;
+     *  inspect `costProvenance` before treating it as observed. */
+    costUsd: number;
+    /** Observed, model-priced estimate, or genuinely uncaptured USD amount.
+     *  Optional only so existing serialized RunRecords remain valid. */
+    costProvenance?: RunCostProvenance;
+    /** Token usage breakdown. */
+    tokenUsage: RunTokenUsage;
+    /** Judge-side metadata, if a judge was used. */
+    judgeMetadata?: RunJudgeMetadata;
+    /** Per-split scores + raw bag. */
+    outcome: RunOutcome;
+    /** Canonical, cross-agent failure class drawn from the shared
+     *  `FAILURE_CLASSES` taxonomy. This is the aggregation key that makes
+     *  "which failure dominates across the whole fleet" answerable in ONE
+     *  vocabulary — every agent classifies against the same enum. Producers
+     *  set it via the substrate classifier; leave unset only when the failure
+     *  genuinely can't be classified. */
+    failureClass?: FailureClass;
+    /** Free-form domain-specific failure detail, scoped UNDER `failureClass`
+     *  (e.g. failureClass='tool_recovery_failure', failureMode='forge_build_unsatisfied').
+     *  The within-agent drill-down; `failureClass` is the cross-agent key. */
+    failureMode?: string;
+    /** Which split this run was drawn from. */
+    splitTag: RunSplitTag;
+    /**
+     * Stable scenario identifier the run was scored against. Optional for
+     * backwards compatibility, but **strongly recommended**: every primitive
+     * that pairs runs by scenario (preferences, paired stats, BT tournament)
+     * keys on this. The campaign artifact populates it canonically; legacy
+     * runs without it fall back to inference from `outcome.raw.scenario_id`
+     * or `experimentId`.
+     */
+    scenarioId?: string;
+    /**
+     * Canonical identity for the agent profile cell that produced this row:
+     * profile artifact hash plus optional harness/model/prompt/reporting
+     * dimensions. Use `agentProfile.cellId` to group persona sweeps and
+     * longitudinal reports by the complete source profile, not by a loose
+     * candidate label or opaque config hash.
+     */
+    agentProfile?: AgentProfileCell;
+}
+
+/**
+ * Rubric predictive validity — does our eval rubric predict deployment
+ * outcomes?
+ *
+ * `correlationStudy` (already in this package) joins a `TraceStore` to an
+ * `OutcomeStore` and computes Pearson + Spearman + bootstrap CI for each
+ * (eval-metric, outcome-metric) pair. That answers "does X correlate with
+ * Y at all." `rubricPredictiveValidity` is the campaign-shaped wrapper
+ * around it: take a sequence of `RunRecord`s (the canonical campaign
+ * artifact) and a `DeploymentOutcomeStore`, join on `runId`, return a
+ * ranked verdict on every rubric whose dimension scores were captured in
+ * `outcome.raw`.
+ *
+ * The point — quoting the methodology doc — is that **without this loop
+ * every rubric is faith-based**. Once it's wired, you know which rubrics
+ * have earned their promotion power and which ones are decoration.
+ *
+ *   const validity = await rubricPredictiveValidity({
+ *     runs: lastQuarter,
+ *     outcomes: shipFlagOutcomeStore,
+ *     outcomeMetrics: ['revenue_lift', 'retention_30d', 'csat'],
+ *     rubrics: ['anti_slop', 'semantic_concept', 'tool_recovery'],
+ *   })
+ *   for (const r of validity.ranked) {
+ *     console.log(`${r.rubric} → ${r.bestOutcome}: ρ=${r.spearman.toFixed(2)}`)
+ *   }
+ *
+ * The function is intentionally read-only. Use the verdict to deprecate
+ * decorative rubrics, re-weight composite scores, or trigger a
+ * recalibration sweep when predictive validity drops below a threshold.
+ */
+
+interface RubricPredictiveValidityInput {
+    /**
+     * Canonical campaign output. Each record's `outcome.raw[<rubricId>]`
+     * provides the eval score; missing keys are silently skipped per pair.
+     */
+    runs: RunRecord[];
+    outcomes: OutcomeStore;
+    /**
+     * Outcome metric names to evaluate against. Each must appear in at
+     * least one `DeploymentOutcome.metrics` keyspace; pairs with too few
+     * joined samples are excluded from the result.
+     */
+    outcomeMetrics: string[];
+    /**
+     * Rubric ids to evaluate. Must appear as keys in `RunRecord.outcome.raw`.
+     * If omitted, every numeric key in `outcome.raw` across the run set is
+     * treated as a rubric.
+     */
+    rubrics?: string[];
+    /** Minimum joined-sample count before a pair is reported. Default 8. */
+    minSamples?: number;
+    /** Bootstrap resamples for CI. Default 500. */
+    bootstrapResamples?: number;
+    /** Random seed for the bootstrap (mulberry32). Default unset (Math.random). */
+    seed?: number;
+    /**
+     * Reduction when multiple outcomes attach to one runId. Default `'latest'`
+     * (most recently captured).
+     */
+    reduction?: 'latest' | 'mean' | 'max';
+}
+interface RubricOutcomePair {
+    rubric: string;
+    outcome: string;
+    n: number;
+    pearson: number;
+    spearman: number;
+    ci95: {
+        low: number;
+        high: number;
+    };
+    /**
+     * Verdict bucket. `load_bearing` ≥ 0.7, `informative` ≥ 0.4,
+     * `decorative` < 0.4 in absolute correlation. A negative correlation
+     * with a desired outcome is also `decorative` — actively misleading
+     * is worse than uninformative.
+     */
+    verdict: 'load_bearing' | 'informative' | 'decorative';
+}
+interface RubricRanking {
+    rubric: string;
+    /** Outcome metric this rubric correlated best with. */
+    bestOutcome: string;
+    spearman: number;
+    pearson: number;
+    n: number;
+    verdict: RubricOutcomePair['verdict'];
+}
+interface RubricPredictiveValidityReport {
+    pairs: RubricOutcomePair[];
+    /** Per-rubric best pair, sorted descending by |spearman|. */
+    ranked: RubricRanking[];
+    joinedSamples: number;
+    skippedRuns: number;
+    /** Rubrics that were declared but never produced a usable score. */
+    rubricsWithoutData: string[];
+}
+declare function rubricPredictiveValidity(input: RubricPredictiveValidityInput): Promise<RubricPredictiveValidityReport>;
+
+/**
+ * Judge calibration — measure judge quality against human gold + bias.
+ *
+ * Workflow:
+ *   1. Build a golden set: {itemId, humanScore}[].
+ *   2. Run candidate judges; each produces {itemId, score}.
+ *   3. `calibrateJudge(golden, candidate)` reports κ + Pearson + MAE.
+ *   4. `calibrateJudgeContinuous(golden, candidate)` adds quadratic-weighted
+ *      κ over the un-rounded [0,1] scores plus ICC(2,1), Pearson, Spearman,
+ *      and bootstrap CIs — use this for fine-grained judges where rounding
+ *      to int discards information (e.g. 0.78 vs 0.81 both round to 1 and
+ *      look "perfectly agreed" to integer κ).
+ *   5. Run bias probes (positional, verbosity, self-preference) to
+ *      detect systematic score inflation.
+ *   6. For N≥2 judges on the same items, `continuousAgreement(scores)`
+ *      reports ICC(2,1) + κ_w + Pearson + Spearman with bootstrap CIs.
+ *
+ * Returns actionable diagnostics, not a single number. Consumers then
+ * decide whether to trust the judge, retrain it, or add a tie-breaker.
+ */
+interface GoldenItem {
+    itemId: string;
+    humanScore: number;
+    /** Optional group used for per-group bias audits (e.g. model-of-output family). */
+    group?: string;
+}
+interface CandidateScore {
+    itemId: string;
+    score: number;
+    /** Optional — enables positional-bias analysis (did order matter?). */
+    positionOfAInput?: 'first' | 'second';
+}
+interface CalibrationResult {
+    n: number;
+    pearson: number;
+    /** Cohen's κ with quadratic weights over integer-rounded scores. */
+    kappa: number;
+    /** Mean absolute error vs human. */
+    mae: number;
+    /** Worst-5 miscalibrations (largest |judge - human|). */
+    worstItems: Array<{
+        itemId: string;
+        judge: number;
+        human: number;
+        delta: number;
+    }>;
+}
+interface ContinuousAgreement {
+    /** Cohen's κ_w with quadratic weights, computed on raw [0,1] scores. */
+    weightedKappa: number;
+    /** ICC(2,1): two-way random effects, absolute agreement, single rater. */
+    icc: number;
+    /** Pearson product-moment correlation (averaged over rater pairs if N>2). */
+    pearson: number;
+    /** Spearman rank correlation (averaged over rater pairs if N>2). */
+    spearman: number;
+    /** 95% bootstrap percentile CIs over items. */
+    ci: {
+        icc: [number, number];
+        weightedKappa: [number, number];
+    };
+    /** Number of complete items (no NaN across raters). */
+    n: number;
+    /** Number of raters. */
+    raters: number;
+}
+interface ContinuousCalibrationResult extends CalibrationResult {
+    /** Cohen's κ_w computed on raw (un-rounded) scores. */
+    weightedKappaContinuous: number;
+    /** ICC(2,1) treating golden + candidate as two raters. */
+    icc: number;
+    spearman: number;
+    ci: {
+        icc: [number, number];
+        weightedKappa: [number, number];
+    };
+}
+
+/**
+ * Series convergence — detects whether a sequence of scalar measurements
+ * is stabilizing, drifting, or noisy.
+ *
+ * Lifted from ADC convergence.ts. The per-turn `ConvergenceTracker` is
+ * about progress *within* a single run; this module is about drift
+ * *across* runs (e.g. "are my nightly eval scores stabilizing?").
+ *
+ * Three signals:
+ *   - stabilized: last K values have low variance (< epsilon) — done
+ *   - drifting:   recent trend is monotonic and beyond noise — regressing or improving
+ *   - noisy:      neither — keep iterating, but flag as untrustworthy for gating
+ */
+interface SeriesConvergenceOptions {
+    /** Window size for "recent" analysis (default 5). */
+    window?: number;
+    /** Coefficient-of-variation threshold below which the window is stabilized (default 0.05 = 5%). */
+    stableCv?: number;
+    /** Minimum monotone run length to call drift (default 3). */
+    driftRun?: number;
+}
+interface SeriesConvergenceResult {
+    state: 'stabilized' | 'drifting-up' | 'drifting-down' | 'noisy' | 'insufficient-data';
+    windowMean: number;
+    windowCv: number;
+    /** Longest monotonic run at the tail of the series (positive for up, negative for down). */
+    tailRun: number;
+    /** True when n ≥ window AND windowCv ≤ stableCv. */
+    stable: boolean;
+}
+
+interface CorpusAgreementPerDimension extends ContinuousAgreement {
+    dimension: string;
+    /** Item IDs that contributed to this dimension's matrix (every judge scored them). */
+    itemIds: string[];
+    /** Judge IDs that contributed to this dimension's matrix. */
+    judgeIds: string[];
+}
+interface CorpusAgreementReport {
+    /** Per-dimension ICC(2,1) + κ_w + Pearson + Spearman + bootstrap CIs. */
+    perDimension: CorpusAgreementPerDimension[];
+    /** Mean ICC across dimensions (NaN if no dimension yielded a finite ICC). */
+    overallIcc: number;
+    /** Mean weighted κ across dimensions (NaN if none finite). */
+    overallWeightedKappa: number;
+    /** Dimensions evaluated (sorted). */
+    dimensions: string[];
+    /** Judges seen across the corpus (sorted). */
+    judgeIds: string[];
+}
+
+/**
+ * Judge sentinel — eval trustworthiness as a continuously measured,
+ * alarmed trend.
+ *
+ * Judges are models; models change underneath us; calibration decays.
+ * This module composes three existing instruments into one loop:
+ *
+ *   snapshot — adapters turn real instrument outputs (`calibrateJudge` /
+ *              `calibrateJudgeContinuous`, `corpusInterRaterAgreement` /
+ *              `continuousAgreement`, a rerun sentinel golden set) into
+ *              `SentinelSnapshot`s
+ *   series   — `analyzeSeries` (src/series-convergence) classifies each
+ *              judge×metric history as stabilized / drifting / noisy
+ *   alarm    — `judgeSentinelReport` turns trends + thresholds into named
+ *              alarms; `evalHealthStamp` is the tiny object a campaign
+ *              attaches to its verdicts
+ *
+ * Alarm conditions:
+ *   - convergence state `drifting-down` on any tracked metric
+ *   - irr below `minIrr`
+ *   - calibrationKappa / sentinelPassRate dropped vs the series baseline
+ *     beyond `maxKappaDrop` / `maxSentinelDrop`
+ *   - judge model changed with no post-change golden-grounded snapshot
+ *     (the silent-upgrade trap: judges agreeing with each other after a
+ *     model swap proves nothing — only re-measuring against gold does)
+ *   - newest snapshot older than `staleAfterDays` relative to `asOf`
+ *
+ * Wire-in: run the snapshot adapters from a post-campaign hook or a
+ * nightly job, append to a `SentinelStore`, then compute
+ * `judgeSentinelReport(await store.history(), { asOf })` with `asOf` =
+ * the campaign timestamp. Attach `evalHealthStamp(report)` to campaign
+ * verdicts: an alarmed sentinel means every downstream verdict carries
+ * an integrity warning until the judge is recalibrated.
+ *
+ * The module is clock-free — every timestamp (`at`, `asOf`) is
+ * caller-supplied ISO-8601, so reports are reproducible.
+ */
+
+declare const SENTINEL_METRIC_NAMES: readonly ["irr", "calibrationKappa", "sentinelPassRate"];
+type SentinelMetricName = (typeof SENTINEL_METRIC_NAMES)[number];
+interface SentinelMetrics {
+    /** Inter-rater reliability — ICC(2,1) from agreement instruments. */
+    irr?: number;
+    /** Weighted κ vs the human golden set (`calibrateJudge*`). */
+    calibrationKappa?: number;
+    /** Fraction of a fixed sentinel golden set the judge still scores within tolerance. */
+    sentinelPassRate?: number;
+}
+interface SentinelSnapshot {
+    /** Caller-supplied ISO-8601 timestamp — the module never reads a clock. */
+    at: string;
+    judgeId: string;
+    /** Exact model identity behind the judge (pin the full version string). */
+    judgeModel: string;
+    /**
+     * Metrics measured at `at`. May be empty: an empty-metrics snapshot is a
+     * model-change marker — it records the new `judgeModel` immediately and
+     * the silent-upgrade alarm stays raised until a golden-grounded snapshot
+     * (calibrationKappa or sentinelPassRate) follows.
+     */
+    metrics: SentinelMetrics;
+}
+/** Identity + timestamp the caller supplies alongside an instrument output. */
+interface SnapshotMeta {
+    at: string;
+    judgeId: string;
+    judgeModel: string;
+}
+/** Shape guard for snapshots crossing a boundary (store append, JSONL load, report input). */
+declare function validateSentinelSnapshot(snapshot: SentinelSnapshot, source?: string): void;
+/**
+ * From `calibrateJudge` / `calibrateJudgeContinuous` output. Prefers the
+ * un-rounded continuous κ_w when the report carries one (integer κ
+ * discards information for [0,1] judges).
+ */
+declare function snapshotFromCalibration(report: CalibrationResult | ContinuousCalibrationResult, meta: SnapshotMeta): SentinelSnapshot;
+/**
+ * From `corpusInterRaterAgreement` (statistics.ts) or `continuousAgreement`
+ * (judge-calibration.ts) output. IRR = ICC(2,1) — the reliability
+ * coefficient both instruments compute. For ensemble-level agreement,
+ * `meta.judgeId` names the ensemble and `meta.judgeModel` pins its
+ * composition (e.g. a joined list of member model strings).
+ */
+declare function snapshotFromAgreement(report: CorpusAgreementReport | ContinuousAgreement, meta: SnapshotMeta): SentinelSnapshot;
+interface SentinelSetOptions {
+    /** Max |judge − human| for an item to count as passed. Default 0.1 (tuned for [0,1] scores). */
+    tolerance?: number;
+}
+/**
+ * From a rerun of a fixed sentinel golden set: the same `GoldenItem`s the
+ * judge was originally calibrated on, re-scored periodically. Pass rate =
+ * fraction of joined items where the judge stays within `tolerance` of the
+ * human score. Items the judge didn't score are excluded from the
+ * denominator; zero joined items is an error, not a 0% pass rate.
+ */
+declare function snapshotFromSentinelSet(scores: CandidateScore[], golden: GoldenItem[], meta: SnapshotMeta, options?: SentinelSetOptions): SentinelSnapshot;
+interface SentinelStore {
+    append(snapshot: SentinelSnapshot): Promise<void>;
+    /** All snapshots (optionally for one judge), in append order. */
+    history(judgeId?: string): Promise<SentinelSnapshot[]>;
+}
+declare function inMemorySentinelStore(initial?: SentinelSnapshot[]): SentinelStore;
+/**
+ * JSONL store — one snapshot per line, appended atomically per call.
+ * A corrupt or shape-invalid line is a loud error naming the file and
+ * line number: a sentinel history that silently drops records would
+ * defeat the drift detection it exists to provide.
+ */
+declare function fileSentinelStore(path: string): SentinelStore;
+interface SentinelThresholds {
+    /** Floor for irr — below it the judge ensemble is unreliable regardless of trend. Default 0.6. */
+    minIrr?: number;
+    /** Max tolerated calibrationKappa drop vs the series baseline. Default 0.15. */
+    maxKappaDrop?: number;
+    /** Max tolerated sentinelPassRate drop vs the series baseline. Default 0.1. */
+    maxSentinelDrop?: number;
+    /** Newest snapshot older than this (days, vs asOf) = the sentinel itself went blind. Default 30. */
+    staleAfterDays?: number;
+}
+interface JudgeSentinelOptions {
+    /** Caller-supplied ISO timestamp staleness is measured against. */
+    asOf: string;
+    thresholds?: SentinelThresholds;
+    /** Forwarded to `analyzeSeries` (window, stableCv, driftRun). */
+    convergence?: SeriesConvergenceOptions;
+}
+interface SentinelTrend {
+    judgeId: string;
+    metric: SentinelMetricName;
+    /** Verbatim `analyzeSeries` state for this judge×metric history. */
+    state: SeriesConvergenceResult['state'];
+    /** Newest value in the series. */
+    current: number;
+    /** Oldest value in the series — the reference the drop thresholds compare against. */
+    baseline: number;
+    /** current − baseline (signed; negative = decayed). */
+    drift: number;
+    alarmed: boolean;
+    reason?: string;
+}
+interface SentinelReport {
+    perJudge: SentinelTrend[];
+    alarms: string[];
+    healthy: boolean;
+    /**
+     * `judgeId:metric` pairs with too few snapshots for the convergence
+     * machine. Named, not counted — a blind spot you can't see is a blind
+     * spot you won't fix. Floor/drop checks still apply to thin series, so
+     * insufficient history alone never masks an alarm.
+     */
+    insufficientHistory: string[];
+}
+declare function judgeSentinelReport(history: SentinelSnapshot[], opts: JudgeSentinelOptions): SentinelReport;
+interface EvalHealthStamp {
+    healthy: boolean;
+    alarms: string[];
+}
+/**
+ * The object a campaign attaches to its verdicts. Compute it from the
+ * sentinel report in a post-campaign hook (or a nightly job feeding the
+ * next day's campaigns) and store it alongside the verdict payload.
+ * `healthy: false` means the judges that produced those verdicts have an
+ * unresolved drift / decay / silent-upgrade / staleness alarm — treat the
+ * verdicts as carrying an integrity warning until recalibration clears it.
+ */
+declare function evalHealthStamp(report: SentinelReport): EvalHealthStamp;
+
+export { type CalibrationBin, type CalibrationOptions, type CalibrationPair, type CalibrationReport, type CorrelationResult, type CorrelationStudyOptions, type CorrelationStudyResult, type DeploymentOutcome, type EvalHealthStamp, type EvalMetricSpec, FileSystemOutcomeStore, type FileSystemOutcomeStoreOptions, InMemoryOutcomeStore, type JudgeSentinelOptions, type OutcomeFilter, type OutcomePair, type OutcomeStore, type RubricOutcomePair, type RubricPredictiveValidityInput, type RubricPredictiveValidityReport, type RubricRanking, SENTINEL_METRIC_NAMES, type SentinelMetricName, type SentinelMetrics, type SentinelReport, type SentinelSetOptions, type SentinelSnapshot, type SentinelStore, type SentinelThresholds, type SentinelTrend, type SnapshotMeta, calibrationCurve, calibrationFromPairs, correlationStudy, evalHealthStamp, fileSentinelStore, inMemorySentinelStore, judgeSentinelReport, rubricPredictiveValidity, snapshotFromAgreement, snapshotFromCalibration, snapshotFromSentinelSet, validateSentinelSnapshot };