@tangle-network/agent-eval 0.48.0 → 0.50.0

package/dist/index-BRxz6qov.d.ts

@@ -0,0 +1,409 @@
+import { M as MutableSurface, m as GateDecision } from './types-Dbj5gu8n.js';
+import { G as GainDistributionBin, P as ParetoFigureSpec } from './summary-report-B7gNRX-r.js';
+import { a as ContinuousAgreement } from './judge-calibration-DilmB3Ml.js';
+
+/**
+ * # InsightReport — the rigorous decision packet for any set of agent runs.
+ *
+ * Returned by `analyzeRuns()` and embedded in `SelfImproveResult.insight` +
+ * the hosted-tier `EvalRunEvent.insightReport`. One shape across two surfaces:
+ *
+ *   - **Customer who has a closed loop** (`selfImprove`): the report ships
+ *     with the loop output. Their dashboard renders ship/hold + lift CI +
+ *     calibration + cluster + Pareto in one packet.
+ *   - **Customer who has observed runs but no loop** (`analyzeRuns` directly):
+ *     same packet from a `RunRecord[]` they already have — production traces,
+ *     approve/reject corpus, CSV gold set.
+ *
+ * Every field is optional except the distributional summary — fields are
+ * populated when the input data supports them:
+ *
+ *   - `lift` requires both baseline and candidate splits to be present.
+ *   - `interRater` requires multi-rater feedback (≥2 raters per run).
+ *   - `judges` populates per-judge stats only when the run records carry
+ *     `outcome.judgeScores`.
+ *   - `failureClusters` requires the optional `analystRegistry` to be wired.
+ *   - `contamination` requires canary scenarios to be passed in.
+ *   - `outcomeCorrelation` requires a downstream outcome signal.
+ *   - `sequential` requires the run set to be ordered (treats them as a
+ *     stream and emits an anytime-valid interim decision).
+ *
+ * Consumers read the `recommendations` array first — that's the
+ * actionable layer, ranked by priority. The numeric sections back it up.
+ */
+
+interface InsightReport {
+    /** Number of runs analyzed. */
+    n: number;
+    /** Composite-score distribution across all runs. Always present. */
+    composite: ScalarDistribution;
+    /** Per-dimension distributions for every dimension that appeared in any
+     *  run's judge scores. Empty when no judge scores were recorded. */
+    perDimension: Record<string, ScalarDistribution>;
+    /** Cost/quality distribution and Pareto frontier. */
+    costQuality: {
+        cost: ScalarDistribution;
+        pareto: ParetoFigureSpec;
+    };
+    /** Per-judge calibration + bias detection. Populated for every judge name
+     *  that appears in `outcome.judgeScores`. Bias fields require either a
+     *  gold reference or multi-rater data. */
+    judges: Record<string, JudgeInsight>;
+    /** Inter-rater agreement when multiple judges scored the same runs.
+     *  Includes pairwise kappa and the specific run ids where raters
+     *  disagree — the cases worth a human meeting. */
+    interRater?: InterRaterInsight;
+    /** Pairwise lift (baseline → candidate) with bootstrap CI. Present when
+     *  `RunRecord.splitTag` includes both `holdout` and search/dev splits,
+     *  or when caller passes an explicit baseline/candidate split. */
+    lift?: LiftInsight;
+    /** Failure clusters with exemplars. Populated when an AnalystRegistry
+     *  is wired in `analyzeRuns({ analyst })`. */
+    failureClusters?: FailureClusterInsight;
+    /** Canary leak count + holdout audit status. Populated when canary
+     *  scenarios are passed in. */
+    contamination?: ContaminationInsight;
+    /** Correlation between judge composite and a downstream outcome the
+     *  caller supplies (engagement, revenue, downstream pass rate, etc.).
+     *  When present, the optional reward model is the model that maps
+     *  judge scores → predicted outcome. */
+    outcomeCorrelation?: OutcomeCorrelationInsight;
+    /** Aggregate release-readiness summary. A consumer needing the full
+     *  substrate `ReleaseConfidenceScorecard` (SLO-axis evaluation,
+     *  ActionableSideInfo bag) calls `evaluateReleaseConfidence()` directly;
+     *  this summary captures the analyzeRuns-derived axes. */
+    release: ReleaseSummary;
+    /** Top-N actionable recommendations, ranked by priority. The packet's
+     *  human-readable layer; the numeric sections are the evidence. */
+    recommendations: Recommendation[];
+}
+/** Distributional summary of a scalar-valued metric. */
+interface ScalarDistribution {
+    /** Sample count after dropping non-finite values. */
+    n: number;
+    mean: number;
+    p50: number;
+    p95: number;
+    stddev: number;
+    min: number;
+    max: number;
+    /** Histogram bins using `agent-eval`'s `gainHistogram` primitive. */
+    histogram: GainDistributionBin[];
+}
+interface JudgeInsight {
+    /** Number of times this judge scored a run. */
+    n: number;
+    /** Mean composite over this judge's runs. */
+    meanScore: number;
+    /** Calibration against a gold reference, when provided. Cohen's κ for
+     *  binary thresholding + continuous agreement metrics. */
+    calibration?: ContinuousAgreement;
+    /** Positional bias — when the judge sees options in different orders,
+     *  do its preferences track the content or the position? */
+    positionalBias?: number;
+    /** Self-preference — when the judge sees its own model's output vs a
+     *  competitor, does it over-pick its own? */
+    selfPreference?: number;
+    /** Verbosity bias — does the judge reward longer outputs regardless of
+     *  quality? */
+    verbosityBias?: number;
+}
+interface InterRaterInsight {
+    /** Number of raters whose scores were aggregated. */
+    raters: number;
+    /** Number of runs every rater scored. */
+    jointlyRated: number;
+    /** Cohen's κ averaged across rater pairs. */
+    kappa: number;
+    /** Pairwise κ per rater pair (key = `"raterA::raterB"`). */
+    perPair: Record<string, number>;
+    /** Run ids where raters disagree the most — the high-value triage list. */
+    disagreementCases: Array<{
+        runId: string;
+        ratings: Array<{
+            rater: string;
+            score: number;
+        }>;
+        range: number;
+    }>;
+}
+interface LiftInsight {
+    baselineMean: number;
+    candidateMean: number;
+    /** Candidate − baseline. */
+    delta: number;
+    /** Lower / upper bound of bootstrap CI on the delta. */
+    ci95: [number, number];
+    /** Paired-t-test p-value. */
+    pValue: number;
+    /** Number of paired observations. */
+    n: number;
+    /** Cohen's d for the delta. */
+    cohensD: number;
+    /** Minimum detectable effect at current n, 80% power. */
+    mde: number;
+    /** Sample size needed to detect the observed delta at 80% power. */
+    requiredN: number;
+}
+interface FailureClusterInsight {
+    /** All clusters identified by the registry, ranked by share descending. */
+    clusters: Array<{
+        id: string;
+        name: string;
+        /** Fraction of failed runs in this cluster, 0..1. */
+        share: number;
+        /** Exemplar `runId`s (≤ 5) the consumer can drill into. */
+        exemplars: string[];
+        /** Short LLM-generated suggested fix when the registry supports it. */
+        suggestedFix?: string;
+    }>;
+    totalFailures: number;
+}
+interface ContaminationInsight {
+    /** Canary phrases that leaked into outputs. */
+    leaks: number;
+    /** Holdout audit verdict — did any holdout-tagged run end up in the
+     *  search/dev pool, or vice versa? */
+    holdoutAuditPassed: boolean;
+    details?: Array<{
+        runId: string;
+        canary: string;
+        matched: string;
+    }>;
+}
+interface OutcomeCorrelationInsight {
+    /** What outcome the consumer is correlating against (e.g.
+     *  `'engagement_rate'`, `'approval_rate'`, `'downstream_pass'`). */
+    metric: string;
+    /** Number of (run, outcome) pairs used. */
+    n: number;
+    /** Pearson correlation between composite score and outcome. */
+    pearson: number;
+    /** Spearman rank correlation — robust to monotonic non-linearity. */
+    spearman: number;
+    /** When present, the simple linear reward model fit to the data. */
+    rewardModel?: {
+        intercept: number;
+        slope: number;
+        r2: number;
+    };
+}
+interface ReleaseSummary {
+    /** Overall verdict across axes — fail if any axis fails, else warn if any
+     *  warns, else pass. */
+    status: 'pass' | 'warn' | 'fail';
+    axes: Array<{
+        name: 'quality-lift' | 'contamination' | 'composite-distribution';
+        status: 'pass' | 'warn' | 'fail';
+        detail: string;
+    }>;
+    /** Free-form issues surfaced beyond the standard axes. Empty by default;
+     *  consumers can post-process to populate. */
+    issues: string[];
+}
+interface Recommendation {
+    priority: 'critical' | 'high' | 'medium' | 'low';
+    kind: 'ship' | 'hold' | 'investigate' | 'fix' | 'recalibrate' | 'expand-corpus';
+    title: string;
+    detail: string;
+    /** Optional pointer back into the report for the evidence. */
+    evidencePath?: string;
+}
+
+/**
+ * # Hosted-tier wire format — the schema that EVERY orchestrator (ours,
+ * a partner's self-hosted one, a future open implementation) must accept.
+ *
+ * **Stability:** every type in this file is committed under semver. New
+ * minors only ADD optional fields. Breaking changes mean a major bump
+ * (`HostedWireVersion` literal increment).
+ *
+ * The wire format is two event streams in one transport:
+ *
+ *   1. **Eval-run events** (`POST /v1/ingest/eval-runs`). Posted when a
+ *      campaign / improvement-loop completes (or per-generation if
+ *      streaming). Carries the structured result + per-cell scores +
+ *      surface diffs the orchestrator stores for the dashboard.
+ *
+ *   2. **Trace spans** (`POST /v1/ingest/traces`). Standard OTLP-shaped
+ *      spans with a few additional attributes so the orchestrator can
+ *      pivot from eval-run → underlying execution. Compatible with any
+ *      OTel collector.
+ *
+ * Both endpoints are authenticated with a bearer token + a tenant id
+ * header. Tenants isolate everything downstream of ingest; no tenant
+ * ever sees another tenant's data.
+ */
+
+declare const HOSTED_WIRE_VERSION: "2026-05-26.v1";
+type HostedWireVersion = typeof HOSTED_WIRE_VERSION;
+/** Every ingest request carries these. */
+interface HostedIngestHeaders {
+    /** Bearer token. The orchestrator validates against the tenant key. */
+    authorization: `Bearer ${string}`;
+    /** Stable tenant id (the orchestrator-side primary key for the tenant). */
+    'x-tangle-tenant-id': string;
+    /** Wire-version pin so the server can reject incompatible payloads. */
+    'x-tangle-wire-version': HostedWireVersion;
+    /** Optional idempotency key for retry-safe ingest. */
+    'idempotency-key'?: string;
+}
+/** Lifecycle stages of an eval-run as the substrate reports them. */
+type EvalRunStatus = 'started' | 'baseline-complete' | 'generation-complete' | 'gate-decided' | 'finished' | 'errored';
+interface EvalRunCellScore {
+    /** Stable scenario id from the consumer's scenario set. */
+    scenarioId: string;
+    /** Repetition index when reps > 1; 0 for the default. */
+    rep: number;
+    /** Composite score across all judges + dimensions for this cell. */
+    compositeMean: number;
+    /** Per-judge → per-dimension scores; null where the judge did not run. */
+    dimensions: Record<string, Record<string, number>>;
+    /** Per-cell error message if the dispatch threw. Null on success. */
+    errorMessage?: string;
+}
+interface EvalRunGenerationSnapshot {
+    /** Generation index. 0 is baseline. */
+    index: number;
+    /** Candidate surface fingerprint (stable hash) — pivot key into the
+     *  trace stream to fetch the underlying execution. */
+    surfaceHash: string;
+    /** The candidate surface itself. May be omitted to avoid PII when the
+     *  consumer prefers not to ship verbatim prompts. */
+    surface?: MutableSurface;
+    /** Per-cell scores for this generation. */
+    cells: EvalRunCellScore[];
+    /** Aggregate composite mean across all cells in this generation. */
+    compositeMean: number;
+    /** Total $ spent across this generation. */
+    costUsd: number;
+    /** Wall-clock duration of this generation. */
+    durationMs: number;
+}
+/**
+ * The top-level eval-run event. One ingest call per logical eval-run;
+ * generations stream in incrementally via repeated calls with the same
+ * `runId`. The orchestrator deduplicates by `(runId, generation.index)`.
+ */
+interface EvalRunEvent {
+    /** Stable run id (the substrate's `runId`). UUID or substrate-generated. */
+    runId: string;
+    /** Where this run was happening — derived from `RunCampaignOptions.runDir`. */
+    runDir: string;
+    /** ISO-8601 timestamp the substrate recorded the event. */
+    timestamp: string;
+    /** Lifecycle stage this event represents. */
+    status: EvalRunStatus;
+    /** Free-form consumer tags (env, branch, model id, etc.). Searchable. */
+    labels: Record<string, string>;
+    /** Baseline campaign snapshot. Present when status >= baseline-complete. */
+    baseline?: EvalRunGenerationSnapshot;
+    /** Per-generation snapshots. Streams in; orchestrator appends. */
+    generations: EvalRunGenerationSnapshot[];
+    /** Final gate decision. Present when status >= gate-decided. */
+    gateDecision?: GateDecision;
+    /** Held-out lift = winner-on-holdout - baseline-on-holdout. */
+    holdoutLift?: number;
+    /** Total $ spent across baseline + every generation. */
+    totalCostUsd: number;
+    /** Total wall-clock duration. */
+    totalDurationMs: number;
+    /** Error message if status === 'errored'. */
+    errorMessage?: string;
+    /** Rigor packet emitted alongside the run — distributional summary,
+     *  paired-bootstrap lift CI, judge stats, inter-rater agreement,
+     *  contamination check, failure clusters (when an analyst is wired),
+     *  outcome correlation (when downstream signal is supplied), and the
+     *  recommendations the dashboard surfaces verbatim. Additive; older
+     *  clients that don't know about this field continue to work. */
+    insightReport?: InsightReport;
+}
+/**
+ * OTel-shape span with a few additional attributes for eval-run pivoting.
+ * Compatible with any OTLP collector — `name`, `traceId`, `spanId`,
+ * `startTimeUnixNano`, `endTimeUnixNano`, `attributes` are stock OTel.
+ */
+interface TraceSpanEvent {
+    traceId: string;
+    spanId: string;
+    parentSpanId?: string;
+    name: string;
+    startTimeUnixNano: number;
+    endTimeUnixNano: number;
+    attributes: Record<string, string | number | boolean>;
+    events?: Array<{
+        timeUnixNano: number;
+        name: string;
+        attributes?: Record<string, string | number | boolean>;
+    }>;
+    status?: {
+        code: 'OK' | 'ERROR' | 'UNSET';
+        message?: string;
+    };
+    /** Pivot back into the eval-run stream. */
+    'tangle.runId'?: string;
+    /** Pivot to the specific generation. */
+    'tangle.generation'?: number;
+    /** Pivot to the specific cell. */
+    'tangle.cellId'?: string;
+    /** Pivot to the specific scenario. */
+    'tangle.scenarioId'?: string;
+}
+interface IngestEvalRunsRequest {
+    wireVersion: HostedWireVersion;
+    events: EvalRunEvent[];
+}
+interface IngestTracesRequest {
+    wireVersion: HostedWireVersion;
+    spans: TraceSpanEvent[];
+}
+interface IngestResponse {
+    /** Accepted events / spans count. */
+    accepted: number;
+    /** Rejected events with reasons (validation failures, dup idempotency key, etc.). */
+    rejected: Array<{
+        index: number;
+        reason: string;
+    }>;
+}
+
+/**
+ * # Hosted-tier ingest client.
+ *
+ * Ships eval-run events + trace spans to any orchestrator (ours, a
+ * partner's self-hosted one, or a future open implementation) that
+ * speaks the wire format in `./types.ts`.
+ *
+ * Three modes:
+ *   - **Ours:** point at `https://orchestrator.tangle.tools/v1`. We
+ *     handle ingest + storage + dashboard.
+ *   - **Self-hosted:** point at whatever URL runs the reference receiver
+ *     from `examples/hosted-ingest-server/`.
+ *   - **Off (default):** when `hostedTenant` is unset, nothing is sent.
+ *     Everything stays local.
+ */
+
+interface HostedTenant {
+    /** Orchestrator endpoint base URL (no trailing slash). Required. */
+    endpoint: string;
+    /** Bearer token issued by the orchestrator. Required. */
+    apiKey: string;
+    /** Tenant id — the orchestrator's primary key for this consumer. Required. */
+    tenantId: string;
+    /** Optional `fetch` override (auth wrappers, custom agent, test mocks). */
+    fetchImpl?: typeof fetch;
+    /** Per-call timeout in ms. Default 30s. */
+    timeoutMs?: number;
+    /** Retries on 5xx / network errors. Default 2. */
+    retries?: number;
+}
+interface HostedClient {
+    ingestEvalRun(event: EvalRunEvent, idempotencyKey?: string): Promise<IngestResponse>;
+    ingestEvalRuns(events: EvalRunEvent[], idempotencyKey?: string): Promise<IngestResponse>;
+    ingestTraces(spans: TraceSpanEvent[], idempotencyKey?: string): Promise<IngestResponse>;
+    readonly tenant: HostedTenant;
+    readonly wireVersion: HostedWireVersion;
+}
+declare function createHostedClient(tenant: HostedTenant): HostedClient;
+
+export { type EvalRunCellScore as E, type FailureClusterInsight as F, type HostedClient as H, type InsightReport as I, type JudgeInsight as J, type LiftInsight as L, type OutcomeCorrelationInsight as O, type Recommendation as R, type ScalarDistribution as S, type TraceSpanEvent as T, type HostedTenant as a, type InterRaterInsight as b, type ReleaseSummary as c, type EvalRunEvent as d, type EvalRunGenerationSnapshot as e, type EvalRunStatus as f, HOSTED_WIRE_VERSION as g, type HostedIngestHeaders as h, type HostedWireVersion as i, type IngestEvalRunsRequest as j, type IngestResponse as k, type IngestTracesRequest as l, createHostedClient as m };