@tangle-network/agent-eval 0.117.1 → 0.118.0

package/dist/insight-report-DY4nDW9Q.d.ts

@@ -1,310 +0,0 @@
-import { G as GainDistributionBin, P as ParetoFigureSpec } from './summary-report-C5bKFfm-.js';
-import { C as ContinuousAgreement } from './judge-calibration-7C-IDmKr.js';
-
-/**
- * # InsightReport — the rigorous decision packet for any set of agent runs.
- *
- * Returned by `analyzeRuns()` and embedded in `SelfImproveResult.insight` +
- * the hosted-tier `EvalRunEvent.insightReport`. One shape across two surfaces:
- *
- *   - **Customer who has a closed loop** (`selfImprove`): the report ships
- *     with the loop output. Their dashboard renders ship/hold + lift CI +
- *     calibration + cluster + Pareto in one packet.
- *   - **Customer who has observed runs but no loop** (`analyzeRuns` directly):
- *     same packet from a `RunRecord[]` they already have — production traces,
- *     approve/reject corpus, CSV gold set.
- *
- * Every field is optional except the distributional summary — fields are
- * populated when the input data supports them:
- *
- *   - `lift` requires both baseline and candidate splits to be present.
- *   - `interRater` requires multi-rater feedback (≥2 raters per run).
- *   - `judges` populates per-judge stats only when the run records carry
- *     `outcome.judgeScores`.
- *   - `failureClusters` requires the optional `analystRegistry` to be wired.
- *   - `contamination` requires canary scenarios to be passed in.
- *   - `outcomeCorrelation` requires a downstream outcome signal.
- *   - `sequential` requires the run set to be ordered (treats them as a
- *     stream and emits an anytime-valid interim decision).
- *
- * Consumers read the `recommendations` array first — that's the
- * actionable layer, ranked by priority. The numeric sections back it up.
- */
-
-interface InsightReport {
-    /** Number of runs analyzed. */
-    n: number;
-    /** Composite-score distribution across all runs. Always present. */
-    composite: ScalarDistribution;
-    /** Per-dimension distributions for every dimension that appeared in any
-     *  run's judge scores. Empty when no judge scores were recorded. */
-    perDimension: Record<string, ScalarDistribution>;
-    /** Cost/quality distribution and Pareto frontier. */
-    costQuality: {
-        cost: ScalarDistribution;
-        pareto: ParetoFigureSpec;
-        /** Cost source coverage. `uncaptured` rows are excluded from the USD
-         *  distribution and Pareto chart; observed and estimated totals remain
-         *  separate so reports never present estimates as billed spend. */
-        provenance?: CostProvenanceSummary;
-        /** Set when the cost/quality view is degraded because the input data
-         *  doesn't fully support it — e.g. all `costUsd` were zero, or only a
-         *  single candidate appears (so the Pareto is a single point). The
-         *  named fields name the degraded sub-view, free-text the reason. */
-        degraded?: {
-            cost?: string;
-            pareto?: string;
-        };
-    };
-    /** Per-judge calibration + bias detection. Populated for every judge name
-     *  that appears in `outcome.judgeScores`. Bias fields require either a
-     *  gold reference or multi-rater data. */
-    judges: Record<string, JudgeInsight>;
-    /** Inter-rater agreement when multiple judges scored the same runs.
-     *  Includes pairwise kappa and the specific run ids where raters
-     *  disagree — the cases worth a human meeting. */
-    interRater?: InterRaterInsight;
-    /** Pairwise lift (baseline → candidate) with bootstrap CI. Present when
-     *  `RunRecord.splitTag` includes both `holdout` and search/dev splits,
-     *  or when caller passes an explicit baseline/candidate split. */
-    lift?: LiftInsight;
-    /** Failure clusters with exemplars. Populated when an AnalystRegistry
-     *  is wired in `analyzeRuns({ analyst })`. */
-    failureClusters?: FailureClusterInsight;
-    /** Canary leak count + holdout audit status. Populated when canary
-     *  scenarios are passed in. */
-    contamination?: ContaminationInsight;
-    /** Correlation between judge composite and a downstream outcome the
-     *  caller supplies (engagement, revenue, downstream pass rate, etc.).
-     *  When present, the optional reward model is the model that maps
-     *  judge scores → predicted outcome. */
-    outcomeCorrelation?: OutcomeCorrelationInsight;
-    /** Aggregate release-readiness summary. A consumer needing the full
-     *  substrate `ReleaseConfidenceScorecard` (SLO-axis evaluation,
-     *  ActionableSideInfo bag) calls `evaluateReleaseConfidence()` directly;
-     *  this summary captures the analyzeRuns-derived axes. */
-    release: ReleaseSummary;
-    /** Delta vs a prior period when `baselineRuns` is passed. Per-metric
-     *  current vs baseline with Welch CI + Cohen's d + significance flag.
-     *  Answers "did my last change help?" — the customer-conversion question.
-     *  Surfaced metrics: composite, cost, duration, tokenUsage, plus any
-     *  per-dimension judge metric present in both windows. */
-    priorPeriodComparison?: PriorPeriodComparison;
-    /** Model-free failure-mode breakdown from `RunRecord.failureMode`, ranked
-     *  by count descending. Present when any run carries a `failureMode`.
-     *  Complements `failureClusters` (LLM-semantic) with the structured tags
-     *  the harness already recorded — actionable with no analyst wired. */
-    failureModes?: FailureModeTally[];
-    /** Top-N actionable recommendations, ranked by priority. The packet's
-     *  human-readable layer; the numeric sections are the evidence. */
-    recommendations: Recommendation[];
-}
-interface CostProvenanceSummary {
-    observed: {
-        n: number;
-        totalUsd: number;
-    };
-    estimated: {
-        n: number;
-        totalUsd: number;
-    };
-    uncaptured: {
-        n: number;
-    };
-    knownFraction: number;
-}
-/** Distributional summary of a scalar-valued metric. */
-interface ScalarDistribution {
-    /** Sample count after dropping non-finite values. */
-    n: number;
-    mean: number;
-    p50: number;
-    p95: number;
-    stddev: number;
-    min: number;
-    max: number;
-    /** Histogram bins using `agent-eval`'s `gainHistogram` primitive. */
-    histogram: GainDistributionBin[];
-    /** Worst-N runs by score, ascending. Populated for the composite
-     *  distribution so the report names the runs a customer should
-     *  inspect first. Undefined when the distribution was computed from a
-     *  raw value list with no run identity (e.g. cost). */
-    tailRuns?: Array<{
-        runId: string;
-        score: number;
-    }>;
-}
-interface JudgeInsight {
-    /** Number of times this judge scored a run. */
-    n: number;
-    /** Mean composite over this judge's runs. */
-    meanScore: number;
-    /** Calibration against a gold reference, when provided. Cohen's κ for
-     *  binary thresholding + continuous agreement metrics. */
-    calibration?: ContinuousAgreement;
-    /** Positional bias — when the judge sees options in different orders,
-     *  do its preferences track the content or the position? */
-    positionalBias?: number;
-    /** Self-preference — when the judge sees its own model's output vs a
-     *  competitor, does it over-pick its own? */
-    selfPreference?: number;
-    /** Verbosity bias — does the judge reward longer outputs regardless of
-     *  quality? */
-    verbosityBias?: number;
-}
-interface InterRaterInsight {
-    /** Number of raters whose scores were aggregated. */
-    raters: number;
-    /** Number of runs every rater scored. */
-    jointlyRated: number;
-    /** Cohen's κ averaged across rater pairs. */
-    kappa: number;
-    /** Pairwise κ per rater pair (key = `"raterA::raterB"`). */
-    perPair: Record<string, number>;
-    /** Run ids where raters disagree the most — the high-value triage list. */
-    disagreementCases: Array<{
-        runId: string;
-        ratings: Array<{
-            rater: string;
-            score: number;
-        }>;
-        range: number;
-    }>;
-}
-interface LiftInsight {
-    baselineMean: number;
-    candidateMean: number;
-    /** Candidate − baseline. */
-    delta: number;
-    /** Lower / upper bound of bootstrap CI on the delta. */
-    ci95: [number, number];
-    /** Paired-t-test p-value. */
-    pValue: number;
-    /** Number of paired observations. */
-    n: number;
-    /** Cohen's d for the delta. */
-    cohensD: number;
-    /** Minimum detectable effect at current n, 80% power. */
-    mde: number;
-    /** Sample size needed to detect the observed delta at 80% power. */
-    requiredN: number;
-}
-interface FailureClusterInsight {
-    /** All clusters identified by the registry, ranked by share descending. */
-    clusters: Array<{
-        id: string;
-        name: string;
-        /** Fraction of failed runs in this cluster, 0..1. */
-        share: number;
-        /** Exemplar `runId`s (≤ 5) the consumer can drill into. */
-        exemplars: string[];
-        /** Short LLM-generated suggested fix when the registry supports it. */
-        suggestedFix?: string;
-    }>;
-    totalFailures: number;
-}
-/** Model-free failure breakdown over the structured `RunRecord.failureMode`
- *  enum. Unlike `failureClusters` (semantic, requires an LLM analyst), this
- *  is computed directly from the tags the harness already recorded — so a
- *  customer ingesting one batch with no judge/analyst still learns which
- *  named failure dominates. */
-interface FailureModeTally {
-    /** The `failureMode` tag. */
-    mode: string;
-    /** Number of runs carrying this tag. */
-    count: number;
-    /** Share of the whole corpus, 0..1. */
-    share: number;
-}
-interface ContaminationInsight {
-    /** Canary phrases that leaked into outputs. */
-    leaks: number;
-    /** Holdout audit verdict — did any holdout-tagged run end up in the
-     *  search/dev pool, or vice versa? */
-    holdoutAuditPassed: boolean;
-    details?: Array<{
-        runId: string;
-        canary: string;
-        matched: string;
-    }>;
-}
-interface OutcomeCorrelationInsight {
-    /** What outcome the consumer is correlating against (e.g.
-     *  `'engagement_rate'`, `'approval_rate'`, `'downstream_pass'`). */
-    metric: string;
-    /** Number of (run, outcome) pairs used. */
-    n: number;
-    /** Pearson correlation between composite score and outcome. */
-    pearson: number;
-    /** Spearman rank correlation — robust to monotonic non-linearity. */
-    spearman: number;
-    /** When present, the simple linear reward model fit to the data. */
-    rewardModel?: {
-        intercept: number;
-        slope: number;
-        r2: number;
-    };
-}
-interface ReleaseSummary {
-    /** Overall verdict across axes — fail if any axis fails, else warn if any
-     *  warns, else pass. */
-    status: 'pass' | 'warn' | 'fail';
-    axes: Array<{
-        name: 'quality-lift' | 'contamination' | 'composite-distribution';
-        status: 'pass' | 'warn' | 'fail';
-        detail: string;
-    }>;
-    /** Free-form issues surfaced beyond the standard axes. Empty by default;
-     *  consumers can post-process to populate. */
-    issues: string[];
-}
-interface MetricDelta {
-    /** Current-period mean. */
-    current: number;
-    /** Baseline-period mean. */
-    baseline: number;
-    /** current - baseline. Positive means improved (or, for cost/duration,
-     *  the consumer-side interpretation: "higher current" — semantic
-     *  direction depends on the metric). */
-    delta: number;
-    /** Welch 95% confidence interval on the delta. Two-sample, unpaired —
-     *  the baseline and current run sets may have different scenarios. */
-    ci95: [number, number];
-    /** Welch t-test p-value (two-sided). */
-    pValue: number;
-    /** Cohen's d (pooled stddev). Effect size, signed. */
-    cohensD: number;
-    /** Sample sizes. */
-    baselineN: number;
-    currentN: number;
-    /** True when p < 0.05 AND |d| >= 0.2 (small-effect threshold). The
-     *  conjunction prevents large-effect-but-noisy and significant-but-
-     *  tiny from triggering recommendations. */
-    significant: boolean;
-}
-interface PriorPeriodComparison {
-    /** Sample counts. */
-    baselineN: number;
-    currentN: number;
-    /** Optional human-readable label — "vs prior 7 days", "vs v3 release". */
-    windowLabel?: string;
-    /** Every metric we could compare. Keys: 'composite', 'cost', 'duration',
-     *  'tokenUsage' for always-present ones; per-dimension keys when both
-     *  windows have judge scores on the same dimension. */
-    metrics: Record<string, MetricDelta>;
-    /** Metric names where current is significantly WORSE than baseline.
-     *  Direction-aware: for cost/duration, higher current = worse. */
-    regressedMetrics: string[];
-    /** Metric names where current is significantly BETTER than baseline. */
-    improvedMetrics: string[];
-}
-interface Recommendation {
-    priority: 'critical' | 'high' | 'medium' | 'low';
-    kind: 'ship' | 'hold' | 'investigate' | 'fix' | 'recalibrate' | 'expand-corpus';
-    title: string;
-    detail: string;
-    /** Optional pointer back into the report for the evidence. */
-    evidencePath?: string;
-}
-
-export type { CostProvenanceSummary as C, FailureClusterInsight as F, InsightReport as I, JudgeInsight as J, LiftInsight as L, OutcomeCorrelationInsight as O, Recommendation as R, ScalarDistribution as S, InterRaterInsight as a, ReleaseSummary as b };

package/dist/integrity-DqlBiLyK.d.ts

@@ -1,81 +0,0 @@
-import { C as CaptureIntegrityError } from './errors-oeQrLqXC.js';
-import { R as RawProviderSink } from './raw-provider-sink-C46HDghv.js';
-import { T as TraceStore } from './store-DGqD0Pyo.js';
-
-/**
- * Run-completion integrity check — at end of run, verify the expected event
- * types were actually captured. The point is the launch-review failure mode:
- * a run *appears* successful but the raw provider events were never written,
- * so a downstream reviewer can't reconstruct what happened.
- *
- * Pattern:
- *
- *   const report = await assertRunCaptured(store, runId, {
- *     llmSpansMin: 1,
- *     judgeSpansMin: 1,
- *     rawSink: providerSink,                  // must have ≥ 1 event for this run
- *     requireRawCoverageOfLlmSpans: true,     // every llm span has matching raw events
- *   })
- *   if (!report.ok) throwIfRunIncomplete(report)  // or mark run failed and continue
- *
- * The function is read-only on the store and returns a structured report;
- * the caller chooses the failure mode (throw, mark run failed, log warning).
- * `throwIfRunIncomplete` is the convenient strict mode.
- */
-
-interface RunIntegrityExpectations {
-    /** Minimum LLM span count. Default 0 (no requirement). */
-    llmSpansMin?: number;
-    /** Minimum judge span count. Default 0. */
-    judgeSpansMin?: number;
-    /** Minimum tool span count. Default 0. */
-    toolSpansMin?: number;
-    /**
-     * Raw provider sink to consult for capture verification. When present,
-     * the check requires at least one raw event for the run.
-     */
-    rawSink?: RawProviderSink;
-    /** Minimum raw provider event count. Default 0; ignored when `rawSink` absent. */
-    rawProviderEventsMin?: number;
-    /**
-     * Every LLM span must have at least one matching raw `request` event
-     * (matched by spanId). Catches the common bug where the structured span
-     * was emitted but the raw HTTP capture was wired to a different sink.
-     */
-    requireRawCoverageOfLlmSpans?: boolean;
-    /** Run outcome must be set (not null/undefined). Default false. */
-    requireOutcome?: boolean;
-}
-type RunIntegrityIssueCode = 'no_run' | 'missing_llm_spans' | 'missing_judge_spans' | 'missing_tool_spans' | 'missing_raw_events' | 'no_raw_sink' | 'orphan_llm_span' | 'missing_outcome';
-interface RunIntegrityIssue {
-    code: RunIntegrityIssueCode;
-    message: string;
-    detail?: Record<string, unknown>;
-}
-interface RunIntegrityReport {
-    ok: boolean;
-    runId: string;
-    llmSpanCount: number;
-    judgeSpanCount: number;
-    toolSpanCount: number;
-    rawProviderEventCount: number;
-    /**
-     * Coverage of LLM spans by raw provider events keyed on spanId.
-     * `total` is the number of LLM spans; `covered` is the count with at
-     * least one matching `request` raw event.
-     */
-    rawSpanCoverage: {
-        covered: number;
-        total: number;
-    };
-    issues: RunIntegrityIssue[];
-}
-declare class RunIntegrityError extends CaptureIntegrityError {
-    readonly report: RunIntegrityReport;
-    constructor(report: RunIntegrityReport);
-}
-declare function assertRunCaptured(store: TraceStore, runId: string, expectations?: RunIntegrityExpectations): Promise<RunIntegrityReport>;
-/** Strict mode: throws `RunIntegrityError` when the report isn't ok. */
-declare function throwIfRunIncomplete(report: RunIntegrityReport): void;
-
-export { type RunIntegrityExpectations as R, type RunIntegrityReport as a, RunIntegrityError as b, type RunIntegrityIssue as c, type RunIntegrityIssueCode as d, assertRunCaptured as e, throwIfRunIncomplete as t };

package/dist/judge-calibration-7C-IDmKr.d.ts

@@ -1,145 +0,0 @@
-/**
- * Judge calibration — measure judge quality against human gold + bias.
- *
- * Workflow:
- *   1. Build a golden set: {itemId, humanScore}[].
- *   2. Run candidate judges; each produces {itemId, score}.
- *   3. `calibrateJudge(golden, candidate)` reports κ + Pearson + MAE.
- *   4. `calibrateJudgeContinuous(golden, candidate)` adds quadratic-weighted
- *      κ over the un-rounded [0,1] scores plus ICC(2,1), Pearson, Spearman,
- *      and bootstrap CIs — use this for fine-grained judges where rounding
- *      to int discards information (e.g. 0.78 vs 0.81 both round to 1 and
- *      look "perfectly agreed" to integer κ).
- *   5. Run bias probes (positional, verbosity, self-preference) to
- *      detect systematic score inflation.
- *   6. For N≥2 judges on the same items, `continuousAgreement(scores)`
- *      reports ICC(2,1) + κ_w + Pearson + Spearman with bootstrap CIs.
- *
- * Returns actionable diagnostics, not a single number. Consumers then
- * decide whether to trust the judge, retrain it, or add a tie-breaker.
- */
-interface GoldenItem {
-    itemId: string;
-    humanScore: number;
-    /** Optional group used for per-group bias audits (e.g. model-of-output family). */
-    group?: string;
-}
-interface CandidateScore {
-    itemId: string;
-    score: number;
-    /** Optional — enables positional-bias analysis (did order matter?). */
-    positionOfAInput?: 'first' | 'second';
-}
-interface CalibrationResult {
-    n: number;
-    pearson: number;
-    /** Cohen's κ with quadratic weights over integer-rounded scores. */
-    kappa: number;
-    /** Mean absolute error vs human. */
-    mae: number;
-    /** Worst-5 miscalibrations (largest |judge - human|). */
-    worstItems: Array<{
-        itemId: string;
-        judge: number;
-        human: number;
-        delta: number;
-    }>;
-}
-/**
- * Measure judge quality against human gold labels: computes Cohen's κ, Pearson correlation, and MAE over matched item ids.
- */
-declare function calibrateJudge(golden: GoldenItem[], candidate: CandidateScore[]): CalibrationResult;
-interface PositionalBiasResult {
-    /**
-     * Score delta (first-position - second-position) averaged across items
-     * presented in both positions. Non-zero = positional bias.
-     */
-    avgDelta: number;
-    n: number;
-}
-/**
- * Feed the same items to the judge twice with A/B swapped and pass all
- * results here. Items that don't appear in both positions are ignored.
- */
-declare function positionalBias(scores: CandidateScore[]): PositionalBiasResult;
-interface VerbosityBiasResult {
-    /** Pearson correlation between output length and score. Strong positive = verbosity bias. */
-    pearson: number;
-    n: number;
-}
-declare function verbosityBias(samples: Array<{
-    outputLen: number;
-    score: number;
-}>): VerbosityBiasResult;
-interface SelfPreferenceResult {
-    /** Mean judge score when judge's family matches output's family. */
-    inFamilyMean: number;
-    outOfFamilyMean: number;
-    deltaMean: number;
-    n: number;
-}
-/**
- * Pass the same scenarios scored with judge-model X grading outputs from
- * model X (in-family) and model Y (out-of-family). Non-zero delta
- * indicates self-preference.
- */
-declare function selfPreference(samples: Array<{
-    score: number;
-    inFamily: boolean;
-}>): SelfPreferenceResult;
-interface ContinuousAgreement {
-    /** Cohen's κ_w with quadratic weights, computed on raw [0,1] scores. */
-    weightedKappa: number;
-    /** ICC(2,1): two-way random effects, absolute agreement, single rater. */
-    icc: number;
-    /** Pearson product-moment correlation (averaged over rater pairs if N>2). */
-    pearson: number;
-    /** Spearman rank correlation (averaged over rater pairs if N>2). */
-    spearman: number;
-    /** 95% bootstrap percentile CIs over items. */
-    ci: {
-        icc: [number, number];
-        weightedKappa: [number, number];
-    };
-    /** Number of complete items (no NaN across raters). */
-    n: number;
-    /** Number of raters. */
-    raters: number;
-}
-interface ContinuousAgreementOptions {
-    /** Bootstrap iterations. Default 1000. Set to 0 to skip CIs (CI = [NaN, NaN]). */
-    bootstrap?: number;
-    /** κ weighting scheme. Default 'quadratic'. */
-    weights?: 'linear' | 'quadratic';
-    /** PRNG seed for reproducible bootstrap. Default 0xC0FFEE. */
-    seed?: number;
-    /** Confidence level for percentile CI. Default 0.95. */
-    ciLevel?: number;
-}
-/**
- * Inter-rater agreement on continuous (typically [0,1]) scores.
- *
- * `scores` has shape [n_items][n_raters]. Rows with any non-finite entry
- * are dropped. Returns NaN metrics if fewer than 2 raters or 2 complete
- * items remain.
- */
-declare function continuousAgreement(scores: number[][], opts?: ContinuousAgreementOptions): ContinuousAgreement;
-interface ContinuousCalibrationResult extends CalibrationResult {
-    /** Cohen's κ_w computed on raw (un-rounded) scores. */
-    weightedKappaContinuous: number;
-    /** ICC(2,1) treating golden + candidate as two raters. */
-    icc: number;
-    spearman: number;
-    ci: {
-        icc: [number, number];
-        weightedKappa: [number, number];
-    };
-}
-/**
- * Drop-in superset of `calibrateJudge` that adds continuous-value
- * agreement metrics. The old fields (n, pearson, kappa, mae, worstItems)
- * are preserved unchanged so existing callers continue to work.
- */
-declare function calibrateJudgeContinuous(golden: GoldenItem[], candidate: CandidateScore[], opts?: ContinuousAgreementOptions): ContinuousCalibrationResult;
-
-export { type ContinuousAgreement as C, type GoldenItem as G, type PositionalBiasResult as P, type SelfPreferenceResult as S, type VerbosityBiasResult as V, type CalibrationResult as a, type ContinuousCalibrationResult as b, type CandidateScore as c, type ContinuousAgreementOptions as d, calibrateJudge as e, calibrateJudgeContinuous as f, continuousAgreement as g, positionalBias as p, selfPreference as s, verbosityBias as v };