@tangle-network/agent-eval 0.32.0 → 0.33.1

package/dist/{index-DPILdKbP.d.ts → index-CN2agEaO.d.ts}

@@ -1,148 +1,8 @@
 import { a as DatasetScenario, c as Dataset, b as DatasetManifest } from './dataset-ueRVTUoY.js';
+import { b as CalibrationResult } from './judge-calibration-DilmB3Ml.js';
 import { O as OutcomeStore } from './outcome-store-D6KWmYvj.js';
 import { T as TraceStore } from './store-Db2Bv8Cf.js';
 
-/**
- * Judge calibration — measure judge quality against human gold + bias.
- *
- * Workflow:
- *   1. Build a golden set: {itemId, humanScore}[].
- *   2. Run candidate judges; each produces {itemId, score}.
- *   3. `calibrateJudge(golden, candidate)` reports κ + Pearson + MAE.
- *   4. `calibrateJudgeContinuous(golden, candidate)` adds quadratic-weighted
- *      κ over the un-rounded [0,1] scores plus ICC(2,1), Pearson, Spearman,
- *      and bootstrap CIs — use this for fine-grained judges where rounding
- *      to int discards information (e.g. 0.78 vs 0.81 both round to 1 and
- *      look "perfectly agreed" to integer κ).
- *   5. Run bias probes (positional, verbosity, self-preference) to
- *      detect systematic score inflation.
- *   6. For N≥2 judges on the same items, `continuousAgreement(scores)`
- *      reports ICC(2,1) + κ_w + Pearson + Spearman with bootstrap CIs.
- *
- * Returns actionable diagnostics, not a single number. Consumers then
- * decide whether to trust the judge, retrain it, or add a tie-breaker.
- */
-interface GoldenItem {
-    itemId: string;
-    humanScore: number;
-    /** Optional group used for per-group bias audits (e.g. model-of-output family). */
-    group?: string;
-}
-interface CandidateScore {
-    itemId: string;
-    score: number;
-    /** Optional — enables positional-bias analysis (did order matter?). */
-    positionOfAInput?: 'first' | 'second';
-}
-interface CalibrationResult {
-    n: number;
-    pearson: number;
-    /** Cohen's κ with quadratic weights over integer-rounded scores. */
-    kappa: number;
-    /** Mean absolute error vs human. */
-    mae: number;
-    /** Worst-5 miscalibrations (largest |judge - human|). */
-    worstItems: Array<{
-        itemId: string;
-        judge: number;
-        human: number;
-        delta: number;
-    }>;
-}
-declare function calibrateJudge(golden: GoldenItem[], candidate: CandidateScore[]): CalibrationResult;
-interface PositionalBiasResult {
-    /**
-     * Score delta (first-position - second-position) averaged across items
-     * presented in both positions. Non-zero = positional bias.
-     */
-    avgDelta: number;
-    n: number;
-}
-/**
- * Feed the same items to the judge twice with A/B swapped and pass all
- * results here. Items that don't appear in both positions are ignored.
- */
-declare function positionalBias(scores: CandidateScore[]): PositionalBiasResult;
-interface VerbosityBiasResult {
-    /** Pearson correlation between output length and score. Strong positive = verbosity bias. */
-    pearson: number;
-    n: number;
-}
-declare function verbosityBias(samples: Array<{
-    outputLen: number;
-    score: number;
-}>): VerbosityBiasResult;
-interface SelfPreferenceResult {
-    /** Mean judge score when judge's family matches output's family. */
-    inFamilyMean: number;
-    outOfFamilyMean: number;
-    deltaMean: number;
-    n: number;
-}
-/**
- * Pass the same scenarios scored with judge-model X grading outputs from
- * model X (in-family) and model Y (out-of-family). Non-zero delta
- * indicates self-preference.
- */
-declare function selfPreference(samples: Array<{
-    score: number;
-    inFamily: boolean;
-}>): SelfPreferenceResult;
-interface ContinuousAgreement {
-    /** Cohen's κ_w with quadratic weights, computed on raw [0,1] scores. */
-    weightedKappa: number;
-    /** ICC(2,1): two-way random effects, absolute agreement, single rater. */
-    icc: number;
-    /** Pearson product-moment correlation (averaged over rater pairs if N>2). */
-    pearson: number;
-    /** Spearman rank correlation (averaged over rater pairs if N>2). */
-    spearman: number;
-    /** 95% bootstrap percentile CIs over items. */
-    ci: {
-        icc: [number, number];
-        weightedKappa: [number, number];
-    };
-    /** Number of complete items (no NaN across raters). */
-    n: number;
-    /** Number of raters. */
-    raters: number;
-}
-interface ContinuousAgreementOptions {
-    /** Bootstrap iterations. Default 1000. Set to 0 to skip CIs (CI = [NaN, NaN]). */
-    bootstrap?: number;
-    /** κ weighting scheme. Default 'quadratic'. */
-    weights?: 'linear' | 'quadratic';
-    /** PRNG seed for reproducible bootstrap. Default 0xC0FFEE. */
-    seed?: number;
-    /** Confidence level for percentile CI. Default 0.95. */
-    ciLevel?: number;
-}
-/**
- * Inter-rater agreement on continuous (typically [0,1]) scores.
- *
- * `scores` has shape [n_items][n_raters]. Rows with any non-finite entry
- * are dropped. Returns NaN metrics if fewer than 2 raters or 2 complete
- * items remain.
- */
-declare function continuousAgreement(scores: number[][], opts?: ContinuousAgreementOptions): ContinuousAgreement;
-interface ContinuousCalibrationResult extends CalibrationResult {
-    /** Cohen's κ_w computed on raw (un-rounded) scores. */
-    weightedKappaContinuous: number;
-    /** ICC(2,1) treating golden + candidate as two raters. */
-    icc: number;
-    spearman: number;
-    ci: {
-        icc: [number, number];
-        weightedKappa: [number, number];
-    };
-}
-/**
- * Drop-in superset of `calibrateJudge` that adds continuous-value
- * agreement metrics. The old fields (n, pearson, kappa, mae, worstItems)
- * are preserved unchanged so existing callers continue to work.
- */
-declare function calibrateJudgeContinuous(golden: GoldenItem[], candidate: CandidateScore[], opts?: ContinuousAgreementOptions): ContinuousCalibrationResult;
-
 /**
  * Red-team battery — adversarial scenario corpus with per-category
  * scorers.
@@ -328,4 +188,4 @@ declare function nistAiRmfReport(ctx: GovernanceContext): Promise<GovernanceRepo
 
 declare function soc2Report(ctx: GovernanceContext): Promise<GovernanceReport>;
 
-export { verbosityBias as A, type ContinuousAgreementOptions as C, DEFAULT_RED_TEAM_CORPUS as D, type EuRiskClass as E, type GoldenItem as G, type PositionalBiasResult as P, type RedTeamCase as R, type SelfPreferenceResult as S, type UseCaseSignals as U, type VerbosityBiasResult as V, type ContinuousAgreement as a, type CalibrationResult as b, type CandidateScore as c, type ContinuousCalibrationResult as d, type GovernanceContext as e, type GovernanceFinding as f, type GovernanceReport as g, type RedTeamCategory as h, type RedTeamFinding as i, type RedTeamPayload as j, type RedTeamReport as k, calibrateJudge as l, calibrateJudgeContinuous as m, classifyEuAiRisk as n, continuousAgreement as o, euAiActReport as p, nistAiRmfReport as q, positionalBias as r, redTeamDataset as s, redTeamReport as t, renderMarkdown as u, scoreRedTeamOutput as v, selfPreference as w, soc2Report as x, summarize as y, toolNamesForRun as z };
+export { DEFAULT_RED_TEAM_CORPUS as D, type EuRiskClass as E, type GovernanceContext as G, type RedTeamCase as R, type UseCaseSignals as U, type GovernanceFinding as a, type GovernanceReport as b, type RedTeamCategory as c, type RedTeamFinding as d, type RedTeamPayload as e, type RedTeamReport as f, classifyEuAiRisk as g, euAiActReport as h, redTeamReport as i, renderMarkdown as j, soc2Report as k, summarize as l, nistAiRmfReport as n, redTeamDataset as r, scoreRedTeamOutput as s, toolNamesForRun as t };

package/dist/{index-BTqhGHJT.d.ts → index-ClMxVqe_.d.ts}

@@ -1,4 +1,4 @@
-import { a as RunSplitTag } from './run-record-BfX5y68A.js';
+import { a as RunSplitTag } from './run-record-YinVdFwu.js';
 
 /**
  * Shared types for the reference benchmark wrappers under