@tangle-network/agent-eval 0.32.0 → 0.33.1

package/dist/judge-calibration-DilmB3Ml.d.ts

@@ -0,0 +1,142 @@
+/**
+ * Judge calibration — measure judge quality against human gold + bias.
+ *
+ * Workflow:
+ *   1. Build a golden set: {itemId, humanScore}[].
+ *   2. Run candidate judges; each produces {itemId, score}.
+ *   3. `calibrateJudge(golden, candidate)` reports κ + Pearson + MAE.
+ *   4. `calibrateJudgeContinuous(golden, candidate)` adds quadratic-weighted
+ *      κ over the un-rounded [0,1] scores plus ICC(2,1), Pearson, Spearman,
+ *      and bootstrap CIs — use this for fine-grained judges where rounding
+ *      to int discards information (e.g. 0.78 vs 0.81 both round to 1 and
+ *      look "perfectly agreed" to integer κ).
+ *   5. Run bias probes (positional, verbosity, self-preference) to
+ *      detect systematic score inflation.
+ *   6. For N≥2 judges on the same items, `continuousAgreement(scores)`
+ *      reports ICC(2,1) + κ_w + Pearson + Spearman with bootstrap CIs.
+ *
+ * Returns actionable diagnostics, not a single number. Consumers then
+ * decide whether to trust the judge, retrain it, or add a tie-breaker.
+ */
+interface GoldenItem {
+    itemId: string;
+    humanScore: number;
+    /** Optional group used for per-group bias audits (e.g. model-of-output family). */
+    group?: string;
+}
+interface CandidateScore {
+    itemId: string;
+    score: number;
+    /** Optional — enables positional-bias analysis (did order matter?). */
+    positionOfAInput?: 'first' | 'second';
+}
+interface CalibrationResult {
+    n: number;
+    pearson: number;
+    /** Cohen's κ with quadratic weights over integer-rounded scores. */
+    kappa: number;
+    /** Mean absolute error vs human. */
+    mae: number;
+    /** Worst-5 miscalibrations (largest |judge - human|). */
+    worstItems: Array<{
+        itemId: string;
+        judge: number;
+        human: number;
+        delta: number;
+    }>;
+}
+declare function calibrateJudge(golden: GoldenItem[], candidate: CandidateScore[]): CalibrationResult;
+interface PositionalBiasResult {
+    /**
+     * Score delta (first-position - second-position) averaged across items
+     * presented in both positions. Non-zero = positional bias.
+     */
+    avgDelta: number;
+    n: number;
+}
+/**
+ * Feed the same items to the judge twice with A/B swapped and pass all
+ * results here. Items that don't appear in both positions are ignored.
+ */
+declare function positionalBias(scores: CandidateScore[]): PositionalBiasResult;
+interface VerbosityBiasResult {
+    /** Pearson correlation between output length and score. Strong positive = verbosity bias. */
+    pearson: number;
+    n: number;
+}
+declare function verbosityBias(samples: Array<{
+    outputLen: number;
+    score: number;
+}>): VerbosityBiasResult;
+interface SelfPreferenceResult {
+    /** Mean judge score when judge's family matches output's family. */
+    inFamilyMean: number;
+    outOfFamilyMean: number;
+    deltaMean: number;
+    n: number;
+}
+/**
+ * Pass the same scenarios scored with judge-model X grading outputs from
+ * model X (in-family) and model Y (out-of-family). Non-zero delta
+ * indicates self-preference.
+ */
+declare function selfPreference(samples: Array<{
+    score: number;
+    inFamily: boolean;
+}>): SelfPreferenceResult;
+interface ContinuousAgreement {
+    /** Cohen's κ_w with quadratic weights, computed on raw [0,1] scores. */
+    weightedKappa: number;
+    /** ICC(2,1): two-way random effects, absolute agreement, single rater. */
+    icc: number;
+    /** Pearson product-moment correlation (averaged over rater pairs if N>2). */
+    pearson: number;
+    /** Spearman rank correlation (averaged over rater pairs if N>2). */
+    spearman: number;
+    /** 95% bootstrap percentile CIs over items. */
+    ci: {
+        icc: [number, number];
+        weightedKappa: [number, number];
+    };
+    /** Number of complete items (no NaN across raters). */
+    n: number;
+    /** Number of raters. */
+    raters: number;
+}
+interface ContinuousAgreementOptions {
+    /** Bootstrap iterations. Default 1000. Set to 0 to skip CIs (CI = [NaN, NaN]). */
+    bootstrap?: number;
+    /** κ weighting scheme. Default 'quadratic'. */
+    weights?: 'linear' | 'quadratic';
+    /** PRNG seed for reproducible bootstrap. Default 0xC0FFEE. */
+    seed?: number;
+    /** Confidence level for percentile CI. Default 0.95. */
+    ciLevel?: number;
+}
+/**
+ * Inter-rater agreement on continuous (typically [0,1]) scores.
+ *
+ * `scores` has shape [n_items][n_raters]. Rows with any non-finite entry
+ * are dropped. Returns NaN metrics if fewer than 2 raters or 2 complete
+ * items remain.
+ */
+declare function continuousAgreement(scores: number[][], opts?: ContinuousAgreementOptions): ContinuousAgreement;
+interface ContinuousCalibrationResult extends CalibrationResult {
+    /** Cohen's κ_w computed on raw (un-rounded) scores. */
+    weightedKappaContinuous: number;
+    /** ICC(2,1) treating golden + candidate as two raters. */
+    icc: number;
+    spearman: number;
+    ci: {
+        icc: [number, number];
+        weightedKappa: [number, number];
+    };
+}
+/**
+ * Drop-in superset of `calibrateJudge` that adds continuous-value
+ * agreement metrics. The old fields (n, pearson, kappa, mae, worstItems)
+ * are preserved unchanged so existing callers continue to work.
+ */
+declare function calibrateJudgeContinuous(golden: GoldenItem[], candidate: CandidateScore[], opts?: ContinuousAgreementOptions): ContinuousCalibrationResult;
+
+export { type ContinuousAgreementOptions as C, type GoldenItem as G, type PositionalBiasResult as P, type SelfPreferenceResult as S, type VerbosityBiasResult as V, type ContinuousAgreement as a, type CalibrationResult as b, type CandidateScore as c, type ContinuousCalibrationResult as d, calibrateJudge as e, calibrateJudgeContinuous as f, continuousAgreement as g, positionalBias as p, selfPreference as s, verbosityBias as v };

package/dist/meta-eval/index.d.ts

@@ -1,8 +1,8 @@
 import { R as Run, T as TraceStore } from '../store-Db2Bv8Cf.js';
 import { a as OutcomeFilter, O as OutcomeStore } from '../outcome-store-D6KWmYvj.js';
 export { D as DeploymentOutcome, F as FileSystemOutcomeStore, b as FileSystemOutcomeStoreOptions, I as InMemoryOutcomeStore } from '../outcome-store-D6KWmYvj.js';
-export { R as RubricOutcomePair, a as RubricPredictiveValidityInput, b as RubricPredictiveValidityReport, c as RubricRanking, r as rubricPredictiveValidity } from '../rubric-predictive-validity-CMHypZ_M.js';
-import '../run-record-BfX5y68A.js';
+export { R as RubricOutcomePair, a as RubricPredictiveValidityInput, b as RubricPredictiveValidityReport, c as RubricRanking, r as rubricPredictiveValidity } from '../rubric-predictive-validity-BvaNwfBE.js';
+import '../run-record-YinVdFwu.js';
 import '../errors-mje_cKOs.js';
 
 /**

package/dist/openapi.json

@@ -2,7 +2,7 @@
   "openapi": "3.1.0",
   "info": {
     "title": "@tangle-network/agent-eval — wire protocol",
-    "version": "0.32.0",
+    "version": "0.33.1",
     "description": "HTTP and stdio RPC interface to agent-eval. The TypeScript runtime is the source of truth; this spec is the contract that cross-language clients (Python, Rust, Go) generate from.\n\nWire-protocol version: 1.0.0. Bumps on breaking changes to request/response schemas.",
     "contact": {
       "name": "Tangle Network",

package/dist/optimization.d.ts

@@ -1,10 +1,10 @@
-export { C as CallbackResearcher, a as CallbackResearcherOptions, b as CampaignFactoryParams, c as CampaignIntegrityPolicy, d as CampaignRunContext, e as CampaignRunOutcome, f as CampaignRunner, g as CampaignScenario, h as CampaignVariant, E as EvalCampaignOptions, i as EvalCampaignResult, j as ExperimentPlan, k as ExperimentResult, F as FailedRun, l as FailureMode, N as NoopResearcher, R as Researcher, S as SteeringChange, r as runEvalCampaign } from './researcher-BRHa5Jxo.js';
+export { C as CallbackResearcher, a as CallbackResearcherOptions, b as CampaignFactoryParams, c as CampaignIntegrityPolicy, d as CampaignRunContext, e as CampaignRunOutcome, f as CampaignRunner, g as CampaignScenario, h as CampaignVariant, E as EvalCampaignOptions, i as EvalCampaignResult, j as ExperimentPlan, k as ExperimentResult, F as FailedRun, l as FailureMode, N as NoopResearcher, R as Researcher, S as SteeringChange, r as runEvalCampaign } from './researcher-CfnL3HEb.js';
 export { F as FeedbackArtifactType, a as FeedbackAttempt, b as FeedbackLabel, c as FeedbackLabelKind, d as FeedbackLabelSource, e as FeedbackOptimizerRow, f as FeedbackOutcome, g as FeedbackReplayAdapter, h as FeedbackReplayResult, i as FeedbackSeverity, j as FeedbackSplitPolicy, k as FeedbackTask, l as FeedbackTrajectory, m as FeedbackTrajectoryFilter, n as FeedbackTrajectoryStore, o as FileSystemFeedbackTrajectoryStore, I as InMemoryFeedbackTrajectoryStore, P as PreferenceMemoryEntry, p as ProposedSideEffect, q as assignFeedbackSplit, r as controlRunToFeedbackTrajectory, s as createFeedbackTrajectory, t as feedbackTrajectoriesToDatasetScenarios, u as feedbackTrajectoriesToOptimizerRows, v as feedbackTrajectoryToDatasetScenario, w as feedbackTrajectoryToOptimizerRow, x as parseFeedbackTrajectoriesJsonl, y as renderPreferenceMemoryMarkdown, z as replayFeedbackTrajectories, A as replayFeedbackTrajectory, B as serializeFeedbackTrajectoriesJsonl, C as summarizePreferenceMemory, D as withAssignedFeedbackSplit } from './feedback-trajectory-iATEAHmc.js';
-export { A as ActionableSideInfo, a as AsiSeverity, D as DEFAULT_MUTATION_PRIMITIVES, E as EvolvableVariant, G as GenerationReport, I as InMemoryTrialCache, M as MultiShotGateConfig, b as MultiShotGateResult, c as MultiShotMutateAdapter, d as MultiShotOptimizationConfig, e as MultiShotOptimizationResult, f as MultiShotRun, g as MultiShotRunInput, h as MultiShotRunner, i as MultiShotScore, j as MultiShotScorer, k as MultiShotSplit, l as MultiShotTrace, m as MultiShotTrialResult, n as MultiShotVariant, o as MutateAdapter, P as PromptEvolutionConfig, p as PromptEvolutionEvent, q as PromptEvolutionResult, R as ReflectionContext, r as ReflectionProposal, S as ScenarioAggregate, s as ScoreAdapter, T as TrialCache, t as TrialResult, u as TrialTrace, V as VariantAggregate, v as buildReflectionPrompt, w as defaultMultiShotObjectives, x as parseReflectionResponse, y as runMultiShotOptimization, z as runPromptEvolution, B as trialTraceFromMultiShotTrial } from './summary-report-D7AQS7eB.js';
+export { A as ActionableSideInfo, a as AsiSeverity, D as DEFAULT_MUTATION_PRIMITIVES, E as EvolvableVariant, G as GenerationReport, I as InMemoryTrialCache, M as MultiShotGateConfig, b as MultiShotGateResult, c as MultiShotMutateAdapter, d as MultiShotOptimizationConfig, e as MultiShotOptimizationResult, f as MultiShotRun, g as MultiShotRunInput, h as MultiShotRunner, i as MultiShotScore, j as MultiShotScorer, k as MultiShotSplit, l as MultiShotTrace, m as MultiShotTrialResult, n as MultiShotVariant, o as MutateAdapter, P as PromptEvolutionConfig, p as PromptEvolutionEvent, q as PromptEvolutionResult, R as ReflectionContext, r as ReflectionProposal, S as ScenarioAggregate, s as ScoreAdapter, T as TrialCache, t as TrialResult, u as TrialTrace, V as VariantAggregate, v as buildReflectionPrompt, w as defaultMultiShotObjectives, x as parseReflectionResponse, y as runMultiShotOptimization, z as runPromptEvolution, B as trialTraceFromMultiShotTrial } from './summary-report-BPJVzIeW.js';
+import './run-record-YinVdFwu.js';
 import './errors-mje_cKOs.js';
 import './integrity-DYR5gWlb.js';
 import './store-Db2Bv8Cf.js';
-import './run-record-BfX5y68A.js';
 import './emitter-DP_cSSiw.js';
 import './control-runtime-BZ_lVLYW.js';
 import './dataset-ueRVTUoY.js';

package/dist/optimization.js

@@ -25,14 +25,14 @@ import {
   summarizePreferenceMemory,
   trialTraceFromMultiShotTrial,
   withAssignedFeedbackSplit
-} from "./chunk-B73G44OH.js";
-import "./chunk-ZN2CMQIW.js";
+} from "./chunk-FT3IAMQR.js";
 import {
   runEvalCampaign
-} from "./chunk-DTEJNZYK.js";
-import "./chunk-M6RZ5LJN.js";
-import "./chunk-CXJOVDJR.js";
-import "./chunk-4L3WJXQJ.js";
+} from "./chunk-SQYRO3BT.js";
+import "./chunk-VXNVVBZO.js";
+import "./chunk-DCZXFOQN.js";
+import "./chunk-TQL7BAOY.js";
+import "./chunk-KHZRNY3F.js";
 import "./chunk-UBPIXOC4.js";
 import "./chunk-PC4UYEBM.js";
 import "./chunk-TVVP3ZZQ.js";

package/dist/pipelines/index.js

@@ -2,13 +2,13 @@ import {
   compareToBaseline,
   computeToolUseMetrics,
   failureClusterView
-} from "../chunk-GVQT44CS.js";
+} from "../chunk-KE7TDJUO.js";
 import {
   buildTrajectory
 } from "../chunk-RZTMDUO7.js";
 import {
   interRaterReliability
-} from "../chunk-4L3WJXQJ.js";
+} from "../chunk-KHZRNY3F.js";
 import {
   aggregateLlm,
   argHash,