@tangle-network/agent-eval 0.27.0 → 0.27.2

package/dist/index.js

@@ -11,7 +11,7 @@ import {
   failureClusterView,
   iqr,
   welchsTTest
-} from "./chunk-JLZQWFV3.js";
+} from "./chunk-K33INZHH.js";
 import {
   exportTrainingData,
   toNdjson
@@ -28,7 +28,7 @@ import {
   pytestTestParser,
   runTestGradedScenario,
   vitestTestParser
-} from "./chunk-OWLAAMME.js";
+} from "./chunk-QHF6EQKK.js";
 import {
   classifyEuAiRisk,
   euAiActReport,
@@ -43,7 +43,7 @@ import {
   knowledgeReadinessTracePayload,
   scoreKnowledgeReadiness,
   userQuestionsForKnowledgeGaps
-} from "./chunk-WWYCWKUM.js";
+} from "./chunk-3CKU6VGU.js";
 import {
   controlFailureClassFromVerification,
   controlRunToRunRecord,
@@ -54,7 +54,7 @@ import {
   runProposeReview,
   runProposeReviewAsControlLoop,
   scoreFromEvals
-} from "./chunk-ZN274SWR.js";
+} from "./chunk-PALJO75S.js";
 import {
   allCriticalPassed,
   objectiveEval,
@@ -62,7 +62,7 @@ import {
   stopOnNoProgress,
   stopOnRepeatedAction,
   subjectiveEval
-} from "./chunk-LSH4MMOZ.js";
+} from "./chunk-NCRFYPS3.js";
 import {
   CallbackResearcher,
   DEFAULT_MUTATION_PRIMITIVES,
@@ -96,7 +96,7 @@ import {
   summarizePreferenceMemory,
   trialTraceFromMultiShotTrial,
   withAssignedFeedbackSplit
-} from "./chunk-WHZMVFUV.js";
+} from "./chunk-SZSBQUIJ.js";
 import {
   RunRecordValidationError,
   isRunRecord,
@@ -111,10 +111,10 @@ import {
   judgeReplayGate,
   releaseTraceEvidenceFromMultiShotTrials,
   renderReleaseReport
-} from "./chunk-RAF443UI.js";
+} from "./chunk-DBIGN5MJ.js";
 import {
   runEvalCampaign
-} from "./chunk-SESZDQPX.js";
+} from "./chunk-RUI6SIHY.js";
 import {
   LlmCallError,
   LlmClient,
@@ -128,7 +128,7 @@ import {
 import {
   evaluateInterimReleaseConfidence,
   pairedEvalueSequence
-} from "./chunk-NU65VQ7M.js";
+} from "./chunk-MAZ26DC7.js";
 import {
   RESEARCH_REPORT_HARD_PAIR_FLOOR,
   benjaminiHochberg,
@@ -141,18 +141,26 @@ import {
   requiredSampleSize,
   researchReport,
   summaryTable
-} from "./chunk-2A5XJB43.js";
+} from "./chunk-5AKPEK5L.js";
 import {
+  calibrateJudge,
+  calibrateJudgeContinuous,
   cohensD,
   confidenceInterval,
+  continuousAgreement,
+  corpusInterRaterAgreement,
+  corpusInterRaterAgreementFromJudgeScores,
   interRaterReliability,
   mannWhitneyU,
   normalizeScores,
   pairedTTest,
   partialCredit,
+  positionalBias,
+  selfPreference,
+  verbosityBias,
   weightedMean,
   wilcoxonSignedRank
-} from "./chunk-I4MBDTY5.js";
+} from "./chunk-R5UQJNKC.js";
 import {
   DEFAULT_REDACTION_RULES,
   FileSystemTraceStore,
@@ -166,7 +174,7 @@ import {
   iterateRawCalls,
   redactString,
   redactValue
-} from "./chunk-K2TPS5LB.js";
+} from "./chunk-4U4BKCXK.js";
 import {
   aggregateLlm,
   argHash,
@@ -208,7 +216,7 @@ import {
   hashJson,
   signManifest,
   verifyManifest
-} from "./chunk-4F5DQN55.js";
+} from "./chunk-VSMTAMNK.js";
 import {
   AgentEvalError,
   CaptureIntegrityError,
@@ -4956,326 +4964,6 @@ function seededShuffle(items, seed) {
   return out;
 }
 
-// src/judge-calibration.ts
-function calibrateJudge(golden, candidate) {
-  const map = /* @__PURE__ */ new Map();
-  for (const g of golden) map.set(g.itemId, { h: g.humanScore, j: NaN });
-  for (const c of candidate) {
-    const entry = map.get(c.itemId);
-    if (entry) entry.j = c.score;
-  }
-  const common = [...map.values()].filter((v) => Number.isFinite(v.j));
-  const n = common.length;
-  if (n < 2) {
-    return { n, pearson: NaN, kappa: NaN, mae: NaN, worstItems: [] };
-  }
-  const humans = common.map((c) => c.h);
-  const judges = common.map((c) => c.j);
-  const pearson = pearsonR(humans, judges);
-  const kappa = weightedKappa(humans.map(Math.round), judges.map(Math.round));
-  const absDiffs = common.map((c) => Math.abs(c.j - c.h));
-  const mae = absDiffs.reduce((a, b) => a + b, 0) / n;
-  const worst2 = [...map.entries()].filter(([, v]) => Number.isFinite(v.j)).map(([itemId, v]) => ({ itemId, judge: v.j, human: v.h, delta: Math.abs(v.j - v.h) })).sort((a, b) => b.delta - a.delta).slice(0, 5);
-  return { n, pearson, kappa, mae, worstItems: worst2 };
-}
-function positionalBias(scores) {
-  const pairs = /* @__PURE__ */ new Map();
-  for (const s of scores) {
-    const slot = pairs.get(s.itemId) ?? {};
-    if (s.positionOfAInput === "first") slot.first = s.score;
-    else if (s.positionOfAInput === "second") slot.second = s.score;
-    pairs.set(s.itemId, slot);
-  }
-  const deltas = [];
-  for (const { first, second } of pairs.values()) {
-    if (first !== void 0 && second !== void 0) deltas.push(first - second);
-  }
-  if (deltas.length === 0) return { avgDelta: 0, n: 0 };
-  return { avgDelta: deltas.reduce((a, b) => a + b, 0) / deltas.length, n: deltas.length };
-}
-function verbosityBias(samples) {
-  const n = samples.length;
-  if (n < 3) return { pearson: NaN, n };
-  return {
-    pearson: pearsonR(
-      samples.map((s) => s.outputLen),
-      samples.map((s) => s.score)
-    ),
-    n
-  };
-}
-function selfPreference(samples) {
-  const inF = samples.filter((s) => s.inFamily).map((s) => s.score);
-  const outF = samples.filter((s) => !s.inFamily).map((s) => s.score);
-  if (inF.length === 0 || outF.length === 0)
-    return { inFamilyMean: 0, outOfFamilyMean: 0, deltaMean: 0, n: 0 };
-  const inMean = inF.reduce((a, b) => a + b, 0) / inF.length;
-  const outMean = outF.reduce((a, b) => a + b, 0) / outF.length;
-  return {
-    inFamilyMean: inMean,
-    outOfFamilyMean: outMean,
-    deltaMean: inMean - outMean,
-    n: samples.length
-  };
-}
-function pearsonR(a, b) {
-  if (a.length !== b.length || a.length < 2) return NaN;
-  const mA = a.reduce((s, v) => s + v, 0) / a.length;
-  const mB = b.reduce((s, v) => s + v, 0) / b.length;
-  let num = 0, dA = 0, dB = 0;
-  for (let i = 0; i < a.length; i++) {
-    const da = a[i] - mA;
-    const db = b[i] - mB;
-    num += da * db;
-    dA += da * da;
-    dB += db * db;
-  }
-  if (dA === 0 || dB === 0) return dA === 0 && dB === 0 ? 1 : 0;
-  return num / Math.sqrt(dA * dB);
-}
-function weightedKappa(a, b) {
-  if (a.length !== b.length || a.length === 0) return NaN;
-  const min = Math.min(...a, ...b);
-  const max = Math.max(...a, ...b);
-  const K = max - min + 1;
-  if (K < 2) return 1;
-  const observed = Array.from({ length: K }, () => new Array(K).fill(0));
-  const rowMarg = new Array(K).fill(0);
-  const colMarg = new Array(K).fill(0);
-  for (let i = 0; i < a.length; i++) {
-    const ai = a[i] - min;
-    const bi = b[i] - min;
-    const row = observed[ai];
-    row[bi] = (row[bi] ?? 0) + 1;
-    rowMarg[ai]++;
-    colMarg[bi]++;
-  }
-  let num = 0;
-  let den = 0;
-  for (let i = 0; i < K; i++) {
-    for (let j = 0; j < K; j++) {
-      const w = (i - j) ** 2 / (K - 1) ** 2;
-      const expected = rowMarg[i] * colMarg[j] / a.length;
-      num += w * observed[i][j];
-      den += w * expected;
-    }
-  }
-  if (den === 0) return 1;
-  return 1 - num / den;
-}
-function continuousAgreement(scores, opts = {}) {
-  const bootstrap = opts.bootstrap ?? 1e3;
-  const weights = opts.weights ?? "quadratic";
-  const seed = opts.seed ?? 12648430;
-  const ciLevel = opts.ciLevel ?? 0.95;
-  const matrix = scores.filter((row) => row.length >= 2 && row.every((v) => Number.isFinite(v)));
-  const raters = matrix[0]?.length ?? 0;
-  const clean = matrix.filter((row) => row.length === raters);
-  const nClean = clean.length;
-  if (nClean < 2 || raters < 2) {
-    return {
-      weightedKappa: NaN,
-      icc: NaN,
-      pearson: NaN,
-      spearman: NaN,
-      ci: { icc: [NaN, NaN], weightedKappa: [NaN, NaN] },
-      n: nClean,
-      raters
-    };
-  }
-  const kappa = continuousWeightedKappa(clean, weights);
-  const icc = icc21(clean);
-  const pearson = avgPairwise(clean, pearsonR);
-  const spearman = avgPairwise(clean, spearmanR);
-  const ciIcc = [NaN, NaN];
-  const ciKappa = [NaN, NaN];
-  if (bootstrap > 0) {
-    const rng = mulberry32(seed);
-    const iccs = [];
-    const kappas = [];
-    for (let b = 0; b < bootstrap; b++) {
-      const sample = new Array(nClean);
-      for (let i = 0; i < nClean; i++) {
-        sample[i] = clean[Math.floor(rng() * nClean)];
-      }
-      const iccB = icc21(sample);
-      const kB = continuousWeightedKappa(sample, weights);
-      if (Number.isFinite(iccB)) iccs.push(iccB);
-      if (Number.isFinite(kB)) kappas.push(kB);
-    }
-    const [lo, hi] = percentileBounds(ciLevel);
-    if (iccs.length > 0) {
-      iccs.sort((a, b) => a - b);
-      ciIcc[0] = quantile(iccs, lo);
-      ciIcc[1] = quantile(iccs, hi);
-    }
-    if (kappas.length > 0) {
-      kappas.sort((a, b) => a - b);
-      ciKappa[0] = quantile(kappas, lo);
-      ciKappa[1] = quantile(kappas, hi);
-    }
-  }
-  return {
-    weightedKappa: kappa,
-    icc,
-    pearson,
-    spearman,
-    ci: { icc: ciIcc, weightedKappa: ciKappa },
-    n: nClean,
-    raters
-  };
-}
-function calibrateJudgeContinuous(golden, candidate, opts = {}) {
-  const base = calibrateJudge(golden, candidate);
-  const map = /* @__PURE__ */ new Map();
-  for (const g of golden) map.set(g.itemId, { h: g.humanScore, j: NaN });
-  for (const c of candidate) {
-    const entry = map.get(c.itemId);
-    if (entry) entry.j = c.score;
-  }
-  const rows = [];
-  for (const v of map.values()) {
-    if (Number.isFinite(v.j)) rows.push([v.h, v.j]);
-  }
-  const agreement = continuousAgreement(rows, opts);
-  return {
-    ...base,
-    weightedKappaContinuous: agreement.weightedKappa,
-    icc: agreement.icc,
-    spearman: agreement.spearman,
-    ci: agreement.ci
-  };
-}
-function continuousWeightedKappa(rows, scheme) {
-  if (rows.length === 0) return NaN;
-  const raters = rows[0].length;
-  if (raters < 2) return NaN;
-  const wFn = scheme === "linear" ? (x, y) => Math.abs(x - y) : (x, y) => (x - y) ** 2;
-  let sum2 = 0;
-  let pairs = 0;
-  for (let r1 = 0; r1 < raters; r1++) {
-    for (let r2 = r1 + 1; r2 < raters; r2++) {
-      const a = rows.map((row) => row[r1]);
-      const b = rows.map((row) => row[r2]);
-      const n = a.length;
-      let obs = 0;
-      for (let i = 0; i < n; i++) obs += wFn(a[i], b[i]);
-      obs /= n;
-      let exp = 0;
-      for (let i = 0; i < n; i++) {
-        for (let j = 0; j < n; j++) exp += wFn(a[i], b[j]);
-      }
-      exp /= n * n;
-      if (exp === 0) {
-        sum2 += obs === 0 ? 1 : 0;
-      } else {
-        sum2 += 1 - obs / exp;
-      }
-      pairs++;
-    }
-  }
-  return pairs === 0 ? NaN : sum2 / pairs;
-}
-function icc21(rows) {
-  const n = rows.length;
-  if (n < 2) return NaN;
-  const k = rows[0].length;
-  if (k < 2) return NaN;
-  const rowMeans = rows.map((row) => row.reduce((s, v) => s + v, 0) / k);
-  const colMeans = new Array(k).fill(0);
-  for (let j = 0; j < k; j++) {
-    let s = 0;
-    for (let i = 0; i < n; i++) s += rows[i][j];
-    colMeans[j] = s / n;
-  }
-  let grand = 0;
-  for (let i = 0; i < n; i++) grand += rowMeans[i];
-  grand /= n;
-  let ssR = 0;
-  for (let i = 0; i < n; i++) ssR += (rowMeans[i] - grand) ** 2;
-  ssR *= k;
-  let ssC = 0;
-  for (let j = 0; j < k; j++) ssC += (colMeans[j] - grand) ** 2;
-  ssC *= n;
-  let ssT = 0;
-  for (let i = 0; i < n; i++) {
-    for (let j = 0; j < k; j++) ssT += (rows[i][j] - grand) ** 2;
-  }
-  const ssE = ssT - ssR - ssC;
-  const dfR = n - 1;
-  const dfC = k - 1;
-  const dfE = (n - 1) * (k - 1);
-  const msR = ssR / dfR;
-  const msC = ssC / dfC;
-  const msE = dfE > 0 ? ssE / dfE : 0;
-  const denom = msR + (k - 1) * msE + k * (msC - msE) / n;
-  if (denom === 0) {
-    return msR === 0 && msE === 0 ? 1 : 0;
-  }
-  return (msR - msE) / denom;
-}
-function avgPairwise(rows, fn) {
-  const k = rows[0]?.length ?? 0;
-  if (k < 2) return NaN;
-  let sum2 = 0;
-  let pairs = 0;
-  for (let i = 0; i < k; i++) {
-    for (let j = i + 1; j < k; j++) {
-      const a = rows.map((row) => row[i]);
-      const b = rows.map((row) => row[j]);
-      const r = fn(a, b);
-      if (Number.isFinite(r)) {
-        sum2 += r;
-        pairs++;
-      }
-    }
-  }
-  return pairs === 0 ? NaN : sum2 / pairs;
-}
-function spearmanR(a, b) {
-  if (a.length !== b.length || a.length < 2) return NaN;
-  return pearsonR(rankWithTies(a), rankWithTies(b));
-}
-function rankWithTies(xs) {
-  const n = xs.length;
-  const indexed = xs.map((v, i2) => ({ v, i: i2 }));
-  indexed.sort((x, y) => x.v - y.v);
-  const ranks = new Array(n).fill(0);
-  let i = 0;
-  while (i < n) {
-    let j = i;
-    while (j + 1 < n && indexed[j + 1].v === indexed[i].v) j++;
-    const avg = (i + j) / 2 + 1;
-    for (let k = i; k <= j; k++) ranks[indexed[k].i] = avg;
-    i = j + 1;
-  }
-  return ranks;
-}
-function mulberry32(seed) {
-  let a = seed >>> 0;
-  return () => {
-    a = a + 1831565813 >>> 0;
-    let t = a;
-    t = Math.imul(t ^ t >>> 15, t | 1);
-    t ^= t + Math.imul(t ^ t >>> 7, t | 61);
-    return ((t ^ t >>> 14) >>> 0) / 4294967296;
-  };
-}
-function percentileBounds(ciLevel) {
-  const tail = (1 - ciLevel) / 2;
-  return [tail, 1 - tail];
-}
-function quantile(sorted, q) {
-  if (sorted.length === 0) return NaN;
-  if (sorted.length === 1) return sorted[0];
-  const pos = q * (sorted.length - 1);
-  const lo = Math.floor(pos);
-  const hi = Math.ceil(pos);
-  if (lo === hi) return sorted[lo];
-  const frac = pos - lo;
-  return sorted[lo] * (1 - frac) + sorted[hi] * frac;
-}
-
 // src/observability.ts
 async function toLangfuseEnvelope(store, runId) {
   const run = await store.getRun(runId);
@@ -6077,7 +5765,7 @@ async function proposeSynthesisTargets(dataset, traceStore, options = {}) {
     runCountByScenario.set(r.scenarioId, (runCountByScenario.get(r.scenarioId) ?? 0) + 1);
   }
   const runCounts = [...runCountByScenario.values()];
-  const p25 = runCounts.length > 0 ? quantile2(runCounts, 0.25) : 0;
+  const p25 = runCounts.length > 0 ? quantile(runCounts, 0.25) : 0;
   for (const s of scenarios) {
     const count = runCountByScenario.get(s.id) ?? 0;
     if (count <= p25 && count < 3) {
@@ -6131,7 +5819,7 @@ async function proposeSynthesisTargets(dataset, traceStore, options = {}) {
   }
   return targets.sort((a, b) => b.priority - a.priority).slice(0, topK);
 }
-function quantile2(xs, p) {
+function quantile(xs, p) {
   const sorted = [...xs].sort((a, b) => a - b);
   const idx = p * (sorted.length - 1);
   const lo = Math.floor(idx);
@@ -9446,6 +9134,8 @@ export {
   controlFailureClassFromVerification,
   controlRunToFeedbackTrajectory,
   controlRunToRunRecord,
+  corpusInterRaterAgreement,
+  corpusInterRaterAgreementFromJudgeScores,
   createAntiSlopJudge,
   createCompositeMutator,
   createCustomJudge,