@tangle-network/agent-eval 0.86.0 → 0.89.0

package/dist/rl.js

@@ -1,22 +1,22 @@
+import {
+  detectRewardHacking,
+  extractVerifiableReward,
+  extractVerifiableRewardsFromRecords,
+  filterDeterministicallyRewarded
+} from "./chunk-5HRORJQY.js";
 import {
   doublyRobust,
   inverseProbabilityWeighting,
   offPolicyEstimateAll,
   selfNormalizedImportanceWeighting
 } from "./chunk-4DIJWVUT.js";
-import {
-  detectRewardHacking,
-  extractVerifiableReward,
-  extractVerifiableRewardsFromRecords,
-  filterDeterministicallyRewarded
-} from "./chunk-YV7J7X5N.js";
 import {
   FileSystemOutcomeStore,
   InMemoryOutcomeStore
 } from "./chunk-3RF76KTD.js";
 import {
   runEvalCampaign
-} from "./chunk-WJL2NJXN.js";
+} from "./chunk-GSH6QNNS.js";
 import "./chunk-CVVHBFGN.js";
 import {
   rubricPredictiveValidity
@@ -24,11 +24,16 @@ import {
 import {
   evaluateInterimReleaseConfidence
 } from "./chunk-MAZ26DC7.js";
-import "./chunk-Z7VFTS2J.js";
+import "./chunk-CY6U5S3X.js";
 import {
   benjaminiHochberg,
   wilcoxonSignedRank
-} from "./chunk-IDVBLYCY.js";
+} from "./chunk-LMZQ2Z4U.js";
+import {
+  observationsFromRunRecords,
+  thompsonCurriculum,
+  varianceBasedCurriculum
+} from "./chunk-VZSRQ272.js";
 import "./chunk-SBCB6VZY.js";
 import "./chunk-PC4UYEBM.js";
 import "./chunk-KWRRMR3J.js";
@@ -612,149 +617,6 @@ function buildPairwiseFromCampaign(input) {
   return outcomes;
 }
 
-// src/rl/active-curriculum.ts
-function varianceBasedCurriculum(observations, candidateCells, opts) {
-  const variancePrior = opts.variancePrior ?? 0.05;
-  const floor = opts.floorPerCell ?? 1;
-  const budget = opts.budget;
-  const grouped = /* @__PURE__ */ new Map();
-  for (const o of observations) {
-    const k = `${o.variantId}::${o.scenarioId}`;
-    const arr = grouped.get(k) ?? [];
-    arr.push(o.score);
-    grouped.set(k, arr);
-  }
-  const cellStats = candidateCells.map((c) => {
-    const k = `${c.variantId}::${c.scenarioId}`;
-    const samples = grouped.get(k) ?? [];
-    const n = samples.length;
-    const mean = n === 0 ? 0.5 : samples.reduce((s, v) => s + v, 0) / n;
-    const variance = n < 2 ? variancePrior : samples.reduce((s, v) => s + (v - mean) ** 2, 0) / (n - 1) + variancePrior;
-    const weight = Math.sqrt(variance) + 1 / Math.sqrt(Math.max(1, n));
-    return { variantId: c.variantId, scenarioId: c.scenarioId, n, mean, variance, weight };
-  });
-  const floorTotal = floor * cellStats.length;
-  if (floorTotal >= budget) {
-    const each = Math.max(1, Math.floor(budget / Math.max(1, cellStats.length)));
-    return cellStats.map((c) => ({
-      variantId: c.variantId,
-      scenarioId: c.scenarioId,
-      count: each,
-      reason: `floor allocation (budget tight; n=${c.n})`
-    }));
-  }
-  const remaining = budget - floorTotal;
-  const totalWeight = cellStats.reduce((s, c) => s + c.weight, 0);
-  return cellStats.map((c) => {
-    const proportional = totalWeight === 0 ? 0 : Math.round(c.weight / totalWeight * remaining);
-    return {
-      variantId: c.variantId,
-      scenarioId: c.scenarioId,
-      count: floor + proportional,
-      reason: `variance ${c.variance.toFixed(3)} (n=${c.n}, mean=${c.mean.toFixed(3)})`
-    };
-  });
-}
-function thompsonCurriculum(observations, candidateCells, opts) {
-  const threshold = opts.decisionThreshold ?? 0.5;
-  const alpha0 = opts.priorAlpha ?? 1;
-  const beta0 = opts.priorBeta ?? 1;
-  const rng = makeRng(opts.seed);
-  const grouped = /* @__PURE__ */ new Map();
-  for (const o of observations) {
-    const k = `${o.variantId}::${o.scenarioId}`;
-    const cur = grouped.get(k) ?? { passes: 0, failures: 0 };
-    const pass = o.pass ?? o.score >= threshold;
-    if (pass) cur.passes += 1;
-    else cur.failures += 1;
-    grouped.set(k, cur);
-  }
-  const stats = candidateCells.map((c) => {
-    const k = `${c.variantId}::${c.scenarioId}`;
-    const cur = grouped.get(k) ?? { passes: 0, failures: 0 };
-    const a = alpha0 + cur.passes;
-    const b = beta0 + cur.failures;
-    const sampled = sampleBeta(a, b, rng);
-    const distance = Math.abs(sampled - threshold);
-    const variance = a * b / ((a + b) ** 2 * (a + b + 1));
-    const sigma = Math.max(0.05, Math.sqrt(variance));
-    const weight = Math.exp(-((distance / sigma) ** 2));
-    return {
-      variantId: c.variantId,
-      scenarioId: c.scenarioId,
-      n: cur.passes + cur.failures,
-      sampled,
-      sigma,
-      weight,
-      a,
-      b
-    };
-  });
-  const totalWeight = stats.reduce((s, c) => s + c.weight, 0);
-  return stats.map((c) => {
-    const proportional = totalWeight === 0 ? 0 : Math.round(c.weight / totalWeight * opts.budget);
-    return {
-      variantId: c.variantId,
-      scenarioId: c.scenarioId,
-      count: Math.max(0, proportional),
-      reason: `Beta(${c.a.toFixed(1)},${c.b.toFixed(1)}) sample=${c.sampled.toFixed(3)} (target ${threshold})`
-    };
-  });
-}
-function observationsFromRunRecords(runs, opts = {}) {
-  const threshold = opts.passThreshold ?? 0.5;
-  const useHoldout = opts.useHoldout ?? true;
-  const out = [];
-  for (const r of runs) {
-    if (!r.scenarioId) continue;
-    const score = useHoldout ? r.outcome.holdoutScore ?? r.outcome.searchScore : r.outcome.searchScore ?? r.outcome.holdoutScore;
-    if (typeof score !== "number" || !Number.isFinite(score)) continue;
-    out.push({
-      variantId: r.candidateId,
-      scenarioId: r.scenarioId,
-      score,
-      pass: score >= threshold
-    });
-  }
-  return out;
-}
-function makeRng(seed) {
-  if (seed === void 0) return Math.random;
-  let s = seed >>> 0;
-  return () => {
-    s = s + 1831565813 >>> 0;
-    let t = s;
-    t = Math.imul(t ^ t >>> 15, t | 1);
-    t ^= t + Math.imul(t ^ t >>> 7, t | 61);
-    return ((t ^ t >>> 14) >>> 0) / 4294967296;
-  };
-}
-function sampleBeta(alpha, beta, rng) {
-  const a = Math.max(1, alpha);
-  const b = Math.max(1, beta);
-  const x = sampleGamma(a, rng);
-  const y = sampleGamma(b, rng);
-  return x / (x + y);
-}
-function sampleGamma(shape, rng) {
-  const d = shape - 1 / 3;
-  const c = 1 / Math.sqrt(9 * d);
-  while (true) {
-    let x;
-    let v;
-    do {
-      const u1 = rng() || 1e-12;
-      const u2 = rng() || 1e-12;
-      x = Math.sqrt(-2 * Math.log(u1)) * Math.cos(2 * Math.PI * u2);
-      v = 1 + c * x;
-    } while (v <= 0);
-    v = v * v * v;
-    const u = rng();
-    if (u < 1 - 0.0331 * x ** 4) return d * v;
-    if (Math.log(u) < 0.5 * x * x + d * (1 - v + Math.log(v))) return d * v;
-  }
-}
-
 // src/rl/adaptation-eval.ts
 async function runAdaptationCurve(opts) {
   const ks = opts.ks ?? [0, 1, 2, 4, 8, 16];
@@ -809,7 +671,7 @@ async function runAdaptationCurve(opts) {
 function compareAdaptationCurves(a, b, opts = {}) {
   const conf = opts.confidence ?? 0.95;
   const resamples = opts.bootstrapResamples ?? 500;
-  const rng = makeRng2(opts.seed);
+  const rng = makeRng(opts.seed);
   const perK = [];
   for (const ap of a.points) {
     const bp = b.points.find((p) => p.k === ap.k);
@@ -841,7 +703,7 @@ function compareAdaptationCurves(a, b, opts = {}) {
 function firstPassK(curve, threshold = 0.5) {
   return curve.points.find((p) => p.passRate >= threshold)?.k ?? null;
 }
-function makeRng2(seed) {
+function makeRng(seed) {
   if (seed === void 0) return Math.random;
   let s = seed >>> 0;
   return () => {
@@ -1661,22 +1523,239 @@ function buildSummary(args) {
   }
   return lines.join(" | ");
 }
+
+// src/rl/sim-fidelity.ts
+var ABSENT_CATEGORY = "(absent)";
+var DEFAULT_MIN_N_PER_FEATURE = 20;
+var DEFAULT_QUANTILE_BUCKETS = 4;
+var REPRESENTATIVE_MIN_FIDELITY = 0.8;
+var TOP_SHIFT_COUNT = 5;
+var defaultBehaviorFeatures = (record) => {
+  const raw = record.outcome?.raw ?? {};
+  const toolErrors = finiteOrNull(raw.tool_errors);
+  const turnsAborted = finiteOrNull(raw.turns_aborted);
+  const completion = record.completion;
+  return {
+    score: finiteOrNull(record.outcome?.holdoutScore) ?? finiteOrNull(record.outcome?.searchScore),
+    failure_class: record.failureClass ?? null,
+    wall_ms: finiteOrNull(record.wallMs),
+    output_tokens: finiteOrNull(record.tokenUsage?.output),
+    turn_count: finiteOrNull(raw.turns_completed) ?? finiteOrNull(raw.assistant_messages),
+    tool_errors: toolErrors,
+    tool_error_recovery: toolErrorRecovery(toolErrors, turnsAborted, record.failureClass),
+    completion_length: typeof completion === "string" ? completion.length : null
+  };
+};
+function toolErrorRecovery(toolErrors, turnsAborted, failureClass) {
+  if (toolErrors === null) return null;
+  if (toolErrors === 0) return "no-tool-errors";
+  const failed = (turnsAborted ?? 0) > 0 || failureClass !== void 0 && failureClass !== "success";
+  return failed ? "unrecovered" : "recovered";
+}
+function finiteOrNull(value) {
+  return typeof value === "number" && Number.isFinite(value) ? value : null;
+}
+function jsDivergence(p, q) {
+  const keys = /* @__PURE__ */ new Set([...Object.keys(p), ...Object.keys(q)]);
+  if (keys.size === 0) {
+    throw new ValidationError("jsDivergence: both histograms are empty");
+  }
+  let pSum = 0;
+  let qSum = 0;
+  for (const key of keys) {
+    const pv = p[key] ?? 0;
+    const qv = q[key] ?? 0;
+    if (!Number.isFinite(pv) || !Number.isFinite(qv) || pv < 0 || qv < 0) {
+      throw new ValidationError(`jsDivergence: negative or non-finite count for category "${key}"`);
+    }
+    pSum += pv;
+    qSum += qv;
+  }
+  if (pSum === 0 || qSum === 0) {
+    throw new ValidationError("jsDivergence: a histogram with zero total mass has no distribution");
+  }
+  let divergence = 0;
+  for (const key of keys) {
+    const pp = (p[key] ?? 0) / pSum;
+    const qp = (q[key] ?? 0) / qSum;
+    const m = (pp + qp) / 2;
+    if (pp > 0) divergence += 0.5 * pp * Math.log2(pp / m);
+    if (qp > 0) divergence += 0.5 * qp * Math.log2(qp / m);
+  }
+  return Math.min(1, Math.max(0, divergence));
+}
+function quantileEdges(values, bucketCount = DEFAULT_QUANTILE_BUCKETS) {
+  if (values.length === 0) {
+    throw new ValidationError("quantileEdges: requires at least one value");
+  }
+  if (!Number.isInteger(bucketCount) || bucketCount < 2) {
+    throw new ValidationError(
+      `quantileEdges: bucketCount must be an integer >= 2, got ${bucketCount}`
+    );
+  }
+  const sorted = [...values].sort((a, b) => a - b);
+  const edges = [];
+  for (let k = 1; k < bucketCount; k++) {
+    const pos = k / bucketCount * (sorted.length - 1);
+    const lo = sorted[Math.floor(pos)];
+    const hi = sorted[Math.ceil(pos)];
+    edges.push(lo + (pos - Math.floor(pos)) * (hi - lo));
+  }
+  return [...new Set(edges)];
+}
+function bucketLabel(value, edges) {
+  let i = 0;
+  while (i < edges.length && value >= edges[i]) i++;
+  const lo = i === 0 ? "-inf" : String(edges[i - 1]);
+  const hi = i === edges.length ? "+inf" : String(edges[i]);
+  return `[${lo},${hi})`;
+}
+function simFidelityReport(simulated, production, opts = {}) {
+  if (simulated.length === 0) {
+    throw new ValidationError("simFidelityReport: simulated records are empty");
+  }
+  if (production.length === 0) {
+    throw new ValidationError("simFidelityReport: production records are empty");
+  }
+  const extract = opts.features ?? defaultBehaviorFeatures;
+  const minN = opts.minNPerFeature ?? DEFAULT_MIN_N_PER_FEATURE;
+  const simMaps = simulated.map(extract);
+  const prodMaps = production.map(extract);
+  const featureNames = [];
+  const seen = /* @__PURE__ */ new Set();
+  for (const map of [...simMaps, ...prodMaps]) {
+    for (const name of Object.keys(map)) {
+      if (!seen.has(name)) {
+        seen.add(name);
+        featureNames.push(name);
+      }
+    }
+  }
+  const perDimension = [];
+  const insufficientData = [];
+  for (const feature of featureNames) {
+    const simVals = simMaps.map((m) => m[feature] ?? null);
+    const prodVals = prodMaps.map((m) => m[feature] ?? null);
+    const nSim = simVals.filter((v) => v !== null).length;
+    const nProd = prodVals.filter((v) => v !== null).length;
+    if (nSim < minN || nProd < minN) {
+      insufficientData.push(feature);
+      continue;
+    }
+    const { sim, prod } = histograms(feature, simVals, prodVals);
+    perDimension.push({
+      feature,
+      divergence: jsDivergence(sim, prod),
+      topShifts: topShifts(sim, simVals.length, prod, prodVals.length),
+      nSim,
+      nProd
+    });
+  }
+  if (perDimension.length === 0) {
+    return { perDimension, fidelity: Number.NaN, insufficientData, verdict: "insufficient-data" };
+  }
+  const fidelity = 1 - perDimension.reduce((sum, d) => sum + d.divergence, 0) / perDimension.length;
+  return {
+    perDimension,
+    fidelity,
+    insufficientData,
+    verdict: fidelity >= REPRESENTATIVE_MIN_FIDELITY ? "representative" : "skewed"
+  };
+}
+function histograms(feature, simVals, prodVals) {
+  const kinds = /* @__PURE__ */ new Set();
+  for (const v of [...simVals, ...prodVals]) {
+    if (v !== null) kinds.add(typeof v);
+  }
+  if (kinds.size > 1) {
+    throw new ValidationError(
+      `simFidelityReport: feature "${feature}" mixes string and number values \u2014 an extractor must return one kind per feature`
+    );
+  }
+  let toCategory;
+  if (kinds.has("number")) {
+    const union = [];
+    for (const v of [...simVals, ...prodVals]) {
+      if (v !== null) union.push(v);
+    }
+    const edges = quantileEdges(union);
+    toCategory = (v) => bucketLabel(v, edges);
+  } else {
+    toCategory = (v) => v;
+  }
+  const count = (vals) => {
+    const hist = {};
+    for (const v of vals) {
+      const key = v === null ? ABSENT_CATEGORY : toCategory(v);
+      hist[key] = (hist[key] ?? 0) + 1;
+    }
+    return hist;
+  };
+  return { sim: count(simVals), prod: count(prodVals) };
+}
+function topShifts(sim, simTotal, prod, prodTotal) {
+  const keys = [.../* @__PURE__ */ new Set([...Object.keys(sim), ...Object.keys(prod)])];
+  const shifts = keys.map((value) => ({
+    value,
+    pSim: (sim[value] ?? 0) / simTotal,
+    pProd: (prod[value] ?? 0) / prodTotal
+  }));
+  shifts.sort((a, b) => {
+    const delta = Math.abs(b.pSim - b.pProd) - Math.abs(a.pSim - a.pProd);
+    return delta !== 0 ? delta : a.value.localeCompare(b.value);
+  });
+  return shifts.slice(0, TOP_SHIFT_COUNT);
+}
+function easyModeCheck(simulated, production, opts = {}) {
+  if (simulated.length === 0) {
+    throw new ValidationError("easyModeCheck: simulated records are empty");
+  }
+  if (production.length === 0) {
+    throw new ValidationError("easyModeCheck: production records are empty");
+  }
+  const threshold = opts.passThreshold ?? 0.5;
+  const tolerance = opts.inflationTolerance ?? 0.1;
+  const passRate = (records, side) => {
+    let passes = 0;
+    for (const r of records) {
+      const score = finiteOrNull(r.outcome?.holdoutScore) ?? finiteOrNull(r.outcome?.searchScore);
+      if (score === null) {
+        throw new ValidationError(
+          `easyModeCheck: ${side} run "${r.runId}" carries neither holdoutScore nor searchScore`
+        );
+      }
+      if (score >= threshold) passes++;
+    }
+    return passes / records.length;
+  };
+  const simPassRate = passRate(simulated, "simulated");
+  const prodPassRate = passRate(production, "production");
+  const gap = simPassRate - prodPassRate;
+  return { simPassRate, prodPassRate, gap, inflated: gap > tolerance };
+}
 export {
+  ABSENT_CATEGORY,
+  DEFAULT_MIN_N_PER_FEATURE,
+  DEFAULT_QUANTILE_BUCKETS,
   FileSystemOutcomeStore,
   InMemoryOutcomeStore,
   PredictiveValidityResearcher,
+  REPRESENTATIVE_MIN_FIDELITY,
   adversarialScenarioSearch,
   appendToCorpus,
   applyEloUpdate,
   bestOfN,
+  bucketLabel,
   buildDatasetFromCorpus,
   buildPairwiseFromCampaign,
   buildRlDataset,
   campaignToRunRecords,
   compareAdaptationCurves,
   datasheetToMarkdown,
+  defaultBehaviorFeatures,
   detectRewardHacking,
   doublyRobust,
+  easyModeCheck,
   extractPreferences,
   extractStepRewards,
   extractVerifiableReward,
@@ -1686,10 +1765,12 @@ export {
   fitBradleyTerry,
   injectIrrelevantClause,
   inverseProbabilityWeighting,
+  jsDivergence,
   observationsFromRunRecords,
   offPolicyEstimateAll,
   paretoFrontier,
   prmTrainingPairs,
+  quantileEdges,
   readCorpus,
   renameVariables,
   runAdaptationCurve,
@@ -1701,6 +1782,7 @@ export {
   selfConsistency,
   selfNormalizedImportanceWeighting,
   shuffleOrder,
+  simFidelityReport,
   stepRewardsToJsonl,
   thompsonCurriculum,
   toAnthropicFormat,