@tangle-network/agent-eval 0.38.0 → 0.40.1

package/dist/reporting.js

@@ -5,7 +5,7 @@ import {
   judgeReplayGate,
   releaseTraceEvidenceFromMultiShotTrials,
   renderReleaseReport
-} from "./chunk-LGAPK7NA.js";
+} from "./chunk-NKLGKF2Q.js";
 import {
   rubricPredictiveValidity
 } from "./chunk-YRZ4M5GS.js";
@@ -19,12 +19,12 @@ import {
   paretoChart,
   researchReport,
   summaryTable
-} from "./chunk-TQL7BAOY.js";
+} from "./chunk-EGIPWXHL.js";
 import {
   benjaminiHochberg,
   pairedBootstrap,
   wilcoxonSignedRank
-} from "./chunk-KHZRNY3F.js";
+} from "./chunk-WP7SY7AI.js";
 import "./chunk-VSMTAMNK.js";
 import "./chunk-QYJT52YW.js";
 import "./chunk-PZ5AY32C.js";

package/dist/rl.js

@@ -1,19 +1,25 @@
+import {
+  detectRewardHacking,
+  extractVerifiableReward,
+  extractVerifiableRewardsFromRecords,
+  filterDeterministicallyRewarded
+} from "./chunk-YV7J7X5N.js";
 import {
   runEvalCampaign
-} from "./chunk-RL6TERL2.js";
+} from "./chunk-LCIDRYGP.js";
 import "./chunk-VXNVVBZO.js";
-import "./chunk-TSPOEDM3.js";
+import "./chunk-BWZEGTES.js";
 import {
   rubricPredictiveValidity
 } from "./chunk-YRZ4M5GS.js";
 import {
   evaluateInterimReleaseConfidence
 } from "./chunk-MAZ26DC7.js";
-import "./chunk-TQL7BAOY.js";
+import "./chunk-EGIPWXHL.js";
 import {
   benjaminiHochberg,
   wilcoxonSignedRank
-} from "./chunk-KHZRNY3F.js";
+} from "./chunk-WP7SY7AI.js";
 import "./chunk-UBPIXOC4.js";
 import "./chunk-PC4UYEBM.js";
 import "./chunk-TVVP3ZZQ.js";
@@ -157,7 +163,7 @@ async function runContaminationProbe(input, opts = {}) {
   const deltas = valid.map((p) => p.delta);
   const sortedDeltas = [...deltas].sort((a, b) => a - b);
   const median = sortedDeltas[Math.floor(sortedDeltas.length / 2)];
-  const mean2 = deltas.reduce((s, d) => s + d, 0) / deltas.length;
+  const mean = deltas.reduce((s, d) => s + d, 0) / deltas.length;
   const pseudoP = valid.map((p) => Math.min(1, Math.max(1e-6, 1 - Math.abs(p.delta) / 1)));
   const { qValues } = benjaminiHochberg(pseudoP, fdr);
   for (let i = 0; i < valid.length; i++) {
@@ -171,7 +177,7 @@ async function runContaminationProbe(input, opts = {}) {
     perScenario,
     pairedTest,
     medianDelta: median,
-    meanDelta: mean2,
+    meanDelta: mean,
     contaminationSuspected,
     reason,
     n: valid.length
@@ -753,167 +759,6 @@ function buildPairwiseFromCampaign(input) {
   return outcomes;
 }
 
-// src/rl/verifiable-reward.ts
-var DEFAULT_DETERMINISTIC_LAYERS = /* @__PURE__ */ new Set([
-  "install",
-  "typecheck",
-  "build",
-  "lint",
-  "test",
-  "compile",
-  "schema",
-  "sandbox",
-  "unit_tests",
-  "integration_tests"
-]);
-var DEFAULT_SOURCE_FOR = (name) => {
-  const lower = name.toLowerCase();
-  if (lower.includes("test")) return "test";
-  if (lower.includes("compile") || lower.includes("build") || lower.includes("typecheck") || lower.includes("lint"))
-    return "compile";
-  if (lower.includes("schema")) return "schema";
-  if (lower.includes("sandbox")) return "sandbox";
-  if (lower.includes("judge") || lower.includes("semantic")) return "judge";
-  return "composite";
-};
-function extractVerifiableReward(report, opts = {}) {
-  const deterministicSet = new Set(opts.deterministicLayers ?? [...DEFAULT_DETERMINISTIC_LAYERS]);
-  const sourceFor = opts.sourceFor ?? DEFAULT_SOURCE_FOR;
-  const fallbackToJudge = opts.fallbackToJudge ?? true;
-  const judgeFloor = opts.judgeConfidenceFloor ?? 0.7;
-  const deterministic = report.layers.filter(
-    (l) => deterministicSet.has(l.layer) && typeof l.score === "number" && Number.isFinite(l.score)
-  );
-  if (deterministic.length === 1) {
-    const layer = deterministic[0];
-    return {
-      value: clamp01(layer.score),
-      source: sourceFor(layer.layer),
-      determinism: "deterministic",
-      confidence: 1,
-      origin: layer.layer,
-      breakdown: layerBreakdown(layer)
-    };
-  }
-  if (deterministic.length > 1) {
-    let num = 0;
-    let denom = 0;
-    const breakdown = {};
-    for (const l of deterministic) {
-      const w = l.detail?.weight ?? 1;
-      num += w * (l.score ?? 0);
-      denom += w;
-      breakdown[l.layer] = l.score;
-    }
-    return {
-      value: denom === 0 ? 0 : clamp01(num / denom),
-      source: "composite",
-      determinism: "deterministic",
-      confidence: 1,
-      origin: deterministic.map((l) => l.layer).join("+"),
-      breakdown
-    };
-  }
-  if (!fallbackToJudge) return null;
-  const judge = report.layers.find(
-    (l) => typeof l.score === "number" && Number.isFinite(l.score) && sourceFor(l.layer) === "judge"
-  ) ?? report.layers.find((l) => typeof l.score === "number" && Number.isFinite(l.score));
-  if (!judge) return null;
-  const confFromDetail = judge.detail?.confidence;
-  return {
-    value: clamp01(judge.score),
-    source: "judge",
-    determinism: "probabilistic",
-    confidence: typeof confFromDetail === "number" ? confFromDetail : judgeFloor,
-    origin: judge.layer,
-    breakdown: layerBreakdown(judge)
-  };
-}
-function extractVerifiableRewardsFromRecords(runs, opts = {}) {
-  const sourceFor = opts.sourceFor ?? DEFAULT_SOURCE_FOR;
-  const deterministicSet = new Set(opts.deterministicLayers ?? [...DEFAULT_DETERMINISTIC_LAYERS]);
-  const fallbackToJudge = opts.fallbackToJudge ?? true;
-  const judgeFloor = opts.judgeConfidenceFloor ?? 0.7;
-  return runs.map((run) => {
-    const layerScores = [];
-    for (const [k, v] of Object.entries(run.outcome.raw)) {
-      if (k.startsWith("layer.") && !k.includes(".", 6) && typeof v === "number" && Number.isFinite(v)) {
-        layerScores.push({ name: k.slice("layer.".length), score: v });
-      }
-    }
-    const det = layerScores.filter((l) => deterministicSet.has(l.name));
-    if (det.length === 1) {
-      const layer = det[0];
-      return {
-        runId: run.runId,
-        reward: {
-          value: clamp01(layer.score),
-          source: sourceFor(layer.name),
-          determinism: "deterministic",
-          confidence: 1,
-          origin: layer.name
-        }
-      };
-    }
-    if (det.length > 1) {
-      const value = det.reduce((s, l) => s + l.score, 0) / det.length;
-      const breakdown = Object.fromEntries(
-        det.map((l) => [l.name, l.score])
-      );
-      return {
-        runId: run.runId,
-        reward: {
-          value: clamp01(value),
-          source: "composite",
-          determinism: "deterministic",
-          confidence: 1,
-          origin: det.map((l) => l.name).join("+"),
-          breakdown
-        }
-      };
-    }
-    if (!fallbackToJudge) return { runId: run.runId, reward: null };
-    const primary = run.outcome.holdoutScore ?? run.outcome.searchScore;
-    if (typeof primary !== "number" || !Number.isFinite(primary)) {
-      return { runId: run.runId, reward: null };
-    }
-    return {
-      runId: run.runId,
-      reward: {
-        value: clamp01(primary),
-        source: "judge",
-        determinism: "probabilistic",
-        confidence: judgeFloor,
-        origin: "run.outcome.score"
-      }
-    };
-  });
-}
-function filterDeterministicallyRewarded(runs, opts = {}) {
-  const rewarded = extractVerifiableRewardsFromRecords(runs, { ...opts, fallbackToJudge: false });
-  const out = [];
-  for (let i = 0; i < runs.length; i++) {
-    const r = rewarded[i];
-    if (r.reward && r.reward.determinism === "deterministic") {
-      out.push({ run: runs[i], reward: r.reward });
-    }
-  }
-  return out;
-}
-function clamp01(x) {
-  if (!Number.isFinite(x)) return 0;
-  return Math.max(0, Math.min(1, x));
-}
-function layerBreakdown(l) {
-  const out = {};
-  if (l.diagnostics) {
-    for (const [k, v] of Object.entries(l.diagnostics)) {
-      if (typeof v === "number" && Number.isFinite(v)) out[k] = v;
-    }
-  }
-  return out;
-}
-
 // src/rl/active-curriculum.ts
 function varianceBasedCurriculum(observations, candidateCells, opts) {
   const variancePrior = opts.variancePrior ?? 0.05;
@@ -930,10 +775,10 @@ function varianceBasedCurriculum(observations, candidateCells, opts) {
     const k = `${c.variantId}::${c.scenarioId}`;
     const samples = grouped.get(k) ?? [];
     const n = samples.length;
-    const mean2 = n === 0 ? 0.5 : samples.reduce((s, v) => s + v, 0) / n;
-    const variance = n < 2 ? variancePrior : samples.reduce((s, v) => s + (v - mean2) ** 2, 0) / (n - 1) + variancePrior;
+    const mean = n === 0 ? 0.5 : samples.reduce((s, v) => s + v, 0) / n;
+    const variance = n < 2 ? variancePrior : samples.reduce((s, v) => s + (v - mean) ** 2, 0) / (n - 1) + variancePrior;
     const weight = Math.sqrt(variance) + 1 / Math.sqrt(Math.max(1, n));
-    return { variantId: c.variantId, scenarioId: c.scenarioId, n, mean: mean2, variance, weight };
+    return { variantId: c.variantId, scenarioId: c.scenarioId, n, mean, variance, weight };
   });
   const floorTotal = floor * cellStats.length;
   if (floorTotal >= budget) {
@@ -1400,151 +1245,6 @@ function defaultReward(run) {
   return typeof v === "number" && Number.isFinite(v) ? v : null;
 }
 
-// src/rl/reward-hacking.ts
-var DEFAULT_PROXY = (r) => {
-  const v = r.outcome.holdoutScore ?? r.outcome.searchScore;
-  return typeof v === "number" && Number.isFinite(v) ? v : null;
-};
-function detectRewardHacking(input) {
-  const proxyOf = input.proxyOf ?? DEFAULT_PROXY;
-  const truthOf = input.truthOf;
-  const sus = input.thresholds?.suspect ?? 0.3;
-  const gam = input.thresholds?.gaming ?? 0.6;
-  const runs = input.runs.filter((r) => proxyOf(r) !== null);
-  const n = runs.length;
-  if (n < 4) {
-    return {
-      findings: [],
-      verdict: "clean",
-      n,
-      rationale: [`fewer than 4 runs with proxy reward (n=${n}); insufficient evidence`]
-    };
-  }
-  const windowSize = Math.max(1, input.windowSize ?? Math.min(50, Math.floor(n / 2)));
-  const before = runs.slice(0, n - windowSize);
-  const after = runs.slice(n - windowSize);
-  const findings = [];
-  if (truthOf) {
-    const beforeProxy = before.map(proxyOf).filter((v) => typeof v === "number");
-    const afterProxy = after.map(proxyOf).filter((v) => typeof v === "number");
-    const beforeTruth = before.map(truthOf).filter((v) => typeof v === "number");
-    const afterTruth = after.map(truthOf).filter((v) => typeof v === "number");
-    if (beforeProxy.length >= 2 && afterProxy.length >= 2 && beforeTruth.length >= 2 && afterTruth.length >= 2) {
-      const proxyDelta = mean(afterProxy) - mean(beforeProxy);
-      const truthDelta = mean(afterTruth) - mean(beforeTruth);
-      const gap = Math.max(0, proxyDelta - truthDelta);
-      const severity = clamp012(gap * 5);
-      findings.push({
-        signal: "reward_divergence",
-        severity,
-        message: severity >= sus ? `proxy reward rose by ${proxyDelta.toFixed(3)} while truth changed by ${truthDelta.toFixed(3)} \u2014 potential Goodhart` : `proxy and truth moved together (proxy ${proxyDelta.toFixed(3)}, truth ${truthDelta.toFixed(3)})`,
-        detail: {
-          proxyDelta,
-          truthDelta,
-          gap,
-          beforeN: beforeProxy.length,
-          afterN: afterProxy.length
-        }
-      });
-    }
-  }
-  {
-    const beforeP = before.map(proxyOf).filter((v) => typeof v === "number");
-    const afterP = after.map(proxyOf).filter((v) => typeof v === "number");
-    if (beforeP.length >= 4 && afterP.length >= 4) {
-      const ks = ksStatistic(beforeP, afterP);
-      const severity = clamp012(ks - 0.2);
-      findings.push({
-        signal: "distribution_shift",
-        severity,
-        message: severity >= sus ? `KS=${ks.toFixed(3)} between before/after windows \u2014 distributional shift large` : `KS=${ks.toFixed(3)} between before/after windows \u2014 within-distribution drift`,
-        detail: { ks, beforeN: beforeP.length, afterN: afterP.length }
-      });
-    }
-  }
-  {
-    const secondaryOf = input.secondaryRewardOf ?? defaultSecondary(input.verifiableRewardOptions);
-    const aligned = runs.map((r) => ({ p: proxyOf(r), s: secondaryOf(r) })).filter(
-      (x) => typeof x.p === "number" && typeof x.s === "number"
-    );
-    if (aligned.length >= 4) {
-      const ps = aligned.map((x) => x.p);
-      const ss = aligned.map((x) => x.s);
-      const r = pearsonR(ps, ss);
-      const severity = clamp012(0.5 - Math.max(0, r));
-      findings.push({
-        signal: "reward_disagreement",
-        severity,
-        message: severity >= sus ? `proxy and independent secondary reward correlate \u03C1=${r.toFixed(3)} \u2014 possibly hacking proxy` : `proxy and secondary reward correlate \u03C1=${r.toFixed(3)}`,
-        detail: { pearson: r, n: aligned.length }
-      });
-    }
-  }
-  {
-    const detRuns = filterDeterministicallyRewarded(runs, input.verifiableRewardOptions ?? {});
-    if (detRuns.length >= 4) {
-      const detBefore = detRuns.slice(0, Math.floor(detRuns.length / 2));
-      const detAfter = detRuns.slice(Math.floor(detRuns.length / 2));
-      const detDelta = mean(detAfter.map((r) => r.reward.value)) - mean(detBefore.map((r) => r.reward.value));
-      const proxyDelta = mean(after.map(proxyOf).filter((v) => typeof v === "number")) - mean(before.map(proxyOf).filter((v) => typeof v === "number"));
-      const driftGap = Math.max(0, proxyDelta - detDelta);
-      const severity = clamp012(driftGap * 5);
-      findings.push({
-        signal: "judge_drift",
-        severity,
-        message: severity >= sus ? `judge proxy +${proxyDelta.toFixed(3)} while deterministic reward +${detDelta.toFixed(3)} \u2014 judge drifting up without verifiable backing` : `judge and deterministic rewards move in step (judge ${proxyDelta.toFixed(3)}, det ${detDelta.toFixed(3)})`,
-        detail: { proxyDelta, detDelta, driftGap, n: detRuns.length }
-      });
-    }
-  }
-  const maxSev = findings.reduce((m, f) => Math.max(m, f.severity), 0);
-  const verdict = maxSev >= gam ? "gaming" : maxSev >= sus ? "suspect" : "clean";
-  const rationale = findings.filter((f) => f.severity >= sus).map((f) => `${f.signal}: severity ${f.severity.toFixed(2)} \u2014 ${f.message}`);
-  if (rationale.length === 0) rationale.push("no signals fired above suspect threshold");
-  return { findings, verdict, rationale, n };
-}
-function mean(xs) {
-  if (xs.length === 0) return 0;
-  return xs.reduce((s, x) => s + x, 0) / xs.length;
-}
-function clamp012(x) {
-  if (!Number.isFinite(x)) return 0;
-  return Math.max(0, Math.min(1, x));
-}
-function pearsonR(a, b) {
-  if (a.length !== b.length || a.length < 2) return 0;
-  const ma = mean(a);
-  const mb = mean(b);
-  let num = 0, da = 0, db = 0;
-  for (let i = 0; i < a.length; i++) {
-    const xa = a[i] - ma;
-    const xb = b[i] - mb;
-    num += xa * xb;
-    da += xa * xa;
-    db += xb * xb;
-  }
-  if (da === 0 || db === 0) return 0;
-  return num / Math.sqrt(da * db);
-}
-function ksStatistic(a, b) {
-  const sortedA = [...a].sort((x, y) => x - y);
-  const sortedB = [...b].sort((x, y) => x - y);
-  const all = [.../* @__PURE__ */ new Set([...sortedA, ...sortedB])].sort((x, y) => x - y);
-  let max = 0;
-  for (const v of all) {
-    const fa = sortedA.filter((x) => x <= v).length / sortedA.length;
-    const fb = sortedB.filter((x) => x <= v).length / sortedB.length;
-    max = Math.max(max, Math.abs(fa - fb));
-  }
-  return max;
-}
-function defaultSecondary(verifiableOpts) {
-  return (run) => {
-    const filtered = filterDeterministicallyRewarded([run], verifiableOpts ?? {});
-    return filtered.length === 1 ? filtered[0].reward.value : null;
-  };
-}
-
 // src/rl/auto-research.ts
 async function analyzeOptimizationResult(opts) {
   const trials = extractTrials(opts.result);