@tangle-network/agent-eval 0.52.0 → 0.54.0

package/dist/contract/index.js

@@ -75,17 +75,21 @@ async function analyzeRuns(opts) {
   const interRater = opts.raterScores ? computeInterRater(opts.raterScores) : void 0;
   const lift = computeLift(runs, opts.baselineCandidateId, opts.candidateCandidateId, split);
   const failureClusters = opts.analyst ? await computeFailureClusters(runs, opts.analyst, split) : void 0;
+  const failureModes = computeFailureModes(runs);
   const contamination = opts.canaryScenarios ? computeContamination(runs, opts.canaryScenarios) : void 0;
   const outcomeCorrelation = opts.outcomeSignal ? computeOutcomeCorrelation(runs, opts.outcomeSignal, split) : void 0;
   const release = buildReleaseScorecard(composite, lift, contamination);
+  const priorPeriodComparison = opts.baselineRuns ? computePriorPeriodComparison(runs, opts.baselineRuns, split, opts.baselineLabel) : void 0;
   const recommendations = buildRecommendations({
     composite,
     judges,
     interRater,
     lift,
     failureClusters,
+    failureModes,
     contamination,
     outcomeCorrelation,
+    priorPeriodComparison,
     threshold
   });
   return {
@@ -100,9 +104,138 @@ async function analyzeRuns(opts) {
     contamination,
     outcomeCorrelation,
     release,
+    ...failureModes ? { failureModes } : {},
+    ...priorPeriodComparison ? { priorPeriodComparison } : {},
     recommendations
   };
 }
+function computeFailureModes(runs) {
+  const counts = /* @__PURE__ */ new Map();
+  for (const r of runs) {
+    const key = r.failureClass ?? r.failureMode;
+    if (key) counts.set(key, (counts.get(key) ?? 0) + 1);
+  }
+  if (counts.size === 0) return void 0;
+  const n = runs.length;
+  return [...counts.entries()].map(([mode, count]) => ({ mode, count, share: n > 0 ? count / n : 0 })).sort((a, b) => b.count - a.count || a.mode.localeCompare(b.mode));
+}
+function computePriorPeriodComparison(current, baseline, split, windowLabel) {
+  if (current.length === 0 || baseline.length === 0) return void 0;
+  const metrics = {};
+  const directions = {};
+  const compositeCurrent = current.map((r) => compositeOf(r, split)).filter(Number.isFinite);
+  const compositeBaseline = baseline.map((r) => compositeOf(r, split)).filter(Number.isFinite);
+  if (compositeCurrent.length > 0 && compositeBaseline.length > 0) {
+    metrics.composite = welchCompare(compositeBaseline, compositeCurrent);
+    directions.composite = "higher-is-better";
+  }
+  const costCurrent = current.map((r) => r.costUsd).filter(Number.isFinite);
+  const costBaseline = baseline.map((r) => r.costUsd).filter(Number.isFinite);
+  if (costCurrent.length > 0 && costBaseline.length > 0) {
+    metrics.cost = welchCompare(costBaseline, costCurrent);
+    directions.cost = "lower-is-better";
+  }
+  const durCurrent = current.map((r) => r.wallMs).filter(Number.isFinite);
+  const durBaseline = baseline.map((r) => r.wallMs).filter(Number.isFinite);
+  if (durCurrent.length > 0 && durBaseline.length > 0) {
+    metrics.duration = welchCompare(durBaseline, durCurrent);
+    directions.duration = "lower-is-better";
+  }
+  const tokCurrent = current.map((r) => (r.tokenUsage.input ?? 0) + (r.tokenUsage.output ?? 0)).filter(Number.isFinite);
+  const tokBaseline = baseline.map((r) => (r.tokenUsage.input ?? 0) + (r.tokenUsage.output ?? 0)).filter(Number.isFinite);
+  if (tokCurrent.length > 0 && tokBaseline.length > 0) {
+    metrics.tokenUsage = welchCompare(tokBaseline, tokCurrent);
+    directions.tokenUsage = "lower-is-better";
+  }
+  const dimsCurrent = collectPerDimension(current);
+  const dimsBaseline = collectPerDimension(baseline);
+  for (const dim of Object.keys(dimsCurrent)) {
+    const b = dimsBaseline[dim];
+    const c = dimsCurrent[dim];
+    if (!b || b.length === 0 || !c || c.length === 0) continue;
+    metrics[`dim.${dim}`] = welchCompare(b, c);
+    directions[`dim.${dim}`] = "higher-is-better";
+  }
+  const regressedMetrics = [];
+  const improvedMetrics = [];
+  for (const [name, delta] of Object.entries(metrics)) {
+    if (!delta.significant) continue;
+    const dir = directions[name] ?? "higher-is-better";
+    const better = dir === "higher-is-better" ? delta.delta > 0 : delta.delta < 0;
+    if (better) improvedMetrics.push(name);
+    else regressedMetrics.push(name);
+  }
+  return {
+    baselineN: baseline.length,
+    currentN: current.length,
+    ...windowLabel ? { windowLabel } : {},
+    metrics,
+    regressedMetrics,
+    improvedMetrics
+  };
+}
+function collectPerDimension(runs) {
+  const out = {};
+  for (const r of runs) {
+    const perDim = r.outcome.judgeScores?.perDimMean;
+    if (!perDim) continue;
+    for (const [dim, value] of Object.entries(perDim)) {
+      if (!Number.isFinite(value)) continue;
+      if (!out[dim]) out[dim] = [];
+      out[dim].push(value);
+    }
+  }
+  return out;
+}
+function welchCompare(baseline, current) {
+  const baselineMean = mean(baseline);
+  const currentMean = mean(current);
+  const baselineVar = sampleVariance(baseline, baselineMean);
+  const currentVar = sampleVariance(current, currentMean);
+  const baselineN = baseline.length;
+  const currentN = current.length;
+  const delta = currentMean - baselineMean;
+  const se = Math.sqrt(baselineVar / baselineN + currentVar / currentN);
+  const halfWidth = 1.96 * (se > 0 ? se : 0);
+  const ci95 = [delta - halfWidth, delta + halfWidth];
+  const t = se > 0 ? delta / se : 0;
+  const pValue = se > 0 ? 2 * (1 - standardNormalCdf(Math.abs(t))) : 1;
+  const pooledStddev = Math.sqrt(
+    ((baselineN - 1) * baselineVar + (currentN - 1) * currentVar) / Math.max(1, baselineN + currentN - 2)
+  );
+  const cohensD2 = pooledStddev > 0 ? delta / pooledStddev : 0;
+  const significant = pValue < 0.05 && Math.abs(cohensD2) >= 0.2;
+  return {
+    current: currentMean,
+    baseline: baselineMean,
+    delta,
+    ci95,
+    pValue,
+    cohensD: cohensD2,
+    baselineN,
+    currentN,
+    significant
+  };
+}
+function sampleVariance(xs, xsMean) {
+  if (xs.length < 2) return 0;
+  let s = 0;
+  for (const x of xs) s += (x - xsMean) ** 2;
+  return s / (xs.length - 1);
+}
+function standardNormalCdf(z) {
+  const a1 = 0.254829592;
+  const a2 = -0.284496736;
+  const a3 = 1.421413741;
+  const a4 = -1.453152027;
+  const a5 = 1.061405429;
+  const p = 0.3275911;
+  const sign = z < 0 ? -1 : 1;
+  const x = Math.abs(z) / Math.SQRT2;
+  const t = 1 / (1 + p * x);
+  const y = 1 - ((((a5 * t + a4) * t + a3) * t + a2) * t + a1) * t * Math.exp(-x * x);
+  return 0.5 * (1 + sign * y);
+}
 function resolveSplit(runs, pref) {
   if (pref !== "auto") return pref;
   const hasHoldout = runs.some((r) => Number.isFinite(r.outcome.holdoutScore));
@@ -492,6 +625,32 @@ function buildReleaseScorecard(composite, lift, contamination) {
 }
 function buildRecommendations(ctx) {
   const out = [];
+  if (ctx.priorPeriodComparison) {
+    const ppc = ctx.priorPeriodComparison;
+    const label = ppc.windowLabel ?? "baseline period";
+    for (const name of ppc.regressedMetrics) {
+      const d = ppc.metrics[name];
+      if (!d) continue;
+      out.push({
+        priority: "critical",
+        kind: "investigate",
+        title: `${name} regressed from ${d.baseline.toFixed(3)} \u2192 ${d.current.toFixed(3)} vs ${label}`,
+        detail: `Welch CI95 = [${d.ci95[0].toFixed(3)}, ${d.ci95[1].toFixed(3)}], p=${d.pValue.toFixed(4)}, Cohen's d=${d.cohensD.toFixed(2)} (n_current=${d.currentN}, n_baseline=${d.baselineN}). The regression is statistically significant at p<0.05 with at-least-small effect size.`,
+        evidencePath: `priorPeriodComparison.metrics.${name}`
+      });
+    }
+    for (const name of ppc.improvedMetrics) {
+      const d = ppc.metrics[name];
+      if (!d) continue;
+      out.push({
+        priority: "low",
+        kind: "ship",
+        title: `${name} improved from ${d.baseline.toFixed(3)} \u2192 ${d.current.toFixed(3)} vs ${label}`,
+        detail: `Welch CI95 = [${d.ci95[0].toFixed(3)}, ${d.ci95[1].toFixed(3)}], p=${d.pValue.toFixed(4)}, Cohen's d=${d.cohensD.toFixed(2)} (n_current=${d.currentN}, n_baseline=${d.baselineN}). Statistically significant improvement worth flagging.`,
+        evidencePath: `priorPeriodComparison.metrics.${name}`
+      });
+    }
+  }
   if (ctx.composite.n > 0) {
     if (ctx.composite.mean < 0.3) {
       const tail = ctx.composite.tailRuns ?? [];
@@ -515,6 +674,18 @@ function buildRecommendations(ctx) {
       });
     }
   }
+  if (ctx.failureModes && ctx.failureModes.length > 0) {
+    const top = ctx.failureModes[0];
+    if (top.count >= 3 && top.share >= 0.15) {
+      out.push({
+        priority: top.share >= 0.25 ? "high" : "medium",
+        kind: "investigate",
+        title: `'${top.mode}' is the dominant failure mode \u2014 ${top.count} runs (${(top.share * 100).toFixed(0)}% of the corpus)`,
+        detail: `The mean composite can look acceptable while one named failure dominates the lower tail. ${top.count} of ${ctx.composite.n} runs failed with '${top.mode}'${ctx.failureModes.length > 1 ? ` (next: '${ctx.failureModes[1].mode}' \xD7${ctx.failureModes[1].count})` : ""}. Fix this cause first.`,
+        evidencePath: "failureModes"
+      });
+    }
+  }
   if (Object.keys(ctx.judges).length === 0 && ctx.composite.n > 0) {
     out.push({
       priority: "medium",