@tangle-network/agent-eval 0.121.0 → 0.122.1

package/dist/{chunk-32BZXMSO.js → chunk-VZ6VKOJT.js}

@@ -25,11 +25,8 @@ import {
   stripFencedJson
 } from "./chunk-NJC7U437.js";
 import {
-  mcnemar,
   pairedBootstrap,
-  pairedRiskDifference,
-  weightedComposite,
-  wilcoxonSignedRank
+  weightedComposite
 } from "./chunk-PJQFMIOX.js";
 import {
   CostLedger
@@ -299,155 +296,6 @@ async function runJudgeChat(tc, input, judgeName, request) {
   return paid.value;
 }
 
-// src/paired-arms.ts
-function pairArms(rows, opts) {
-  const { baselineArm, treatmentArm } = opts;
-  if (baselineArm === treatmentArm) {
-    throw new ValidationError(
-      `pairArms: baselineArm and treatmentArm are both '${baselineArm}' \u2014 an arm cannot be compared to itself`
-    );
-  }
-  const byArm = /* @__PURE__ */ new Map();
-  const armsSeen = /* @__PURE__ */ new Set();
-  for (const row of rows) {
-    armsSeen.add(row.arm);
-    if (row.arm !== baselineArm && row.arm !== treatmentArm) continue;
-    const byKey = byArm.get(row.arm) ?? /* @__PURE__ */ new Map();
-    const group = byKey.get(row.pairKey) ?? [];
-    group.push(row);
-    byKey.set(row.pairKey, group);
-    byArm.set(row.arm, byKey);
-  }
-  for (const arm of [baselineArm, treatmentArm]) {
-    if (!byArm.has(arm)) {
-      const seen = [...armsSeen].sort().join(", ") || "<none>";
-      throw new ValidationError(`pairArms: no rows for arm '${arm}' (arms present: ${seen})`);
-    }
-  }
-  const baselineByKey = byArm.get(baselineArm);
-  const treatmentByKey = byArm.get(treatmentArm);
-  const allKeys = [.../* @__PURE__ */ new Set([...baselineByKey.keys(), ...treatmentByKey.keys()])].sort();
-  const pairs = [];
-  const unpairedBaseline = [];
-  const unpairedTreatment = [];
-  for (const pairKey of allKeys) {
-    const b = baselineByKey.get(pairKey) ?? [];
-    const t = treatmentByKey.get(pairKey) ?? [];
-    if (b.length <= 1 && t.length <= 1) {
-      if (b.length === 1 && t.length === 1) {
-        pairs.push({ pairKey, repIndex: 0, baseline: b[0], treatment: t[0] });
-      } else {
-        unpairedBaseline.push(...b);
-        unpairedTreatment.push(...t);
-      }
-      continue;
-    }
-    const bByRep = indexByRepKey(b, pairKey, baselineArm);
-    const tByRep = indexByRepKey(t, pairKey, treatmentArm);
-    const repKeys = [.../* @__PURE__ */ new Set([...bByRep.keys(), ...tByRep.keys()])].sort();
-    let repIndex = 0;
-    for (const repKey of repKeys) {
-      const baseline = bByRep.get(repKey);
-      const treatment = tByRep.get(repKey);
-      if (baseline !== void 0 && treatment !== void 0) {
-        pairs.push({ pairKey, repIndex: repIndex++, baseline, treatment });
-      } else if (baseline !== void 0) {
-        unpairedBaseline.push(baseline);
-      } else if (treatment !== void 0) {
-        unpairedTreatment.push(treatment);
-      }
-    }
-  }
-  return { pairs, unpairedBaseline, unpairedTreatment };
-}
-function indexByRepKey(group, pairKey, arm) {
-  const byRep = /* @__PURE__ */ new Map();
-  for (const row of group) {
-    if (row.repKey === void 0) {
-      throw new ValidationError(
-        `pairArms: pairKey '${pairKey}' has multiple reps in an arm, but a row in arm '${arm}' is missing repKey \u2014 multi-rep items require an explicit repKey on every row so reps pair by identity (pairing reps by outcome or by index would bias the paired statistics)`
-      );
-    }
-    if (byRep.has(row.repKey)) {
-      throw new ValidationError(
-        `pairArms: duplicate repKey '${row.repKey}' for pairKey '${pairKey}' in arm '${arm}' \u2014 (pairKey, repKey) must uniquely identify a rep within an arm`
-      );
-    }
-    byRep.set(row.repKey, row);
-  }
-  return byRep;
-}
-function comparePairedArms(rows, opts) {
-  const { pairs, unpairedBaseline, unpairedTreatment } = pairArms(rows, opts);
-  let correctness = null;
-  const baselinePass = [];
-  const treatmentPass = [];
-  for (const pair of pairs) {
-    if (pair.baseline.pass === void 0 || pair.treatment.pass === void 0) continue;
-    baselinePass.push(pair.baseline.pass ? 1 : 0);
-    treatmentPass.push(pair.treatment.pass ? 1 : 0);
-  }
-  if (baselinePass.length > 0) {
-    const mc = mcnemar(baselinePass, treatmentPass);
-    correctness = {
-      b10: mc.b,
-      b01: mc.c,
-      mcnemar: mc,
-      riskDifference: pairedRiskDifference(baselinePass, treatmentPass)
-    };
-  }
-  const metricNames = opts.metricNames ?? [
-    ...new Set(
-      pairs.flatMap((p) => [
-        ...Object.keys(p.baseline.metrics ?? {}),
-        ...Object.keys(p.treatment.metrics ?? {})
-      ])
-    )
-  ].sort();
-  const metricDeltas = metricNames.map((name) => {
-    const before = [];
-    const after = [];
-    let nMissing = 0;
-    for (const pair of pairs) {
-      const b = metricValue(pair.baseline, name);
-      const t = metricValue(pair.treatment, name);
-      if (b === void 0 || t === void 0) {
-        nMissing++;
-        continue;
-      }
-      before.push(b);
-      after.push(t);
-    }
-    const bootstrapCi = before.length === 0 ? null : pairedBootstrap(before, after, opts.bootstrap);
-    return {
-      name,
-      n: before.length,
-      nMissing,
-      medianDelta: bootstrapCi === null ? Number.NaN : bootstrapCi.median,
-      meanDelta: bootstrapCi === null ? Number.NaN : bootstrapCi.mean,
-      bootstrapCi,
-      wilcoxon: before.length === 0 ? null : wilcoxonSignedRank(before, after)
-    };
-  });
-  return {
-    nPairs: pairs.length,
-    nUnpairedBaseline: unpairedBaseline.length,
-    nUnpairedTreatment: unpairedTreatment.length,
-    correctness,
-    metricDeltas
-  };
-}
-function metricValue(row, name) {
-  const v = row.metrics?.[name];
-  if (v === void 0) return void 0;
-  if (!Number.isFinite(v)) {
-    throw new ValidationError(
-      `comparePairedArms: non-finite value for metric '${name}' on pairKey '${row.pairKey}' (arm '${row.arm}'): ${v}`
-    );
-  }
-  return v;
-}
-
 // src/pareto.ts
 function dominates(a, b, objectives) {
   let strictlyBetter = false;
@@ -2256,6 +2104,7 @@ function recoverFrom(slice) {
 }
 
 // src/reflective-mutation.ts
+var EMITTED_EVIDENCE_MAX_CHARS = 2e3;
 var DEFAULT_MUTATION_PRIMITIVES = [
   'Strengthen an imperative ("should" \u2192 "must")',
   "Add a concrete example pulled from a missed-golden phrase",
@@ -2287,7 +2136,7 @@ function buildReflectionPrompt(ctx) {
       );
       if (trial.failureNote) {
         sections.push("");
-        sections.push(`**Why it scored low:** ${truncate(trial.failureNote, 600)}`);
+        sections.push(`**Why it scored low:** ${truncate(trial.failureNote, 1500)}`);
       }
       const missed = (trial.expectations ?? []).filter((e) => !e.matched);
       if (missed.length > 0) {
@@ -2301,7 +2150,7 @@ function buildReflectionPrompt(ctx) {
         sections.push("");
         sections.push("**What the agent emitted:**");
         sections.push("```");
-        sections.push(truncate(trial.emitted, 600));
+        sections.push(truncate(trial.emitted, EMITTED_EVIDENCE_MAX_CHARS));
         sections.push("```");
       }
       sections.push("");
@@ -2712,7 +2561,11 @@ function buildEvidence(ctx, evidenceK, baseTarget) {
     score: s.composite,
     // The judge's "why it scored low" — grounds the reflection on real failure
     // patterns instead of blind rephrasing. Generalizable by the judge contract.
-    ...s.notes ? { failureNote: s.notes } : {}
+    ...s.notes ? { failureNote: s.notes } : {},
+    // The candidate's raw output for the scenario — buildReflectionPrompt
+    // renders it as the "what the agent emitted" block, so the reflection sees
+    // the actual wrong output, not just the score.
+    ...s.emitted ? { emitted: s.emitted } : {}
   });
   const top = byScore.slice(0, evidenceK).map(toTrace);
   const bottom = byScore.slice(-evidenceK).reverse().map(toTrace);
@@ -2737,6 +2590,7 @@ function campaignBreakdown(campaign) {
   const dimCounts = {};
   const byScenario = /* @__PURE__ */ new Map();
   const notesByScenario = /* @__PURE__ */ new Map();
+  const emittedByScenario = /* @__PURE__ */ new Map();
   for (const cell of campaign.cells) {
     const judgeScores = Object.values(cell.judgeScores).filter(
       (score) => score.failed !== true && Number.isFinite(score.composite)
@@ -2746,6 +2600,15 @@ function campaignBreakdown(campaign) {
     const arr = byScenario.get(cell.scenarioId) ?? [];
     arr.push(cellComposite);
     byScenario.set(cell.scenarioId, arr);
+    if (typeof cell.artifact === "string" && cell.artifact.trim().length > 0) {
+      const prev = emittedByScenario.get(cell.scenarioId);
+      if (!prev || cellComposite < prev.composite) {
+        emittedByScenario.set(cell.scenarioId, {
+          composite: cellComposite,
+          text: cell.artifact.slice(0, EMITTED_EVIDENCE_MAX_CHARS)
+        });
+      }
+    }
     for (const s of judgeScores) {
       if (s.notes?.trim()) {
         const set = notesByScenario.get(cell.scenarioId) ?? /* @__PURE__ */ new Set();
@@ -2769,10 +2632,12 @@ function campaignBreakdown(campaign) {
   const scenarios = [...byScenario.entries()].map(([scenarioId, comps]) => {
     const notesSet = notesByScenario.get(scenarioId);
     const notes = notesSet && notesSet.size > 0 ? [...notesSet].join(" | ") : void 0;
+    const emitted = emittedByScenario.get(scenarioId)?.text;
     return {
       scenarioId,
       composite: comps.reduce((a, b) => a + b, 0) / comps.length,
-      ...notes ? { notes } : {}
+      ...notes ? { notes } : {},
+      ...emitted ? { emitted } : {}
     };
   });
   return { dimensions, scenarios };
@@ -3811,8 +3676,6 @@ export {
   adversarialJudge,
   createCustomJudge,
   defaultJudges,
-  pairArms,
-  comparePairedArms,
   recoverTruncatedJson,
   dominates,
   paretoFrontier,
@@ -3875,4 +3738,4 @@ export {
   provenanceSpansPath,
   emitLoopProvenance
 };
-//# sourceMappingURL=chunk-32BZXMSO.js.map
+//# sourceMappingURL=chunk-VZ6VKOJT.js.map