@tangle-network/agent-eval 0.120.4 → 0.122.0

package/dist/{chunk-7QBFOJ3E.js → chunk-VZ6VKOJT.js}

@@ -1,7 +1,6 @@
 import {
   assertCampaignSplitIdentity,
   campaignCoverage,
-  campaignSplitDigest,
   canonicalJson,
   contentHash,
   createRunCostLedger,
@@ -10,11 +9,11 @@ import {
   resolveRunDir,
   runCampaign,
   summarizeAgentReceiptIntegrity
-} from "./chunk-SYV364BL.js";
+} from "./chunk-XDIRG3TO.js";
 import {
   clamp01,
   validatePolicyEditCandidateRecord
-} from "./chunk-5CVUPHJ4.js";
+} from "./chunk-S5TT5R3L.js";
 import {
   detectRewardHacking
 } from "./chunk-ARU2PZFM.js";
@@ -26,15 +25,12 @@ import {
   stripFencedJson
 } from "./chunk-NJC7U437.js";
 import {
-  mcnemar,
   pairedBootstrap,
-  pairedRiskDifference,
-  weightedComposite,
-  wilcoxonSignedRank
+  weightedComposite
 } from "./chunk-PJQFMIOX.js";
 import {
   CostLedger
-} from "./chunk-ZMXDQ4K7.js";
+} from "./chunk-JHOJHHU7.js";
 import {
   DEFAULT_REDACTION_RULES
 } from "./chunk-GGE4NNQT.js";
@@ -300,155 +296,6 @@ async function runJudgeChat(tc, input, judgeName, request) {
   return paid.value;
 }
 
-// src/paired-arms.ts
-function pairArms(rows, opts) {
-  const { baselineArm, treatmentArm } = opts;
-  if (baselineArm === treatmentArm) {
-    throw new ValidationError(
-      `pairArms: baselineArm and treatmentArm are both '${baselineArm}' \u2014 an arm cannot be compared to itself`
-    );
-  }
-  const byArm = /* @__PURE__ */ new Map();
-  const armsSeen = /* @__PURE__ */ new Set();
-  for (const row of rows) {
-    armsSeen.add(row.arm);
-    if (row.arm !== baselineArm && row.arm !== treatmentArm) continue;
-    const byKey = byArm.get(row.arm) ?? /* @__PURE__ */ new Map();
-    const group = byKey.get(row.pairKey) ?? [];
-    group.push(row);
-    byKey.set(row.pairKey, group);
-    byArm.set(row.arm, byKey);
-  }
-  for (const arm of [baselineArm, treatmentArm]) {
-    if (!byArm.has(arm)) {
-      const seen = [...armsSeen].sort().join(", ") || "<none>";
-      throw new ValidationError(`pairArms: no rows for arm '${arm}' (arms present: ${seen})`);
-    }
-  }
-  const baselineByKey = byArm.get(baselineArm);
-  const treatmentByKey = byArm.get(treatmentArm);
-  const allKeys = [.../* @__PURE__ */ new Set([...baselineByKey.keys(), ...treatmentByKey.keys()])].sort();
-  const pairs = [];
-  const unpairedBaseline = [];
-  const unpairedTreatment = [];
-  for (const pairKey of allKeys) {
-    const b = baselineByKey.get(pairKey) ?? [];
-    const t = treatmentByKey.get(pairKey) ?? [];
-    if (b.length <= 1 && t.length <= 1) {
-      if (b.length === 1 && t.length === 1) {
-        pairs.push({ pairKey, repIndex: 0, baseline: b[0], treatment: t[0] });
-      } else {
-        unpairedBaseline.push(...b);
-        unpairedTreatment.push(...t);
-      }
-      continue;
-    }
-    const bByRep = indexByRepKey(b, pairKey, baselineArm);
-    const tByRep = indexByRepKey(t, pairKey, treatmentArm);
-    const repKeys = [.../* @__PURE__ */ new Set([...bByRep.keys(), ...tByRep.keys()])].sort();
-    let repIndex = 0;
-    for (const repKey of repKeys) {
-      const baseline = bByRep.get(repKey);
-      const treatment = tByRep.get(repKey);
-      if (baseline !== void 0 && treatment !== void 0) {
-        pairs.push({ pairKey, repIndex: repIndex++, baseline, treatment });
-      } else if (baseline !== void 0) {
-        unpairedBaseline.push(baseline);
-      } else if (treatment !== void 0) {
-        unpairedTreatment.push(treatment);
-      }
-    }
-  }
-  return { pairs, unpairedBaseline, unpairedTreatment };
-}
-function indexByRepKey(group, pairKey, arm) {
-  const byRep = /* @__PURE__ */ new Map();
-  for (const row of group) {
-    if (row.repKey === void 0) {
-      throw new ValidationError(
-        `pairArms: pairKey '${pairKey}' has multiple reps in an arm, but a row in arm '${arm}' is missing repKey \u2014 multi-rep items require an explicit repKey on every row so reps pair by identity (pairing reps by outcome or by index would bias the paired statistics)`
-      );
-    }
-    if (byRep.has(row.repKey)) {
-      throw new ValidationError(
-        `pairArms: duplicate repKey '${row.repKey}' for pairKey '${pairKey}' in arm '${arm}' \u2014 (pairKey, repKey) must uniquely identify a rep within an arm`
-      );
-    }
-    byRep.set(row.repKey, row);
-  }
-  return byRep;
-}
-function comparePairedArms(rows, opts) {
-  const { pairs, unpairedBaseline, unpairedTreatment } = pairArms(rows, opts);
-  let correctness = null;
-  const baselinePass = [];
-  const treatmentPass = [];
-  for (const pair of pairs) {
-    if (pair.baseline.pass === void 0 || pair.treatment.pass === void 0) continue;
-    baselinePass.push(pair.baseline.pass ? 1 : 0);
-    treatmentPass.push(pair.treatment.pass ? 1 : 0);
-  }
-  if (baselinePass.length > 0) {
-    const mc = mcnemar(baselinePass, treatmentPass);
-    correctness = {
-      b10: mc.b,
-      b01: mc.c,
-      mcnemar: mc,
-      riskDifference: pairedRiskDifference(baselinePass, treatmentPass)
-    };
-  }
-  const metricNames = opts.metricNames ?? [
-    ...new Set(
-      pairs.flatMap((p) => [
-        ...Object.keys(p.baseline.metrics ?? {}),
-        ...Object.keys(p.treatment.metrics ?? {})
-      ])
-    )
-  ].sort();
-  const metricDeltas = metricNames.map((name) => {
-    const before = [];
-    const after = [];
-    let nMissing = 0;
-    for (const pair of pairs) {
-      const b = metricValue(pair.baseline, name);
-      const t = metricValue(pair.treatment, name);
-      if (b === void 0 || t === void 0) {
-        nMissing++;
-        continue;
-      }
-      before.push(b);
-      after.push(t);
-    }
-    const bootstrapCi = before.length === 0 ? null : pairedBootstrap(before, after, opts.bootstrap);
-    return {
-      name,
-      n: before.length,
-      nMissing,
-      medianDelta: bootstrapCi === null ? Number.NaN : bootstrapCi.median,
-      meanDelta: bootstrapCi === null ? Number.NaN : bootstrapCi.mean,
-      bootstrapCi,
-      wilcoxon: before.length === 0 ? null : wilcoxonSignedRank(before, after)
-    };
-  });
-  return {
-    nPairs: pairs.length,
-    nUnpairedBaseline: unpairedBaseline.length,
-    nUnpairedTreatment: unpairedTreatment.length,
-    correctness,
-    metricDeltas
-  };
-}
-function metricValue(row, name) {
-  const v = row.metrics?.[name];
-  if (v === void 0) return void 0;
-  if (!Number.isFinite(v)) {
-    throw new ValidationError(
-      `comparePairedArms: non-finite value for metric '${name}' on pairKey '${row.pairKey}' (arm '${row.arm}'): ${v}`
-    );
-  }
-  return v;
-}
-
 // src/pareto.ts
 function dominates(a, b, objectives) {
   let strictlyBetter = false;
@@ -2257,6 +2104,7 @@ function recoverFrom(slice) {
 }
 
 // src/reflective-mutation.ts
+var EMITTED_EVIDENCE_MAX_CHARS = 2e3;
 var DEFAULT_MUTATION_PRIMITIVES = [
   'Strengthen an imperative ("should" \u2192 "must")',
   "Add a concrete example pulled from a missed-golden phrase",
@@ -2288,7 +2136,7 @@ function buildReflectionPrompt(ctx) {
       );
       if (trial.failureNote) {
         sections.push("");
-        sections.push(`**Why it scored low:** ${truncate(trial.failureNote, 600)}`);
+        sections.push(`**Why it scored low:** ${truncate(trial.failureNote, 1500)}`);
       }
       const missed = (trial.expectations ?? []).filter((e) => !e.matched);
       if (missed.length > 0) {
@@ -2302,7 +2150,7 @@ function buildReflectionPrompt(ctx) {
         sections.push("");
         sections.push("**What the agent emitted:**");
         sections.push("```");
-        sections.push(truncate(trial.emitted, 600));
+        sections.push(truncate(trial.emitted, EMITTED_EVIDENCE_MAX_CHARS));
         sections.push("```");
       }
       sections.push("");
@@ -2713,7 +2561,11 @@ function buildEvidence(ctx, evidenceK, baseTarget) {
     score: s.composite,
     // The judge's "why it scored low" — grounds the reflection on real failure
     // patterns instead of blind rephrasing. Generalizable by the judge contract.
-    ...s.notes ? { failureNote: s.notes } : {}
+    ...s.notes ? { failureNote: s.notes } : {},
+    // The candidate's raw output for the scenario — buildReflectionPrompt
+    // renders it as the "what the agent emitted" block, so the reflection sees
+    // the actual wrong output, not just the score.
+    ...s.emitted ? { emitted: s.emitted } : {}
   });
   const top = byScore.slice(0, evidenceK).map(toTrace);
   const bottom = byScore.slice(-evidenceK).reverse().map(toTrace);
@@ -2738,6 +2590,7 @@ function campaignBreakdown(campaign) {
   const dimCounts = {};
   const byScenario = /* @__PURE__ */ new Map();
   const notesByScenario = /* @__PURE__ */ new Map();
+  const emittedByScenario = /* @__PURE__ */ new Map();
   for (const cell of campaign.cells) {
     const judgeScores = Object.values(cell.judgeScores).filter(
       (score) => score.failed !== true && Number.isFinite(score.composite)
@@ -2747,6 +2600,15 @@ function campaignBreakdown(campaign) {
     const arr = byScenario.get(cell.scenarioId) ?? [];
     arr.push(cellComposite);
     byScenario.set(cell.scenarioId, arr);
+    if (typeof cell.artifact === "string" && cell.artifact.trim().length > 0) {
+      const prev = emittedByScenario.get(cell.scenarioId);
+      if (!prev || cellComposite < prev.composite) {
+        emittedByScenario.set(cell.scenarioId, {
+          composite: cellComposite,
+          text: cell.artifact.slice(0, EMITTED_EVIDENCE_MAX_CHARS)
+        });
+      }
+    }
     for (const s of judgeScores) {
       if (s.notes?.trim()) {
         const set = notesByScenario.get(cell.scenarioId) ?? /* @__PURE__ */ new Set();
@@ -2770,10 +2632,12 @@ function campaignBreakdown(campaign) {
   const scenarios = [...byScenario.entries()].map(([scenarioId, comps]) => {
     const notesSet = notesByScenario.get(scenarioId);
     const notes = notesSet && notesSet.size > 0 ? [...notesSet].join(" | ") : void 0;
+    const emitted = emittedByScenario.get(scenarioId)?.text;
     return {
       scenarioId,
       composite: comps.reduce((a, b) => a + b, 0) / comps.length,
-      ...notes ? { notes } : {}
+      ...notes ? { notes } : {},
+      ...emitted ? { emitted } : {}
     };
   });
   return { dimensions, scenarios };
@@ -2797,32 +2661,23 @@ async function runOptimization(opts) {
       "runOptimization: promoteTopK must be 1 because the loop has one global incumbent"
     );
   }
-  const requireJudgeScore = (opts.judges?.length ?? 0) > 0;
-  const reps = opts.reps ?? 1;
-  const premeasuredBaseline = opts.premeasuredBaseline;
-  const baselineCampaign = premeasuredBaseline ? validatedPremeasuredBaseline({
-    input: premeasuredBaseline,
-    baselineSurface: opts.baselineSurface,
-    scenarios: opts.scenarios,
-    reps,
-    seed: opts.seed ?? 42
-  }) : await runCampaign({
+  const baselineCampaign = await runCampaign({
     ...opts,
     costLedger,
     costPhase: "search.baseline",
     dispatch: (scenario, ctx) => opts.dispatchWithSurface(opts.baselineSurface, scenario, ctx),
     runDir: `${opts.runDir}/baseline`
   });
+  const requireJudgeScore = (opts.judges?.length ?? 0) > 0;
   const baselineCoverage = campaignCoverage(
     baselineCampaign.cells,
     opts.scenarios,
-    reps,
+    opts.reps ?? 1,
     requireJudgeScore
   );
   if (!baselineCoverage.complete) {
-    const label = opts.premeasuredBaseline ? "premeasured baseline" : "baseline";
     throw new Error(
-      `runOptimization: ${label} is incomplete (${baselineCoverage.scorableCellIds.length}/${baselineCoverage.expectedCellIds.length} designed cells scorable) \u2014 ${formatCoverageFailures(baselineCoverage)}. Refusing to optimize against an incomplete incumbent.`
+      `runOptimization: baseline is incomplete (${baselineCoverage.scorableCellIds.length}/${baselineCoverage.expectedCellIds.length} designed cells scorable) \u2014 ${formatCoverageFailures(baselineCoverage)}. Refusing to optimize against an incomplete incumbent.`
     );
   }
   const generations = [];
@@ -2846,7 +2701,7 @@ async function runOptimization(opts) {
   if (opts.analyzeGeneration && opts.maxGenerations > 0 && baselineCampaign.cells.length > 0) {
     const fresh = await opts.analyzeGeneration({
       generation: -1,
-      runDir: baselineCampaign.runDir,
+      runDir: `${opts.runDir}/baseline`,
       candidates: [
         { surfaceHash: winnerSurfaceHash, campaign: baselineCampaign, composite: winnerComposite }
       ],
@@ -2997,38 +2852,6 @@ async function runOptimization(opts) {
     cost: costLedger.summary()
   };
 }
-function validatedPremeasuredBaseline(args) {
-  const { input } = args;
-  if (input.surfaceHash !== surfaceHash(args.baselineSurface)) {
-    throw new Error(
-      "runOptimization: premeasured baseline surface hash does not match baselineSurface"
-    );
-  }
-  const campaign = input.campaign;
-  if (campaign.reps !== args.reps) {
-    throw new Error(
-      `runOptimization: premeasured baseline reps ${campaign.reps} do not match requested reps ${args.reps}`
-    );
-  }
-  if (campaign.seed !== args.seed) {
-    throw new Error(
-      `runOptimization: premeasured baseline seed ${campaign.seed} does not match requested seed ${args.seed}`
-    );
-  }
-  try {
-    assertCampaignSplitIdentity(campaign.scenarios, campaign.reps, campaign.splitDigest);
-  } catch (error) {
-    throw new Error(
-      `runOptimization: premeasured baseline has an invalid retained split identity \u2014 ${error instanceof Error ? error.message : String(error)}`
-    );
-  }
-  if (campaign.splitDigest !== campaignSplitDigest(args.scenarios, args.reps)) {
-    throw new Error(
-      "runOptimization: premeasured baseline split does not match the requested scenarios"
-    );
-  }
-  return campaign;
-}
 function toParetoParent(surface, hash, campaign, generation, label, rationale) {
   const objectives = {};
   for (const { scenarioId, composite } of campaignBreakdown(campaign).scenarios) {
@@ -3123,7 +2946,7 @@ async function runImprovementLoop(opts) {
   const dispatchTimeoutMs = opts.dispatchTimeoutMs ?? DEFAULT_DISPATCH_TIMEOUT_MS;
   const optimization = await runOptimization({ ...opts, dispatchTimeoutMs, costLedger });
   const winnerIsBaseline = optimization.winnerSurfaceHash === surfaceHash(opts.baselineSurface);
-  const { runCampaign: runCampaign2 } = await import("./run-campaign-75RTPVV5.js");
+  const { runCampaign: runCampaign2 } = await import("./run-campaign-HNFPJET4.js");
   const baselineOnHoldout = await runCampaign2({
     ...opts,
     costLedger,
@@ -3853,8 +3676,6 @@ export {
   adversarialJudge,
   createCustomJudge,
   defaultJudges,
-  pairArms,
-  comparePairedArms,
   recoverTruncatedJson,
   dominates,
   paretoFrontier,
@@ -3917,4 +3738,4 @@ export {
   provenanceSpansPath,
   emitLoopProvenance
 };
-//# sourceMappingURL=chunk-7QBFOJ3E.js.map
+//# sourceMappingURL=chunk-VZ6VKOJT.js.map