@tangle-network/agent-eval 0.126.7 → 0.127.0

package/dist/{chunk-E3IADPLG.js → chunk-PRB53E3J.js}

@@ -7,7 +7,7 @@ import {
   pairHoldout,
   recoverTruncatedJson,
   surfaceContentHash
-} from "./chunk-BTACVO2W.js";
+} from "./chunk-65QETEKG.js";
 import {
   SearchLedgerConflictError,
   SearchLedgerError,
@@ -19,29 +19,33 @@ import {
   runCampaign,
   summarizeBackendIntegrity,
   withSearchLedgerFileLock
-} from "./chunk-ZVCHKKOP.js";
+} from "./chunk-WED5KLZQ.js";
 import {
   Mutex
 } from "./chunk-WGXIEX7P.js";
+import {
+  campaignCellToRunRecord
+} from "./chunk-ZPUEEM5U.js";
+import {
+  comparePairedArms
+} from "./chunk-DPUHNQLN.js";
 import {
   eProcess,
-  mcnemar,
   mulberry32,
-  pairedBootstrap,
-  pairedRiskDifference,
-  wilcoxonSignedRank
-} from "./chunk-PJQFMIOX.js";
+  pairedBootstrap
+} from "./chunk-MHELPNRP.js";
 import {
   CostLedger
 } from "./chunk-WS3NZZQQ.js";
 import {
   analyzeTraces
-} from "./chunk-NUKSVU3W.js";
+} from "./chunk-5DTSBUL2.js";
 import {
   buildAgentProfileCell,
   modelHasSnapshot,
+  runTaskScore,
   validateRunRecord
-} from "./chunk-IILEIWGW.js";
+} from "./chunk-NL33OXEP.js";
 import {
   canonicalize
 } from "./chunk-VSMTAMNK.js";
@@ -112,155 +116,6 @@ function failureModeRecallJudge(opts = {}) {
   };
 }
 
-// src/paired-arms.ts
-function pairArms(rows, opts) {
-  const { baselineArm, treatmentArm } = opts;
-  if (baselineArm === treatmentArm) {
-    throw new ValidationError(
-      `pairArms: baselineArm and treatmentArm are both '${baselineArm}' \u2014 an arm cannot be compared to itself`
-    );
-  }
-  const byArm = /* @__PURE__ */ new Map();
-  const armsSeen = /* @__PURE__ */ new Set();
-  for (const row of rows) {
-    armsSeen.add(row.arm);
-    if (row.arm !== baselineArm && row.arm !== treatmentArm) continue;
-    const byKey = byArm.get(row.arm) ?? /* @__PURE__ */ new Map();
-    const group = byKey.get(row.pairKey) ?? [];
-    group.push(row);
-    byKey.set(row.pairKey, group);
-    byArm.set(row.arm, byKey);
-  }
-  for (const arm of [baselineArm, treatmentArm]) {
-    if (!byArm.has(arm)) {
-      const seen = [...armsSeen].sort().join(", ") || "<none>";
-      throw new ValidationError(`pairArms: no rows for arm '${arm}' (arms present: ${seen})`);
-    }
-  }
-  const baselineByKey = byArm.get(baselineArm);
-  const treatmentByKey = byArm.get(treatmentArm);
-  const allKeys = [.../* @__PURE__ */ new Set([...baselineByKey.keys(), ...treatmentByKey.keys()])].sort();
-  const pairs = [];
-  const unpairedBaseline = [];
-  const unpairedTreatment = [];
-  for (const pairKey of allKeys) {
-    const b = baselineByKey.get(pairKey) ?? [];
-    const t = treatmentByKey.get(pairKey) ?? [];
-    if (b.length <= 1 && t.length <= 1) {
-      if (b.length === 1 && t.length === 1) {
-        pairs.push({ pairKey, repIndex: 0, baseline: b[0], treatment: t[0] });
-      } else {
-        unpairedBaseline.push(...b);
-        unpairedTreatment.push(...t);
-      }
-      continue;
-    }
-    const bByRep = indexByRepKey(b, pairKey, baselineArm);
-    const tByRep = indexByRepKey(t, pairKey, treatmentArm);
-    const repKeys = [.../* @__PURE__ */ new Set([...bByRep.keys(), ...tByRep.keys()])].sort();
-    let repIndex = 0;
-    for (const repKey of repKeys) {
-      const baseline = bByRep.get(repKey);
-      const treatment = tByRep.get(repKey);
-      if (baseline !== void 0 && treatment !== void 0) {
-        pairs.push({ pairKey, repIndex: repIndex++, baseline, treatment });
-      } else if (baseline !== void 0) {
-        unpairedBaseline.push(baseline);
-      } else if (treatment !== void 0) {
-        unpairedTreatment.push(treatment);
-      }
-    }
-  }
-  return { pairs, unpairedBaseline, unpairedTreatment };
-}
-function indexByRepKey(group, pairKey, arm) {
-  const byRep = /* @__PURE__ */ new Map();
-  for (const row of group) {
-    if (row.repKey === void 0) {
-      throw new ValidationError(
-        `pairArms: pairKey '${pairKey}' has multiple reps in an arm, but a row in arm '${arm}' is missing repKey \u2014 multi-rep items require an explicit repKey on every row so reps pair by identity (pairing reps by outcome or by index would bias the paired statistics)`
-      );
-    }
-    if (byRep.has(row.repKey)) {
-      throw new ValidationError(
-        `pairArms: duplicate repKey '${row.repKey}' for pairKey '${pairKey}' in arm '${arm}' \u2014 (pairKey, repKey) must uniquely identify a rep within an arm`
-      );
-    }
-    byRep.set(row.repKey, row);
-  }
-  return byRep;
-}
-function comparePairedArms(rows, opts) {
-  const { pairs, unpairedBaseline, unpairedTreatment } = pairArms(rows, opts);
-  let correctness = null;
-  const baselinePass = [];
-  const treatmentPass = [];
-  for (const pair of pairs) {
-    if (pair.baseline.pass === void 0 || pair.treatment.pass === void 0) continue;
-    baselinePass.push(pair.baseline.pass ? 1 : 0);
-    treatmentPass.push(pair.treatment.pass ? 1 : 0);
-  }
-  if (baselinePass.length > 0) {
-    const mc = mcnemar(baselinePass, treatmentPass);
-    correctness = {
-      b10: mc.b,
-      b01: mc.c,
-      mcnemar: mc,
-      riskDifference: pairedRiskDifference(baselinePass, treatmentPass)
-    };
-  }
-  const metricNames = opts.metricNames ?? [
-    ...new Set(
-      pairs.flatMap((p) => [
-        ...Object.keys(p.baseline.metrics ?? {}),
-        ...Object.keys(p.treatment.metrics ?? {})
-      ])
-    )
-  ].sort();
-  const metricDeltas = metricNames.map((name) => {
-    const before = [];
-    const after = [];
-    let nMissing = 0;
-    for (const pair of pairs) {
-      const b = metricValue(pair.baseline, name);
-      const t = metricValue(pair.treatment, name);
-      if (b === void 0 || t === void 0) {
-        nMissing++;
-        continue;
-      }
-      before.push(b);
-      after.push(t);
-    }
-    const bootstrapCi = before.length === 0 ? null : pairedBootstrap(before, after, opts.bootstrap);
-    return {
-      name,
-      n: before.length,
-      nMissing,
-      medianDelta: bootstrapCi === null ? Number.NaN : bootstrapCi.median,
-      meanDelta: bootstrapCi === null ? Number.NaN : bootstrapCi.mean,
-      bootstrapCi,
-      wilcoxon: before.length === 0 ? null : wilcoxonSignedRank(before, after)
-    };
-  });
-  return {
-    nPairs: pairs.length,
-    nUnpairedBaseline: unpairedBaseline.length,
-    nUnpairedTreatment: unpairedTreatment.length,
-    correctness,
-    metricDeltas
-  };
-}
-function metricValue(row, name) {
-  const v = row.metrics?.[name];
-  if (v === void 0) return void 0;
-  if (!Number.isFinite(v)) {
-    throw new ValidationError(
-      `comparePairedArms: non-finite value for metric '${name}' on pairKey '${row.pairKey}' (arm '${row.arm}'): ${v}`
-    );
-  }
-  return v;
-}
-
 // src/campaign/cross-surface-context.ts
 function validateCrossSurfaceInput(input) {
   assertNonEmptyUnique(input.taskOrder, "taskOrder");
@@ -1528,7 +1383,7 @@ function neutralizationGate(options) {
           contributingGates: [
             {
               name: "neutralizationGate",
-              passed: false,
+              status: "fail",
               detail: { candidateLift: cand.lift, neutralizedLift: neut.lift, n: cand.n }
             }
           ],
@@ -1548,7 +1403,7 @@ function neutralizationGate(options) {
         contributingGates: [
           {
             name: "neutralizationGate",
-            passed,
+            status: passed ? "pass" : "fail",
             detail: {
               candidateLift: cand.lift,
               neutralizedLift: neut.lift,
@@ -1754,14 +1609,20 @@ function sequentialPairedGate(options) {
         return {
           decision: "need_more_work",
           reasons: [`${name}: no paired holdout observations \u2014 nothing to test`],
-          contributingGates: [{ name, passed: false, detail }]
+          contributingGates: [{ name, status: "not_evaluated", detail }]
         };
       }
       const decision = last.decision === "promote" ? "ship" : last.decision === "continue" ? "need_more_work" : "hold";
       return {
         decision,
         reasons: [`${name}: ${last.reason}`],
-        contributingGates: [{ name, passed: decision === "ship", detail }],
+        contributingGates: [
+          {
+            name,
+            status: decision === "ship" ? "pass" : decision === "need_more_work" ? "not_evaluated" : "fail",
+            detail
+          }
+        ],
         delta: meanDelta
       };
     }
@@ -1785,7 +1646,11 @@ function sequentialDecide(options = {}) {
         `sequentialDecide: generation ${record.generationIndex} has no candidates \u2014 cannot extract a top candidate`
       );
     }
-    return record.candidates.reduce((best, c) => c.composite > best.composite ? c : best);
+    return record.candidates.reduce((best, candidate) => {
+      if (best.composite === null) return candidate;
+      if (candidate.composite === null) return best;
+      return candidate.composite > best.composite ? candidate : best;
+    });
   };
   const decide = ({ history }) => {
     if (stopped) return stopped;
@@ -2726,10 +2591,6 @@ function sha(input) {
 function mean2(xs) {
   return xs.length === 0 ? 0 : xs.reduce((a, b) => a + b, 0) / xs.length;
 }
-function cellComposite(cell) {
-  const composites = Object.values(cell.judgeScores).map((s) => s.composite);
-  return composites.length === 0 ? 0 : mean2(composites);
-}
 function requireResolvedModel(cell, profileId) {
   const resolved = cell.resolvedModel?.trim();
   if (!resolved) {
@@ -2749,60 +2610,17 @@ function buildRunRecord(args) {
   const profileId = agentProfileId(profile);
   const declaredModel = agentProfileModelId(profile);
   const model = declaredModel === HARNESS_NATIVE_MODEL ? requireResolvedModel(cell, profileId) : declaredModel;
-  const composite = cellComposite(cell);
-  const raw = { composite };
-  const perJudge = {};
-  const dimAccum = {};
-  const notes = [];
-  for (const [judgeName, js] of Object.entries(cell.judgeScores)) {
-    perJudge[judgeName] = { ...js.dimensions };
-    for (const [dim, value] of Object.entries(js.dimensions)) {
-      raw[`${judgeName}.${dim}`] = value;
-      dimAccum[dim] ??= [];
-      dimAccum[dim].push(value);
-    }
-    if (js.notes) notes.push(`${judgeName}: ${js.notes}`);
-  }
-  const perDimMean = {};
-  for (const [dim, values] of Object.entries(dimAccum)) perDimMean[dim] = mean2(values);
-  const costUsd = cell.costUsd;
-  raw.cost_usd = costUsd;
-  raw.cost_estimated = cell.costEstimated ? 1 : 0;
-  raw.tokens_input = cell.tokenUsage.input;
-  raw.tokens_output = cell.tokenUsage.output;
-  if (typeof cell.tokenUsage.cached === "number") raw.tokens_cached = cell.tokenUsage.cached;
-  raw.latency_ms = cell.durationMs;
-  if (costUsd > 0) {
-    raw.tokens_per_dollar = (cell.tokenUsage.input + cell.tokenUsage.output) / costUsd;
-  }
-  if (composite > 0.01) raw.cost_per_quality = costUsd / composite;
-  const outcome = splitTag === "holdout" ? { holdoutScore: composite, raw } : { searchScore: composite, raw };
-  if (Object.keys(perJudge).length > 0) {
-    outcome.judgeScores = {
-      perJudge,
-      perDimMean,
-      composite,
-      ...notes.length > 0 ? { notes: notes.join(" | ") } : {}
-    };
-  }
-  const record = {
+  const record = campaignCellToRunRecord(cell, {
     runId: `${matrixId}:${profileId}:${cell.cellId}`,
     experimentId,
     candidateId: profileId,
-    seed: cell.seed,
     model,
     promptHash: profileHash,
     configHash,
     commitSha,
-    wallMs: cell.durationMs,
-    costUsd,
-    tokenUsage: cell.tokenUsage,
-    outcome,
     splitTag,
-    scenarioId: cell.scenarioId,
-    ...args.agentProfileCell ? { agentProfile: args.agentProfileCell } : {},
-    ...cell.error ? { failureMode: cell.error } : {}
-  };
+    agentProfile: args.agentProfileCell
+  });
   if (args.corpusText && args.scenario) {
     try {
       const text = args.corpusText(cell.artifact, args.scenario);
@@ -2842,10 +2660,13 @@ async function runProfileMatrix(opts) {
         configHash: profileHash,
         commitSha: opts.commitSha,
         wallMs: 0,
-        costUsd: 0,
+        costUsd: null,
+        costProvenance: { kind: "uncaptured", usd: null },
         tokenUsage: { input: 0, output: 0 },
-        outcome: splitTag === "holdout" ? { holdoutScore: 0, raw: {} } : { searchScore: 0, raw: {} },
-        splitTag
+        terminalOutcome: "succeeded",
+        outcome: { raw: { execution_error_count: 0 } },
+        splitTag,
+        scenarioId: "recordability-probe"
       });
     } catch (err) {
       throw new ProfileMatrixError(
@@ -2920,7 +2741,9 @@ async function runProfileMatrix(opts) {
       // one harness, so the first record's model is representative).
       model: declaredModel === HARNESS_NATIVE_MODEL ? profileRecords[0]?.model ?? declaredModel : declaredModel,
       records: profileRecords.length,
-      meanComposite: mean2(profileRecords.map(compositeOf)),
+      meanComposite: meanOrNull2(
+        profileRecords.map(scoreOf).filter((score) => score !== void 0)
+      ),
       totalCostUsd,
       integrity: summarizeBackendIntegrity(profileRecords)
     };
@@ -2937,16 +2760,21 @@ async function runProfileMatrix(opts) {
   const byPersona = opts.personaOf ? rollupByPersona(records, opts.scenarios, opts.personaOf) : void 0;
   return { matrixId, experimentId, records, byProfile, byScenario, byPersona, integrity, campaigns };
 }
-function compositeOf(r) {
-  return r.outcome.holdoutScore ?? r.outcome.searchScore ?? 0;
+function scoreOf(r) {
+  return runTaskScore(r);
+}
+function meanOrNull2(values) {
+  return values.length === 0 ? null : mean2(values);
 }
 function rollup(records, keyOf) {
   const groups = /* @__PURE__ */ new Map();
   for (const r of records) {
     const key = keyOf(r);
     if (key === void 0) continue;
+    const score = scoreOf(r);
+    if (score === void 0) continue;
     const arr = groups.get(key) ?? [];
-    arr.push(compositeOf(r));
+    arr.push(score);
     groups.set(key, arr);
   }
   const out = {};
@@ -3508,7 +3336,7 @@ function replayEntries(entries, campaignId) {
         const generationOperation = event.plan.operations.find(
           (operation) => operation.operationId === slot.generationOperationId
         );
-        if (!generationOperation || generationOperation.kind !== "candidate-generation") {
+        if (generationOperation?.kind !== "candidate-generation") {
           throw new SearchLedgerIntegrityError(
             `candidate slot ${slot.slotId} references unplanned candidate-generation operation ${slot.generationOperationId}`
           );
@@ -4585,8 +4413,6 @@ function resolveWorktreePath(surface, worktreeDir) {
 }
 
 export {
-  pairArms,
-  comparePairedArms,
   completionVerdict,
   verifyCompletion,
   parseCorrectnessResponse,
@@ -4634,4 +4460,4 @@ export {
   verifyCodeSurface,
   resolveWorktreePath
 };
-//# sourceMappingURL=chunk-E3IADPLG.js.map
+//# sourceMappingURL=chunk-PRB53E3J.js.map