@tangle-network/agent-eval 0.118.3 → 0.119.1

package/dist/contract/index.js

@@ -13,7 +13,7 @@ import {
 import {
   analyzeRuns,
   summarizeExecution
-} from "../chunk-7V3QDWNL.js";
+} from "../chunk-XJ7JVCHB.js";
 import {
   REFERENCE_EQUIVALENCE_INPUT_LIMITS,
   REFERENCE_EQUIVALENCE_JUDGE_VERSION,
@@ -26,6 +26,7 @@ import {
   evolutionaryProposer,
   gepaProposer,
   heldOutGate,
+  pairArms,
   paretoPolicy,
   paretoSignificanceGate,
   powerPreflight,
@@ -34,20 +35,21 @@ import {
   runReferenceEquivalenceJudge,
   surfaceContentHash,
   surfaceHash
-} from "../chunk-QWRLW4CT.js";
+} from "../chunk-PSXJ32SR.js";
 import {
+  canonicalJson,
   createRunCostLedger,
   fsCampaignStorage,
   inMemoryCampaignStorage,
   resolveRunDir,
   runCampaign
-} from "../chunk-EGNRE7VA.js";
+} from "../chunk-PAHNGS65.js";
 import {
   buildDefaultAnalystRegistry,
   createChatClient
-} from "../chunk-267UCQWI.js";
+} from "../chunk-QWMPPZ3X.js";
 import "../chunk-HHWE3POT.js";
-import "../chunk-VNLWDTDA.js";
+import "../chunk-S5TT5R3L.js";
 import {
   FileSystemOutcomeStore,
   InMemoryOutcomeStore
@@ -55,8 +57,10 @@ import {
 import "../chunk-ARU2PZFM.js";
 import "../chunk-NJC7U437.js";
 import "../chunk-DPZAEKA6.js";
-import "../chunk-PJQFMIOX.js";
-import "../chunk-MLHRFWQK.js";
+import {
+  pairedBootstrap
+} from "../chunk-PJQFMIOX.js";
+import "../chunk-JHOJHHU7.js";
 import "../chunk-VI2UW6B6.js";
 import {
   recordAggregateMeasurements,
@@ -65,7 +69,7 @@ import {
 import "../chunk-PXE2VKMX.js";
 import "../chunk-ZET2UAYW.js";
 import "../chunk-GGE4NNQT.js";
-import "../chunk-6QIM2EAP.js";
+import "../chunk-PXD6ZFNY.js";
 import "../chunk-PC4UYEBM.js";
 import {
   parseRunRecordSafe
@@ -536,6 +540,277 @@ function requirePositiveInteger(value, field) {
   return value;
 }
 
+// src/contract/measured-comparison.ts
+import { createHash as createHash2 } from "crypto";
+import {
+  agentImprovementMeasuredComparisonSchema
+} from "@tangle-network/agent-interface";
+function measuredComparisonFromSelfImproveResult(options) {
+  const { result } = options;
+  const power = result.power;
+  if (!power) throw new Error("agent improvement comparison requires heldout power analysis");
+  if (result.provenance.gate.reasons.length === 0) {
+    throw new Error("agent improvement comparison requires measured decision reasons");
+  }
+  const pairs = pairMeasuredCells(
+    result.raw.baselineOnHoldout.cells,
+    result.raw.winnerOnHoldout.cells
+  );
+  const composite = measuredObjective(
+    {
+      kind: "objective",
+      name: "composite",
+      direction: "higher-is-better",
+      unit: "score"
+    },
+    pairs,
+    measuredComposite
+  );
+  assertMeasuredNumber(result.lift, composite.delta, "heldout lift");
+  assertMeasuredNumber(result.baseline.compositeMean, composite.baseline, "heldout baseline");
+  assertMeasuredNumber(result.winner.compositeMean, composite.candidate, "heldout candidate");
+  assertMeasuredNumber(result.provenance.heldOutLift, composite.delta, "provenance heldout lift");
+  if (result.gateDecision !== result.provenance.gate.decision || power.n !== composite.n || power.confidence !== 0.95) {
+    throw new Error("agent improvement measurement sources do not agree");
+  }
+  return agentImprovementMeasuredComparisonSchema.parse({
+    schemaVersion: 1,
+    kind: "agent-improvement-measured-comparison",
+    benchmark: options.benchmark,
+    baselineProfileDigest: options.baselineProfileDigest,
+    candidateBundleDigest: options.candidateBundleDigest,
+    overall: {
+      name: "composite",
+      baseline: composite.baseline,
+      candidate: composite.candidate,
+      delta: composite.delta,
+      confidenceInterval: composite.confidenceInterval,
+      n: composite.n,
+      direction: "higher-is-better",
+      unit: "score"
+    },
+    objectives: measuredObjectives(pairs),
+    ...result.winner.label || result.winner.rationale ? {
+      candidate: {
+        ...result.winner.label ? { label: result.winner.label } : {},
+        ...result.winner.rationale ? { rationale: result.winner.rationale } : {}
+      }
+    } : {},
+    decision: {
+      outcome: result.gateDecision,
+      reasons: result.provenance.gate.reasons,
+      contributingChecks: result.provenance.gate.contributingGates.map((check) => ({
+        name: check.name,
+        passed: check.passed
+      }))
+    },
+    power: {
+      sufficient: power.scaleAssumed && !power.underpowered,
+      n: power.n,
+      minimumDetectableDelta: power.mde,
+      confidenceLevel: power.confidence,
+      scaleAssumed: power.scaleAssumed,
+      sharedScorerChannel: power.sharedChannelCaveat !== void 0,
+      reason: power.recommendation
+    },
+    provenance: {
+      kind: "agent-eval-loop",
+      schema: result.provenance.schema,
+      runId: result.provenance.runId,
+      recordDigest: digest(result.provenance),
+      baselineContentHash: result.provenance.baselineContentHash,
+      candidateContentHash: result.provenance.winnerContentHash
+    },
+    diff: result.diff,
+    evaluation: {
+      generationsExplored: result.generationsExplored,
+      durationMs: result.durationMs,
+      totalCostUsd: result.totalCostUsd
+    }
+  });
+}
+function measuredObjectives(pairs) {
+  const qualityColumns = /* @__PURE__ */ new Map();
+  for (const [baseline, candidate] of pairs) {
+    for (const cell of [baseline, candidate]) {
+      for (const [objective, score] of Object.entries(cell.judgeScores)) {
+        if (score.failed) continue;
+        qualityColumns.set(`objective:${objective}`, {
+          kind: "objective",
+          name: objective,
+          direction: "higher-is-better",
+          unit: "score"
+        });
+        for (const name of Object.keys(score.dimensions)) {
+          qualityColumns.set(`dimension:${objective}:${name}`, {
+            kind: "dimension",
+            objective,
+            name,
+            direction: "higher-is-better",
+            unit: "score"
+          });
+        }
+      }
+    }
+  }
+  return [
+    ...[...qualityColumns.entries()].sort(([left], [right]) => left.localeCompare(right)).map(
+      ([, column]) => measuredObjective(column, pairs, (cell) => measuredQuality(cell, column))
+    ),
+    measuredCostObjective(pairs),
+    measuredObjective(
+      {
+        kind: "latency",
+        name: "latency",
+        direction: "lower-is-better",
+        unit: "milliseconds"
+      },
+      pairs,
+      (cell) => cell.durationMs
+    )
+  ];
+}
+function measuredCostObjective(pairs) {
+  const cells = pairs.flat();
+  for (const cell of cells) finiteMeasuredValue(cell.costUsd, "cost:cost");
+  const reported = cells.some(
+    (cell) => cell.costUsd !== 0 || cell.tokenUsage !== void 0 && (cell.tokenUsage.input > 0 || cell.tokenUsage.output > 0)
+  );
+  if (!reported) {
+    return {
+      kind: "cost",
+      name: "cost",
+      availability: "unavailable",
+      reason: "heldout cells did not report model usage or cost",
+      direction: "lower-is-better",
+      unit: "usd"
+    };
+  }
+  return measuredObjective(
+    {
+      kind: "cost",
+      name: "cost",
+      direction: "lower-is-better",
+      unit: "usd"
+    },
+    pairs,
+    (cell) => cell.costUsd
+  );
+}
+function measuredComposite(cell) {
+  const values = Object.values(cell.judgeScores).filter((score) => !score.failed).map((score) => score.composite).filter(Number.isFinite);
+  if (values.length === 0) {
+    throw new Error(`heldout cell ${measuredCellKey(cell)} has no successful composite score`);
+  }
+  return measuredMean(values);
+}
+function pairMeasuredCells(baselineCells, candidateCells) {
+  const errors = [...baselineCells, ...candidateCells].filter((cell) => cell.error);
+  if (errors.length > 0) {
+    throw new Error(
+      `measured objectives cannot publish ${errors.length} errored heldout cells: ${errors.map((cell) => measuredCellKey(cell)).join(", ")}`
+    );
+  }
+  const rows = [
+    ...baselineCells.map((cell) => ({
+      pairKey: cell.scenarioId,
+      repKey: String(cell.rep),
+      arm: "baseline",
+      cell
+    })),
+    ...candidateCells.map((cell) => ({
+      pairKey: cell.scenarioId,
+      repKey: String(cell.rep),
+      arm: "candidate",
+      cell
+    }))
+  ];
+  const paired = pairArms(rows, { baselineArm: "baseline", treatmentArm: "candidate" });
+  if (paired.pairs.length === 0 || paired.unpairedBaseline.length > 0 || paired.unpairedTreatment.length > 0) {
+    throw new Error("measured objectives require the same non-empty paired heldout cells");
+  }
+  return paired.pairs.map(
+    (pair) => [pair.baseline.cell, pair.treatment.cell]
+  );
+}
+function measuredObjective(identity, pairs, value) {
+  const key = measuredObjectiveKey(identity);
+  const baseline = pairs.map(([cell]) => finiteMeasuredValue(value(cell), key));
+  const candidate = pairs.map(([, cell]) => finiteMeasuredValue(value(cell), key));
+  const interval = pairedBootstrap(baseline, candidate, {
+    confidence: 0.95,
+    resamples: 2e3,
+    statistic: "mean",
+    seed: measuredSeed(key)
+  });
+  const baselineMean = measuredMean(baseline);
+  const candidateMean = measuredMean(candidate);
+  return {
+    ...identity,
+    availability: "measured",
+    baseline: baselineMean,
+    candidate: candidateMean,
+    delta: candidateMean - baselineMean,
+    confidenceInterval: {
+      level: interval.confidence,
+      lower: interval.low,
+      upper: interval.high,
+      method: "paired-bootstrap",
+      statistic: "mean",
+      resamples: interval.resamples
+    },
+    n: interval.n
+  };
+}
+function measuredQuality(cell, column) {
+  const objective = column.kind === "objective" ? column.name : column.objective;
+  const score = cell.judgeScores[objective];
+  if (!score || score.failed) {
+    throw new Error(
+      `heldout cell ${measuredCellKey(cell)} is missing measured objective '${objective}'`
+    );
+  }
+  const value = column.kind === "objective" ? score.composite : score.dimensions[column.name];
+  if (value === void 0) {
+    throw new Error(
+      `heldout cell ${measuredCellKey(cell)} is missing '${objective}' dimension '${column.name}'`
+    );
+  }
+  return finiteMeasuredValue(value, measuredObjectiveKey(column));
+}
+function measuredObjectiveKey(identity) {
+  return identity.kind === "dimension" ? `${identity.kind}:${identity.objective}:${identity.name}` : `${identity.kind}:${identity.name}`;
+}
+function measuredCellKey(cell) {
+  return `${cell.scenarioId}:${cell.rep}`;
+}
+function finiteMeasuredValue(value, name) {
+  if (!Number.isFinite(value)) throw new Error(`measured objective '${name}' is not finite`);
+  return value;
+}
+function measuredMean(values) {
+  if (values.length === 0) throw new Error("measured objective has no paired values");
+  return values.reduce((sum, value) => sum + value, 0) / values.length;
+}
+function measuredSeed(value) {
+  let seed = 2166136261;
+  for (const byte of Buffer.from(value, "utf8")) {
+    seed = Math.imul(seed ^ byte, 16777619) >>> 0;
+  }
+  return seed;
+}
+function assertMeasuredNumber(actual, expected, name) {
+  const tolerance = Number.EPSILON * Math.max(1, Math.abs(actual), Math.abs(expected)) * 8;
+  if (!Number.isFinite(actual) || !Number.isFinite(expected) || Math.abs(actual - expected) > tolerance) {
+    throw new Error(`${name} does not agree across the measured comparison`);
+  }
+}
+function digest(value) {
+  const json = JSON.stringify(value);
+  if (json === void 0) throw new Error("agent improvement provenance is not serializable");
+  return `sha256:${createHash2("sha256").update(canonicalJson(JSON.parse(json))).digest("hex")}`;
+}
+
 // src/contract/eval-reporting-suite.ts
 import { mkdir, writeFile } from "fs/promises";
 import { dirname, join as join2 } from "path";
@@ -1074,6 +1349,7 @@ export {
   gepaProposer,
   heldOutGate,
   inMemoryCampaignStorage,
+  measuredComparisonFromSelfImproveResult,
   paretoPolicy,
   paretoSignificanceGate,
   parseAgentTrace,