@tangle-network/agent-eval 0.77.0 → 0.80.0

package/dist/contract/index.d.ts

@@ -1,15 +1,15 @@
-import { S as Scenario, M as MutableSurface, b as DispatchContext, a as JudgeConfig, I as ImprovementDriver, G as Gate } from '../types-Bba0vl1V.js';
-export { C as CampaignAggregates, c as CampaignArtifactWriter, d as CampaignCellResult, e as CampaignCostMeter, f as CampaignResult, g as CampaignTraceWriter, h as CodeSurface, D as Dispatch, i as GateContext, j as GateDecision, k as GateResult, l as GenerationCandidate, m as GenerationRecord, n as JudgeDimension, J as JudgeScore, o as Mutator, O as OptimizerConfig, p as SessionScript } from '../types-Bba0vl1V.js';
-import { L as LoopProvenanceRecord } from '../provenance-B-TFszPW.js';
-export { A as AxisEvidence, a as AxisVerdict, B as BuildEvidenceVectorOptions, D as DefaultProductionGateOptions, E as EvidenceVector, b as EvolutionaryDriverOptions, H as HeldOutGateOptions, O as ObjectiveSource, P as ParetoSignificanceGateOptions, c as PromotionObjective, d as PromotionPolicy, R as RunEvalOptions, e as buildEvidenceVector, f as composeGate, g as defaultProductionGate, h as evolutionaryDriver, i as heldOutGate, p as paretoPolicy, j as paretoSignificanceGate, r as runEval } from '../provenance-B-TFszPW.js';
-import { C as CampaignStorage, R as RunImprovementLoopResult } from '../run-improvement-loop-BqYH2vCR.js';
-export { G as GepaDriverOptions, a as RunCampaignOptions, b as RunImprovementLoopOptions, f as fsCampaignStorage, g as gepaDriver, i as inMemoryCampaignStorage, r as runCampaign, c as runImprovementLoop } from '../run-improvement-loop-BqYH2vCR.js';
-export { D as DeploymentOutcome, F as FileSystemOutcomeStore, b as FileSystemOutcomeStoreOptions, I as InMemoryOutcomeStore, O as OutcomeStore } from '../outcome-store-D6KWmYvj.js';
-import { HostedTenant, TraceSpanEvent } from '../hosted/index.js';
-import { R as RunRecord, b as RunSplitTag } from '../run-record-BgTFzO2r.js';
-import { I as InsightReport } from '../insight-report-Df3lxYXM.js';
-export { F as FailureClusterInsight, a as InterRaterInsight, J as JudgeInsight, L as LiftInsight, O as OutcomeCorrelationInsight, R as Recommendation, b as ReleaseSummary, S as ScalarDistribution } from '../insight-report-Df3lxYXM.js';
-import { a as AnalystRegistry } from '../registry-DuVYiTvw.js';
+import { S as Scenario, M as MutableSurface, b as DispatchContext, a as JudgeConfig, I as ImprovementDriver, G as Gate, c as GateDecision } from '../types-4mm2msnR.js';
+export { C as CampaignAggregates, d as CampaignArtifactWriter, e as CampaignCellResult, f as CampaignCostMeter, g as CampaignResult, h as CampaignTraceWriter, i as CodeSurface, D as Dispatch, j as GateContext, k as GateResult, l as GenerationCandidate, m as GenerationRecord, n as JudgeDimension, J as JudgeScore, o as Mutator, O as OptimizerConfig, p as SessionScript } from '../types-4mm2msnR.js';
+import { L as LoopProvenanceRecord } from '../provenance-jG-Gngg8.js';
+export { A as AxisEvidence, a as AxisVerdict, B as BuildEvidenceVectorOptions, D as DefaultProductionGateOptions, E as EvidenceVector, b as EvolutionaryDriverOptions, H as HeldOutGateOptions, O as ObjectiveSource, P as ParetoSignificanceGateOptions, c as PromotionObjective, d as PromotionPolicy, R as RunEvalOptions, e as buildEvidenceVector, f as composeGate, g as defaultProductionGate, h as evolutionaryDriver, i as heldOutGate, p as paretoPolicy, j as paretoSignificanceGate, r as runEval } from '../provenance-jG-Gngg8.js';
+import { C as CampaignStorage, R as RunImprovementLoopResult } from '../run-improvement-loop-BAl_aVOZ.js';
+export { G as GepaDriverOptions, a as RunCampaignOptions, b as RunImprovementLoopOptions, f as fsCampaignStorage, g as gepaDriver, i as inMemoryCampaignStorage, r as runCampaign, c as runImprovementLoop } from '../run-improvement-loop-BAl_aVOZ.js';
+export { D as DeploymentOutcome, F as FileSystemOutcomeStore, a as FileSystemOutcomeStoreOptions, I as InMemoryOutcomeStore, b as OutcomeStore } from '../outcome-store-rnXLEqSn.js';
+import { HostedTenant, EvalRunCellScore, EvalRunGenerationSnapshot, EvalRunEvent, TraceSpanEvent } from '../hosted/index.js';
+import { R as RunRecord, b as RunSplitTag } from '../run-record-sItO5ftF.js';
+import { I as InsightReport } from '../insight-report-dlpEzQDi.js';
+export { F as FailureClusterInsight, a as InterRaterInsight, J as JudgeInsight, L as LiftInsight, O as OutcomeCorrelationInsight, R as Recommendation, b as ReleaseSummary, S as ScalarDistribution } from '../insight-report-dlpEzQDi.js';
+import { a as AnalystRegistry } from '../registry-BK0Zee01.js';
 import { a as DatasetScenario } from '../dataset-B2kL-fSM.js';
 import '../red-team-DW9Ca_tj.js';
 import '../store-CKUAgsJz.js';
@@ -22,9 +22,9 @@ import '@tangle-network/tcloud';
 import '../llm-client-DbjLfz-K.js';
 import '../errors-Dwqw-T_m.js';
 import '../raw-provider-sink-C46HDghv.js';
-import '../summary-report-ByiOUrHj.js';
+import '../summary-report-BTaXq1TS.js';
 import '../failure-cluster-CL7IVgkJ.js';
-import '../types-CRD68aH7.js';
+import '../types-DRvV0zRo.js';
 import '../store-GmBE2pZZ.js';
 
 /**
@@ -351,6 +351,119 @@ interface AnalyzeRunsOptions {
 }
 declare function analyzeRuns(opts: AnalyzeRunsOptions): Promise<InsightReport>;
 
+/**
+ * # `@tangle-network/agent-eval/contract` — eval-run diff primitive.
+ *
+ * The substrate side of the v-N-versus-v-N+1 dashboard view. Given two
+ * `EvalRunEvent`s (or two `EvalRunGenerationSnapshot`s from one run), this
+ * returns a normalised diff: per-cell composite + per-judge/per-dimension
+ * deltas, surface-hash change, aggregate cost + duration shifts.
+ *
+ * Consumed by:
+ *   - The hosted-tier dashboard (intelligence-web) — renders v3 vs v4
+ *     comparisons of cells × judges × dimensions.
+ *   - CI reporting — emits a "shipped: composite +0.07, cost +$1.20" line
+ *     in PR review for autonomous-improvement runs.
+ *   - Any downstream consumer that needs "what actually changed" without
+ *     reimplementing the matching + arithmetic.
+ *
+ * Cells are matched on the natural composite key `(scenarioId, rep)`.
+ * Unmatched cells surface as `removed` / `added` so callers can tell
+ * "this cell got worse" from "this cell wasn't run."
+ */
+
+/** Per-dimension delta. `before` / `after` are null when the judge did not
+ *  emit a value for that side. `delta` is `after - before`; null when
+ *  either side is null. */
+interface EvalDimensionDelta {
+    before: number | null;
+    after: number | null;
+    delta: number | null;
+}
+/** Per-cell delta, keyed on `(scenarioId, rep)`. */
+interface EvalCellScoreDelta {
+    scenarioId: string;
+    rep: number;
+    compositeBefore: number;
+    compositeAfter: number;
+    compositeDelta: number;
+    /** Per-judge → per-dimension deltas. Outer key = judge name from
+     *  `EvalRunCellScore.dimensions`; inner key = dimension name. */
+    dimensions: Record<string, Record<string, EvalDimensionDelta>>;
+}
+/** Diff between two generation snapshots — the unit the dashboard renders
+ *  for a single "v3 vs v4" comparison. */
+interface EvalGenerationDiff {
+    beforeIndex: number;
+    afterIndex: number;
+    beforeSurfaceHash: string;
+    afterSurfaceHash: string;
+    surfaceChanged: boolean;
+    /** Cells present in both snapshots, matched on `(scenarioId, rep)`. */
+    matched: EvalCellScoreDelta[];
+    /** Cells present in `before` but missing from `after`. */
+    removed: EvalRunCellScore[];
+    /** Cells present in `after` but missing from `before`. */
+    added: EvalRunCellScore[];
+    /** Aggregate composite mean across all cells in the snapshot. */
+    compositeBefore: number;
+    compositeAfter: number;
+    compositeDelta: number;
+    costUsdBefore: number;
+    costUsdAfter: number;
+    costUsdDelta: number;
+    durationMsBefore: number;
+    durationMsAfter: number;
+    durationMsDelta: number;
+}
+/** Diff between two full eval-runs. Includes both baseline-vs-baseline and
+ *  winner-vs-winner generation diffs when both sides expose them, plus
+ *  run-level metadata. */
+interface EvalRunDiff {
+    beforeRunId: string;
+    afterRunId: string;
+    beforeTimestamp: string;
+    afterTimestamp: string;
+    beforeGateDecision: GateDecision | null;
+    afterGateDecision: GateDecision | null;
+    beforeHoldoutLift: number | null;
+    afterHoldoutLift: number | null;
+    holdoutLiftDelta: number | null;
+    beforeTotalCostUsd: number;
+    afterTotalCostUsd: number;
+    totalCostUsdDelta: number;
+    beforeTotalDurationMs: number;
+    afterTotalDurationMs: number;
+    totalDurationMsDelta: number;
+    /** Baseline-vs-baseline diff. Null when either run has no baseline. */
+    baselineDiff: EvalGenerationDiff | null;
+    /** Highest-index-generation comparison. Null when either run has no
+     *  recorded generations (e.g. baseline-only or errored before any
+     *  generation completed). */
+    winnersDiff: EvalGenerationDiff | null;
+}
+/**
+ * Diff two generation snapshots. Cells are matched on `(scenarioId, rep)`;
+ * unmatched cells surface in `added` / `removed`. Aggregate fields are
+ * recomputed from the snapshot's stored fields, not re-derived from cells —
+ * this keeps the diff consistent with whatever aggregation the substrate
+ * actually reported.
+ */
+declare function diffGenerations(before: EvalRunGenerationSnapshot, after: EvalRunGenerationSnapshot): EvalGenerationDiff;
+/**
+ * Diff two full eval-runs. Produces baseline-vs-baseline and
+ * winner-vs-winner generation diffs when both sides expose them, plus
+ * run-level cost / lift / gate-decision deltas.
+ */
+declare function diffRuns(before: EvalRunEvent, after: EvalRunEvent): EvalRunDiff;
+/**
+ * Within-run baseline → winning-generation diff. The natural "what did the
+ * improvement loop produce" view for a single run. Returns null when the
+ * run never reached a generation past baseline (errored early, or the gate
+ * shipped the baseline as-is).
+ */
+declare function diffRunBaselineToWinner(run: EvalRunEvent): EvalGenerationDiff | null;
+
 /**
  * `fromAgentTrace` — provenance correlation from Cursor's Agent Trace spec
  * (https://github.com/cursor/agent-trace, RFC v0.1.0).
@@ -567,4 +680,4 @@ interface FromOtelSpansOptions {
 }
 declare function fromOtelSpans(opts: FromOtelSpansOptions): RunRecord[];
 
-export { type AgentTraceContributor, type AgentTraceContributorType, type AgentTraceConversation, type AgentTraceFile, type AgentTraceIndex, type AgentTraceRange, type AgentTraceRecord, type AnalyzeRunsOptions, type AuthoringProvenance, CampaignStorage, DispatchContext, type FeedbackTableMeta, type FeedbackTableRow, type FromFeedbackTableOptions, type FromFeedbackTableResult, type FromOtelSpansOptions, Gate, ImprovementDriver, InsightReport, JudgeConfig, MutableSurface, type PartitionByAuthoringModelResult, RunImprovementLoopResult, Scenario, type SelfImproveBudget, type SelfImproveLlm, type SelfImproveOptions, type SelfImproveProgressEvent, type SelfImproveResult, analyzeRuns, fromFeedbackTable, fromOtelSpans, parseAgentTrace, partitionRunsByAuthoringModel, selfImprove };
+export { type AgentTraceContributor, type AgentTraceContributorType, type AgentTraceConversation, type AgentTraceFile, type AgentTraceIndex, type AgentTraceRange, type AgentTraceRecord, type AnalyzeRunsOptions, type AuthoringProvenance, CampaignStorage, DispatchContext, type EvalCellScoreDelta, type EvalDimensionDelta, type EvalGenerationDiff, type EvalRunDiff, type FeedbackTableMeta, type FeedbackTableRow, type FromFeedbackTableOptions, type FromFeedbackTableResult, type FromOtelSpansOptions, Gate, GateDecision, ImprovementDriver, InsightReport, JudgeConfig, MutableSurface, type PartitionByAuthoringModelResult, RunImprovementLoopResult, Scenario, type SelfImproveBudget, type SelfImproveLlm, type SelfImproveOptions, type SelfImproveProgressEvent, type SelfImproveResult, analyzeRuns, diffGenerations, diffRunBaselineToWinner, diffRuns, fromFeedbackTable, fromOtelSpans, parseAgentTrace, partitionRunsByAuthoringModel, selfImprove };

package/dist/contract/index.js

@@ -9,7 +9,7 @@ import {
   paretoPolicy,
   paretoSignificanceGate,
   runEval
-} from "../chunk-XPILG2CA.js";
+} from "../chunk-GXHLRXDI.js";
 import {
   checkCanaries
 } from "../chunk-SHTXZ4O2.js";
@@ -19,7 +19,7 @@ import {
   heldOutGate,
   runImprovementLoop,
   surfaceContentHash
-} from "../chunk-JYE3WOTE.js";
+} from "../chunk-RPLZ4OIB.js";
 import {
   fsCampaignStorage,
   inMemoryCampaignStorage,
@@ -1087,6 +1087,119 @@ function cellsToRunRecords(cells, candidateId, runId, surface) {
   });
 }
 
+// src/contract/diff.ts
+function keyForCell(cell) {
+  return JSON.stringify([cell.scenarioId, cell.rep]);
+}
+function diffDimensions(before, after) {
+  const out = {};
+  const judges = /* @__PURE__ */ new Set([...Object.keys(before), ...Object.keys(after)]);
+  for (const judge of judges) {
+    const beforeDims = before[judge] ?? {};
+    const afterDims = after[judge] ?? {};
+    const dims = /* @__PURE__ */ new Set([...Object.keys(beforeDims), ...Object.keys(afterDims)]);
+    const judgeOut = {};
+    for (const dim of dims) {
+      const rawBefore = beforeDims[dim];
+      const rawAfter = afterDims[dim];
+      const b = typeof rawBefore === "number" && Number.isFinite(rawBefore) ? rawBefore : null;
+      const a = typeof rawAfter === "number" && Number.isFinite(rawAfter) ? rawAfter : null;
+      judgeOut[dim] = {
+        before: b,
+        after: a,
+        delta: b !== null && a !== null ? a - b : null
+      };
+    }
+    out[judge] = judgeOut;
+  }
+  return out;
+}
+function diffGenerations(before, after) {
+  const beforeMap = new Map(before.cells.map((c) => [keyForCell(c), c]));
+  const afterMap = new Map(after.cells.map((c) => [keyForCell(c), c]));
+  const matched = [];
+  const removed = [];
+  const added = [];
+  for (const [key, beforeCell] of beforeMap) {
+    const afterCell = afterMap.get(key);
+    if (!afterCell) {
+      removed.push(beforeCell);
+      continue;
+    }
+    matched.push({
+      scenarioId: beforeCell.scenarioId,
+      rep: beforeCell.rep,
+      compositeBefore: beforeCell.compositeMean,
+      compositeAfter: afterCell.compositeMean,
+      compositeDelta: afterCell.compositeMean - beforeCell.compositeMean,
+      dimensions: diffDimensions(beforeCell.dimensions, afterCell.dimensions)
+    });
+  }
+  for (const [key, afterCell] of afterMap) {
+    if (!beforeMap.has(key)) added.push(afterCell);
+  }
+  return {
+    beforeIndex: before.index,
+    afterIndex: after.index,
+    beforeSurfaceHash: before.surfaceHash,
+    afterSurfaceHash: after.surfaceHash,
+    surfaceChanged: before.surfaceHash !== after.surfaceHash,
+    matched,
+    removed,
+    added,
+    compositeBefore: before.compositeMean,
+    compositeAfter: after.compositeMean,
+    compositeDelta: after.compositeMean - before.compositeMean,
+    costUsdBefore: before.costUsd,
+    costUsdAfter: after.costUsd,
+    costUsdDelta: after.costUsd - before.costUsd,
+    durationMsBefore: before.durationMs,
+    durationMsAfter: after.durationMs,
+    durationMsDelta: after.durationMs - before.durationMs
+  };
+}
+function winnerOf(run) {
+  if (run.generations.length === 0) return null;
+  let winner = run.generations[0];
+  for (const gen of run.generations) {
+    if (gen.index > winner.index) winner = gen;
+  }
+  return winner;
+}
+function diffRuns(before, after) {
+  const beforeWinner = winnerOf(before);
+  const afterWinner = winnerOf(after);
+  const baselineDiff = before.baseline && after.baseline ? diffGenerations(before.baseline, after.baseline) : null;
+  const winnersDiff = beforeWinner && afterWinner ? diffGenerations(beforeWinner, afterWinner) : null;
+  const beforeLift = before.holdoutLift ?? null;
+  const afterLift = after.holdoutLift ?? null;
+  return {
+    beforeRunId: before.runId,
+    afterRunId: after.runId,
+    beforeTimestamp: before.timestamp,
+    afterTimestamp: after.timestamp,
+    beforeGateDecision: before.gateDecision ?? null,
+    afterGateDecision: after.gateDecision ?? null,
+    beforeHoldoutLift: beforeLift,
+    afterHoldoutLift: afterLift,
+    holdoutLiftDelta: beforeLift !== null && afterLift !== null ? afterLift - beforeLift : null,
+    beforeTotalCostUsd: before.totalCostUsd,
+    afterTotalCostUsd: after.totalCostUsd,
+    totalCostUsdDelta: after.totalCostUsd - before.totalCostUsd,
+    beforeTotalDurationMs: before.totalDurationMs,
+    afterTotalDurationMs: after.totalDurationMs,
+    totalDurationMsDelta: after.totalDurationMs - before.totalDurationMs,
+    baselineDiff,
+    winnersDiff
+  };
+}
+function diffRunBaselineToWinner(run) {
+  if (!run.baseline) return null;
+  const winner = winnerOf(run);
+  if (!winner || winner.index === run.baseline.index) return null;
+  return diffGenerations(run.baseline, winner);
+}
+
 // src/contract/intake/agent-trace.ts
 function rangeLines(r) {
   return Math.max(0, r.end_line - r.start_line + 1);
@@ -1328,6 +1441,9 @@ export {
   buildEvidenceVector,
   composeGate,
   defaultProductionGate,
+  diffGenerations,
+  diffRunBaselineToWinner,
+  diffRuns,
   evolutionaryDriver,
   fromFeedbackTable,
   fromOtelSpans,