@tangle-network/agent-eval 0.179.0 → 0.181.0

package/dist/{define-agent-eval-Cx4Ls9ta.d.ts → define-agent-eval-CwOWWQt_.d.ts}

@@ -1,22 +1,36 @@
 import { c as CostLedgerHandle, f as CostLedgerSummary, m as CostReceipt } from "./cost-ledger-DbQdN3nO.js";
-import { _ as ProposalFinding } from "./types-DzuaM493.js";
-import { B as PowerPreflight, ft as EvidenceReceipt, it as CampaignEvidenceContext } from "./statistical-heldout-0La5ZTlv.js";
-import { H as SurfaceProposer, R as Scenario, S as JudgeConfig, a as CampaignResult, d as DispatchContext, j as MutableSurface, k as LabeledScenarioStore, p as Gate, v as GateResult } from "./types-C34V4Vto.js";
-import { $n as LoopProvenanceRecord, Ii as PremeasuredOptimizationBaseline, Li as RunOptimizationOptions, Pi as RunImprovementLoopResult, Zr as OptimizationMethod, aa as SearchHistoryReceipt, co as RunEvalOptions, do as CampaignCellRetryPolicy, ea as SearchHistoryAdmissionOptions, fo as RunCampaignOptions, ni as OptimizationMethodProvenance, ra as SearchHistoryCoverageRow, ri as OptimizationMethodResult, so as openAutoPr, ui as ComparisonCost, wo as CampaignStorage } from "./index-BxWvILU8.js";
-import { o as InsightReport } from "./insight-report-DETqPc_A.js";
-import { n as HostedTenant } from "./client-CuQgX33c.js";
+import { _ as ProposalFinding } from "./types-D7gEdPoQ.js";
+import { B as PowerPreflight, Mt as EvaluationClaim, ft as EvidenceReceipt, it as CampaignEvidenceContext } from "./statistical-heldout-CpVd6FmY.js";
+import { H as SurfaceProposer, R as Scenario, S as JudgeConfig, a as CampaignResult, d as DispatchContext, j as MutableSurface, k as LabeledScenarioStore, p as Gate, v as GateResult } from "./types-CS0qk_Yp.js";
+import { Ao as FinalEvidenceUse, Fi as RunImprovementLoopResult, Li as PremeasuredOptimizationBaseline, Qr as OptimizationMethod, Ri as RunOptimizationOptions, To as CampaignStorage, Yn as LoopProvenanceRecord, co as openAutoPr, di as ComparisonCost, fo as CampaignCellRetryPolicy, ia as SearchHistoryCoverageRow, ii as OptimizationMethodResult, ko as FinalEvidencePolicy, lo as RunEvalOptions, oa as SearchHistoryReceipt, or as CampaignComparisonUnits, po as RunCampaignOptions, ri as OptimizationMethodProvenance, ta as SearchHistoryAdmissionOptions } from "./index-Bp_6sj3x.js";
+import { o as InsightReport } from "./insight-report-D1qa0HWs.js";
+import { n as HostedTenant } from "./client-kh2jOjTK.js";
 //#region src/campaign/presets/run-final-comparison.d.ts
 interface FinalComparisonOptions<TScenario extends Scenario, TArtifact> extends Omit<RunCampaignOptions<TScenario, TArtifact>, 'dispatch'> {
   baselineSurface: MutableSurface;
   winnerSurface: MutableSurface;
   dispatchWithSurface: (surface: MutableSurface, scenario: TScenario, ctx: Parameters<RunCampaignOptions<TScenario, TArtifact>['dispatch']>[1]) => Promise<TArtifact>;
   gate: Gate<TArtifact, TScenario>;
+  claim?: EvaluationClaim;
+  finalEvidence?: FinalEvidencePolicy;
   holdout?: 'measured' | 'deferred';
   label?: string;
   neutralize?: (winner: MutableSurface, baseline: MutableSurface) => MutableSurface;
 }
 /** Compare a search-selected surface without changing the search's selection. */
 declare function runFinalComparison<TScenario extends Scenario, TArtifact>(opts: FinalComparisonOptions<TScenario, TArtifact>): Promise<{
+  claim?: {
+    use: "certification" | "comparison" | "development";
+    population: {
+      id: string;
+      description: string;
+    };
+    samplingFrame: string;
+    independentUnit: string;
+    generalization: "fixed-roster" | "new-units";
+    minimumEffect?: number | undefined;
+  } | undefined;
+  finalEvidence?: FinalEvidenceUse | undefined;
   baselineOnHoldout: CampaignResult<TArtifact, TScenario>;
   winnerOnHoldout: CampaignResult<TArtifact, TScenario>;
   neutralizedOnHoldout?: CampaignResult<TArtifact, TScenario> | undefined;
@@ -37,6 +51,7 @@ interface SelfImproveMethodProvenance {
   winnerContentHash: string;
   diff: string;
   optimizationMethod: NonNullable<SelfImproveProposerResult<Scenario, unknown>['optimization']>;
+  claim?: EvaluationClaim;
   evidence: {
     trainSplitDigest: `sha256:${string}`;
     selectionSplitDigest: `sha256:${string}`;
@@ -45,6 +60,7 @@ interface SelfImproveMethodProvenance {
     winnerCampaignDigest: `sha256:${string}`;
     costReceiptsDigest: `sha256:${string}`;
     neutralizedCampaignDigest?: `sha256:${string}`;
+    holdoutObservations?: CampaignComparisonUnits;
   };
   gate: Awaited<ReturnType<typeof runFinalComparison>>['gateResult'];
   holdout?: 'deferred';
@@ -119,7 +135,7 @@ interface SelfImproveBudget {
    *  static holdout scenario and recording a meaningless lift. Unless
    *  `holdoutScenarios` reserves an explicit set, ALL scenarios train. */
   holdout?: 'measured' | 'deferred';
-  /** Per-scenario replicates per cell — raises bootstrap-CI tightness. Default 1. */
+  /** Repeated executions per scenario. A claim's independent-unit count stays unchanged. Default 1. */
   reps?: number;
   /** DEPTH dial forwarded to the proposer's `propose()` as
    *  `ctx.maxImprovementShots` — max iterations an agentic candidate generator
@@ -327,9 +343,15 @@ interface SelfImproveOptions<TScenario extends Scenario, TArtifact> extends Sear
   searchLedger?: RunOptimizationOptions<TScenario, TArtifact>['searchLedger'];
   /** Complete-method final measurement receipts; authority and environment remain caller-owned. */
   evidence?: CampaignEvidenceContext;
+  /** Declare the population and independent units without requiring fresh data. */
+  claim?: EvaluationClaim;
+  /** Opt into durable fresh-evidence consumption for this final comparison. */
+  finalEvidence?: FinalEvidencePolicy;
 }
 interface SelfImproveProposerResult<TScenario extends Scenario, TArtifact> {
   mode: 'proposer';
+  claim?: EvaluationClaim;
+  finalEvidence?: FinalEvidenceUse;
   /** Composite mean across all scenarios, baseline run. When
    *  `budget.holdout === 'deferred'` this is measured on the improvement
    *  (search) split — no holdout campaign ran. */
@@ -395,10 +417,9 @@ interface SelfImproveProposerResult<TScenario extends Scenario, TArtifact> {
    * Hosted-tier dashboards render this as the v3-vs-v4 decision view.
    */
   insight: InsightReport;
-  /** Minimum-detectable-lift analysis from the baseline holdout cells: could this
-   *  budget have shipped ANY plausible effect? Absent when the baseline produced
-   *  fewer than 3 scored holdout cells. See `powerPreflight` for the standalone
-   *  pre-run version (run `gate: 'none'` first, budget the real search after). */
+  /** Approximate detectable lift from baseline holdout observations.
+   *  Declared independent units determine n and baseline variance.
+   *  Absent with fewer than three observations or a deferred holdout. */
   power?: PowerPreflight;
   /**
    * Raw substrate result for advanced inspection — full per-generation
@@ -502,4 +523,4 @@ interface DefinedAgentEval<TScenario extends Scenario, TArtifact> {
 declare function defineAgentEval<TScenario extends Scenario, TArtifact>(defaults: DefineAgentEvalOptions<TScenario, TArtifact>): DefinedAgentEval<TScenario, TArtifact>;
 //#endregion
 export { SelfImproveMethodResult as _, DefinedAgentEval as a, SelfImproveMethodOptions as c, SelfImproveProposerOptions as d, SelfImproveProposerResult as f, SelfImproveMethodProvenance as g, selfImprove as h, DefineAgentEvalOptions as i, SelfImproveOptions as l, SelfImproveRunError as m, AgentEvalEvaluateOptions as n, defineAgentEval as o, SelfImproveResult as p, AgentEvalImproveOptions as r, SelfImproveBudget as s, AgentEvalAgent as t, SelfImproveProgressEvent as u };
-//# sourceMappingURL=define-agent-eval-Cx4Ls9ta.d.ts.map
+//# sourceMappingURL=define-agent-eval-CwOWWQt_.d.ts.map

package/dist/define-agent-eval-CwOWWQt_.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"define-agent-eval-CwOWWQt_.d.ts","names":[],"sources":["../src/campaign/presets/run-final-comparison.ts","../src/contract/self-improve-method.ts","../src/contract/self-improve.ts","../src/contract/define-agent-eval.ts"],"mappings":";;;;;;;;UAaiB,uBAAuB,kBAAkB,UAAU,mBAC1D,KAAK,mBAAmB,WAAW;EAC3C,iBAAiB;EACjB,eAAe;EACf,sBACE,SAAS,gBACT,UAAU,WACV,KAAK,WAAW,mBAAmB,WAAW,+BAC3C,QAAQ;EACb,MAAM,KAAK,WAAW;EACtB,QAAQ;EACR,gBAAgB;EAChB;EACA;EACA,cAAc,QAAQ,gBAAgB,UAAU,mBAAmB;;;iBAI/C,mBAAmB,kBAAkB,UAAU,WACnE,MAAM,uBAAuB,WAAW,aAAU;;;;;;;;;;;;kBAA5C;;;;;;;;;;;UCKS;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,oBAAoB,YAAY,0BAA0B;EAC1D,QAAQ;EACR;IACE;IACA;IACA;IACA;IACA;IACA;IACA;IACA,sBAAsB;;EAExB,MAAM,QAAQ,kBAAkB;EAChC;EACA;EACA;EACA;EACA,MAAM;EACN;EACA;;UAGe,wBAAwB,kBAAkB,UAAU,mBAC3D,KACN,0BAA0B,WAAW;EAWvC;IAAa,UAAU;IAAiB,QAAQ;;EAChD,uBAAuB;EACvB;;EAEA,UAAU,0BAA0B,WAAW;EAC/C,QAAQ,KAAK,0BAA0B,WAAW;IAChD;;EAEF,YAAY;EACZ,cAAc,YAAY,0BAA0B,WAAW;;EAE/D,MAAM;;EAEN,YAAY;EACZ,UAAU,0BAA0B,WAAW;EAC/C,KAAK,QAAQ,kBAAkB,mBAAmB,WAAW;IAC3D;IACA,iBAAiB,0BAA0B,WAAW;IACtD,eAAe,0BAA0B,WAAW;IACpD;IACA,QAAQ;IACR,MAAM;IACN,WAAW,kBAAkB;;;;;UCnBhB;;;EAGf;;;;EAIA;;EAEA;;EAEA;;;EAGA;;;EAGA;;;;EAIA;;EAEA,mBAAmB;;;;;;;;;EASnB;;EAEA;;;;;EAKA;;KAGU;EACN;EAA0B;;EAC1B;EAA4B;EAAuB;;EACnD;EAA4B;EAAe;;EAC3C;EAA8B;EAAe;EAAuB;;EAGpE;EAAsB;EAAkB;;EACxC;EAAyB;EAAW;EAAY;EAAa;;UAElD,mBAAmB,kBAAkB,UAAU,mBACtD;;;;;;;;;;;;;;EAcR,QAAQ,SAAS,gBAAgB,UAAU,WAAW,KAAK,oBAAoB,QAAQ;;;;;;;EAQvF;;EAEA;;;EAIA,WAAW;;EAGX,OAAO,YAAY,WAAW;;;EAI9B,iBAAiB;;EAGjB,SAAS;;;;;;;;;;;;;EAcT,sBAAsB,gCAAgC,WAAW;;;;;EAMjE,WAAW,gBAAgB;;;;;;EAO3B,SAAS,mBAAmB,WAAW;;;EAIvC,qBAAqB;;;EAIrB,OAAO,KAAK,WAAW;;;;;;;;EASvB,cAAc,eAAe,gBAAgB,iBAAiB,mBAAmB;;;;EAKjF,UAAU;;;;EAKV;;;EAIA,gBAAgB,QAAQ,uBAAuB;;;;;EAM/C,iBAAiB;IACf,UAAU;IACV;IACA;;;;EAKF;;;;;;;EAQA,YAAY;;;EAIZ,cAAc,OAAO;;;;EAKrB;EACA;EACA;;;;;;;;;;;;;;EAeA,eAAe;;;EAIf,eAAe;;;;EAKf,eAAe;;EAGf;;;;;;;;EASA;;;;;;;;;EAUA,oBAAoB,uBAAuB,WAAW;;;EAItD,WAAW;;;;;;;EAQX,mBAAmB,uBAAuB,WAAW;;;;;;EAOrD,eAAe,uBAAuB,WAAW;;;;EAKjD,eAAe,uBAAuB,WAAW;;EAEjD,WAAW;;EAEX,QAAQ;;EAER,gBAAgB;;UAGD,0BAA0B,kBAAkB,UAAU;EACrE;EACA,QAAQ;EACR,gBAAgB;;;;EAIhB;IACE;IACA,aAAa;;;;;EAKf;IACE;IACA,aAAa;IACb,SAAS;;;IAGT;;;;IAIA;;;;;;EAMF;;;EAGA;;;;EAIA,YAAY;;EAEZ;;;EAGA;;EAEA;;EAEA;;EAEA,MAAM;;;EAGN,UAAU;;;EAGV,gBAAgB;;EAEhB;IACE;IACA,MAAM;IACN;IACA,aAAa;;;;;;;;EAQf,SAAS;;;;EAIT,QAAQ;;;;;;EAMR,KAAK,yBAAyB,WAAW;;KAG/B,kBAAkB,kBAAkB,UAAU,aACtD,0BAA0B,WAAW,aACrC,wBAAwB,WAAW;KAE3B,yBAAyB,kBAAkB,UAAU,aAAa,KAC5E,mBAAmB,WAAW;EAG9B,QAAQ,mBAAmB,WAAW;EACtC;EACA,gBAAgB,QAAQ;;KAGd,2BAA2B,kBAAkB,UAAU,aAAa,KAC9E,mBAAmB,WAAW;EAG9B;EACA,gBAAgB,QAAQ;;;cAIb,4BAA4B;WAC9B,MAAM;WACN,UAAU;EAEnB,YAAY,gBAAgB,QAAQ;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAsNtB,YAAY,kBAAkB,UAAU,WACtD,MAAM,yBAAyB,WAAW,aACzC,QAAQ,wBAAwB,WAAW;iBAC9B,YAAY,kBAAkB,UAAU,WACtD,MAAM,2BAA2B,WAAW,aAC3C,QAAQ,0BAA0B,WAAW;iBAChC,YAAY,kBAAkB,UAAU,WACtD,MAAM,mBAAmB,WAAW,aACnC,QAAQ,kBAAkB,WAAW;;;KC/oB5B,eAAe,kBAAkB,UAAU,cACrD,SAAS,gBACT,UAAU,WACV,KAAK,oBACF,QAAQ;KAED,uBAAuB,kBAAkB,UAAU,aAAa,mBAC1E,WACA;UAGe,yBAAyB,kBAAkB,UAAU,mBAC5D,KACN,eAAe,WAAW;;EAI5B,YAAY;;EAEZ,UAAU;;EAEV,QAAQ,eAAe,WAAW;;EAElC,QAAQ,YAAY,WAAW;;EAE/B,SAAS,YAAY,WAAW;;EAEhC;;KAGU,wBAAwB,kBAAkB,UAAU,aAAa,KAC3E,QAAQ,mBAAmB,WAAW;EAGtC,SAAS,QAAQ;EACjB,eAAe,QAAQ;;UAGR,iBAAiB,kBAAkB,UAAU;;WAEnD,oBAAoB;;WAEpB,iBAAiB;;;;;EAK1B,SACE,OAAO,yBAAyB,WAAW,aAC1C,QAAQ,eAAe,WAAW;;;;;;;EAOrC,QACE,OAAO,wBAAwB,WAAW,aACzC,QAAQ,kBAAkB,WAAW;;;;;;;;;iBAU1B,gBAAgB,kBAAkB,UAAU,WAC1D,UAAU,uBAAuB,WAAW,aAC3C,iBAAiB,WAAW"}

package/dist/{define-agent-eval-Dzidv34q.js → define-agent-eval-Ddu33JH9.js}

@@ -1,20 +1,20 @@
 import { s as ValidationError } from "./errors-Dngq5h35.js";
 import { a as spearmanR, r as pearsonR } from "./descriptive-1V17A-qa.js";
-import { r as continuousAgreement } from "./judge-calibration-BnpVKtnb.js";
+import { r as continuousAgreement } from "./judge-calibration-DYmaBtJr.js";
 import { i as pairedCohensDz } from "./effect-sizes-DiH8MGOH.js";
 import { i as requiredPairedSampleSize, r as pairedMde } from "./power-and-mde-B8F2RdcD.js";
 import { r as pairRunRecords } from "./paired-arms-D4aeIHUy.js";
 import { o as pairedTTest, r as pairedBootstrap } from "./paired-tests-C8iCsioC.js";
 import { r as welchsTTest } from "./baseline-BC-eBZ7U.js";
+import { a as campaignCellToRunRecord, i as campaignCellTaskScore, n as campaignCellExecutionEvidence, r as campaignCellJudgeDimensions } from "./run-record-Br-Yzt_k.js";
+import { B as surfaceDispatchRef, N as defineEvaluationClaim, T as pairedCampaignComposites, V as surfaceHash, Z as campaignSplitDigest, f as campaignMeasurementDigest, h as loopProvenanceArgsFromResult, k as powerPreflight, m as emitLoopProvenance, p as canonicalDigest, t as createCampaignEvidenceReceipt, tt as aggregatePairedHoldout, z as surfaceContentHash } from "./campaign-evidence-B8oF9xQ6.js";
 import "./query-D1nLIKt7.js";
 import { r as observedSplitScore } from "./reward-nw2xZGZG.js";
-import { c as validateRunRecord, i as modelHasSnapshot } from "./run-record-DQpSf7t-.js";
-import { r as paretoChart } from "./summary-report-B16xy9Kd.js";
-import { C as assertSearchHistoryAdmissionOptions, I as runFinalComparison, L as openAutoPr, R as defaultProductionGate, W as runEval, d as combineComparisonCosts, f as costFromLedgerSummary, h as runImprovementLoop, q as resolveRunDir, u as executeOptimizationMethod } from "./llm-judge-v80Kmu9g.js";
-import { a as campaignCellToRunRecord, i as campaignCellTaskScore, n as campaignCellExecutionEvidence, r as campaignCellJudgeDimensions } from "./run-record-CR63CpHK.js";
-import { H as surfaceContentHash, O as powerPreflight, U as surfaceDispatchRef, W as surfaceHash, et as campaignSplitDigest, f as campaignMeasurementDigest, h as loopProvenanceArgsFromResult, m as emitLoopProvenance, p as canonicalDigest, t as createCampaignEvidenceReceipt } from "./campaign-evidence-D8DBLqLI.js";
-import { C as inMemoryCampaignStorage, S as fsCampaignStorage, x as createRunCostLedger } from "./external-optimizer-subprocess-q3VzlGAO.js";
-import { t as createHostedClient } from "./client-BlLY6o2w.js";
+import { c as validateRunRecord, i as modelHasSnapshot } from "./run-record-DualPTn2.js";
+import { r as paretoChart } from "./summary-report-e-MaOAHV.js";
+import { C as assertSearchHistoryAdmissionOptions, G as assertIndependentEvaluationSplit, I as runFinalComparison, J as reserveFinalEvidence, K as captureFinalEvidencePolicy, L as openAutoPr, Q as resolveRunDir, R as captureJudge, Y as runEval, d as combineComparisonCosts, f as costFromLedgerSummary, h as runImprovementLoop, q as evaluationUnitMap, u as executeOptimizationMethod, z as defaultProductionGate } from "./llm-judge-DEFZeSiu.js";
+import { C as inMemoryCampaignStorage, S as fsCampaignStorage, x as createRunCostLedger } from "./external-optimizer-subprocess-D4dzUBZI.js";
+import { t as createHostedClient } from "./client-CXE-U1SA.js";
 //#region src/contamination-guard.ts
 function checkCanaries(output, scenarios) {
 	const leaks = [];
@@ -83,7 +83,7 @@ async function analyzeRuns(opts) {
 	};
 	const judges = computeJudgeInsights(runs);
 	const interRater = opts.raterScores ? computeInterRater(opts.raterScores) : void 0;
-	const lift = computeLift(runs, opts.baselineCandidateId, opts.candidateCandidateId, split);
+	const lift = computeLift(runs, opts.baselineCandidateId, opts.candidateCandidateId, split, opts.independentUnitByScenarioId);
 	const failureClusters = opts.analyst ? await computeFailureClusters(runs, opts.analyst, split) : void 0;
 	const failureClasses = computeFailureClasses(runs, split);
 	const contamination = opts.canaryScenarios ? computeContamination(runs, opts.canaryScenarios) : void 0;
@@ -614,7 +614,7 @@ function computeInterRater(ratings) {
 		disagreementCases
 	};
 }
-function computeLift(runs, baselineId, candidateId, split) {
+function computeLift(runs, baselineId, candidateId, split, independentUnitByScenarioId) {
 	let bId = baselineId;
 	let cId = candidateId;
 	if (!bId || !cId) {
@@ -633,9 +633,20 @@ function computeLift(runs, baselineId, candidateId, split) {
 	const candidate = runs.filter((r) => r.candidateId === cId);
 	if (baseline.length === 0 || candidate.length === 0) return void 0;
 	const pairing = pairRunRecords(baseline.filter((run) => Number.isFinite(compositeOf(run, split))), candidate.filter((run) => Number.isFinite(compositeOf(run, split))));
-	const pairedBaseline = pairing.pairs.map((pair) => compositeOf(pair.baseline, split));
-	const pairedCandidate = pairing.pairs.map((pair) => compositeOf(pair.treatment, split));
+	let pairedBaseline = pairing.pairs.map((pair) => compositeOf(pair.baseline, split));
+	let pairedCandidate = pairing.pairs.map((pair) => compositeOf(pair.treatment, split));
 	if (pairedBaseline.length === 0) return void 0;
+	const grouped = independentUnitByScenarioId ? aggregatePairedHoldout({
+		before: pairedBaseline,
+		after: pairedCandidate,
+		cellIds: pairing.pairs.map((pair, index) => `${pair.pairKey}:${index}`)
+	}, new Map(pairing.pairs.map((pair) => [pair.pairKey, independentUnitByScenarioId.get(pair.baseline.scenarioId)]))) : void 0;
+	if (grouped) {
+		pairedBaseline = grouped.before;
+		pairedCandidate = grouped.after;
+	}
+	const reverseInferredArms = grouped !== void 0 && (!baselineId || !candidateId) && mean(pairedBaseline) > mean(pairedCandidate);
+	if (reverseInferredArms) [pairedBaseline, pairedCandidate] = [pairedCandidate, pairedBaseline];
 	const baselineMean = mean(pairedBaseline);
 	const candidateMean = mean(pairedCandidate);
 	const delta = candidateMean - baselineMean;
@@ -663,10 +674,14 @@ function computeLift(runs, baselineId, candidateId, split) {
 		ci95: [bootstrap.low, bootstrap.high],
 		pValue: tTest.p,
 		n: pairedBaseline.length,
+		...grouped ? {
+			pairedRunN: pairing.pairs.length,
+			independentUnitIds: grouped.unitIds
+		} : {},
 		minimumRequired: 20,
 		decisionEligible: bootstrap.gateEligible,
-		unpairedBaseline: pairing.unpairedBaseline.length,
-		unpairedCandidate: pairing.unpairedTreatment.length,
+		unpairedBaseline: (reverseInferredArms ? pairing.unpairedTreatment : pairing.unpairedBaseline).length,
+		unpairedCandidate: (reverseInferredArms ? pairing.unpairedBaseline : pairing.unpairedTreatment).length,
 		cohensD: d,
 		mde,
 		requiredN
@@ -682,29 +697,26 @@ async function computeFailureClusters(runs, analyst, split) {
 		totalFailures: 0
 	};
 	const clusters = /* @__PURE__ */ new Map();
+	const recordCluster = (key, runId) => {
+		const cluster = clusters.get(key) ?? {
+			exemplars: [],
+			runs: 0
+		};
+		cluster.runs += 1;
+		if (cluster.exemplars.length < 5 && !cluster.exemplars.includes(runId)) cluster.exemplars.push(runId);
+		clusters.set(key, cluster);
+	};
 	for (const run of failed) try {
 		const result = await analyst.run(run.runId, { runRecord: run });
-		for (const finding of result.findings) {
-			const key = finding.area || finding.analyst_id || "unclassified";
-			const c = clusters.get(key) ?? {
-				exemplars: [],
-				share: 0
-			};
-			if (c.exemplars.length < 5) c.exemplars.push(run.runId);
-			clusters.set(key, c);
-		}
+		const keys = new Set(result.findings.map((finding) => finding.area || finding.analyst_id || "unclassified"));
+		for (const key of keys) recordCluster(key, run.runId);
 	} catch {
-		const c = clusters.get("analyst-error") ?? {
-			exemplars: [],
-			share: 0
-		};
-		if (c.exemplars.length < 5) c.exemplars.push(run.runId);
-		clusters.set("analyst-error", c);
+		recordCluster("analyst-error", run.runId);
 	}
 	const clusterList = [...clusters.entries()].map(([id, c]) => ({
 		id,
 		name: id,
-		share: c.exemplars.length / failed.length,
+		share: c.runs / failed.length,
 		exemplars: c.exemplars
 	}));
 	clusterList.sort((a, b) => b.share - a.share);
@@ -966,16 +978,20 @@ function buildRecommendations(ctx) {
 }
 //#endregion
 //#region src/contract/self-improve-reporting.ts
-function meanComposite(byScenario) {
-	const perScenario = {};
-	const values = [];
-	for (const [id, agg] of Object.entries(byScenario)) {
-		perScenario[id] = agg.meanComposite;
-		values.push(agg.meanComposite);
-	}
+function pairedCompositeSummary(baseline, winner, independentUnitByScenarioId) {
+	const scores = pairedCampaignComposites(baseline, winner, independentUnitByScenarioId);
+	const perScenario = (campaign) => Object.fromEntries(Object.entries(campaign.aggregates.byScenario).map(([id, aggregate]) => [id, aggregate.meanComposite]));
 	return {
-		compositeMean: values.length === 0 ? 0 : values.reduce((s, v) => s + v, 0) / values.length,
-		perScenario
+		baseline: {
+			compositeMean: scores.beforeMean,
+			perScenario: perScenario(baseline)
+		},
+		winner: {
+			compositeMean: scores.afterMean,
+			perScenario: perScenario(winner)
+		},
+		baselineComposites: scores.before,
+		observations: scores.observations
 	};
 }
 /** 32-bit FNV-1a over raw UTF-16 code units, rendered as hex for a cell key.
@@ -1029,10 +1045,7 @@ async function runSelfImproveMethod(args) {
 	const evidenceContext = opts.evidence === void 0 ? void 0 : structuredClone(opts.evidence);
 	if (evidenceContext && budget.holdout === "deferred") throw new Error("selfImprove: evidence receipts require measured holdout");
 	const baselineSurface = structuredClone(opts.baselineSurface);
-	const judge = {
-		...opts.judge,
-		dimensions: opts.judge.dimensions.map((dimension) => ({ ...dimension }))
-	};
+	const judge = opts.judge;
 	const { selected, cost: methodCost, history } = await executeOptimizationMethod({
 		method: opts.method,
 		storage,
@@ -1080,9 +1093,12 @@ async function runSelfImproveMethod(args) {
 	}]);
 	const gate = opts.gate ?? defaultProductionGate({
 		holdoutScenarios: holdout,
-		deltaThreshold: .05
+		deltaThreshold: opts.claim?.minimumEffect ?? .05,
+		independentUnitByScenarioId: opts.claim ? evaluationUnitMap(opts.claim, holdout) : void 0
 	});
 	const comparison = await runFinalComparison({
+		claim: opts.claim,
+		finalEvidence: opts.finalEvidence,
 		baselineSurface,
 		winnerSurface: selected.winnerSurface,
 		scenarios: holdout,
@@ -1135,12 +1151,15 @@ async function runSelfImproveMethod(args) {
 		})
 	} : void 0;
 	const deferred = comparison.holdout === "deferred";
-	const baseline = deferred ? null : meanComposite(comparison.baselineOnHoldout.aggregates.byScenario);
-	const winner = deferred ? null : meanComposite(comparison.winnerOnHoldout.aggregates.byScenario);
+	const report = deferred ? null : pairedCompositeSummary(comparison.baselineOnHoldout, comparison.winnerOnHoldout, opts.claim ? evaluationUnitMap(opts.claim, holdout) : void 0);
+	const baseline = report?.baseline ?? null;
+	const winner = report?.winner ?? null;
 	const lift = baseline && winner ? winner.compositeMean - baseline.compositeMean : void 0;
 	const insight = deferred ? void 0 : await analyzeRuns({
 		runs: [...cellsToRunRecords(comparison.baselineOnHoldout.cells, "baseline", runDir, baselineSurface, "holdout", opts.model), ...comparison.winnerOnHoldout === comparison.baselineOnHoldout ? [] : cellsToRunRecords(comparison.winnerOnHoldout.cells, "winner", runDir, selected.winnerSurface, "holdout", opts.model)],
 		baselineCandidateId: "baseline",
+		independentUnitByScenarioId: opts.claim ? evaluationUnitMap(opts.claim, holdout) : void 0,
+		decisionThreshold: opts.claim?.minimumEffect ?? .05,
 		...comparison.winnerOnHoldout === comparison.baselineOnHoldout ? {} : { candidateCandidateId: "winner" }
 	});
 	const cost = totalCost();
@@ -1161,6 +1180,7 @@ async function runSelfImproveMethod(args) {
 		winnerContentHash: surfaceContentHash(selected.winnerSurface),
 		diff: comparison.promotedDiff,
 		optimizationMethod: optimization,
+		...opts.claim ? { claim: opts.claim } : {},
 		evidence: {
 			trainSplitDigest: campaignSplitDigest(train, budget.reps ?? 1),
 			selectionSplitDigest: campaignSplitDigest(selection, budget.reps ?? 1),
@@ -1168,6 +1188,7 @@ async function runSelfImproveMethod(args) {
 			baselineCampaignDigest: campaignMeasurementDigest(comparison.baselineOnHoldout),
 			winnerCampaignDigest: campaignMeasurementDigest(comparison.winnerOnHoldout),
 			costReceiptsDigest: canonicalDigest(receipts),
+			...report ? { holdoutObservations: report.observations } : {},
 			...comparison.neutralizedOnHoldout ? { neutralizedCampaignDigest: campaignMeasurementDigest(comparison.neutralizedOnHoldout) } : {}
 		},
 		gate: comparison.gateResult,
@@ -1204,6 +1225,8 @@ async function runSelfImproveMethod(args) {
 	const result = {
 		...evidence ? { evidence } : {},
 		mode: "method",
+		...opts.claim ? { claim: opts.claim } : {},
+		...comparison.finalEvidence ? { finalEvidence: comparison.finalEvidence } : {},
 		searchHistoryCoverage: history,
 		baseline,
 		winner: {
@@ -1236,7 +1259,7 @@ async function runSelfImproveMethod(args) {
 		}
 	};
 	if (opts.hostedTenant) {
-		const snapshot = (index, surface, campaign) => ({
+		const snapshot = (index, surface, campaign, compositeMean) => ({
 			index,
 			surfaceHash: surfaceHash(surface),
 			surface,
@@ -1252,7 +1275,7 @@ async function runSelfImproveMethod(args) {
 					...cell.error ? { errorMessage: cell.error } : {}
 				};
 			}),
-			compositeMean: deferred ? null : meanComposite(campaign.aggregates.byScenario).compositeMean,
+			compositeMean,
 			costUsd: campaign.aggregates.cost.totalCostUsd,
 			durationMs: campaign.durationMs
 		});
@@ -1266,8 +1289,8 @@ async function runSelfImproveMethod(args) {
 					...opts.hostedLabels,
 					mode: "method"
 				},
-				baseline: snapshot(0, baselineSurface, comparison.baselineOnHoldout),
-				generations: [snapshot(1, selected.winnerSurface, comparison.winnerOnHoldout)],
+				baseline: snapshot(0, baselineSurface, comparison.baselineOnHoldout, baseline?.compositeMean ?? null),
+				generations: [snapshot(1, selected.winnerSurface, comparison.winnerOnHoldout, winner?.compositeMean ?? null)],
 				gateDecision: result.gateDecision,
 				...lift === void 0 ? {} : { holdoutLift: lift },
 				totalCostUsd: result.totalCostUsd,
@@ -1308,7 +1331,7 @@ function assertSelfImproveSearchMode(opts) {
 	if (budget?.populationSize !== void 0) throw new Error("selfImprove: method owns its candidates; budget.populationSize applies only to proposer mode");
 	if (budget?.candidateConcurrency !== void 0 || budget?.maxImprovementShots !== void 0 || opts.analyzeGeneration !== void 0 || opts.findings !== void 0 || opts.selectParent !== void 0 || opts.premeasuredBaseline !== void 0 || opts.selectionRankKey !== void 0 || opts.searchLedger !== void 0) throw new Error("selfImprove: candidateConcurrency, maxImprovementShots, analyzeGeneration, findings, selectParent, premeasuredBaseline, selectionRankKey, and searchLedger apply only to proposer mode");
 }
-function splitMethodPartitions(searchScenarios, explicitSelection, fraction) {
+function splitMethodPartitions(searchScenarios, explicitSelection, fraction, unitByScenario) {
 	if (!Number.isFinite(fraction) || fraction <= 0 || fraction >= 1) throw new Error("selfImprove: budget.selectionFraction must be in (0, 1)");
 	const byId = /* @__PURE__ */ new Map();
 	for (const scenario of searchScenarios) {
@@ -1325,12 +1348,23 @@ function splitMethodPartitions(searchScenarios, explicitSelection, fraction) {
 		}
 		const train = searchScenarios.filter((scenario) => !selectionIds.has(scenario.id));
 		if (train.length === 0) throw new Error("selfImprove: method train split is empty");
+		if (unitByScenario) {
+			const selectedUnits = new Set([...selectionIds].map((id) => unitByScenario.get(id)));
+			if (train.some((scenario) => selectedUnits.has(unitByScenario.get(scenario.id)))) throw new Error("selfImprove: training and selection share independent units");
+		}
 		return {
 			train,
 			selection: explicitSelection.map((scenario) => byId.get(scenario.id))
 		};
 	}
 	if (searchScenarios.length < 2) throw new Error("selfImprove: method requires at least two non-final scenarios");
+	if (unitByScenario) {
+		const split = splitTrainHoldout(searchScenarios, fraction, unitByScenario);
+		return {
+			train: split.train,
+			selection: split.holdout
+		};
+	}
 	const sorted = [...searchScenarios].sort((a, b) => stableScenarioHash(a.id) - stableScenarioHash(b.id));
 	const count = Math.max(1, Math.min(sorted.length - 1, Math.round(sorted.length * fraction)));
 	return {
@@ -1353,11 +1387,18 @@ function stableScenarioHash(value) {
 	}
 	return hash;
 }
-/**
-* Deterministic train/holdout split by a stable hash of `scenario.id`,
-* so the same scenario set always splits the same way across runs.
-*/
-function splitTrainHoldout(scenarios, fraction) {
+/** Deterministic split by scenario identity, or by source identity for new-unit claims. */
+function splitTrainHoldout(scenarios, fraction, unitByScenario) {
+	if (unitByScenario) {
+		const units = [...new Set(scenarios.map((scenario) => unitByScenario.get(scenario.id)))].sort((a, b) => stableScenarioHash(a) - stableScenarioHash(b) || (a < b ? -1 : a > b ? 1 : 0));
+		if (units.length < 2) throw new Error("selfImprove: splitting needs at least two independent units");
+		const count = Math.max(1, Math.min(units.length - 1, Math.round(units.length * fraction)));
+		const finalUnits = new Set(units.slice(0, count));
+		return {
+			holdout: scenarios.filter((scenario) => finalUnits.has(unitByScenario.get(scenario.id))),
+			train: scenarios.filter((scenario) => !finalUnits.has(unitByScenario.get(scenario.id)))
+		};
+	}
 	const sorted = [...scenarios].sort((a, b) => stableScenarioHash(a.id) - stableScenarioHash(b.id));
 	const nHoldout = Math.max(1, Math.min(sorted.length - 1, Math.round(sorted.length * fraction)));
 	return {
@@ -1378,6 +1419,17 @@ function winnerSearchCampaign(result) {
 	return result.baselineCampaign;
 }
 async function selfImprove(opts) {
+	opts = {
+		...opts,
+		judge: captureJudge(opts.judge),
+		...opts.claim || opts.finalEvidence ? {
+			claim: opts.claim && defineEvaluationClaim(opts.claim),
+			finalEvidence: opts.finalEvidence && captureFinalEvidencePolicy(opts.finalEvidence),
+			scenarios: structuredClone(opts.scenarios),
+			selectionScenarios: opts.selectionScenarios && structuredClone(opts.selectionScenarios),
+			budget: opts.budget && structuredClone(opts.budget)
+		} : {}
+	};
 	const startedAt = Date.now();
 	const runDir = resolveRunDir(opts.runDir ?? (opts.method ? `.agent-eval/runs/self-improve-${startedAt}` : `mem://selfImprove-${startedAt}`));
 	const storage = opts.storage ?? (runDir.startsWith("mem://") ? inMemoryCampaignStorage() : fsCampaignStorage());
@@ -1400,6 +1452,8 @@ async function runSelfImprove(opts, costLedger, startedAt, runDir, storage) {
 	const holdoutMode = budget.holdout ?? "measured";
 	const holdoutDeferred = holdoutMode === "deferred";
 	const expectUsage = opts.expectUsage ?? "assert";
+	const unitByScenario = opts.claim ? evaluationUnitMap(opts.claim, opts.scenarios) : void 0;
+	const splitUnitByScenario = opts.claim?.generalization === "new-units" ? unitByScenario : void 0;
 	const explicitHoldout = budget.holdoutScenarios;
 	const { train, holdout } = explicitHoldout ? {
 		train: opts.scenarios.filter((s) => !explicitHoldout.some((h) => h.id === s.id)),
@@ -1407,11 +1461,16 @@ async function runSelfImprove(opts, costLedger, startedAt, runDir, storage) {
 	} : holdoutDeferred ? {
 		train: opts.scenarios,
 		holdout: []
-	} : splitTrainHoldout(opts.scenarios, holdoutFraction);
+	} : splitTrainHoldout(opts.scenarios, holdoutFraction, splitUnitByScenario);
 	if (train.length === 0) throw new Error("selfImprove: train split is empty. Reduce holdoutFraction or pass more scenarios.");
 	if (holdout.length === 0 && !holdoutDeferred) throw new Error("selfImprove: holdout split is empty. Pass more scenarios.");
+	if (opts.claim?.generalization === "new-units") assertIndependentEvaluationSplit(opts.claim, holdout, train);
+	if (opts.finalEvidence) {
+		if (holdoutDeferred) throw new Error("final evidence requires measured holdout");
+		if (opts.method) await reserveFinalEvidence(opts.finalEvidence, opts.claim, holdout, train);
+	}
 	if (opts.method) {
-		const partitions = splitMethodPartitions(train, opts.selectionScenarios, budget.selectionFraction ?? .25);
+		const partitions = splitMethodPartitions(train, opts.selectionScenarios, budget.selectionFraction ?? .25, splitUnitByScenario);
 		return runSelfImproveMethod({
 			opts: {
 				...opts,
@@ -1435,7 +1494,8 @@ async function runSelfImprove(opts, costLedger, startedAt, runDir, storage) {
 	};
 	const gate = opts.gate ?? defaultProductionGate({
 		holdoutScenarios: holdout,
-		deltaThreshold: .05
+		deltaThreshold: opts.claim?.minimumEffect ?? .05,
+		independentUnitByScenarioId: opts.claim ? evaluationUnitMap(opts.claim, holdout) : void 0
 	});
 	if (opts.onProgress) opts.onProgress({
 		kind: "baseline.started",
@@ -1455,6 +1515,8 @@ async function runSelfImprove(opts, costLedger, startedAt, runDir, storage) {
 		reps: budget.reps,
 		maxImprovementShots: budget.maxImprovementShots,
 		holdoutScenarios: holdout,
+		claim: opts.claim,
+		finalEvidence: opts.finalEvidence,
 		holdout: holdoutMode,
 		gate,
 		neutralize: opts.neutralize,
@@ -1480,16 +1542,16 @@ async function runSelfImprove(opts, costLedger, startedAt, runDir, storage) {
 	const reportSplit = holdoutDeferred ? "search" : "holdout";
 	const reportBaselineCampaign = holdoutDeferred ? result.baselineCampaign : result.baselineOnHoldout;
 	const reportWinnerCampaign = holdoutDeferred ? winnerSearchCampaign(result) : result.winnerOnHoldout;
-	const baseline = meanComposite(reportBaselineCampaign.aggregates.byScenario);
-	const winnerStats = meanComposite(reportWinnerCampaign.aggregates.byScenario);
+	const reportUnitMap = opts.claim ? evaluationUnitMap(opts.claim, holdoutDeferred ? train : holdout) : void 0;
+	const report = pairedCompositeSummary(reportBaselineCampaign, reportWinnerCampaign, reportUnitMap);
+	const baseline = report.baseline;
+	const winnerStats = report.winner;
 	let power;
-	const baselineHoldoutComposites = result.baselineOnHoldout.cells.filter((cell) => !cell.error).map((cell) => {
-		const scores = Object.values(cell.judgeScores);
-		return scores.length === 0 ? NaN : scores.reduce((sum, s) => sum + s.composite, 0) / scores.length;
-	}).filter((v) => Number.isFinite(v));
+	const baselineHoldoutComposites = holdoutDeferred ? [] : report.baselineComposites;
 	if (baselineHoldoutComposites.length >= 3) {
 		power = powerPreflight({
 			baselineComposites: baselineHoldoutComposites,
+			deltaThreshold: opts.claim?.minimumEffect ?? .05,
 			sharedScorerChannel: true
 		});
 		if (opts.onProgress) opts.onProgress({
@@ -1518,6 +1580,8 @@ async function runSelfImprove(opts, costLedger, startedAt, runDir, storage) {
 	const insight = await analyzeRuns({
 		runs: [...cellsToRunRecords(reportBaselineCampaign.cells, "baseline", runDir, opts.baselineSurface, reportSplit, opts.model), ...reportWinnerCampaign === reportBaselineCampaign ? [] : cellsToRunRecords(reportWinnerCampaign.cells, "winner", runDir, result.winnerSurface, reportSplit, opts.model)],
 		baselineCandidateId: "baseline",
+		independentUnitByScenarioId: reportUnitMap,
+		decisionThreshold: opts.claim?.minimumEffect ?? .05,
 		...reportWinnerCampaign === reportBaselineCampaign ? {} : { candidateCandidateId: "winner" }
 	});
 	const durationMs = Date.now() - startedAt;
@@ -1530,13 +1594,16 @@ async function runSelfImprove(opts, costLedger, startedAt, runDir, storage) {
 			result,
 			costReceipts: costLedger.list(),
 			totalCostUsd: totalCost,
-			totalDurationMs: durationMs
+			totalDurationMs: durationMs,
+			independentUnitByScenarioId: holdoutDeferred ? void 0 : reportUnitMap
 		}),
 		storage,
 		hostedClient: opts.hostedTenant ? createHostedClient(opts.hostedTenant) : void 0
 	});
 	if (opts.onProvenance) opts.onProvenance(provenance);
 	const summary = {
+		...opts.claim ? { claim: opts.claim } : {},
+		...result.finalEvidence ? { finalEvidence: result.finalEvidence } : {},
 		mode: "proposer",
 		baseline,
 		winner: {
@@ -1714,4 +1781,4 @@ function requirePositiveInteger(value, field) {
 //#endregion
 export { summarizeExecution as a, analyzeRuns as i, SelfImproveRunError as n, checkCanaries as o, selfImprove as r, defineAgentEval as t };
 
-//# sourceMappingURL=define-agent-eval-Dzidv34q.js.map
+//# sourceMappingURL=define-agent-eval-Ddu33JH9.js.map