@tangle-network/agent-eval 0.175.0 → 0.176.0

package/dist/{define-agent-eval-V1jQyCDR.d.ts → define-agent-eval-CCbl8k2E.d.ts}

@@ -1,8 +1,8 @@
 import { c as CostLedgerHandle, f as CostLedgerSummary, m as CostReceipt } from "./cost-ledger-DbQdN3nO.js";
 import { _ as ProposalFinding } from "./types-DN2WdT5S.js";
-import { B as PowerPreflight } from "./statistical-heldout-Cqb73yE9.js";
+import { B as PowerPreflight, ft as EvidenceReceipt, it as CampaignEvidenceContext } from "./statistical-heldout-Z9NROFFS.js";
 import { H as SurfaceProposer, R as Scenario, S as JudgeConfig, a as CampaignResult, d as DispatchContext, j as MutableSurface, k as LabeledScenarioStore, p as Gate, v as GateResult } from "./types-BJz2CPTM.js";
-import { $r as OptimizationMethodProvenance, Ni as PremeasuredOptimizationBaseline, Pi as RunOptimizationOptions, Yr as OptimizationMethod, Zn as LoopProvenanceRecord, ao as CampaignCellRetryPolicy, ei as OptimizationMethodResult, ji as RunImprovementLoopResult, no as RunEvalOptions, oo as RunCampaignOptions, si as ComparisonCost, ta as SearchHistoryReceipt, to as openAutoPr, vo as CampaignStorage } from "./index-DKXuBPXf.js";
+import { $r as OptimizationMethodProvenance, Ni as PremeasuredOptimizationBaseline, Pi as RunOptimizationOptions, Yr as OptimizationMethod, Zi as SearchHistoryAdmissionOptions, Zn as LoopProvenanceRecord, ao as RunEvalOptions, co as CampaignCellRetryPolicy, ea as SearchHistoryCoverageRow, ei as OptimizationMethodResult, io as openAutoPr, ji as RunImprovementLoopResult, lo as RunCampaignOptions, na as SearchHistoryReceipt, si as ComparisonCost, xo as CampaignStorage } from "./index-Bg6OT2Dd.js";
 import { o as InsightReport } from "./insight-report-DETqPc_A.js";
 import { n as HostedTenant } from "./client-vyYQg3bm.js";
 //#region src/campaign/presets/run-final-comparison.d.ts
@@ -56,6 +56,11 @@ interface SelfImproveMethodProvenance {
   totalDurationMs: number;
 }
 interface SelfImproveMethodResult<TScenario extends Scenario, TArtifact> extends Omit<SelfImproveProposerResult<TScenario, TArtifact>, 'mode' | 'baseline' | 'winner' | 'provenance' | 'generationsExplored' | 'cost' | 'raw' | 'optimization' | 'insight'> {
+  evidence?: {
+    baseline: EvidenceReceipt;
+    winner: EvidenceReceipt;
+  };
+  searchHistoryCoverage: SearchHistoryCoverageRow;
   mode: 'method';
   /** No final measurement exists when holdout is deferred. */
   baseline: SelfImproveProposerResult<TScenario, TArtifact>['baseline'] | null;
@@ -149,7 +154,7 @@ type SelfImproveProgressEvent = {
   mde: number;
   underpowered: boolean;
 };
-interface SelfImproveOptions<TScenario extends Scenario, TArtifact> {
+interface SelfImproveOptions<TScenario extends Scenario, TArtifact> extends SearchHistoryAdmissionOptions {
   /**
    * Your agent — a function that takes the current `MutableSurface`
    * (typically a system prompt the loop is optimizing) plus the
@@ -320,6 +325,8 @@ interface SelfImproveOptions<TScenario extends Scenario, TArtifact> {
    *  return the bounded receipt on `searchHistory`. See
    *  `RunOptimizationOptions.searchLedger`. */
   searchLedger?: RunOptimizationOptions<TScenario, TArtifact>['searchLedger'];
+  /** Complete-method final measurement receipts; authority and environment remain caller-owned. */
+  evidence?: CampaignEvidenceContext;
 }
 interface SelfImproveProposerResult<TScenario extends Scenario, TArtifact> {
   mode: 'proposer';
@@ -495,4 +502,4 @@ interface DefinedAgentEval<TScenario extends Scenario, TArtifact> {
 declare function defineAgentEval<TScenario extends Scenario, TArtifact>(defaults: DefineAgentEvalOptions<TScenario, TArtifact>): DefinedAgentEval<TScenario, TArtifact>;
 //#endregion
 export { SelfImproveMethodResult as _, DefinedAgentEval as a, SelfImproveMethodOptions as c, SelfImproveProposerOptions as d, SelfImproveProposerResult as f, SelfImproveMethodProvenance as g, selfImprove as h, DefineAgentEvalOptions as i, SelfImproveOptions as l, SelfImproveRunError as m, AgentEvalEvaluateOptions as n, defineAgentEval as o, SelfImproveResult as p, AgentEvalImproveOptions as r, SelfImproveBudget as s, AgentEvalAgent as t, SelfImproveProgressEvent as u };
-//# sourceMappingURL=define-agent-eval-V1jQyCDR.d.ts.map
+//# sourceMappingURL=define-agent-eval-CCbl8k2E.d.ts.map

package/dist/define-agent-eval-CCbl8k2E.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"define-agent-eval-CCbl8k2E.d.ts","names":[],"sources":["../src/campaign/presets/run-final-comparison.ts","../src/contract/self-improve-method.ts","../src/contract/self-improve.ts","../src/contract/define-agent-eval.ts"],"mappings":";;;;;;;;UAMiB,uBAAuB,kBAAkB,UAAU,mBAC1D,KAAK,mBAAmB,WAAW;EAC3C,iBAAiB;EACjB,eAAe;EACf,sBACE,SAAS,gBACT,UAAU,WACV,KAAK,WAAW,mBAAmB,WAAW,+BAC3C,QAAQ;EACb,MAAM,KAAK,WAAW;EACtB;EACA;EACA,cAAc,QAAQ,gBAAgB,UAAU,mBAAmB;;;iBAI/C,mBAAmB,kBAAkB,UAAU,WACnE,MAAM,uBAAuB,WAAW,aAAU;;;;;;;;;;;UCWnC;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,oBAAoB,YAAY,0BAA0B;EAC1D;IACE;IACA;IACA;IACA;IACA;IACA;IACA;;EAEF,MAAM,QAAQ,kBAAkB;EAChC;EACA;EACA;EACA;EACA,MAAM;EACN;EACA;;UAGe,wBAAwB,kBAAkB,UAAU,mBAC3D,KACN,0BAA0B,WAAW;EAWvC;IAAa,UAAU;IAAiB,QAAQ;;EAChD,uBAAuB;EACvB;;EAEA,UAAU,0BAA0B,WAAW;EAC/C,QAAQ,KAAK,0BAA0B,WAAW;IAChD;;EAEF,YAAY;EACZ,cAAc,YAAY,0BAA0B,WAAW;;EAE/D,MAAM;;EAEN,YAAY;EACZ,UAAU,0BAA0B,WAAW;EAC/C,KAAK,QAAQ,kBAAkB,mBAAmB,WAAW;IAC3D;IACA,iBAAiB,0BAA0B,WAAW;IACtD,eAAe,0BAA0B,WAAW;IACpD;IACA,QAAQ;IACR,MAAM;IACN,WAAW,kBAAkB;;;;;UCxBhB;;;EAGf;;;;EAIA;;EAEA;;EAEA;;;EAGA;;;EAGA;;;;EAIA;;EAEA,mBAAmB;;;;;;;;;EASnB;;EAEA;;;;;EAKA;;KAGU;EACN;EAA0B;;EAC1B;EAA4B;EAAuB;;EACnD;EAA4B;EAAe;;EAC3C;EAA8B;EAAe;EAAuB;;EAGpE;EAAsB;EAAkB;;EACxC;EAAyB;EAAW;EAAY;EAAa;;UAElD,mBAAmB,kBAAkB,UAAU,mBACtD;;;;;;;;;;;;;;EAcR,QAAQ,SAAS,gBAAgB,UAAU,WAAW,KAAK,oBAAoB,QAAQ;;;;;;;EAQvF;;EAEA;;;EAIA,WAAW;;EAGX,OAAO,YAAY,WAAW;;;EAI9B,iBAAiB;;EAGjB,SAAS;;;;;;;;;;;;;EAcT,sBAAsB,gCAAgC,WAAW;;;;;EAMjE,WAAW,gBAAgB;;;;;;EAO3B,SAAS,mBAAmB,WAAW;;;EAIvC,qBAAqB;;;EAIrB,OAAO,KAAK,WAAW;;;;;;;;EASvB,cAAc,eAAe,gBAAgB,iBAAiB,mBAAmB;;;;EAKjF,UAAU;;;;EAKV;;;EAIA,gBAAgB,QAAQ,uBAAuB;;;;;EAM/C,iBAAiB;IACf,UAAU;IACV;IACA;;;;EAKF;;;;;;;EAQA,YAAY;;;EAIZ,cAAc,OAAO;;;;EAKrB;EACA;EACA;;;;;;;;;;;;;;EAeA,eAAe;;;EAIf,eAAe;;;;EAKf,eAAe;;EAGf;;;;;;;;EASA;;;;;;;;;EAUA,oBAAoB,uBAAuB,WAAW;;;EAItD,WAAW;;;;;;;EAQX,mBAAmB,uBAAuB,WAAW;;;;;;EAOrD,eAAe,uBAAuB,WAAW;;;;EAKjD,eAAe,uBAAuB,WAAW;;EAEjD,WAAW;;UAGI,0BAA0B,kBAAkB,UAAU;EACrE;;;;EAIA;IACE;IACA,aAAa;;;;;EAKf;IACE;IACA,aAAa;IACb,SAAS;;;IAGT;;;;IAIA;;;;;;EAMF;;;EAGA;;;;EAIA,YAAY;;EAEZ;;;EAGA;;EAEA;;EAEA;;EAEA,MAAM;;;EAGN,UAAU;;;EAGV,gBAAgB;;EAEhB;IACE;IACA,MAAM;IACN;IACA,aAAa;;;;;;;;EAQf,SAAS;;;;;EAKT,QAAQ;;;;;;EAMR,KAAK,yBAAyB,WAAW;;KAG/B,kBAAkB,kBAAkB,UAAU,aACtD,0BAA0B,WAAW,aACrC,wBAAwB,WAAW;KAE3B,yBAAyB,kBAAkB,UAAU,aAAa,KAC5E,mBAAmB,WAAW;EAG9B,QAAQ,mBAAmB,WAAW;EACtC;EACA,gBAAgB,QAAQ;;KAGd,2BAA2B,kBAAkB,UAAU,aAAa,KAC9E,mBAAmB,WAAW;EAG9B;EACA,gBAAgB,QAAQ;;;cAIb,4BAA4B;WAC9B,MAAM;WACN,UAAU;EAEnB,YAAY,gBAAgB,QAAQ;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAgMtB,YAAY,kBAAkB,UAAU,WACtD,MAAM,yBAAyB,WAAW,aACzC,QAAQ,wBAAwB,WAAW;iBAC9B,YAAY,kBAAkB,UAAU,WACtD,MAAM,2BAA2B,WAAW,aAC3C,QAAQ,0BAA0B,WAAW;iBAChC,YAAY,kBAAkB,UAAU,WACtD,MAAM,mBAAmB,WAAW,aACnC,QAAQ,kBAAkB,WAAW;;;KC1mB5B,eAAe,kBAAkB,UAAU,cACrD,SAAS,gBACT,UAAU,WACV,KAAK,oBACF,QAAQ;KAED,uBAAuB,kBAAkB,UAAU,aAAa,mBAC1E,WACA;UAGe,yBAAyB,kBAAkB,UAAU,mBAC5D,KACN,eAAe,WAAW;;EAI5B,YAAY;;EAEZ,UAAU;;EAEV,QAAQ,eAAe,WAAW;;EAElC,QAAQ,YAAY,WAAW;;EAE/B,SAAS,YAAY,WAAW;;EAEhC;;KAGU,wBAAwB,kBAAkB,UAAU,aAAa,KAC3E,QAAQ,mBAAmB,WAAW;EAGtC,SAAS,QAAQ;EACjB,eAAe,QAAQ;;UAGR,iBAAiB,kBAAkB,UAAU;;WAEnD,oBAAoB;;WAEpB,iBAAiB;;;;;EAK1B,SACE,OAAO,yBAAyB,WAAW,aAC1C,QAAQ,eAAe,WAAW;;;;;;;EAOrC,QACE,OAAO,wBAAwB,WAAW,aACzC,QAAQ,kBAAkB,WAAW;;;;;;;;;iBAU1B,gBAAgB,kBAAkB,UAAU,WAC1D,UAAU,uBAAuB,WAAW,aAC3C,iBAAiB,WAAW"}

package/dist/{define-agent-eval-ox5McL6e.js → define-agent-eval-DEMsu5eA.js}

@@ -8,12 +8,12 @@ import { o as pairedTTest, r as pairedBootstrap } from "./paired-tests-C8iCsioC.
 import { r as welchsTTest } from "./baseline-BC-eBZ7U.js";
 import "./query-D1nLIKt7.js";
 import { r as observedSplitScore } from "./reward-nw2xZGZG.js";
-import { c as validateRunRecord, i as modelHasSnapshot } from "./run-record-ZIsR9Fif.js";
-import { r as paretoChart } from "./summary-report-Bgh8CpNK.js";
-import { J as defaultProductionGate, K as runFinalComparison, T as runImprovementLoop, _ as campaignMeasurementDigest, at as surfaceDispatchRef, b as loopProvenanceArgsFromResult, c as assertOptimizationResult, h as createMethodCostScope, it as surfaceContentHash, m as costFromLedgerSummary, ot as surfaceHash, p as combineComparisonCosts, q as openAutoPr, st as runEval, ut as resolveRunDir, v as canonicalDigest, vt as campaignSplitDigest, y as emitLoopProvenance } from "./llm-judge-DmNaBrXB.js";
-import { c as campaignCellTaskScore, l as campaignCellToRunRecord, o as campaignCellExecutionEvidence, s as campaignCellJudgeDimensions } from "./reward-hacking-CKW4teig.js";
-import { C as inMemoryCampaignStorage, S as fsCampaignStorage, x as createRunCostLedger } from "./external-optimizer-subprocess-CQi27uEI.js";
-import { t as powerPreflight } from "./power-preflight-CFXm0Vjo.js";
+import { c as validateRunRecord, i as modelHasSnapshot } from "./run-record-DQpSf7t-.js";
+import { r as paretoChart } from "./summary-report-B16xy9Kd.js";
+import { C as assertSearchHistoryAdmissionOptions, I as runFinalComparison, L as openAutoPr, R as defaultProductionGate, W as runEval, d as combineComparisonCosts, f as costFromLedgerSummary, h as runImprovementLoop, q as resolveRunDir, u as executeOptimizationMethod } from "./llm-judge-DliimmRb.js";
+import { a as campaignCellToRunRecord, i as campaignCellTaskScore, n as campaignCellExecutionEvidence, r as campaignCellJudgeDimensions } from "./run-record-CR63CpHK.js";
+import { H as surfaceContentHash, O as powerPreflight, U as surfaceDispatchRef, W as surfaceHash, et as campaignSplitDigest, f as campaignMeasurementDigest, h as loopProvenanceArgsFromResult, m as emitLoopProvenance, p as canonicalDigest, t as createCampaignEvidenceReceipt } from "./campaign-evidence-D8DBLqLI.js";
+import { C as inMemoryCampaignStorage, S as fsCampaignStorage, x as createRunCostLedger } from "./external-optimizer-subprocess-q3VzlGAO.js";
 import { t as createHostedClient } from "./client-BlLY6o2w.js";
 //#region src/contamination-guard.ts
 function checkCanaries(output, scenarios) {
@@ -1026,39 +1026,41 @@ async function runSelfImproveMethod(args) {
 	const { opts, train, selection, holdout, costLedger, storage, runDir, startedAt } = args;
 	const budget = opts.budget ?? {};
 	if (opts.autoOnPromote === "pr" && (!opts.ghOwner || !opts.ghRepo)) throw new Error("selfImprove: autoOnPromote='pr' requires ghOwner + ghRepo");
+	const evidenceContext = opts.evidence === void 0 ? void 0 : structuredClone(opts.evidence);
+	if (evidenceContext && budget.holdout === "deferred") throw new Error("selfImprove: evidence receipts require measured holdout");
 	const baselineSurface = structuredClone(opts.baselineSurface);
 	const judge = {
 		...opts.judge,
-		dimensions: opts.judge.dimensions.map((dimension) => Object.freeze({ ...dimension }))
+		dimensions: opts.judge.dimensions.map((dimension) => ({ ...dimension }))
 	};
-	Object.freeze(judge.dimensions);
-	Object.freeze(judge);
-	const methodCostScope = createMethodCostScope(costLedger, opts.method.name);
-	const method = await opts.method.optimize(Object.freeze({
-		baselineSurface: structuredClone(baselineSurface),
-		trainScenarios: Object.freeze(train.map((scenario) => structuredClone(scenario))),
-		selectionScenarios: Object.freeze(selection.map((scenario) => structuredClone(scenario))),
-		dispatchWithSurface: opts.agent,
-		judges: Object.freeze([judge]),
-		runDir: `${runDir}/optimization/${opts.method.name.replace(/[^a-zA-Z0-9._-]/g, "_")}`,
-		seed: 42,
-		runOptions: Object.freeze({
-			storage,
-			maxConcurrency: budget.maxConcurrency ?? 2,
-			reps: budget.reps,
-			dispatchRef: opts.dispatchRef,
-			dispatchTimeoutMs: opts.dispatchTimeoutMs,
-			cellRetry: opts.cellRetry,
-			cellPlacement: opts.cellPlacement,
-			labeledStore: opts.labeledStore,
-			captureSource: opts.captureSource,
-			expectUsage: opts.expectUsage ?? "assert"
-		}),
-		costLedger: methodCostScope.ledger
-	}));
-	assertOptimizationResult(opts.method.name, method);
-	const selected = structuredClone(method);
-	const methodCost = methodCostScope.reconcile(selected.cost);
+	const { selected, cost: methodCost, history } = await executeOptimizationMethod({
+		method: opts.method,
+		storage,
+		searchHistoryPolicy: opts.searchHistoryPolicy,
+		searchHistoryVerification: opts.searchHistoryVerification,
+		input: {
+			baselineSurface,
+			trainScenarios: train,
+			selectionScenarios: selection,
+			dispatchWithSurface: opts.agent,
+			judges: [judge],
+			runDir: `${runDir}/optimization/${opts.method.name.replace(/[^a-zA-Z0-9._-]/g, "_")}`,
+			seed: 42,
+			runOptions: {
+				storage,
+				maxConcurrency: budget.maxConcurrency ?? 2,
+				reps: budget.reps,
+				dispatchRef: opts.dispatchRef,
+				dispatchTimeoutMs: opts.dispatchTimeoutMs,
+				cellRetry: opts.cellRetry,
+				cellPlacement: opts.cellPlacement,
+				labeledStore: opts.labeledStore,
+				captureSource: opts.captureSource,
+				expectUsage: opts.expectUsage ?? "assert"
+			},
+			costLedger
+		}
+	});
 	const finalPhase = "selfImprove.method-final";
 	const finalCost = () => combineComparisonCosts([
 		"holdout.baseline",
@@ -1120,6 +1122,18 @@ async function runSelfImproveMethod(args) {
 		expectUsage: opts.expectUsage ?? "assert",
 		label: "selfImprove"
 	});
+	const evidence = evidenceContext ? {
+		baseline: createCampaignEvidenceReceipt({
+			campaign: comparison.baselineOnHoldout,
+			surface: baselineSurface,
+			context: evidenceContext
+		}),
+		winner: createCampaignEvidenceReceipt({
+			campaign: comparison.winnerOnHoldout,
+			surface: selected.winnerSurface,
+			context: evidenceContext
+		})
+	} : void 0;
 	const deferred = comparison.holdout === "deferred";
 	const baseline = deferred ? null : meanComposite(comparison.baselineOnHoldout.aggregates.byScenario);
 	const winner = deferred ? null : meanComposite(comparison.winnerOnHoldout.aggregates.byScenario);
@@ -1188,7 +1202,9 @@ async function runSelfImproveMethod(args) {
 		ghRepo: opts.ghRepo
 	}) : void 0;
 	const result = {
+		...evidence ? { evidence } : {},
 		mode: "method",
+		searchHistoryCoverage: history,
 		baseline,
 		winner: {
 			surface: selected.winnerSurface,
@@ -1279,6 +1295,8 @@ var SelfImproveRunError = class extends Error {
 	}
 };
 function assertSelfImproveSearchMode(opts) {
+	assertSearchHistoryAdmissionOptions(opts);
+	if (!opts.method && (opts.searchHistoryPolicy !== void 0 || opts.searchHistoryVerification !== void 0 || opts.evidence !== void 0)) throw new Error("selfImprove: search history admission requires method");
 	if (opts.method && opts.proposer) throw new Error("selfImprove: method and proposer are mutually exclusive");
 	if (!opts.method) {
 		if (opts.selectionScenarios !== void 0) throw new Error("selfImprove: selectionScenarios requires method");
@@ -1696,4 +1714,4 @@ function requirePositiveInteger(value, field) {
 //#endregion
 export { summarizeExecution as a, analyzeRuns as i, SelfImproveRunError as n, checkCanaries as o, selfImprove as r, defineAgentEval as t };
 
-//# sourceMappingURL=define-agent-eval-ox5McL6e.js.map
+//# sourceMappingURL=define-agent-eval-DEMsu5eA.js.map