@tangle-network/agent-eval 0.133.2 → 0.133.3

package/dist/{skillopt-optimization-method-CF6a327Q.js → skillopt-optimization-method-vvJ4bMNI.js}

@@ -3,10 +3,10 @@ import { c as costForTokenPricing, i as CostLedger, t as CostAccountingIncomplet
 import { f as maximumChargeForLlmRequest, l as costReceiptFromLlm, m as stripFencedJson, u as costReceiptFromLlmError } from "./llm-client-ClPW-dWB.js";
 import { i as combineAbortSignals, r as clamp01 } from "./run-score-iEEAWiBY.js";
 import { n as mapConcurrent } from "./concurrency-MUjT7VjM.js";
-import { a as confidenceInterval, j as weightedComposite, v as pairedBootstrap } from "./statistics-DWM_AyLe.js";
+import { C as pairedBootstrap, D as pairedSignTest, I as weightedComposite, u as confidenceInterval } from "./statistics-RwRNu2__.js";
 import { t as DEFAULT_REDACTION_RULES } from "./redact-7Aq1ukl-.js";
-import { a as campaignCellExecutionEvidence, l as projectCampaignCellQuality, t as detectRewardHacking } from "./reward-hacking-Dl2UBzej.js";
-import { a as tryWithLedgerFileLock, i as appendLedgerLine, m as tryAcquireAtomicFileLock, p as probeAtomicFileLock } from "./ledger-core-CPZfcrC2.js";
+import { a as campaignCellExecutionEvidence, l as projectCampaignCellQuality, t as detectRewardHacking } from "./reward-hacking-DCdRK9TY.js";
+import { a as appendLedgerLine, h as tryAcquireAtomicFileLock, m as probeAtomicFileLock, o as tryWithLedgerFileLock } from "./ledger-core-DAKFKRzi.js";
 import { createRequire } from "node:module";
 import { z } from "zod";
 import { appendFileSync, existsSync, readFileSync, writeFileSync } from "node:fs";
@@ -155,6 +155,51 @@ function assertBackendReport(report, opts) {
 	return report;
 }
 //#endregion
+//#region src/paired-delta-test.ts
+/** Smallest all-positive sample that can clear a one-sided exact sign test. */
+function minimumPairsForPairedDeltaTest(confidence = .95) {
+	if (!Number.isFinite(confidence) || confidence <= 0 || confidence >= 1) throw new Error(`minimumPairsForPairedDeltaTest: confidence must be in (0,1), got ${confidence}`);
+	const oneSidedAlpha = (1 - confidence) / 2;
+	return Math.ceil(Math.log2(1 / oneSidedAlpha));
+}
+/**
+* Tests whether a paired candidate-minus-baseline delta clears a threshold.
+*
+* At 20 or more pairs, the percentile bootstrap lower bound carries the
+* decision. Below that point the interval is descriptive only, so the function
+* switches to a pre-registered one-sided exact sign test. The exact path is
+* deliberately conservative: it requires both a point estimate above the
+* threshold and enough consistently positive paired differences.
+*/
+function pairedDeltaTest(before, after, options = {}) {
+	const threshold = options.threshold ?? 0;
+	if (!Number.isFinite(threshold)) throw new Error(`pairedDeltaTest: threshold must be finite, got ${threshold}`);
+	const exactMinimum = minimumPairsForPairedDeltaTest(options.confidence ?? .95);
+	const requestedMinimum = options.minPairs ?? exactMinimum;
+	if (!Number.isInteger(requestedMinimum) || requestedMinimum < 1) throw new Error(`pairedDeltaTest: minPairs must be a positive integer, got ${requestedMinimum}`);
+	const minimumPairs = Math.max(requestedMinimum, exactMinimum);
+	const bootstrap = pairedBootstrap(before, after, options);
+	const sufficient = bootstrap.n >= minimumPairs;
+	if (bootstrap.gateEligible) return {
+		bootstrap,
+		method: "bootstrap-ci",
+		pValue: null,
+		minimumPairs,
+		sufficient,
+		significant: sufficient && bootstrap.low > threshold
+	};
+	const exact = pairedSignTest(before.map((value, index) => after[index] - value - threshold), "greater");
+	const estimate = options.statistic === "mean" ? bootstrap.mean : bootstrap.median;
+	return {
+		bootstrap,
+		method: "exact-sign",
+		pValue: exact.pValue,
+		minimumPairs,
+		sufficient,
+		significant: sufficient && estimate > threshold && exact.pValue <= (1 - bootstrap.confidence) / 2
+	};
+}
+//#endregion
 //#region src/json-recovery.ts
 /**
 * Truncation-tolerant JSON recovery — shared by every parser that reads JSON
@@ -5251,23 +5296,24 @@ function pairHoldout(candidate, baseline, scenarioIds, select) {
 }
 /** Significance of the held-out composite lift: ship only when the paired
 *  bootstrap CI lower bound on (candidate − baseline) exceeds `deltaThreshold`
-*  (default 0 ⇒ "confidently positive"). Below `minProductiveRuns` paired
-*  observations there is not enough evidence to claim significance → not
-*  significant (`fewRuns`). Interpret `deltaThreshold` in the judge's native
-*  composite scale. */
+*  (default 0 ⇒ "confidently positive"). At small n, where the percentile
+*  bootstrap is descriptive only, a pre-registered exact sign test carries
+*  the decision. Interpret `deltaThreshold` in the judge's native scale. */
 function heldoutSignificance(paired, opts = {}) {
 	const deltaThreshold = opts.deltaThreshold ?? 0;
-	const minProductiveRuns = opts.minProductiveRuns ?? 3;
 	const confidence = opts.confidence ?? .95;
 	const resamples = opts.resamples ?? 2e3;
 	const seed = opts.seed ?? 1337;
 	const statistic = opts.statistic ?? "mean";
-	const bootstrap = pairedBootstrap(paired.before, paired.after, {
+	const decision = pairedDeltaTest(paired.before, paired.after, {
 		confidence,
 		resamples,
 		statistic,
-		seed
+		seed,
+		threshold: deltaThreshold,
+		minPairs: opts.minProductiveRuns
 	});
+	const bootstrap = decision.bootstrap;
 	const medianBootstrap = statistic === "median" ? bootstrap : pairedBootstrap(paired.before, paired.after, {
 		confidence,
 		resamples,
@@ -5282,14 +5328,18 @@ function heldoutSignificance(paired, opts = {}) {
 		if (Math.abs(after - before) < 1e-9) ties += 1;
 	}
 	const tieFraction = n === 0 ? 0 : ties / n;
-	const fewRuns = n < minProductiveRuns;
+	const fewRuns = !decision.sufficient;
+	const significant = decision.significant;
 	return {
 		paired,
 		bootstrap,
 		medianBootstrap,
 		tieFraction,
 		n,
-		significant: !fewRuns && bootstrap.low > deltaThreshold,
+		minimumRequired: decision.minimumPairs,
+		decisionMethod: decision.method,
+		pValue: decision.pValue,
+		significant,
 		fewRuns
 	};
 }
@@ -5317,10 +5367,17 @@ function dimensionRegressions(candidate, baseline, scenarioIds, criticalDimensio
 			statistic: "median",
 			seed: opts.seed ?? 1337
 		});
+		const regression = pairedDeltaTest(paired.after, paired.before, {
+			confidence: opts.confidence ?? .95,
+			resamples: opts.resamples ?? 2e3,
+			statistic: "median",
+			seed: opts.seed ?? 1337,
+			threshold: tolerance
+		});
 		out.push({
 			dimension: dim,
 			bootstrap,
-			regressed: bootstrap.low < -tolerance,
+			regressed: regression.significant,
 			tolerance,
 			n: paired.before.length
 		});
@@ -5395,7 +5452,7 @@ function defaultProductionGate(options) {
 				if (sig.fewRuns) requiredUnavailable.add("heldout-significance");
 				if (!heldoutPass) {
 					const tieNote = sig.tieFraction >= .4 ? `; ${(sig.tieFraction * 100).toFixed(0)}% tied scenarios` : "";
-					reasons.push(sig.fewRuns ? `held-out: only ${sig.n} paired runs (< ${minProductiveRuns}) — too few to claim significance` : `held-out CI.low ${sig.bootstrap.low.toFixed(3)} ≤ threshold ${deltaThreshold} (${heldoutStatistic} Δ ${delta.toFixed(3)}, ${(sig.bootstrap.confidence * 100).toFixed(0)}% CI [${sig.bootstrap.low.toFixed(3)}, ${sig.bootstrap.high.toFixed(3)}]${tieNote})`);
+					reasons.push(sig.fewRuns ? `held-out: only ${sig.n} paired runs (< ${sig.minimumRequired}) — too few to claim significance` : `held-out CI.low ${sig.bootstrap.low.toFixed(3)} ≤ threshold ${deltaThreshold} (${heldoutStatistic} Δ ${delta.toFixed(3)}, ${(sig.bootstrap.confidence * 100).toFixed(0)}% CI [${sig.bootstrap.low.toFixed(3)}, ${sig.bootstrap.high.toFixed(3)}]${tieNote})`);
 				}
 			}
 			const dimensionsProvided = options.criticalDimensions !== void 0;
@@ -5617,7 +5674,7 @@ function heldOutGate(options) {
 			const ci = `${(sig.bootstrap.confidence * 100).toFixed(0)}% CI [${sig.bootstrap.low.toFixed(3)}, ${sig.bootstrap.high.toFixed(3)}]`;
 			return {
 				decision: passed ? "ship" : "hold",
-				reasons: passed ? [`held-out mean Δ ${delta.toFixed(3)}, CI.low ${sig.bootstrap.low.toFixed(3)} > ${deltaThreshold} (${ci}, n=${sig.n}${tieNote})`] : [sig.fewRuns ? `held-out: only ${sig.n} paired runs — too few to claim significance` : `held-out mean Δ ${delta.toFixed(3)}, CI.low ${sig.bootstrap.low.toFixed(3)} ≤ ${deltaThreshold} (${ci}, n=${sig.n}${tieNote})`],
+				reasons: passed ? [`held-out mean Δ ${delta.toFixed(3)}, CI.low ${sig.bootstrap.low.toFixed(3)} > ${deltaThreshold} (${ci}, n=${sig.n}${tieNote})`] : [sig.fewRuns ? `held-out: only ${sig.n} paired runs; ${sig.minimumRequired} required — too few to claim significance` : `held-out mean Δ ${delta.toFixed(3)}, CI.low ${sig.bootstrap.low.toFixed(3)} ≤ ${deltaThreshold} (${ci}, n=${sig.n}${tieNote})`],
 				contributingGates: [{
 					name: "heldOutGate",
 					status,
@@ -5686,6 +5743,30 @@ function powerPreflight(opts) {
 //#endregion
 //#region src/campaign/gates/promotion-policy.ts
 /**
+* Promotion policy over the evidence VECTOR — the substrate's answer to "never
+* collapse the multi-objective promotion decision into one scalar." A
+* `defaultProductionGate` is one opinionated composition; this module factors
+* the decision into two reusable pieces so MANY policies can compete over the
+* SAME evidence (the quant-desk pattern: one evidence bus, plural strategies):
+*
+*   buildEvidenceVector(ctx, objectives, opts) -> EvidenceVector   // the bus
+*   PromotionPolicy = (ev: EvidenceVector) => GateResult           // a strategy
+*   paretoPolicy(ev)                                               // the default strategy
+*   paretoSignificanceGate(options): Gate                          // bus + policy as a Gate
+*
+* The Pareto policy is SYMMETRIC multi-objective: every objective is BOTH a
+* potential gain source AND a safety floor (unlike `defaultProductionGate`,
+* where only `composite` can win and `criticalDimensions` are pure floors). A
+* candidate ships iff it weakly DOMINATES the baseline at the confidence level —
+* no objective credibly worse (CI floor breach) AND at least one objective
+* credibly better (CI gain). Insufficient evidence on ANY axis -> need_more_work
+* (NOT folded into hold: "gather more reps" and "reject" are different actions).
+*
+* Cost/latency are NOT CI axes here — `GateContext` carries only an aggregate
+* per-side cost, no per-cell observation vector to bootstrap. Treat them as hard
+* constraints (compose with a budget gate via `composeGate`), not faked CIs.
+*/
+/**
 * The Evidence Bus. For each objective, pair candidate vs baseline by full
 * cellId and bootstrap a CI on the good-direction paired delta. Reuses the
 * exact `pairHoldout` + `pairedBootstrap` machinery the held-out gate uses, so
@@ -5693,7 +5774,6 @@ function powerPreflight(opts) {
 */
 function buildEvidenceVector(ctx, objectives, opts = {}) {
 	if (objectives.length === 0) throw new Error("buildEvidenceVector: at least 1 objective required");
-	const minProductiveRuns = opts.minProductiveRuns ?? 3;
 	const confidence = opts.confidence ?? .95;
 	const resamples = opts.resamples ?? 2e3;
 	const seed = opts.seed ?? 1337;
@@ -5708,22 +5788,37 @@ function buildEvidenceVector(ctx, objectives, opts = {}) {
 			select = (s) => s.dimensions[dim];
 		}
 		const paired = pairHoldout(ctx.judgeScores, baseline, scenarioIds, select);
-		const bootstrap = pairedBootstrap(obj.direction === "maximize" ? paired.before : paired.after, obj.direction === "maximize" ? paired.after : paired.before, {
+		const before = obj.direction === "maximize" ? paired.before : paired.after;
+		const after = obj.direction === "maximize" ? paired.after : paired.before;
+		const n = paired.before.length;
+		const floorTolerance = obj.floorTolerance ?? .05 * detectScale([...paired.before, ...paired.after]);
+		const gainThreshold = obj.gainThreshold ?? 0;
+		const improvement = pairedDeltaTest(before, after, {
 			confidence,
 			resamples,
 			statistic: "median",
-			seed
+			seed,
+			threshold: gainThreshold,
+			minPairs: opts.minProductiveRuns
 		});
-		const n = paired.before.length;
-		const floorTolerance = obj.floorTolerance ?? .05 * detectScale([...paired.before, ...paired.after]);
-		const gainThreshold = obj.gainThreshold ?? 0;
-		const verdict = n < minProductiveRuns ? "few_runs" : bootstrap.low < -floorTolerance ? "regressed" : bootstrap.low > gainThreshold ? "improved" : "flat";
+		const regression = pairedDeltaTest(after, before, {
+			confidence,
+			resamples,
+			statistic: "median",
+			seed,
+			threshold: floorTolerance,
+			minPairs: opts.minProductiveRuns
+		});
+		const bootstrap = improvement.bootstrap;
+		const verdict = !improvement.sufficient ? "few_runs" : regression.significant ? "regressed" : improvement.significant ? "improved" : "flat";
 		axes.push({
 			name: obj.name,
 			source: obj.source,
 			direction: obj.direction,
 			bootstrap,
 			n,
+			minimumRequired: improvement.minimumPairs,
+			decisionMethod: improvement.method,
 			gainThreshold,
 			floorTolerance,
 			verdict
@@ -7717,6 +7812,6 @@ function skillOptOptimizationMethod(config) {
 	};
 }
 //#endregion
-export { SEARCH_LEDGER_FILE_CONTEXT as $, acquireSingleRunLock as A, dominates as At, surfaceContentHash as B, JudgeParseError as Bt, detectScale as C, llmJudge as Ct, runCanaries as D, fileVerdictCache as Dt, pairHoldout as E, contentHash as Et, assertCodeSurfaceIdentity as F, BackendIntegrityError as Ft, DEFAULT_MUTATION_PRIMITIVES as G, campaignBreakdown as H, assertComponentSurface as I, assertRealAgentReceipts as It, planCampaignRun as J, buildReflectionPrompt as K, codeSurfaceIdentityMaterial as L, assertRealBackend as Lt, compareOptimizationMethods as M, paretoFrontierWithCrowding as Mt, costFromLedgerSummary as N, scalarScore as Nt, composeGate as O, inMemoryVerdictCache as Ot, optimizationTokenUsageFromSummary as P, recoverTruncatedJson as Pt, inMemoryCampaignStorage as Q, componentSurfaceIdentityMaterial as R, summarizeAgentReceiptIntegrity as Rt, defaultProductionGate as S, hashScenarios as St, heldoutSignificance as T, canonicalJson as Tt, campaignMeanComposite as U, surfaceHash as V, compareRankKeys as W, createRunCostLedger as X, runCampaign as Y, fsCampaignStorage as Z, buildEvidenceVector as _, redTeamReport as _t, emitLoopProvenance as a, assertCampaignDesign as at, powerPreflight as b, Dataset as bt, provenanceRecordPath as c, campaignSplitDigest as ct, runImprovementLoop as d, REFERENCE_EQUIVALENCE_INPUT_LIMITS as dt, SearchLedgerConflictError as et, runOptimization as f, REFERENCE_EQUIVALENCE_JUDGE_VERSION as ft, gepaOptimizationMethod as g, redTeamDataset as gt, labelTrustRank as h, DEFAULT_RED_TEAM_CORPUS as ht, canonicalDigest as i, tangleTracesRoot as it, assertOptimizationResult as j, paretoFrontier as jt, externalTextOptimizationMethod as k, crowdingDistance as kt, provenanceSpansPath as l, campaignSplitDigestFromIdentities as lt, isProposedCandidate as m, runReferenceEquivalenceJudge as mt, buildLoopProvenanceRecord as n, SearchLedgerIntegrityError as nt, loopProvenanceArgsFromResult as o, assertCampaignSplitIdentity as ot, runEval as p, createReferenceEquivalenceJudge as pt, parseReflectionResponse as q, campaignMeasurementDigest as r, resolveRunDir as rt, loopProvenanceSpans as s, campaignScenarioIdentity as st, skillOptOptimizationMethod as t, SearchLedgerError as tt, verifyLoopProvenanceRecord as u, openAutoPr as ut, paretoPolicy as v, scoreRedTeamOutput as vt, dimensionRegressions as w, cachedJudge as wt, heldOutGate as x, HoldoutLockedError as xt, paretoSignificanceGate as y, toolNamesForRun as yt, renderSurfaceDiff as z, summarizeBackendIntegrity as zt };
+export { SEARCH_LEDGER_FILE_CONTEXT as $, acquireSingleRunLock as A, dominates as At, surfaceContentHash as B, summarizeAgentReceiptIntegrity as Bt, detectScale as C, llmJudge as Ct, runCanaries as D, fileVerdictCache as Dt, pairHoldout as E, contentHash as Et, assertCodeSurfaceIdentity as F, minimumPairsForPairedDeltaTest as Ft, DEFAULT_MUTATION_PRIMITIVES as G, campaignBreakdown as H, JudgeParseError as Ht, assertComponentSurface as I, pairedDeltaTest as It, planCampaignRun as J, buildReflectionPrompt as K, codeSurfaceIdentityMaterial as L, BackendIntegrityError as Lt, compareOptimizationMethods as M, paretoFrontierWithCrowding as Mt, costFromLedgerSummary as N, scalarScore as Nt, composeGate as O, inMemoryVerdictCache as Ot, optimizationTokenUsageFromSummary as P, recoverTruncatedJson as Pt, inMemoryCampaignStorage as Q, componentSurfaceIdentityMaterial as R, assertRealAgentReceipts as Rt, defaultProductionGate as S, hashScenarios as St, heldoutSignificance as T, canonicalJson as Tt, campaignMeanComposite as U, surfaceHash as V, summarizeBackendIntegrity as Vt, compareRankKeys as W, createRunCostLedger as X, runCampaign as Y, fsCampaignStorage as Z, buildEvidenceVector as _, redTeamReport as _t, emitLoopProvenance as a, assertCampaignDesign as at, powerPreflight as b, Dataset as bt, provenanceRecordPath as c, campaignSplitDigest as ct, runImprovementLoop as d, REFERENCE_EQUIVALENCE_INPUT_LIMITS as dt, SearchLedgerConflictError as et, runOptimization as f, REFERENCE_EQUIVALENCE_JUDGE_VERSION as ft, gepaOptimizationMethod as g, redTeamDataset as gt, labelTrustRank as h, DEFAULT_RED_TEAM_CORPUS as ht, canonicalDigest as i, tangleTracesRoot as it, assertOptimizationResult as j, paretoFrontier as jt, externalTextOptimizationMethod as k, crowdingDistance as kt, provenanceSpansPath as l, campaignSplitDigestFromIdentities as lt, isProposedCandidate as m, runReferenceEquivalenceJudge as mt, buildLoopProvenanceRecord as n, SearchLedgerIntegrityError as nt, loopProvenanceArgsFromResult as o, assertCampaignSplitIdentity as ot, runEval as p, createReferenceEquivalenceJudge as pt, parseReflectionResponse as q, campaignMeasurementDigest as r, resolveRunDir as rt, loopProvenanceSpans as s, campaignScenarioIdentity as st, skillOptOptimizationMethod as t, SearchLedgerError as tt, verifyLoopProvenanceRecord as u, openAutoPr as ut, paretoPolicy as v, scoreRedTeamOutput as vt, dimensionRegressions as w, cachedJudge as wt, heldOutGate as x, HoldoutLockedError as xt, paretoSignificanceGate as y, toolNamesForRun as yt, renderSurfaceDiff as z, assertRealBackend as zt };
 
-//# sourceMappingURL=skillopt-optimization-method-CF6a327Q.js.map
+//# sourceMappingURL=skillopt-optimization-method-vvJ4bMNI.js.map