@tangle-network/agent-eval 0.217.1 → 0.218.1

package/dist/contract/index.js

@@ -4,20 +4,20 @@ import { r as summarizeExecution, t as analyzeRuns } from "../analyze-runs-DqPJa
 import { i as pairedBootstrap } from "../mean-BbA_JCzW.js";
 import { t as decidePairedPromotion } from "../paired-promotion-decision-D0SHb-ZL.js";
 import { a as heldoutSignificance } from "../statistical-heldout-Bksv92Th.js";
-import { B as defaultProductionGate, Q as runCampaign, Z as runEval, i as transientDispatchFailure, o as clamp01, p as runImprovementLoop, t as llmJudge } from "../llm-judge-C8jMTNNG.js";
+import { B as defaultProductionGate, Q as runCampaign, Z as runEval, i as transientDispatchFailure, o as clamp01, p as runImprovementLoop, t as llmJudge } from "../llm-judge-Djxh-Uhs.js";
 import { i as isModelPriced, n as estimateCost } from "../metrics-BKaPBtIG.js";
 import { t as CostLedger } from "../cost-ledger-D0352Z4d.js";
-import { z as campaignSplitDigest } from "../campaign-evidence-CQYw8_0H.js";
-import { o as mapConcurrentRange } from "../journal-CR4QikYp.js";
-import { M as fsCampaignStorage, N as inMemoryCampaignStorage } from "../external-optimizer-subprocess-XIHBOSKt.js";
+import { z as campaignSplitDigest } from "../campaign-evidence-DunzcJGv.js";
+import { o as mapConcurrentRange } from "../journal-C7om8Gum.js";
+import { M as fsCampaignStorage, N as inMemoryCampaignStorage } from "../external-optimizer-subprocess-B5gdLRZv.js";
 import { a as makeProposalFinding } from "../types-DqFMNTgK.js";
 import { n as isRecord } from "../json-BhdsJaOQ.js";
-import { n as SelfImproveRunError, r as selfImprove, t as defineAgentEval } from "../define-agent-eval-BDxNlP3D.js";
-import { t as buildDefaultAnalystRegistry } from "../default-registry-ClYjq3QE.js";
+import { n as SelfImproveRunError, r as selfImprove, t as defineAgentEval } from "../define-agent-eval-BGdFKlrc.js";
+import { t as buildDefaultAnalystRegistry } from "../default-registry-BJ-Pq6yL.js";
 import { t as extractUsage } from "../extract-usage-BZLU01kR.js";
-import { t as createChatClient } from "../chat-client-ldqxYYTN.js";
+import { t as createChatClient } from "../chat-client-B2pXlPdx.js";
 import { n as InMemoryOutcomeStore, t as FileSystemOutcomeStore } from "../outcome-store-DUcMj-3D.js";
-import { c as composeGate, l as externalTextOptimizationMethod, n as compareOptimizationMethods, r as gepaOptimizationMethod, s as heldOutGate, t as skillOptOptimizationMethod } from "../skillopt-optimization-method-kyL94-ri.js";
+import { c as composeGate, l as externalTextOptimizationMethod, n as compareOptimizationMethods, r as gepaOptimizationMethod, s as heldOutGate, t as skillOptOptimizationMethod } from "../skillopt-optimization-method-5dlex7MJ.js";
 import { n as paretoPolicy, r as paretoSignificanceGate, t as buildEvidenceVector } from "../promotion-policy-6A8osGr_.js";
 import { t as fromOtelSpans } from "../otel-spans-CdN909k1.js";
 import { createHash } from "node:crypto";
@@ -369,6 +369,11 @@ function assertKnownCost(cost, label) {
 * only validates the projected measurements and derives their shared decision.
 */
 function evaluatePairedMeasurements(options) {
+	return pairedVerdict(budgetAndChecks(runTotals(dimensionRegressions(pairedEstimates(validatePairedInput({ options }))))));
+}
+/** The checked policy and measurements, their cell ids, shared dimensions and score arrays. */
+function validatePairedInput(prev) {
+	const { options } = prev;
 	if (options.measurements.length === 0) throw new Error("paired measurement evaluation requires at least one paired cell");
 	const preparationCost = options.preparationCost ?? {
 		usd: 0,
@@ -384,6 +389,28 @@ function evaluatePairedMeasurements(options) {
 	const baselineScores = measurements.map((measurement) => measurement.baseline.score);
 	const candidateScores = measurements.map((measurement) => measurement.candidate.score);
 	const { confidenceLevel: confidence, resamples, bootstrapSeed, deltaThreshold, minProductiveRuns, budgetUsd, criticalDimensions, regressionTolerance } = policy;
+	return {
+		...prev,
+		preparationCost,
+		policy,
+		measurements,
+		cellIds,
+		dimensions,
+		baselineScores,
+		candidateScores,
+		confidence,
+		resamples,
+		bootstrapSeed,
+		deltaThreshold,
+		minProductiveRuns,
+		budgetUsd,
+		criticalDimensions,
+		regressionTolerance
+	};
+}
+/** Held-out significance, the overall, cost and latency estimates, and the observed power. */
+function pairedEstimates(prev) {
+	const { options, measurements, cellIds, dimensions, baselineScores, candidateScores, confidence, resamples, bootstrapSeed, deltaThreshold, minProductiveRuns } = prev;
 	const significance = heldoutSignificance({
 		before: baselineScores,
 		after: candidateScores,
@@ -456,6 +483,19 @@ function evaluatePairedMeasurements(options) {
 		confidence,
 		sharedScorerChannel: options.sharedScorerChannel
 	});
+	return {
+		...prev,
+		significance,
+		overall,
+		objectives,
+		cost,
+		latency,
+		power
+	};
+}
+/** Critical dimensions that regressed, on either burden of proof. */
+function dimensionRegressions(prev) {
+	const { measurements, dimensions, confidence, resamples, bootstrapSeed, minProductiveRuns, criticalDimensions, regressionTolerance, objectives, power } = prev;
 	const powerSufficient = power.sufficient;
 	const guardedDimensions = new Set(criticalDimensions);
 	const missingCriticalDimensions = criticalDimensions.filter((dimension) => !dimensions.includes(dimension));
@@ -473,9 +513,22 @@ function evaluatePairedMeasurements(options) {
 		}).promote;
 	}));
 	const regressions = objectives.filter((objective) => objective.kind === "dimension" && guardedDimensions.has(objective.name) && objective.availability === "measured" && (objective.confidenceInterval.lower < -regressionTolerance || regressionProven.has(objective.name)));
+	return {
+		...prev,
+		powerSufficient,
+		guardedDimensions,
+		missingCriticalDimensions,
+		regressionProven,
+		regressions
+	};
+}
+/** Run completion, per-task regressions and the measurement cost. */
+function runTotals(prev) {
+	const { options, measurements } = prev;
 	const measurementCostUsd = measurements.reduce((sum, measurement) => sum + measurement.baseline.costUsd + measurement.candidate.costUsd, 0);
 	const measurementWorkDurationMs = measurements.reduce((sum, measurement) => sum + measurement.baseline.latencyMs + measurement.candidate.latencyMs, 0);
-	const incompleteRuns = measurements.flatMap((measurement) => [measurement.baseline, measurement.candidate]).filter((run) => !run.completed);
+	const completedRuns = measurements.flatMap((measurement) => [measurement.baseline, measurement.candidate]);
+	const incompleteRuns = completedRuns.filter((run) => !run.completed);
 	const regressedTasks = measurements.filter((measurement) => !measurement.candidate.passed && measurement.baseline.passed);
 	const sharedTaskFailures = measurements.filter((measurement) => !measurement.candidate.passed && !measurement.baseline.passed);
 	const repairedTasks = measurements.filter((measurement) => measurement.candidate.passed && !measurement.baseline.passed);
@@ -484,7 +537,22 @@ function evaluatePairedMeasurements(options) {
 		provenance: measurements.every((measurement) => measurement.baseline.costProvenance === "observed" && measurement.candidate.costProvenance === "observed") ? "observed" : "estimated"
 	};
 	const measurementCost = options.measurementCost ?? derivedMeasurementCost;
-	assertKnownCost(measurementCost, "paired measurement cost");
+	return {
+		...prev,
+		measurementCostUsd,
+		measurementWorkDurationMs,
+		completedRuns,
+		incompleteRuns,
+		regressedTasks,
+		sharedTaskFailures,
+		repairedTasks,
+		derivedMeasurementCost,
+		measurementCost
+	};
+}
+/** Total cost against the budget, and the checks the verdict is drawn from. */
+function budgetAndChecks(prev) {
+	const { options, preparationCost, budgetUsd, significance, powerSufficient, missingCriticalDimensions, regressions, incompleteRuns, regressedTasks, derivedMeasurementCost, measurementCost } = prev;
 	if (options.measurementCost !== void 0 && (measurementCost.provenance !== derivedMeasurementCost.provenance || !numbersApproximatelyEqual(measurementCost.usd, derivedMeasurementCost.usd))) throw new Error("paired measurement cost does not match its signed receipts");
 	const totalCost = {
 		usd: preparationCost.usd + measurementCost.usd,
@@ -519,6 +587,18 @@ function evaluatePairedMeasurements(options) {
 	];
 	const shipped = checks.every((check) => check.passed);
 	const hardFailure = incompleteRuns.length > 0 || regressedTasks.length > 0 || regressions.length > 0 || missingCriticalDimensions.length > 0 || !budgetPassed;
+	return {
+		...prev,
+		totalCost,
+		budgetPassed,
+		checks,
+		shipped,
+		hardFailure
+	};
+}
+/** The verdict, its reasons and the evaluation record. */
+function pairedVerdict(prev) {
+	const { options, baselineScores, confidence, deltaThreshold, budgetUsd, significance, overall, objectives, power, powerSufficient, missingCriticalDimensions, regressions, measurementWorkDurationMs, incompleteRuns, regressedTasks, sharedTaskFailures, repairedTasks, measurementCost, totalCost, budgetPassed, checks, shipped, hardFailure } = prev;
 	const reasons = [
 		...significance.significant ? [] : [significanceReason(significance, deltaThreshold)],
 		...powerSufficient || significance.fewRuns ? [] : [power.reason],