@tangle-network/agent-eval 0.190.1 → 0.191.0

package/dist/contract/index.js

@@ -4,21 +4,21 @@ import { t as decidePairedPromotion } from "../paired-promotion-decision-6cn3bLc
 import { a as heldoutSignificance } from "../statistical-heldout-GcLIPxk4.js";
 import { n as summarizeExecution, t as analyzeRuns } from "../analyze-runs-SCSWYpy0.js";
 import { a as parseRunRecordSafe } from "../run-record-BRlM99rX.js";
-import { Y as defaultProductionGate, a as transientDispatchFailure, at as runEval, c as compareOptimizationMethods, h as runImprovementLoop, ot as runCampaign, t as llmJudge } from "../llm-judge-BnxAv7ed.js";
+import { J as runEval, R as defaultProductionGate, Y as runCampaign, a as transientDispatchFailure, c as compareOptimizationMethods, h as runImprovementLoop, t as llmJudge } from "../llm-judge-NJOHM0iA.js";
 import { i as isModelPriced, n as estimateCost } from "../metrics-DOMpkoeb.js";
 import { i as CostLedger } from "../cost-ledger-_IbkejCF.js";
-import { Q as campaignSplitDigest } from "../campaign-evidence-D40-LHIL.js";
+import { Y as campaignSplitDigest } from "../campaign-evidence-BqjlTM3P.js";
 import { m as mapConcurrentRange } from "../journal-c3qY0SDP.js";
-import { C as inMemoryCampaignStorage, S as fsCampaignStorage } from "../external-optimizer-subprocess-kbcPu5po.js";
-import { n as SelfImproveRunError, r as selfImprove, t as defineAgentEval } from "../define-agent-eval-DPizhHfy.js";
-import { t as buildDefaultAnalystRegistry } from "../default-registry-16I-Bxlf.js";
-import { t as extractUsage } from "../extract-usage-BrQ8mCLX.js";
+import { C as inMemoryCampaignStorage, S as fsCampaignStorage } from "../external-optimizer-subprocess-B234V9V0.js";
+import { n as SelfImproveRunError, r as selfImprove, t as defineAgentEval } from "../define-agent-eval-BE43YTm4.js";
+import { t as buildDefaultAnalystRegistry } from "../default-registry-xMLX160E.js";
+import { t as extractUsage } from "../extract-usage-DFATEuWW.js";
 import { i as makeProposalFinding } from "../types-CW5b90JH.js";
 import { t as createChatClient } from "../chat-client-B_ItsSBw.js";
 import { n as InMemoryOutcomeStore, t as FileSystemOutcomeStore } from "../outcome-store-BXlkwMPR.js";
-import { c as composeGate, f as REFERENCE_EQUIVALENCE_INPUT_LIMITS, h as runReferenceEquivalenceJudge, l as externalTextOptimizationMethod, m as createReferenceEquivalenceJudge, n as gepaOptimizationMethod, p as REFERENCE_EQUIVALENCE_JUDGE_VERSION, s as heldOutGate, t as skillOptOptimizationMethod } from "../skillopt-optimization-method-l2vDBtKp.js";
+import { c as composeGate, f as REFERENCE_EQUIVALENCE_INPUT_LIMITS, h as runReferenceEquivalenceJudge, l as externalTextOptimizationMethod, m as createReferenceEquivalenceJudge, n as gepaOptimizationMethod, p as REFERENCE_EQUIVALENCE_JUDGE_VERSION, s as heldOutGate, t as skillOptOptimizationMethod } from "../skillopt-optimization-method-DwO3TOAV.js";
 import { n as paretoPolicy, r as paretoSignificanceGate, t as buildEvidenceVector } from "../promotion-policy-CfMNUro2.js";
-import { t as fromOtelSpans } from "../otel-spans-CQY1pNiN.js";
+import { t as fromOtelSpans } from "../otel-spans-DS7t1t1J.js";
 import { createHash } from "node:crypto";
 import { agentCandidateBenchmarkSuiteSchema, agentCandidateBenchmarkTaskSchema, agentCandidateBundleSchema, agentCandidateEvaluationPolicySchema, agentCandidateExperimentSchema, agentImprovementMeasuredComparisonSchema, agentProfileImprovementExperimentSchema, agentProfileImprovementMeasuredComparisonSchema, agentProfileImprovementRunCellSchema, agentProfileImprovementRunReceiptSchema, agentProfileImprovementSuiteInputsSchema, agentProfileImprovementSuiteSchema, agentProfileImprovementTaskSchema, candidateExecutionEvidenceSchema, canonicalCandidateDigest, canonicalCandidateJson, numbersApproximatelyEqual, omitTopLevelDigest } from "@tangle-network/agent-interface";
 import { createReadStream } from "node:fs";
@@ -1318,136 +1318,6 @@ function resolveWriteTarget(write, sourcePath) {
 	return sourcePath.endsWith(".json") || sourcePath.endsWith(".jsonl") ? join(dirname(sourcePath), ANALYSIS_ARTIFACT) : join(sourcePath, ANALYSIS_ARTIFACT);
 }
 //#endregion
-//#region src/contract/diff.ts
-function keyForCell(cell) {
-	return JSON.stringify([cell.scenarioId, cell.rep]);
-}
-/** Build the per-dimension delta map for a matched cell. Each judge name +
-*  dimension name encountered on EITHER side appears in the result. */
-function diffDimensions(before, after) {
-	const out = {};
-	const judges = /* @__PURE__ */ new Set([...Object.keys(before), ...Object.keys(after)]);
-	for (const judge of judges) {
-		const beforeDims = before[judge] ?? {};
-		const afterDims = after[judge] ?? {};
-		const dims = /* @__PURE__ */ new Set([...Object.keys(beforeDims), ...Object.keys(afterDims)]);
-		const judgeOut = {};
-		for (const dim of dims) {
-			const rawBefore = beforeDims[dim];
-			const rawAfter = afterDims[dim];
-			const b = typeof rawBefore === "number" && Number.isFinite(rawBefore) ? rawBefore : null;
-			const a = typeof rawAfter === "number" && Number.isFinite(rawAfter) ? rawAfter : null;
-			judgeOut[dim] = {
-				before: b,
-				after: a,
-				delta: b !== null && a !== null ? a - b : null
-			};
-		}
-		out[judge] = judgeOut;
-	}
-	return out;
-}
-/**
-* Diff two generation snapshots. Cells are matched on `(scenarioId, rep)`;
-* unmatched cells surface in `added` / `removed`. Aggregate fields are
-* recomputed from the snapshot's stored fields, not re-derived from cells —
-* this keeps the diff consistent with whatever aggregation the substrate
-* actually reported.
-*/
-function diffGenerations(before, after) {
-	const beforeMap = new Map(before.cells.map((c) => [keyForCell(c), c]));
-	const afterMap = new Map(after.cells.map((c) => [keyForCell(c), c]));
-	const matched = [];
-	const removed = [];
-	const added = [];
-	for (const [key, beforeCell] of beforeMap) {
-		const afterCell = afterMap.get(key);
-		if (!afterCell) {
-			removed.push(beforeCell);
-			continue;
-		}
-		matched.push({
-			scenarioId: beforeCell.scenarioId,
-			rep: beforeCell.rep,
-			compositeBefore: beforeCell.compositeMean,
-			compositeAfter: afterCell.compositeMean,
-			compositeDelta: beforeCell.compositeMean === null || afterCell.compositeMean === null ? null : afterCell.compositeMean - beforeCell.compositeMean,
-			dimensions: diffDimensions(beforeCell.dimensions, afterCell.dimensions)
-		});
-	}
-	for (const [key, afterCell] of afterMap) if (!beforeMap.has(key)) added.push(afterCell);
-	return {
-		beforeIndex: before.index,
-		afterIndex: after.index,
-		beforeSurfaceHash: before.surfaceHash,
-		afterSurfaceHash: after.surfaceHash,
-		surfaceChanged: before.surfaceHash !== after.surfaceHash,
-		matched,
-		removed,
-		added,
-		compositeBefore: before.compositeMean,
-		compositeAfter: after.compositeMean,
-		compositeDelta: before.compositeMean === null || after.compositeMean === null ? null : after.compositeMean - before.compositeMean,
-		costUsdBefore: before.costUsd,
-		costUsdAfter: after.costUsd,
-		costUsdDelta: after.costUsd - before.costUsd,
-		durationMsBefore: before.durationMs,
-		durationMsAfter: after.durationMs,
-		durationMsDelta: after.durationMs - before.durationMs
-	};
-}
-/** Highest-index generation, or null if the run recorded none. */
-function winnerOf(run) {
-	if (run.generations.length === 0) return null;
-	let winner = run.generations[0];
-	for (const gen of run.generations) if (gen.index > winner.index) winner = gen;
-	return winner;
-}
-/**
-* Diff two full eval-runs. Produces baseline-vs-baseline and
-* winner-vs-winner generation diffs when both sides expose them, plus
-* run-level cost / lift / gate-decision deltas.
-*/
-function diffRuns(before, after) {
-	const beforeWinner = winnerOf(before);
-	const afterWinner = winnerOf(after);
-	const baselineDiff = before.baseline && after.baseline ? diffGenerations(before.baseline, after.baseline) : null;
-	const winnersDiff = beforeWinner && afterWinner ? diffGenerations(beforeWinner, afterWinner) : null;
-	const beforeLift = before.holdoutLift ?? null;
-	const afterLift = after.holdoutLift ?? null;
-	return {
-		beforeRunId: before.runId,
-		afterRunId: after.runId,
-		beforeTimestamp: before.timestamp,
-		afterTimestamp: after.timestamp,
-		beforeGateDecision: before.gateDecision ?? null,
-		afterGateDecision: after.gateDecision ?? null,
-		beforeHoldoutLift: beforeLift,
-		afterHoldoutLift: afterLift,
-		holdoutLiftDelta: beforeLift !== null && afterLift !== null ? afterLift - beforeLift : null,
-		beforeTotalCostUsd: before.totalCostUsd,
-		afterTotalCostUsd: after.totalCostUsd,
-		totalCostUsdDelta: after.totalCostUsd - before.totalCostUsd,
-		beforeTotalDurationMs: before.totalDurationMs,
-		afterTotalDurationMs: after.totalDurationMs,
-		totalDurationMsDelta: after.totalDurationMs - before.totalDurationMs,
-		baselineDiff,
-		winnersDiff
-	};
-}
-/**
-* Within-run baseline → winning-generation diff. The natural "what did the
-* improvement loop produce" view for a single run. Returns null when the
-* run never reached a generation past baseline (errored early, or the gate
-* shipped the baseline as-is).
-*/
-function diffRunBaselineToWinner(run) {
-	if (!run.baseline) return null;
-	const winner = winnerOf(run);
-	if (!winner || winner.index === run.baseline.index) return null;
-	return diffGenerations(run.baseline, winner);
-}
-//#endregion
 //#region src/contract/intake/agent-trace.ts
 function rangeLines(r) {
 	return Math.max(0, r.end_line - r.start_line + 1);
@@ -3351,6 +3221,6 @@ function fromFeedbackTable(opts) {
 	};
 }
 //#endregion
-export { FileSystemOutcomeStore, InMemoryOutcomeStore, REFERENCE_EQUIVALENCE_INPUT_LIMITS, REFERENCE_EQUIVALENCE_JUDGE_VERSION, SelfImproveRunError, analyzeRuns, buildDefaultAnalystRegistry, buildEvidenceVector, campaignSplitDigest, compareOptimizationMethods, composeGate, createChatClient, createReferenceEquivalenceJudge, defaultProductionGate, defineAgentEval, diffGenerations, diffRunBaselineToWinner, diffRuns, evalReportingSuite, evaluatePairedMeasurements, externalTextOptimizationMethod, fromClaudeCodeSession, fromCodexSession, fromFeedbackTable, fromKimiCodeSession, fromOpenCodeSession, fromOtelSpans, fromPiSession, fromRunRecordDir, fsCampaignStorage, gepaOptimizationMethod, heldOutGate, inMemoryCampaignStorage, llmJudge, makeProposalFinding, measuredComparisonFromAgentProfileImprovementExperiment, measuredComparisonFromCandidateExperiment, observeCodeAgentSession, observeCodeAgentStore, paretoPolicy, paretoSignificanceGate, parseAgentTrace, parseCodeAgentJsonl, parseCodeAgentJsonlFile, partitionRunsByAuthoringModel, runAgentProfileImprovementExperiment, runCampaign, runCandidateExperiment, runEval, runImprovementLoop, runReferenceEquivalenceJudge, sealAgentProfileImprovementExperiment, sealAgentProfileImprovementSuite, sealAgentProfileImprovementTask, sealCandidateBenchmarkSuite, sealCandidateBenchmarkTask, sealCandidateExperiment, selfImprove, skillOptOptimizationMethod, streamCodeAgentJsonlFile, summarizeExecution, transientDispatchFailure, verifyAgentProfileImprovementExperimentComparison, verifyCandidateExperiment, verifyCandidateExperimentComparison };
+export { FileSystemOutcomeStore, InMemoryOutcomeStore, REFERENCE_EQUIVALENCE_INPUT_LIMITS, REFERENCE_EQUIVALENCE_JUDGE_VERSION, SelfImproveRunError, analyzeRuns, buildDefaultAnalystRegistry, buildEvidenceVector, campaignSplitDigest, compareOptimizationMethods, composeGate, createChatClient, createReferenceEquivalenceJudge, defaultProductionGate, defineAgentEval, evalReportingSuite, evaluatePairedMeasurements, externalTextOptimizationMethod, fromClaudeCodeSession, fromCodexSession, fromFeedbackTable, fromKimiCodeSession, fromOpenCodeSession, fromOtelSpans, fromPiSession, fromRunRecordDir, fsCampaignStorage, gepaOptimizationMethod, heldOutGate, inMemoryCampaignStorage, llmJudge, makeProposalFinding, measuredComparisonFromAgentProfileImprovementExperiment, measuredComparisonFromCandidateExperiment, observeCodeAgentSession, observeCodeAgentStore, paretoPolicy, paretoSignificanceGate, parseAgentTrace, parseCodeAgentJsonl, parseCodeAgentJsonlFile, partitionRunsByAuthoringModel, runAgentProfileImprovementExperiment, runCampaign, runCandidateExperiment, runEval, runImprovementLoop, runReferenceEquivalenceJudge, sealAgentProfileImprovementExperiment, sealAgentProfileImprovementSuite, sealAgentProfileImprovementTask, sealCandidateBenchmarkSuite, sealCandidateBenchmarkTask, sealCandidateExperiment, selfImprove, skillOptOptimizationMethod, streamCodeAgentJsonlFile, summarizeExecution, transientDispatchFailure, verifyAgentProfileImprovementExperimentComparison, verifyCandidateExperiment, verifyCandidateExperimentComparison };
 
 //# sourceMappingURL=index.js.map