@tangle-network/agent-eval 0.144.7 → 0.144.8

package/dist/cli.js

@@ -1,6 +1,6 @@
 #!/usr/bin/env node
 import { o as runRolloutReleaseCli } from "./hf-dataset-XggBupCr.js";
-import { n as runAnalystBenchmarkCommand } from "./benchmark-command-BCafwNrf.js";
+import { n as runAnalystBenchmarkCommand } from "./benchmark-command-BKENp2s5.js";
 import { a as runRpcBatch, o as runRpcOnce, p as handleVersion, r as startServerAsync, s as buildOpenApi } from "./server-iu0ede49.js";
 import { writeFileSync } from "node:fs";
 //#region src/cli-config.ts

package/dist/contract/index.js

@@ -7,7 +7,7 @@ import { nt as classifyOtlpSpanRole, rt as isOtlpModelCall } from "../kind-facto
 import { s as makeProposalFinding } from "../usage-receipt-EVI8B8Xu.js";
 import { i as inMemoryCampaignStorage, n as createRunCostLedger, r as fsCampaignStorage } from "../single-run-lock-BMQEv1wG.js";
 import { m as mapConcurrentRange } from "../ledger-core-DXZIqu17.js";
-import { $ as runReferenceEquivalenceJudge, H as resolveRunDir, M as surfaceContentHash, N as surfaceHash, Q as createReferenceEquivalenceJudge, S as assertOptimizationResult, V as runCampaign, X as REFERENCE_EQUIVALENCE_INPUT_LIMITS, Z as REFERENCE_EQUIVALENCE_JUDGE_VERSION, _ as heldOutGate, a as emitLoopProvenance, b as composeGate, ct as llmJudge, d as runImprovementLoop, g as gepaOptimizationMethod, o as loopProvenanceArgsFromResult, p as runEval, q as campaignSplitDigest, t as skillOptOptimizationMethod, v as defaultProductionGate, w as compareOptimizationMethods, x as externalTextOptimizationMethod } from "../skillopt-optimization-method-CQwZ-ZX8.js";
+import { $ as runReferenceEquivalenceJudge, H as resolveRunDir, M as surfaceContentHash, N as surfaceHash, Q as createReferenceEquivalenceJudge, S as assertOptimizationResult, V as runCampaign, X as REFERENCE_EQUIVALENCE_INPUT_LIMITS, Z as REFERENCE_EQUIVALENCE_JUDGE_VERSION, _ as heldOutGate, a as emitLoopProvenance, b as composeGate, ct as llmJudge, d as runImprovementLoop, g as gepaOptimizationMethod, o as loopProvenanceArgsFromResult, p as runEval, q as campaignSplitDigest, t as skillOptOptimizationMethod, v as defaultProductionGate, w as compareOptimizationMethods, x as externalTextOptimizationMethod } from "../skillopt-optimization-method-CkaI2ly4.js";
 import { E as pairedBootstrap } from "../statistics-ByxzSiOM.js";
 import { i as parseRunRecordSafe, r as modelHasSnapshot } from "../run-record-DqOw5X6_.js";
 import { c as heldoutSignificance, i as powerPreflight, n as paretoPolicy, r as paretoSignificanceGate, t as buildEvidenceVector, u as decidePairedPromotion } from "../promotion-policy-CrLrmys8.js";

package/dist/index.js

@@ -19,7 +19,7 @@ import { a as RunCritic, d as defaultIsMaterial, f as diffFindings, h as defineT
 import { a as inMemoryVerdictCache, i as fileVerdictCache, n as canonicalJson, r as contentHash, t as cachedJudge } from "./verdict-cache-BCcOh0kF.js";
 import { t as createDspyRlmTraceEngine } from "./dspy-rlm-engine-BiN49gK6.js";
 import { f as Mutex, p as mapConcurrent } from "./ledger-core-DXZIqu17.js";
-import { $ as runReferenceEquivalenceJudge, L as DEFAULT_MUTATION_PRIMITIVES, M as surfaceContentHash, Q as createReferenceEquivalenceJudge, R as buildReflectionPrompt, X as REFERENCE_EQUIVALENCE_INPUT_LIMITS, Z as REFERENCE_EQUIVALENCE_JUDGE_VERSION, _t as assertRealBackend, at as Dataset, bt as JudgeParseError, ct as llmJudge, dt as paretoFrontier, et as DEFAULT_RED_TEAM_CORPUS, ft as paretoFrontierWithCrowding, gt as assertRealAgentReceipts, ht as BackendIntegrityError, it as toolNamesForRun, lt as crowdingDistance, nt as redTeamReport, ot as HoldoutLockedError, pt as scalarScore, rt as scoreRedTeamOutput, st as hashScenarios, tt as redTeamDataset, ut as dominates, vt as summarizeAgentReceiptIntegrity, y as runCanaries, yt as summarizeBackendIntegrity, z as parseReflectionResponse } from "./skillopt-optimization-method-CQwZ-ZX8.js";
+import { $ as runReferenceEquivalenceJudge, L as DEFAULT_MUTATION_PRIMITIVES, M as surfaceContentHash, Q as createReferenceEquivalenceJudge, R as buildReflectionPrompt, X as REFERENCE_EQUIVALENCE_INPUT_LIMITS, Z as REFERENCE_EQUIVALENCE_JUDGE_VERSION, _t as assertRealBackend, at as Dataset, bt as JudgeParseError, ct as llmJudge, dt as paretoFrontier, et as DEFAULT_RED_TEAM_CORPUS, ft as paretoFrontierWithCrowding, gt as assertRealAgentReceipts, ht as BackendIntegrityError, it as toolNamesForRun, lt as crowdingDistance, nt as redTeamReport, ot as HoldoutLockedError, pt as scalarScore, rt as scoreRedTeamOutput, st as hashScenarios, tt as redTeamDataset, ut as dominates, vt as summarizeAgentReceiptIntegrity, y as runCanaries, yt as summarizeBackendIntegrity, z as parseReflectionResponse } from "./skillopt-optimization-method-CkaI2ly4.js";
 import { $ as selfPreference, A as pairedRiskDifference, B as requiredSampleSize, C as mulberry32, D as pairedCohensDz, E as pairedBootstrap, F as partialCredit, G as wilson, H as weightedComposite, I as passAtK, J as normalCdf, K as studentTCdf, L as pearsonR, M as pairedRiskDifferenceScore, N as pairedSignTest, O as pairedDeltaTieFraction, P as pairedTTest, Q as positionalBias, R as ranks, S as mcnemarRequiredN, T as pairedBinaryScale, U as weightedMean, V as spearmanR, W as wilcoxonSignedRank, X as calibrateJudgeContinuous, Y as calibrateJudge, Z as continuousAgreement, _ as interpretCliffs, a as MANN_WHITNEY_EXACT_MAX_WORK, b as mcnemar, c as bonferroni, d as confidenceInterval, et as verbosityBias, f as corpusInterRaterAgreement, g as interRaterReliability, h as holm, i as MANN_WHITNEY_EXACT_MAX_STATES, j as pairedRiskDifferenceExact, k as pairedMde, l as cliffsDelta, m as eProcess, n as DECISION_PAIRED_DELTA_STATISTIC, o as WILCOXON_EXACT_MAX_N, p as corpusInterRaterAgreementFromJudgeScores, r as DEFAULT_PERMUTATIONS, s as benjaminiHochberg, t as BOOTSTRAP_GATE_MIN_N, u as cohensD, v as isBinaryOutcomeVector, w as normalizeScores, x as mcnemarPower, y as mannWhitneyU, z as requiredPairedSampleSize } from "./statistics-ByxzSiOM.js";
 import { n as pairArms, r as pairRunRecords, t as comparePairedArms } from "./paired-arms-iZ08VFMN.js";
 import { a as improvementVerdict, i as gitProvenanceReader, n as computeExperimentStats, o as inMemoryExperimentStore, r as fileExperimentStore, s as clusteredPairedBinary, t as ExperimentTracker } from "./experiment-tracker-CnRICnMl.js";
@@ -36,7 +36,7 @@ import { C as rollupSupervisorRuns, D as isUnavailable, E as SUPERVISOR_RUN_SCHE
 import { A as TRACE_ANALYST_ACTOR_DESCRIPTION, C as domainEvidencePattern, D as tokenizeDomainWords, E as scoreTraceInsightReadiness, O as traceAnalystOnRunComplete, S as describeTraceInsightScope, T as planTraceInsightQuestions, _ as otlpToTraceRunRecords, a as convertTraceStoresToOtlp, b as buildTraceInsightPrompt, c as otelRunCompleteHook, d as captureFetchToRawSink, f as ToolTraceMissingError, g as otlpToRunRecords, h as otlpRowsToTraceRunRecords, i as iterateRawCalls, j as TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION, k as analyzeTraces, l as OTEL_AGENT_EVAL_SCOPE, m as otlpRowsToRunRecords, n as ReplayCacheMissError, o as createOtelExporter, p as toolSpansToTraceAnalysisStore, r as createReplayFetch, s as createOtelTracingStore, t as ReplayCache, u as exportRunAsOtlp, v as flattenOtlpExportToNdjson, w as inferDomainKeywords, x as defaultTraceInsightPanel, y as buildTraceInsightContext } from "./replay-DyBLaKFc.js";
 import { i as otlpTextToTraceAnalysisStore, n as OtlpFileTraceStore } from "./store-otlp-CKtTpRhv.js";
 import { n as extractUsageFromResponse, r as extractUsageFromSse, t as extractUsage } from "./extract-usage-BW27f3XW.js";
-import { B as harnessAxisOf, F as HARNESS_NATIVE_MODEL, G as parseCorrectnessResponse, H as completionVerdict, I as agentProfileHash, K as verifyCompletion, L as agentProfileId, P as CODING_HARNESSES, R as agentProfileModelId, U as createLlmCorrectnessChecker, V as extractProducedState, W as createTokenRecallChecker, z as expandProfileAxes } from "./campaign-Tdy3h62h.js";
+import { B as harnessAxisOf, F as HARNESS_NATIVE_MODEL, G as parseCorrectnessResponse, H as completionVerdict, I as agentProfileHash, K as verifyCompletion, L as agentProfileId, P as CODING_HARNESSES, R as agentProfileModelId, U as createLlmCorrectnessChecker, V as extractProducedState, W as createTokenRecallChecker, z as expandProfileAxes } from "./campaign--HVSuvV0.js";
 import { n as iqr, r as welchsTTest, t as compareToBaseline } from "./baseline-C-GocmIW.js";
 import { a as judgeSpans, c as runsForScenario, i as hasCapturedToolArgs, l as toolSpans, n as argHash, o as llmSpans, r as groupBy, s as runFailureClass, t as aggregateLlm } from "./query-Di7eEQ79.js";
 import { a as checkBehavioralCanary, i as canaryLeakView, o as checkCanaries, r as HoldoutAuditor, s as runBehavioralCanaries, t as analyzeRuns } from "./analyze-runs-BNNK7irB.js";
@@ -48,7 +48,7 @@ import { i as redactValue, n as REDACTION_VERSION, r as redactString, t as DEFAU
 import { n as DEFAULT_RULES, r as classifyFailure, t as computeToolUseMetrics } from "./tool-use-metrics-DEGMKycK.js";
 import { t as analyzeSeries } from "./series-convergence-CjO2QdRW.js";
 import { n as runCounterfactual, t as attributeCounterfactuals } from "./counterfactual-CWPTrMH7.js";
-import { _ as deterministicSplit, g as BENCHMARK_SPLIT_SEED, t as benchmarks_exports } from "./benchmarks-CDSolHq7.js";
+import { _ as deterministicSplit, g as BENCHMARK_SPLIT_SEED, t as benchmarks_exports } from "./benchmarks-BlPmjd88.js";
 import { t as runEvalCampaign } from "./eval-campaign-DNjCvAm-.js";
 import { n as pairedEvalueSequence, t as evaluateInterimReleaseConfidence } from "./sequential-Br0mAPHA.js";
 import { accessSync, appendFileSync, constants, cpSync, existsSync, mkdirSync, promises, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs";

package/dist/openapi.json

@@ -2,7 +2,7 @@
   "openapi": "3.1.0",
   "info": {
     "title": "@tangle-network/agent-eval — wire protocol",
-    "version": "0.144.7",
+    "version": "0.144.8",
     "description": "HTTP and stdio RPC interface to agent-eval. The TypeScript runtime is the source of truth; this spec is the contract that cross-language clients (Python, Rust, Go) generate from.\n\nWire-protocol version: 1.0.0. Bumps on breaking changes to request/response schemas.",
     "contact": {
       "name": "Tangle Network",

package/dist/{skillopt-optimization-method-CQwZ-ZX8.js → skillopt-optimization-method-CkaI2ly4.js}

@@ -5207,6 +5207,7 @@ async function runOptimization(opts) {
 	const selectionRankKey = opts.selectionRankKey ?? ((campaign) => [campaignMeanComposite(campaign)]);
 	let winnerSurface = baselineSurface;
 	let winnerSurfaceHash = surfaceHash(baselineSurface);
+	const admittedCandidateHashes = /* @__PURE__ */ new Set([winnerSurfaceHash]);
 	let winnerComposite = campaignMeanComposite(baselineCampaign);
 	let winnerRankKey = selectionRankKey(baselineCampaign);
 	assertFiniteRankKey(winnerRankKey, "selectionRankKey for baseline");
@@ -5255,11 +5256,19 @@ async function runOptimization(opts) {
 		if (!Array.isArray(proposed)) throw new TypeError("runOptimization: proposer must return an array");
 		const proposalSnapshot = immutableProposalSnapshot(proposed, "candidate outputs");
 		if (proposalSnapshot.length === 0) break;
-		const surfaceResults = await mapConcurrent(proposalSnapshot.map((p) => isProposedCandidate(p) ? p : {
+		const candidates = proposalSnapshot.map((p) => isProposedCandidate(p) ? p : {
 			surface: p,
 			label: "",
 			rationale: ""
-		}), candidateConcurrency, async ({ surface, label, rationale }, i) => {
+		});
+		const generationHashes = /* @__PURE__ */ new Set();
+		for (const { surface } of candidates) {
+			const hash = surfaceHash(surface);
+			if (admittedCandidateHashes.has(hash) || generationHashes.has(hash)) throw new Error(`runOptimization: duplicate candidate surface hash "${hash}" in generation ${gen}; candidate surfaces must be unique`);
+			generationHashes.add(hash);
+		}
+		for (const hash of generationHashes) admittedCandidateHashes.add(hash);
+		const surfaceResults = await mapConcurrent(candidates, candidateConcurrency, async ({ surface, label, rationale }, i) => {
 			const hash = surfaceHash(surface);
 			const campaign = await runCampaign({
 				...opts,
@@ -6520,4 +6529,4 @@ function skillOptOptimizationMethod(config) {
 //#endregion
 export { runReferenceEquivalenceJudge as $, componentSurfaceIdentityMaterial as A, planCampaignRun as B, combineComparisonCosts as C, assertCodeSurfaceIdentity as D, optimizationTokenUsageFromSummary as E, campaignMeanComposite as F, assertCampaignSplitIdentity as G, resolveRunDir as H, compareRankKeys as I, campaignSplitDigestFromIdentities as J, campaignScenarioIdentity as K, DEFAULT_MUTATION_PRIMITIVES as L, surfaceContentHash as M, surfaceHash as N, assertComponentSurface as O, campaignBreakdown as P, createReferenceEquivalenceJudge as Q, buildReflectionPrompt as R, assertOptimizationResult as S, costFromLedgerSummary as T, tangleTracesRoot as U, runCampaign as V, assertCampaignDesign as W, REFERENCE_EQUIVALENCE_INPUT_LIMITS as X, openAutoPr as Y, REFERENCE_EQUIVALENCE_JUDGE_VERSION as Z, heldOutGate as _, assertRealBackend as _t, emitLoopProvenance as a, Dataset as at, composeGate as b, JudgeParseError as bt, provenanceRecordPath as c, llmJudge as ct, runImprovementLoop as d, paretoFrontier as dt, DEFAULT_RED_TEAM_CORPUS as et, runOptimization as f, paretoFrontierWithCrowding as ft, gepaOptimizationMethod as g, assertRealAgentReceipts as gt, labelTrustRank as h, BackendIntegrityError as ht, canonicalDigest as i, toolNamesForRun as it, renderSurfaceDiff as j, codeSurfaceIdentityMaterial as k, provenanceSpansPath as l, crowdingDistance as lt, isProposedCandidate as m, recoverTruncatedJson as mt, buildLoopProvenanceRecord as n, redTeamReport as nt, loopProvenanceArgsFromResult as o, HoldoutLockedError as ot, runEval as p, scalarScore as pt, campaignSplitDigest as q, campaignMeasurementDigest as r, scoreRedTeamOutput as rt, loopProvenanceSpans as s, hashScenarios as st, skillOptOptimizationMethod as t, redTeamDataset as tt, verifyLoopProvenanceRecord as u, dominates as ut, defaultProductionGate as v, summarizeAgentReceiptIntegrity as vt, compareOptimizationMethods as w, externalTextOptimizationMethod as x, runCanaries as y, summarizeBackendIntegrity as yt, parseReflectionResponse as z };
 
-//# sourceMappingURL=skillopt-optimization-method-CQwZ-ZX8.js.map
+//# sourceMappingURL=skillopt-optimization-method-CkaI2ly4.js.map