@tangle-network/agent-eval 0.144.11 → 0.144.12

package/dist/cli.js

@@ -1,6 +1,6 @@
 #!/usr/bin/env node
 import { o as runRolloutReleaseCli } from "./hf-dataset-XggBupCr.js";
-import { n as runAnalystBenchmarkCommand } from "./benchmark-command-BteMFN62.js";
+import { n as runAnalystBenchmarkCommand } from "./benchmark-command-B_80Ddg2.js";
 import { a as runRpcBatch, o as runRpcOnce, p as handleVersion, r as startServerAsync, s as buildOpenApi } from "./server-iu0ede49.js";
 import { writeFileSync } from "node:fs";
 //#region src/cli-config.ts

package/dist/contract/index.js

@@ -7,7 +7,7 @@ import { nt as isOtlpModelCall, tt as classifyOtlpSpanRole } from "../kind-facto
 import { c as makeProposalFinding } from "../usage-receipt-t7vAzCRQ.js";
 import { a as inMemoryCampaignStorage, i as fsCampaignStorage, r as createRunCostLedger } from "../single-run-lock-DFWHEB09.js";
 import { m as mapConcurrentRange } from "../ledger-core-DXZIqu17.js";
-import { $ as REFERENCE_EQUIVALENCE_INPUT_LIMITS, F as surfaceHash, P as surfaceContentHash, S as assertOptimizationResult, U as runCampaign, W as resolveRunDir, X as campaignSplitDigest, _ as heldOutGate, a as emitLoopProvenance, b as composeGate, d as runImprovementLoop, dt as llmJudge, et as REFERENCE_EQUIVALENCE_JUDGE_VERSION, g as gepaOptimizationMethod, nt as runReferenceEquivalenceJudge, o as loopProvenanceArgsFromResult, p as runEval, t as skillOptOptimizationMethod, tt as createReferenceEquivalenceJudge, v as defaultProductionGate, w as compareOptimizationMethods, x as externalTextOptimizationMethod } from "../skillopt-optimization-method-CQdVeM8k.js";
+import { $ as REFERENCE_EQUIVALENCE_INPUT_LIMITS, F as surfaceHash, P as surfaceContentHash, S as assertOptimizationResult, U as runCampaign, W as resolveRunDir, X as campaignSplitDigest, _ as heldOutGate, a as emitLoopProvenance, b as composeGate, d as runImprovementLoop, dt as llmJudge, et as REFERENCE_EQUIVALENCE_JUDGE_VERSION, g as gepaOptimizationMethod, nt as runReferenceEquivalenceJudge, o as loopProvenanceArgsFromResult, p as runEval, t as skillOptOptimizationMethod, tt as createReferenceEquivalenceJudge, v as defaultProductionGate, w as compareOptimizationMethods, x as externalTextOptimizationMethod } from "../skillopt-optimization-method-5_mDBmlL.js";
 import { E as pairedBootstrap } from "../statistics-ByxzSiOM.js";
 import { i as parseRunRecordSafe, r as modelHasSnapshot } from "../run-record-BmSPWXJR.js";
 import { c as heldoutSignificance, i as powerPreflight, n as paretoPolicy, r as paretoSignificanceGate, t as buildEvidenceVector, u as decidePairedPromotion } from "../promotion-policy-CrLrmys8.js";

package/dist/index.js

@@ -19,7 +19,7 @@ import { a as RunCritic, d as defaultIsMaterial, f as diffFindings, h as defineT
 import { a as inMemoryVerdictCache, i as fileVerdictCache, n as canonicalJson, r as contentHash, t as cachedJudge } from "./verdict-cache-BCcOh0kF.js";
 import { t as createDspyRlmTraceEngine } from "./dspy-rlm-engine-DbTk4JdR.js";
 import { f as Mutex, p as mapConcurrent } from "./ledger-core-DXZIqu17.js";
-import { $ as REFERENCE_EQUIVALENCE_INPUT_LIMITS, B as buildReflectionPrompt, Ct as JudgeParseError, P as surfaceContentHash, St as summarizeBackendIntegrity, V as parseReflectionResponse, at as redTeamReport, bt as assertRealBackend, ct as Dataset, dt as llmJudge, et as REFERENCE_EQUIVALENCE_JUDGE_VERSION, ft as crowdingDistance, gt as scalarScore, ht as paretoFrontierWithCrowding, it as redTeamDataset, lt as HoldoutLockedError, mt as paretoFrontier, nt as runReferenceEquivalenceJudge, ot as scoreRedTeamOutput, pt as dominates, rt as DEFAULT_RED_TEAM_CORPUS, st as toolNamesForRun, tt as createReferenceEquivalenceJudge, ut as hashScenarios, vt as BackendIntegrityError, xt as summarizeAgentReceiptIntegrity, y as runCanaries, yt as assertRealAgentReceipts, z as DEFAULT_MUTATION_PRIMITIVES } from "./skillopt-optimization-method-CQdVeM8k.js";
+import { $ as REFERENCE_EQUIVALENCE_INPUT_LIMITS, B as buildReflectionPrompt, Ct as JudgeParseError, P as surfaceContentHash, St as summarizeBackendIntegrity, V as parseReflectionResponse, at as redTeamReport, bt as assertRealBackend, ct as Dataset, dt as llmJudge, et as REFERENCE_EQUIVALENCE_JUDGE_VERSION, ft as crowdingDistance, gt as scalarScore, ht as paretoFrontierWithCrowding, it as redTeamDataset, lt as HoldoutLockedError, mt as paretoFrontier, nt as runReferenceEquivalenceJudge, ot as scoreRedTeamOutput, pt as dominates, rt as DEFAULT_RED_TEAM_CORPUS, st as toolNamesForRun, tt as createReferenceEquivalenceJudge, ut as hashScenarios, vt as BackendIntegrityError, xt as summarizeAgentReceiptIntegrity, y as runCanaries, yt as assertRealAgentReceipts, z as DEFAULT_MUTATION_PRIMITIVES } from "./skillopt-optimization-method-5_mDBmlL.js";
 import { $ as selfPreference, A as pairedRiskDifference, B as requiredSampleSize, C as mulberry32, D as pairedCohensDz, E as pairedBootstrap, F as partialCredit, G as wilson, H as weightedComposite, I as passAtK, J as normalCdf, K as studentTCdf, L as pearsonR, M as pairedRiskDifferenceScore, N as pairedSignTest, O as pairedDeltaTieFraction, P as pairedTTest, Q as positionalBias, R as ranks, S as mcnemarRequiredN, T as pairedBinaryScale, U as weightedMean, V as spearmanR, W as wilcoxonSignedRank, X as calibrateJudgeContinuous, Y as calibrateJudge, Z as continuousAgreement, _ as interpretCliffs, a as MANN_WHITNEY_EXACT_MAX_WORK, b as mcnemar, c as bonferroni, d as confidenceInterval, et as verbosityBias, f as corpusInterRaterAgreement, g as interRaterReliability, h as holm, i as MANN_WHITNEY_EXACT_MAX_STATES, j as pairedRiskDifferenceExact, k as pairedMde, l as cliffsDelta, m as eProcess, n as DECISION_PAIRED_DELTA_STATISTIC, o as WILCOXON_EXACT_MAX_N, p as corpusInterRaterAgreementFromJudgeScores, r as DEFAULT_PERMUTATIONS, s as benjaminiHochberg, t as BOOTSTRAP_GATE_MIN_N, u as cohensD, v as isBinaryOutcomeVector, w as normalizeScores, x as mcnemarPower, y as mannWhitneyU, z as requiredPairedSampleSize } from "./statistics-ByxzSiOM.js";
 import { n as pairArms, r as pairRunRecords, t as comparePairedArms } from "./paired-arms-iZ08VFMN.js";
 import { a as improvementVerdict, i as gitProvenanceReader, n as computeExperimentStats, o as inMemoryExperimentStore, r as fileExperimentStore, s as clusteredPairedBinary, t as ExperimentTracker } from "./experiment-tracker-CnRICnMl.js";
@@ -36,7 +36,7 @@ import { C as rollupSupervisorRuns, D as isUnavailable, E as SUPERVISOR_RUN_SCHE
 import { A as TRACE_ANALYST_ACTOR_DESCRIPTION, C as domainEvidencePattern, D as tokenizeDomainWords, E as scoreTraceInsightReadiness, O as traceAnalystOnRunComplete, S as describeTraceInsightScope, T as planTraceInsightQuestions, _ as otlpToTraceRunRecords, a as convertTraceStoresToOtlp, b as buildTraceInsightPrompt, c as otelRunCompleteHook, d as captureFetchToRawSink, f as ToolTraceMissingError, g as otlpToRunRecords, h as otlpRowsToTraceRunRecords, i as iterateRawCalls, j as TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION, k as analyzeTraces, l as OTEL_AGENT_EVAL_SCOPE, m as otlpRowsToRunRecords, n as ReplayCacheMissError, o as createOtelExporter, p as toolSpansToTraceAnalysisStore, r as createReplayFetch, s as createOtelTracingStore, t as ReplayCache, u as exportRunAsOtlp, v as flattenOtlpExportToNdjson, w as inferDomainKeywords, x as defaultTraceInsightPanel, y as buildTraceInsightContext } from "./replay-CohS93nE.js";
 import { i as otlpTextToTraceAnalysisStore, n as OtlpFileTraceStore } from "./store-otlp-Dw8PPIlL.js";
 import { n as extractUsageFromResponse, r as extractUsageFromSse, t as extractUsage } from "./extract-usage-CdZdoj1s.js";
-import { B as harnessAxisOf, F as HARNESS_NATIVE_MODEL, G as parseCorrectnessResponse, H as completionVerdict, I as agentProfileHash, K as verifyCompletion, L as agentProfileId, P as CODING_HARNESSES, R as agentProfileModelId, U as createLlmCorrectnessChecker, V as extractProducedState, W as createTokenRecallChecker, z as expandProfileAxes } from "./campaign-C2TTzQII.js";
+import { B as harnessAxisOf, F as HARNESS_NATIVE_MODEL, G as parseCorrectnessResponse, H as completionVerdict, I as agentProfileHash, K as verifyCompletion, L as agentProfileId, P as CODING_HARNESSES, R as agentProfileModelId, U as createLlmCorrectnessChecker, V as extractProducedState, W as createTokenRecallChecker, z as expandProfileAxes } from "./campaign-B0eo1ZEU.js";
 import { n as iqr, r as welchsTTest, t as compareToBaseline } from "./baseline-C-GocmIW.js";
 import { a as judgeSpans, c as runsForScenario, i as hasCapturedToolArgs, l as toolSpans, n as argHash, o as llmSpans, r as groupBy, s as runFailureClass, t as aggregateLlm } from "./query-Di7eEQ79.js";
 import { a as checkBehavioralCanary, i as canaryLeakView, o as checkCanaries, r as HoldoutAuditor, s as runBehavioralCanaries, t as analyzeRuns } from "./analyze-runs-C30yljDJ.js";
@@ -48,7 +48,7 @@ import { i as redactValue, n as REDACTION_VERSION, r as redactString, t as DEFAU
 import { n as DEFAULT_RULES, r as classifyFailure, t as computeToolUseMetrics } from "./tool-use-metrics-DEGMKycK.js";
 import { t as analyzeSeries } from "./series-convergence-CjO2QdRW.js";
 import { n as runCounterfactual, t as attributeCounterfactuals } from "./counterfactual-CWPTrMH7.js";
-import { _ as deterministicSplit, g as BENCHMARK_SPLIT_SEED, t as benchmarks_exports } from "./benchmarks-Dzs8CKb1.js";
+import { _ as deterministicSplit, g as BENCHMARK_SPLIT_SEED, t as benchmarks_exports } from "./benchmarks-DZFX_WHN.js";
 import { t as runEvalCampaign } from "./eval-campaign-B_7wcnav.js";
 import { n as pairedEvalueSequence, t as evaluateInterimReleaseConfidence } from "./sequential-Br0mAPHA.js";
 import { accessSync, appendFileSync, constants, cpSync, existsSync, mkdirSync, promises, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs";

package/dist/openapi.json

@@ -2,7 +2,7 @@
   "openapi": "3.1.0",
   "info": {
     "title": "@tangle-network/agent-eval — wire protocol",
-    "version": "0.144.11",
+    "version": "0.144.12",
     "description": "HTTP and stdio RPC interface to agent-eval. The TypeScript runtime is the source of truth; this spec is the contract that cross-language clients (Python, Rust, Go) generate from.\n\nWire-protocol version: 1.0.0. Bumps on breaking changes to request/response schemas.",
     "contact": {
       "name": "Tangle Network",

package/dist/{skillopt-optimization-method-CQdVeM8k.js → skillopt-optimization-method-5_mDBmlL.js}

@@ -5321,10 +5321,15 @@ function gepaOptimizationMethod(config) {
 				maxEvidenceChars,
 				describeArtifact: config.describeArtifact
 			});
+			let attemptEvaluationCount = 0;
 			const callback = await startExternalOptimizerCallback({
 				token: randomBytes(32).toString("hex"),
 				maxEvaluations: evaluationLimit,
-				acceptEvaluation: () => runBudget.acceptEvaluation(),
+				acceptEvaluation: () => {
+					if (runBudget.acceptEvaluation() === void 0) return void 0;
+					attemptEvaluationCount += 1;
+					return attemptEvaluationCount;
+				},
 				evaluate,
 				observe: observationLog.observe,
 				...config.evaluationCallbackLimits ? { limits: config.evaluationCallbackLimits } : {},
@@ -6749,10 +6754,15 @@ function skillOptOptimizationMethod(config) {
 				maxEvidenceChars,
 				describeArtifact: config.describeArtifact
 			});
+			let attemptEvaluationCount = 0;
 			const callback = await startExternalOptimizerCallback({
 				token: randomBytes(32).toString("hex"),
 				maxEvaluations: config.maxEvaluations,
-				acceptEvaluation: () => runBudget.acceptEvaluation(),
+				acceptEvaluation: () => {
+					if (runBudget.acceptEvaluation() === void 0) return void 0;
+					attemptEvaluationCount += 1;
+					return attemptEvaluationCount;
+				},
 				evaluate,
 				observe: observationLog.observe,
 				...config.evaluationCallbackLimits ? { limits: config.evaluationCallbackLimits } : {},
@@ -6909,4 +6919,4 @@ function skillOptOptimizationMethod(config) {
 //#endregion
 export { REFERENCE_EQUIVALENCE_INPUT_LIMITS as $, assertComponentSurface as A, buildReflectionPrompt as B, combineComparisonCosts as C, JudgeParseError as Ct, readGepaCandidatePopulationArtifact as D, optimizationTokenUsageFromSummary as E, surfaceHash as F, tangleTracesRoot as G, planCampaignRun as H, campaignBreakdown as I, assertCampaignSplitIdentity as J, readExternalOptimizerObservationArtifact as K, campaignMeanComposite as L, componentSurfaceIdentityMaterial as M, renderSurfaceDiff as N, decodeExternalTextCandidate as O, surfaceContentHash as P, openAutoPr as Q, compareRankKeys as R, assertOptimizationResult as S, summarizeBackendIntegrity as St, costFromLedgerSummary as T, runCampaign as U, parseReflectionResponse as V, resolveRunDir as W, campaignSplitDigest as X, campaignScenarioIdentity as Y, campaignSplitDigestFromIdentities as Z, heldOutGate as _, recoverTruncatedJson as _t, emitLoopProvenance as a, redTeamReport as at, composeGate as b, assertRealBackend as bt, provenanceRecordPath as c, Dataset as ct, runImprovementLoop as d, llmJudge as dt, REFERENCE_EQUIVALENCE_JUDGE_VERSION as et, runOptimization as f, crowdingDistance as ft, gepaOptimizationMethod as g, scalarScore as gt, labelTrustRank as h, paretoFrontierWithCrowding as ht, canonicalDigest as i, redTeamDataset as it, codeSurfaceIdentityMaterial as j, assertCodeSurfaceIdentity as k, provenanceSpansPath as l, HoldoutLockedError as lt, isProposedCandidate as m, paretoFrontier as mt, buildLoopProvenanceRecord as n, runReferenceEquivalenceJudge as nt, loopProvenanceArgsFromResult as o, scoreRedTeamOutput as ot, runEval as p, dominates as pt, assertCampaignDesign as q, campaignMeasurementDigest as r, DEFAULT_RED_TEAM_CORPUS as rt, loopProvenanceSpans as s, toolNamesForRun as st, skillOptOptimizationMethod as t, createReferenceEquivalenceJudge as tt, verifyLoopProvenanceRecord as u, hashScenarios as ut, defaultProductionGate as v, BackendIntegrityError as vt, compareOptimizationMethods as w, externalTextOptimizationMethod as x, summarizeAgentReceiptIntegrity as xt, runCanaries as y, assertRealAgentReceipts as yt, DEFAULT_MUTATION_PRIMITIVES as z };
 
-//# sourceMappingURL=skillopt-optimization-method-CQdVeM8k.js.map
+//# sourceMappingURL=skillopt-optimization-method-5_mDBmlL.js.map