@tangle-network/agent-eval 0.21.0 → 0.23.0

package/dist/index.js

@@ -19,7 +19,7 @@ import {
   stopOnNoProgress,
   stopOnRepeatedAction,
   subjectiveEval
-} from "./chunk-ARZ6BEV6.js";
+} from "./chunk-V5QSWN7L.js";
 import {
   CallbackResearcher,
   DEFAULT_MUTATION_PRIMITIVES,
@@ -53,49 +53,111 @@ import {
   summarizePreferenceMemory,
   trialTraceFromMultiShotTrial,
   withAssignedFeedbackSplit
-} from "./chunk-HRZELXCR.js";
+} from "./chunk-VQQSPGSM.js";
 import {
   RunRecordValidationError,
   isRunRecord,
   parseRunRecordSafe,
   roundTripRunRecord,
   validateRunRecord
-} from "./chunk-YUFXO3TU.js";
+} from "./chunk-QBW3YBTR.js";
 import {
-  RESEARCH_REPORT_HARD_PAIR_FLOOR,
   assertReleaseConfidence,
   bootstrapCi,
-  canonicalize,
-  evaluateHypothesis,
   evaluateReleaseConfidence,
-  gainHistogram,
-  hashJson,
   judgeReplayGate,
-  paretoChart,
   releaseTraceEvidenceFromMultiShotTrials,
-  renderReleaseReport,
-  researchReport,
-  signManifest,
-  summaryTable,
-  verifyManifest
-} from "./chunk-3IX6QTB7.js";
+  renderReleaseReport
+} from "./chunk-7EAUOUQS.js";
+import {
+  PredictiveValidityResearcher,
+  adversarialScenarioSearch,
+  analyzeOptimizationResult,
+  applyEloUpdate,
+  bestOfN,
+  buildPairwiseFromCampaign,
+  compareAdaptationCurves,
+  detectRewardHacking,
+  doublyRobust,
+  extractPreferences,
+  extractStepRewards,
+  extractVerifiableReward,
+  extractVerifiableRewardsFromRecords,
+  filterDeterministicallyRewarded,
+  firstPassK,
+  fitBradleyTerry,
+  injectIrrelevantClause,
+  inverseProbabilityWeighting,
+  observationsFromRunRecords,
+  offPolicyEstimateAll,
+  prmTrainingPairs,
+  renameVariables,
+  runAdaptationCurve,
+  runComputeCurve,
+  runContaminationProbe,
+  runRLCampaign,
+  runwiseStepRewardSummary,
+  selfConsistency,
+  selfNormalizedImportanceWeighting,
+  shuffleOrder,
+  stepRewardsToJsonl,
+  thompsonCurriculum,
+  toAnthropicFormat,
+  toDpoJsonl,
+  toDpoRows,
+  toGrpoJsonl,
+  toGrpoRows,
+  toPrmJsonl,
+  toPrmRows,
+  toSftJsonl,
+  toSftRows,
+  toTRLFormat,
+  trialToRunRecord,
+  trialsToRunRecords,
+  varianceBasedCurriculum,
+  variantAggregateToRunRecord,
+  verificationReportToRunRecord
+} from "./chunk-LZKIOBG2.js";
+import {
+  runEvalCampaign
+} from "./chunk-EXGR4XEM.js";
+import {
+  LlmCallError,
+  LlmClient,
+  LlmRouteAssertionError,
+  assertLlmRoute,
+  callLlm,
+  callLlmJson,
+  probeLlm,
+  stripFencedJson
+} from "./chunk-KAO3Q65R.js";
+import {
+  evaluateInterimReleaseConfidence,
+  pairedEvalueSequence,
+  rubricPredictiveValidity
+} from "./chunk-AXHNWLIX.js";
 import {
+  RESEARCH_REPORT_HARD_PAIR_FLOOR,
   benjaminiHochberg,
   bhAdjust,
   bonferroni,
   cohensD,
   confidenceInterval,
+  gainHistogram,
   interRaterReliability,
   mannWhitneyU,
   normalizeScores,
   pairedBootstrap,
   pairedTTest,
   pairedWilcoxon,
+  paretoChart,
   partialCredit,
   requiredSampleSize,
+  researchReport,
+  summaryTable,
   weightedMean,
   wilcoxonSignedRank
-} from "./chunk-KRR4VMH7.js";
+} from "./chunk-IOXMGMHQ.js";
 import {
   DEFAULT_REDACTION_RULES,
   DEFAULT_TRACE_ANALYST_BUDGETS,
@@ -105,7 +167,8 @@ import {
   OTEL_AGENT_EVAL_SCOPE,
   OtlpFileTraceStore,
   REDACTION_VERSION,
-  RunIntegrityError,
+  ReplayCache,
+  ReplayCacheMissError,
   SpanNotFoundError,
   TRACE_ANALYST_ACTOR_DESCRIPTION,
   TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION,
@@ -117,10 +180,10 @@ import {
   aggregateLlm,
   analyzeTraces,
   argHash,
-  assertRunCaptured,
   buildTraceAnalystTools,
   buildTraceInsightContext,
   buildTraceInsightPrompt,
+  createReplayFetch,
   defaultTraceInsightPanel,
   describeTraceInsightScope,
   domainEvidencePattern,
@@ -132,6 +195,7 @@ import {
   isRetrievalSpan,
   isSandboxSpan,
   isToolSpan,
+  iterateRawCalls,
   judgeSpans,
   llmSpans,
   planTraceInsightQuestions,
@@ -140,33 +204,34 @@ import {
   runFailureClass,
   runsForScenario,
   scoreTraceInsightReadiness,
-  throwIfRunIncomplete,
   tokenizeDomainWords,
   toolSpans,
   traceAnalystFunctionGroup,
   traceAnalystOnRunComplete
-} from "./chunk-WOK2RTWG.js";
-import {
-  TraceEmitter,
-  llmSpanFromProvider
-} from "./chunk-5IIQKMD5.js";
+} from "./chunk-4W4NCYM2.js";
 import {
-  LlmCallError,
-  LlmClient,
-  LlmRouteAssertionError,
-  assertLlmRoute,
-  callLlm,
-  callLlmJson,
-  probeLlm,
-  stripFencedJson
-} from "./chunk-3GN6U53I.js";
+  RunIntegrityError,
+  assertRunCaptured,
+  throwIfRunIncomplete
+} from "./chunk-QUKKGHTZ.js";
 import {
   FileSystemRawProviderSink,
   InMemoryRawProviderSink,
   NoopRawProviderSink,
   defaultProviderRedactor,
   providerFromBaseUrl
-} from "./chunk-SNUHRBDL.js";
+} from "./chunk-SQQLHODJ.js";
+import {
+  TraceEmitter,
+  llmSpanFromProvider
+} from "./chunk-5IIQKMD5.js";
+import {
+  canonicalize,
+  evaluateHypothesis,
+  hashJson,
+  signManifest,
+  verifyManifest
+} from "./chunk-6M774GY6.js";
 import "./chunk-PZ5AY32C.js";
 
 // src/client.ts
@@ -10486,12 +10551,15 @@ export {
   OTEL_AGENT_EVAL_SCOPE,
   OtlpFileTraceStore,
   PairwiseSteeringOptimizer,
+  PredictiveValidityResearcher,
   PrmGrader,
   ProductClient,
   ProjectRegistry,
   PromptRegistry,
   REDACTION_VERSION,
   RESEARCH_REPORT_HARD_PAIR_FLOOR,
+  ReplayCache,
+  ReplayCacheMissError,
   RunCritic,
   RunIntegrityError,
   RunRecordValidationError,
@@ -10513,12 +10581,15 @@ export {
   UNIVERSAL_FINDERS,
   acquisitionPlansForKnowledgeGaps,
   adversarialJudge,
+  adversarialScenarioSearch,
   aggregateLlm,
   aggregateRunScore,
   allCriticalPassed,
   analyzeAntiSlop,
+  analyzeOptimizationResult,
   analyzeSeries,
   analyzeTraces,
+  applyEloUpdate,
   argHash,
   assertLlmRoute,
   assertReleaseConfidence,
@@ -10528,12 +10599,14 @@ export {
   deterministicSplit as benchmarkDeterministicSplit,
   benchmarks_exports as benchmarks,
   benjaminiHochberg,
+  bestOfN,
   bhAdjust,
   bisect,
   blockingKnowledgeEval,
   bonferroni,
   bootstrapCi,
   budgetBreachView,
+  buildPairwiseFromCampaign,
   buildReflectionPrompt,
   buildReviewerPrompt,
   buildTraceAnalystTools,
@@ -10559,6 +10632,7 @@ export {
   coherenceJudge,
   collectionPreserved,
   commitBisect,
+  compareAdaptationCurves,
   compareReferenceReplay,
   compareToBaseline,
   compilerJudge,
@@ -10580,6 +10654,7 @@ export {
   createFeedbackTrajectory,
   createIntentMatchJudge,
   createLlmReviewer,
+  createReplayFetch,
   createSandboxCodeMutator,
   createSandboxPool,
   createSemanticConceptJudge,
@@ -10594,15 +10669,18 @@ export {
   defaultTraceInsightPanel,
   deployGateLayer,
   describeTraceInsightScope,
+  detectRewardHacking,
   distillPlaybook,
   domainEvidencePattern,
   dominates,
+  doublyRobust,
   estimateCost,
   estimateTokens,
   euAiActReport,
   evaluateActionPolicy,
   evaluateContract,
   evaluateHypothesis,
+  evaluateInterimReleaseConfidence,
   evaluateOracles,
   evaluateReleaseConfidence,
   executeScenario,
@@ -10612,6 +10690,10 @@ export {
   exportTrainingData,
   extractAssetUrls,
   extractErrorCount,
+  extractPreferences,
+  extractStepRewards,
+  extractVerifiableReward,
+  extractVerifiableRewardsFromRecords,
   failureClusterView,
   feedbackTrajectoriesToDatasetScenarios,
   feedbackTrajectoriesToOptimizerRows,
@@ -10619,12 +10701,15 @@ export {
   feedbackTrajectoryToOptimizerRow,
   fileContains,
   fileExists,
+  filterDeterministicallyRewarded,
   findAutoMatchNoExpectation,
   findConstructorCwdDropped,
   findFallbackToPass,
   findLiteralTruePass,
   findSkipCountsAsPass,
   firstDivergenceView,
+  firstPassK,
+  fitBradleyTerry,
   flowLayer,
   formatBenchmarkReport,
   formatDriverReport,
@@ -10640,12 +10725,14 @@ export {
   inMemoryReferenceReplayStore,
   inMemoryReviewStore,
   inferDomainKeywords,
+  injectIrrelevantClause,
   integrationAsi,
   integrationGateEvals,
   integrationInvokeFailedPayload,
   integrationManifestResolvedPayload,
   integrationManifestValidatedPayload,
   interRaterReliability,
+  inverseProbabilityWeighting,
   iqr,
   isJudgeSpan,
   isLlmSpan,
@@ -10654,6 +10741,7 @@ export {
   isRunRecord,
   isSandboxSpan,
   isToolSpan,
+  iterateRawCalls,
   jestTestParser,
   jsonHasKeys,
   jsonShape,
@@ -10680,8 +10768,11 @@ export {
   normalizeScores,
   notBlocked,
   objectiveEval,
+  observationsFromRunRecords,
+  offPolicyEstimateAll,
   outputLengthRubric,
   pairedBootstrap,
+  pairedEvalueSequence,
   pairedTTest,
   pairedWilcoxon,
   paraphraseRobustness,
@@ -10701,6 +10792,7 @@ export {
   printDriverSummary,
   prmBestOfN,
   prmEnsembleBestOfN,
+  prmTrainingPairs,
   probeLlm,
   promptBisect,
   proposeSynthesisTargets,
@@ -10716,6 +10808,7 @@ export {
   regexMatches,
   regressionView,
   releaseTraceEvidenceFromMultiShotTrials,
+  renameVariables,
   renderMarkdown,
   renderMarkdownReport,
   renderPlaybookMarkdown,
@@ -10733,12 +10826,17 @@ export {
   roundTripRunRecord,
   rowCount,
   rowWhere,
+  rubricPredictiveValidity,
+  runAdaptationCurve,
   runAgentControlLoop,
   runAssertions,
   runBehavioralCanaries,
   runCanaries,
+  runComputeCurve,
+  runContaminationProbe,
   runCounterfactual,
   runE2EWorkflow,
+  runEvalCampaign,
   runExpectations,
   runFailureClass,
   runHarnessExperiment,
@@ -10751,11 +10849,13 @@ export {
   runPromptEvolution,
   runProposeReview,
   runProposeReviewAsControlLoop,
+  runRLCampaign,
   runReferenceReplay,
   runSelfPlay,
   runSemanticConceptJudge,
   runTestGradedScenario,
   runsForScenario,
+  runwiseStepRewardSummary,
   scalarScore,
   scanForMuffledGates,
   scoreAllProjects,
@@ -10768,12 +10868,16 @@ export {
   scoreTraceInsightReadiness,
   securityJudge,
   selectHarnessVariant,
+  selfConsistency,
+  selfNormalizedImportanceWeighting,
   selfPreference,
   sentenceReorderMutator,
   serializeFeedbackTrajectoriesJsonl,
+  shuffleOrder,
   signManifest,
   soc2Report,
   statusAdvanced,
+  stepRewardsToJsonl,
   stopOnNoProgress,
   stopOnRepeatedAction,
   stripFencedJson,
@@ -10785,10 +10889,21 @@ export {
   summaryTable,
   testJudge,
   textInSnapshot,
+  thompsonCurriculum,
   throwIfRunIncomplete,
+  toAnthropicFormat,
+  toDpoJsonl,
+  toDpoRows,
+  toGrpoJsonl,
+  toGrpoRows,
   toLangfuseEnvelope,
   toNdjson,
+  toPrmJsonl,
+  toPrmRows,
   toPrometheusText,
+  toSftJsonl,
+  toSftRows,
+  toTRLFormat,
   tokenizeDomainWords,
   toolIntentAlignmentRubric,
   toolNamesForRun,
@@ -10798,12 +10913,17 @@ export {
   toolWasteView,
   traceAnalystFunctionGroup,
   traceAnalystOnRunComplete,
+  trialToRunRecord,
   trialTraceFromMultiShotTrial,
+  trialsToRunRecords,
   typoMutator,
   urlContains,
   userQuestionsForKnowledgeGaps,
   validateRunRecord,
+  varianceBasedCurriculum,
+  variantAggregateToRunRecord,
   verbosityBias,
+  verificationReportToRunRecord,
   verifyManifest,
   visualDiff,
   viteDeployRunner,