@tangle-network/agent-eval 0.20.12 → 0.21.0

package/dist/index.js

@@ -19,7 +19,7 @@ import {
   stopOnNoProgress,
   stopOnRepeatedAction,
   subjectiveEval
-} from "./chunk-MCMV7DUL.js";
+} from "./chunk-ARZ6BEV6.js";
 import {
   CallbackResearcher,
   DEFAULT_MUTATION_PRIMITIVES,
@@ -53,7 +53,7 @@ import {
   summarizePreferenceMemory,
   trialTraceFromMultiShotTrial,
   withAssignedFeedbackSplit
-} from "./chunk-HKYRWNHV.js";
+} from "./chunk-HRZELXCR.js";
 import {
   RunRecordValidationError,
   isRunRecord,
@@ -62,16 +62,23 @@ import {
   validateRunRecord
 } from "./chunk-YUFXO3TU.js";
 import {
+  RESEARCH_REPORT_HARD_PAIR_FLOOR,
   assertReleaseConfidence,
   bootstrapCi,
+  canonicalize,
+  evaluateHypothesis,
   evaluateReleaseConfidence,
   gainHistogram,
+  hashJson,
   judgeReplayGate,
   paretoChart,
   releaseTraceEvidenceFromMultiShotTrials,
   renderReleaseReport,
-  summaryTable
-} from "./chunk-IKFVX537.js";
+  researchReport,
+  signManifest,
+  summaryTable,
+  verifyManifest
+} from "./chunk-3IX6QTB7.js";
 import {
   benjaminiHochberg,
   bhAdjust,
@@ -88,7 +95,7 @@ import {
   requiredSampleSize,
   weightedMean,
   wilcoxonSignedRank
-} from "./chunk-ODFINDLQ.js";
+} from "./chunk-KRR4VMH7.js";
 import {
   DEFAULT_REDACTION_RULES,
   DEFAULT_TRACE_ANALYST_BUDGETS,
@@ -98,6 +105,7 @@ import {
   OTEL_AGENT_EVAL_SCOPE,
   OtlpFileTraceStore,
   REDACTION_VERSION,
+  RunIntegrityError,
   SpanNotFoundError,
   TRACE_ANALYST_ACTOR_DESCRIPTION,
   TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION,
@@ -109,6 +117,7 @@ import {
   aggregateLlm,
   analyzeTraces,
   argHash,
+  assertRunCaptured,
   buildTraceAnalystTools,
   buildTraceInsightContext,
   buildTraceInsightPrompt,
@@ -131,22 +140,33 @@ import {
   runFailureClass,
   runsForScenario,
   scoreTraceInsightReadiness,
+  throwIfRunIncomplete,
   tokenizeDomainWords,
   toolSpans,
-  traceAnalystFunctionGroup
-} from "./chunk-KWUAAIHR.js";
+  traceAnalystFunctionGroup,
+  traceAnalystOnRunComplete
+} from "./chunk-WOK2RTWG.js";
 import {
   TraceEmitter,
   llmSpanFromProvider
-} from "./chunk-PKCVBYTQ.js";
+} from "./chunk-5IIQKMD5.js";
 import {
   LlmCallError,
   LlmClient,
+  LlmRouteAssertionError,
+  assertLlmRoute,
   callLlm,
   callLlmJson,
   probeLlm,
   stripFencedJson
-} from "./chunk-75MCTH7P.js";
+} from "./chunk-3GN6U53I.js";
+import {
+  FileSystemRawProviderSink,
+  InMemoryRawProviderSink,
+  NoopRawProviderSink,
+  defaultProviderRedactor,
+  providerFromBaseUrl
+} from "./chunk-SNUHRBDL.js";
 import "./chunk-PZ5AY32C.js";
 
 // src/client.ts
@@ -4847,7 +4867,7 @@ var Dataset = class _Dataset {
    * Write to disk for contamination-verifiable archives.
    */
   toJsonl() {
-    return this.scenarios.slice().sort((a, b) => a.id.localeCompare(b.id)).map((s) => JSON.stringify(canonicalize(s))).join("\n") + "\n";
+    return this.scenarios.slice().sort((a, b) => a.id.localeCompare(b.id)).map((s) => JSON.stringify(canonicalize2(s))).join("\n") + "\n";
   }
   static fromJsonl(jsonl, manifest) {
     const scenarios = [];
@@ -4860,18 +4880,18 @@ var Dataset = class _Dataset {
   }
 };
 async function hashScenarios(scenarios) {
-  const canonical = scenarios.slice().sort((a, b) => a.id.localeCompare(b.id)).map(canonicalize);
+  const canonical = scenarios.slice().sort((a, b) => a.id.localeCompare(b.id)).map(canonicalize2);
   const text = JSON.stringify(canonical);
   const bytes = new TextEncoder().encode(text);
   const digest = await globalThis.crypto.subtle.digest("SHA-256", bytes);
   return Array.from(new Uint8Array(digest)).map((b) => b.toString(16).padStart(2, "0")).join("");
 }
-function canonicalize(v) {
+function canonicalize2(v) {
   if (v === null || typeof v !== "object") return v;
-  if (Array.isArray(v)) return v.map(canonicalize);
+  if (Array.isArray(v)) return v.map(canonicalize2);
   const keys = Object.keys(v).sort();
   const out = {};
-  for (const k of keys) out[k] = canonicalize(v[k]);
+  for (const k of keys) out[k] = canonicalize2(v[k]);
   return out;
 }
 function seededShuffle(items, seed) {
@@ -6978,51 +6998,6 @@ function attributeStep(op, prmA, prmB) {
   };
 }
 
-// src/pre-registration.ts
-function canonicalize2(v) {
-  if (v === null || typeof v !== "object") return v;
-  if (Array.isArray(v)) return v.map(canonicalize2);
-  const keys = Object.keys(v).sort();
-  const out = {};
-  for (const k of keys) out[k] = canonicalize2(v[k]);
-  return out;
-}
-async function hashJson(obj) {
-  const canonical = canonicalize2(obj);
-  const bytes = new TextEncoder().encode(JSON.stringify(canonical));
-  const digest = await globalThis.crypto.subtle.digest("SHA-256", bytes);
-  return Array.from(new Uint8Array(digest)).map((b) => b.toString(16).padStart(2, "0")).join("");
-}
-async function signManifest(m) {
-  const hash = await hashJson(m);
-  return { ...m, contentHash: hash, algo: "sha256-content" };
-}
-async function verifyManifest(m) {
-  const { contentHash, algo: _algo, ...rest } = m;
-  void _algo;
-  const resigned = await signManifest(rest);
-  return resigned.contentHash === contentHash;
-}
-async function evaluateHypothesis(manifest, observed) {
-  if (!await verifyManifest(manifest)) {
-    throw new Error("evaluateHypothesis: manifest content hash mismatch (tampered)");
-  }
-  const reasons = [];
-  const directionOk = manifest.direction === "increase" ? observed.effect > 0 : observed.effect < 0;
-  if (!directionOk) reasons.push("wrong_direction");
-  if (Math.abs(observed.effect) < manifest.minEffect) reasons.push("effect_too_small");
-  if (observed.pValue >= manifest.alpha) reasons.push("not_significant");
-  if (observed.n < manifest.preRegisteredN) reasons.push("undersampled");
-  return {
-    manifest,
-    observedN: observed.n,
-    observedEffect: observed.effect,
-    observedPValue: observed.pValue,
-    confirmed: reasons.length === 0,
-    rejectionReasons: reasons
-  };
-}
-
 // src/self-play.ts
 async function runSelfPlay(proposer, scorer, targets, options = {}) {
   if (targets.length < 2) throw new Error("runSelfPlay: at least 2 targets required (need a difference to measure)");
@@ -10481,6 +10456,7 @@ export {
   FileSystemExperimentStore,
   FileSystemFeedbackTrajectoryStore,
   FileSystemOutcomeStore,
+  FileSystemRawProviderSink,
   FileSystemTraceStore,
   HeldOutGate,
   HoldoutAuditor,
@@ -10489,6 +10465,7 @@ export {
   InMemoryExperimentStore,
   InMemoryFeedbackTrajectoryStore,
   InMemoryOutcomeStore,
+  InMemoryRawProviderSink,
   InMemoryTraceStore,
   InMemoryTrialCache,
   InMemoryWorkspaceInspector,
@@ -10497,12 +10474,14 @@ export {
   LineageRecorder,
   LlmCallError,
   LlmClient,
+  LlmRouteAssertionError,
   LockedJsonlAppender,
   MODEL_PRICING,
   MetricsCollector,
   MultiLayerVerifier,
   MutationTelemetry,
   Mutex,
+  NoopRawProviderSink,
   NoopResearcher,
   OTEL_AGENT_EVAL_SCOPE,
   OtlpFileTraceStore,
@@ -10512,7 +10491,9 @@ export {
   ProjectRegistry,
   PromptRegistry,
   REDACTION_VERSION,
+  RESEARCH_REPORT_HARD_PAIR_FLOOR,
   RunCritic,
+  RunIntegrityError,
   RunRecordValidationError,
   SEMANTIC_CONCEPT_JUDGE_VERSION,
   SandboxHarness,
@@ -10539,7 +10520,9 @@ export {
   analyzeSeries,
   analyzeTraces,
   argHash,
+  assertLlmRoute,
   assertReleaseConfidence,
+  assertRunCaptured,
   assignFeedbackSplit,
   attributeCounterfactuals,
   deterministicSplit as benchmarkDeterministicSplit,
@@ -10563,7 +10546,7 @@ export {
   callLlm,
   callLlmJson,
   canaryLeakView,
-  canonicalize2 as canonicalize,
+  canonicalize,
   causalAttribution,
   checkBehavioralCanary,
   checkCanaries,
@@ -10606,6 +10589,7 @@ export {
   decideReferenceReplayRunPromotion,
   defaultJudges,
   defaultMultiShotObjectives,
+  defaultProviderRedactor,
   defaultReferenceReplayMatcher,
   defaultTraceInsightPanel,
   deployGateLayer,
@@ -10720,6 +10704,7 @@ export {
   probeLlm,
   promptBisect,
   proposeSynthesisTargets,
+  providerFromBaseUrl,
   pytestTestParser,
   redTeamDataset,
   redTeamReport,
@@ -10742,6 +10727,7 @@ export {
   replayScorerOverCorpus,
   replayTraceThroughJudge,
   requiredSampleSize,
+  researchReport,
   resetLockedAppendersForTesting,
   resumeBuilderSession,
   roundTripRunRecord,
@@ -10799,6 +10785,7 @@ export {
   summaryTable,
   testJudge,
   textInSnapshot,
+  throwIfRunIncomplete,
   toLangfuseEnvelope,
   toNdjson,
   toPrometheusText,
@@ -10810,6 +10797,7 @@ export {
   toolSuccessRubric,
   toolWasteView,
   traceAnalystFunctionGroup,
+  traceAnalystOnRunComplete,
   trialTraceFromMultiShotTrial,
   typoMutator,
   urlContains,