@tangle-network/agent-eval 0.30.0 → 0.31.0

package/dist/index.js

@@ -11,7 +11,7 @@ import {
   failureClusterView,
   iqr,
   welchsTTest
-} from "./chunk-K33INZHH.js";
+} from "./chunk-GVQT44CS.js";
 import {
   exportTrainingData,
   toNdjson
@@ -28,7 +28,7 @@ import {
   pytestTestParser,
   runTestGradedScenario,
   vitestTestParser
-} from "./chunk-QHF6EQKK.js";
+} from "./chunk-YTMXBHFM.js";
 import {
   classifyEuAiRisk,
   euAiActReport,
@@ -54,7 +54,7 @@ import {
   runProposeReview,
   runProposeReviewAsControlLoop,
   scoreFromEvals
-} from "./chunk-PALJO75S.js";
+} from "./chunk-XEL6UP7C.js";
 import {
   allCriticalPassed,
   objectiveEval,
@@ -96,14 +96,14 @@ import {
   summarizePreferenceMemory,
   trialTraceFromMultiShotTrial,
   withAssignedFeedbackSplit
-} from "./chunk-SZSBQUIJ.js";
+} from "./chunk-Y2CPBYKH.js";
 import {
   RunRecordValidationError,
   isRunRecord,
   parseRunRecordSafe,
   roundTripRunRecord,
   validateRunRecord
-} from "./chunk-NLMNWKVM.js";
+} from "./chunk-WSI4K3WB.js";
 import {
   assertReleaseConfidence,
   bootstrapCi,
@@ -111,10 +111,10 @@ import {
   judgeReplayGate,
   releaseTraceEvidenceFromMultiShotTrials,
   renderReleaseReport
-} from "./chunk-DBIGN5MJ.js";
+} from "./chunk-WGXZAQLR.js";
 import {
   runEvalCampaign
-} from "./chunk-RUI6SIHY.js";
+} from "./chunk-75ZREHD7.js";
 import {
   LlmCallError,
   LlmClient,
@@ -124,7 +124,7 @@ import {
   callLlmJson,
   probeLlm,
   stripFencedJson
-} from "./chunk-4S4BM3QQ.js";
+} from "./chunk-M6RZ5LJN.js";
 import {
   evaluateInterimReleaseConfidence,
   pairedEvalueSequence
@@ -141,7 +141,7 @@ import {
   requiredSampleSize,
   researchReport,
   summaryTable
-} from "./chunk-5AKPEK5L.js";
+} from "./chunk-CXJOVDJR.js";
 import {
   calibrateJudge,
   calibrateJudgeContinuous,
@@ -160,24 +160,43 @@ import {
   verbosityBias,
   weightedMean,
   wilcoxonSignedRank
-} from "./chunk-R5UQJNKC.js";
+} from "./chunk-4L3WJXQJ.js";
 import {
   DEFAULT_REDACTION_RULES,
+  DEFAULT_TRACE_ANALYST_BUDGETS,
   FileSystemTraceStore,
   InMemoryTraceStore,
   OTEL_AGENT_EVAL_SCOPE,
+  OtlpFileTraceStore,
   REDACTION_VERSION,
   ReplayCache,
   ReplayCacheMissError,
+  SpanNotFoundError,
+  TRACE_ANALYST_ACTOR_DESCRIPTION,
+  TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION,
+  TRACE_ANALYST_SUBAGENT_DESCRIPTION,
+  TRACE_ANALYST_TRUNCATION_MARKER_PREFIX,
   TraceFileMissingError,
+  TraceNotFoundError,
   analyzeTraces,
   buildTraceAnalystTools,
+  buildTraceInsightContext,
+  buildTraceInsightPrompt,
   createReplayFetch,
+  defaultTraceInsightPanel,
+  describeTraceInsightScope,
+  domainEvidencePattern,
   exportRunAsOtlp,
+  inferDomainKeywords,
   iterateRawCalls,
+  planTraceInsightQuestions,
   redactString,
-  redactValue
-} from "./chunk-UW4NOOZI.js";
+  redactValue,
+  scoreTraceInsightReadiness,
+  tokenizeDomainWords,
+  traceAnalystFunctionGroup,
+  traceAnalystOnRunComplete
+} from "./chunk-HIO4UIS5.js";
 import {
   aggregateLlm,
   argHash,
@@ -201,7 +220,7 @@ import {
   RunIntegrityError,
   assertRunCaptured,
   throwIfRunIncomplete
-} from "./chunk-KTGTIOFD.js";
+} from "./chunk-UBPIXOC4.js";
 import {
   FileSystemRawProviderSink,
   InMemoryRawProviderSink,
@@ -229,7 +248,7 @@ import {
   ReplayError,
   ValidationError,
   VerificationError
-} from "./chunk-NG236HPC.js";
+} from "./chunk-QYJT52YW.js";
 import "./chunk-PZ5AY32C.js";
 
 // src/run-score.ts
@@ -3237,6 +3256,107 @@ function suggestionForManifest(input) {
   return "No action required.";
 }
 
+// src/integrity/backend-integrity.ts
+var BackendIntegrityError = class extends AgentEvalError {
+  constructor(message, report) {
+    super("backend_integrity", message);
+    this.report = report;
+  }
+  report;
+};
+function isStubRecord(rec) {
+  return rec.tokenUsage.input === 0 && rec.tokenUsage.output === 0;
+}
+function isUncostedRecord(rec) {
+  return rec.tokenUsage.output > 0 && rec.costUsd === 0;
+}
+function summarizeBackendIntegrity(records) {
+  const totalRecords = records.length;
+  let stubRecords = 0;
+  let realRecords = 0;
+  let uncostedRecords = 0;
+  let totalInputTokens = 0;
+  let totalOutputTokens = 0;
+  let totalCostUsd = 0;
+  for (const rec of records) {
+    totalInputTokens += rec.tokenUsage.input;
+    totalOutputTokens += rec.tokenUsage.output;
+    totalCostUsd += rec.costUsd;
+    if (isStubRecord(rec)) stubRecords++;
+    else realRecords++;
+    if (isUncostedRecord(rec)) uncostedRecords++;
+  }
+  const verdict = totalRecords === 0 ? "stub" : stubRecords === totalRecords ? "stub" : stubRecords === 0 ? "real" : "mixed";
+  const diagnosis = buildDiagnosis({
+    totalRecords,
+    stubRecords,
+    realRecords,
+    uncostedRecords,
+    totalInputTokens,
+    totalOutputTokens,
+    totalCostUsd,
+    verdict
+  });
+  return {
+    totalRecords,
+    stubRecords,
+    realRecords,
+    uncostedRecords,
+    totalInputTokens,
+    totalOutputTokens,
+    totalCostUsd,
+    verdict,
+    diagnosis
+  };
+}
+function buildDiagnosis(r) {
+  if (r.totalRecords === 0) {
+    return "no records \u2014 eval produced zero runs; backend likely failed before first turn";
+  }
+  if (r.verdict === "stub") {
+    return [
+      `all ${r.totalRecords} records have zero token usage \u2014 the LLM backend was never called.`,
+      "common causes: --backend sandbox without a sandbox bridge running; stub model returning hard-coded strings;",
+      "auth misconfigured so requests were silently dropped before the LLM. Re-run with --backend tcloud and TANGLE_API_KEY set,",
+      "or boot the cli-bridge / sandbox before invoking the eval."
+    ].join(" ");
+  }
+  if (r.verdict === "mixed") {
+    const pct = (r.stubRecords / r.totalRecords * 100).toFixed(0);
+    return [
+      `${r.stubRecords}/${r.totalRecords} records (${pct}%) have zero token usage \u2014 the backend partially failed.`,
+      "common causes: rate-limit cascade (429s after the first N personas);",
+      "transient auth expiry mid-run; provider outage. Treat the affected records as missing data, not agent failures."
+    ].join(" ");
+  }
+  if (r.uncostedRecords > 0) {
+    const pct = (r.uncostedRecords / r.totalRecords * 100).toFixed(0);
+    return [
+      `${r.totalRecords} records with real LLM activity (in=${r.totalInputTokens}, out=${r.totalOutputTokens} tokens).`,
+      `${r.uncostedRecords} (${pct}%) have output tokens but costUsd=0 \u2014 cost ledger is mis-wired (no input-token`,
+      "propagation from the runtime stream into RunRecord)."
+    ].join(" ");
+  }
+  return `${r.totalRecords} records with real LLM activity (in=${r.totalInputTokens}, out=${r.totalOutputTokens} tokens, $${r.totalCostUsd.toFixed(4)}).`;
+}
+function assertRealBackend(records, opts = {}) {
+  const report = summarizeBackendIntegrity(records);
+  const allowMixed = opts.allowMixed ?? true;
+  if (report.verdict === "stub") {
+    throw new BackendIntegrityError(
+      `backend-integrity: ran against a stub or unconfigured backend \u2014 ${report.diagnosis}`,
+      report
+    );
+  }
+  if (!allowMixed && report.verdict === "mixed") {
+    throw new BackendIntegrityError(
+      `backend-integrity: partial backend failure rejected \u2014 ${report.diagnosis}`,
+      report
+    );
+  }
+  return report;
+}
+
 // src/judges.ts
 function createDomainExpertJudge(domain) {
   return async (tc, { scenario, turns }) => {
@@ -10299,6 +10419,7 @@ export {
   AnalystRegistry,
   AxGepaSteeringOptimizer,
   BENCHMARK_SPLIT_SEED,
+  BackendIntegrityError,
   BenchmarkRunner,
   BudgetBreachError,
   BudgetGuard,
@@ -10320,6 +10441,7 @@ export {
   DEFAULT_RED_TEAM_CORPUS,
   DEFAULT_RUN_SCORE_WEIGHTS,
   DEFAULT_SEVERITY_WEIGHTS,
+  DEFAULT_TRACE_ANALYST_BUDGETS,
   DEFAULT_TRACE_ANALYST_KINDS,
   Dataset,
   DockerSandboxDriver,
@@ -10367,6 +10489,7 @@ export {
   NoopResearcher,
   NotFoundError,
   OTEL_AGENT_EVAL_SCOPE,
+  OtlpFileTraceStore,
   PairwiseSteeringOptimizer,
   ProductClient,
   PromptRegistry,
@@ -10383,10 +10506,17 @@ export {
   SEMANTIC_CONCEPT_JUDGE_VERSION,
   SandboxHarness,
   ScenarioRegistry,
+  SpanNotFoundError,
   SubprocessSandboxDriver,
+  TRACE_ANALYST_ACTOR_DESCRIPTION,
+  TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION,
+  TRACE_ANALYST_SUBAGENT_DESCRIPTION,
+  TRACE_ANALYST_TRUNCATION_MARKER_PREFIX,
   TRACE_SCHEMA_VERSION,
   TokenCounter,
   TraceEmitter,
+  TraceFileMissingError,
+  TraceNotFoundError,
   TrialTelemetry,
   UNIVERSAL_FINDERS,
   ValidationError,
@@ -10399,8 +10529,10 @@ export {
   allCriticalPassed,
   analyzeAntiSlop,
   analyzeSeries,
+  analyzeTraces,
   argHash,
   assertLlmRoute,
+  assertRealBackend,
   assertReleaseConfidence,
   assertRunCaptured,
   assignFeedbackSplit,
@@ -10415,6 +10547,9 @@ export {
   bootstrapCi,
   buildReflectionPrompt,
   buildReviewerPrompt,
+  buildTraceAnalystTools,
+  buildTraceInsightContext,
+  buildTraceInsightPrompt,
   buildTraceToolsForGroup,
   buildTrajectory,
   byteLengthRange,
@@ -10479,10 +10614,13 @@ export {
   defaultMultiShotObjectives,
   defaultProviderRedactor,
   defaultReferenceReplayMatcher,
+  defaultTraceInsightPanel,
   deployGateLayer,
+  describeTraceInsightScope,
   diffFindings,
   discoverPersonas,
   distillPlaybook,
+  domainEvidencePattern,
   dominates,
   estimateCost,
   estimateTokens,
@@ -10526,6 +10664,7 @@ export {
   httpGithubClient,
   inMemoryReferenceReplayStore,
   inMemoryReviewStore,
+  inferDomainKeywords,
   integrationAsi,
   integrationGateEvals,
   integrationInvokeFailedPayload,
@@ -10583,6 +10722,7 @@ export {
   partialCredit,
   passOrthogonality,
   pixelDeltaRatio,
+  planTraceInsightQuestions,
   politenessPrefixMutator,
   positionalBias,
   printDriverSummary,
@@ -10651,6 +10791,7 @@ export {
   scoreKnowledgeReadiness,
   scoreRedTeamOutput,
   scoreReferenceReplay,
+  scoreTraceInsightReadiness,
   securityJudge,
   selectHarnessVariant,
   selfPreference,
@@ -10664,6 +10805,7 @@ export {
   stripFencedJson,
   subjectiveEval,
   summarize,
+  summarizeBackendIntegrity,
   summarizeHarnessResults,
   summarizePreferenceMemory,
   summaryTable,
@@ -10672,8 +10814,11 @@ export {
   throwIfRunIncomplete,
   toLangfuseEnvelope,
   toPrometheusText,
+  tokenizeDomainWords,
   toolNamesForRun,
   toolSpans,
+  traceAnalystFunctionGroup,
+  traceAnalystOnRunComplete,
   trialTraceFromMultiShotTrial,
   typoMutator,
   urlContains,