@tangle-network/agent-eval 0.116.0 → 0.117.1

package/dist/index.js

@@ -9,12 +9,12 @@ import {
   checkBehavioralCanary,
   checkCanaries,
   runBehavioralCanaries
-} from "./chunk-J6P6PK2R.js";
+} from "./chunk-FQNLDL4D.js";
 import {
   BENCHMARK_SPLIT_SEED,
   benchmarks_exports,
   deterministicSplit
-} from "./chunk-3LXTCTWL.js";
+} from "./chunk-JSDVRFAP.js";
 import {
   DEFAULT_RULES,
   buildTrajectory,
@@ -23,7 +23,7 @@ import {
   computeToolUseMetrics,
   iqr,
   welchsTTest
-} from "./chunk-NYFUT3B3.js";
+} from "./chunk-ODVOOEWQ.js";
 import {
   analyzeSeries
 } from "./chunk-BOD4O7OF.js";
@@ -40,40 +40,45 @@ import {
 import {
   CODING_HARNESSES,
   HARNESS_NATIVE_MODEL,
-  JudgeParseError,
-  adversarialJudge,
   agentProfileHash,
   agentProfileId,
   agentProfileModelId,
-  codeExecutionJudge,
-  coherenceJudge,
   comparePairedArms,
   completionVerdict,
-  createCustomJudge,
-  createDomainExpertJudge,
   createLlmCorrectnessChecker,
   createTokenRecallChecker,
-  defaultJudges,
   expandProfileAxes,
   extractProducedState,
   harnessAxisOf,
-  llmJudge,
   pairArms,
   parseCorrectnessResponse,
   verifyCompletion
-} from "./chunk-GSW3OBHK.js";
+} from "./chunk-JSJZ4PJ6.js";
 import {
   DEFAULT_MUTATION_PRIMITIVES,
   DEFAULT_RED_TEAM_CORPUS,
   Dataset,
   HoldoutLockedError,
+  JudgeParseError,
+  REFERENCE_EQUIVALENCE_INPUT_LIMITS,
+  REFERENCE_EQUIVALENCE_JUDGE_VERSION,
+  adversarialJudge,
   buildReflectionPrompt,
   campaignMeanComposite,
+  codeExecutionJudge,
+  coherenceJudge,
+  costReceiptFromTCloud,
+  createCustomJudge,
+  createDomainExpertJudge,
+  createReferenceEquivalenceJudge,
   crowdingDistance,
+  defaultJudges,
   dominates,
   gepaProposer,
   hashScenarios,
   heldOutGate,
+  llmJudge,
+  maximumChargeForTCloudRequest,
   paretoFrontier,
   paretoFrontierWithCrowding,
   parseReflectionResponse,
@@ -81,20 +86,12 @@ import {
   redTeamReport,
   runCanaries,
   runImprovementLoop,
+  runReferenceEquivalenceJudge,
   scalarScore,
   scoreRedTeamOutput,
   surfaceContentHash,
   toolNamesForRun
-} from "./chunk-3274WNK7.js";
-import {
-  MODEL_PRICING,
-  MetricsCollector,
-  TokenCounter,
-  estimateCost,
-  estimateTokens,
-  isModelPriced,
-  resolveModelPricing
-} from "./chunk-VI2UW6B6.js";
+} from "./chunk-HQPHZGL6.js";
 import {
   BackendIntegrityError,
   assertRealBackend,
@@ -104,7 +101,7 @@ import {
   fileVerdictCache,
   inMemoryVerdictCache,
   summarizeBackendIntegrity
-} from "./chunk-FAOEFFRT.js";
+} from "./chunk-IDZTTFRR.js";
 import {
   DEFAULT_COMPLEXITY_WEIGHTS,
   FindingsStore,
@@ -114,24 +111,23 @@ import {
   SKILL_USAGE_ANALYST,
   SkillUsageAnalyst,
   createAnalystAi,
-  createChatClient,
   createSemanticConceptJudge,
   defaultIsMaterial,
   diffFindings,
   runSemanticConceptJudge
-} from "./chunk-NBSS5NDZ.js";
+} from "./chunk-CCZIVI3F.js";
 import {
   buildDefaultAnalystRegistry,
-  computeTraceMetrics
-} from "./chunk-7GKEAIAD.js";
+  computeTraceMetrics,
+  createChatClient
+} from "./chunk-VF3XSYTI.js";
+import "./chunk-HHWE3POT.js";
 import {
-  DEFAULT_RUN_SCORE_WEIGHTS,
-  Mutex,
-  aggregateRunScore,
-  clamp01
-} from "./chunk-MPHTT5HE.js";
+  Mutex
+} from "./chunk-3YYRZDON.js";
 import {
   AnalystRegistry,
+  DEFAULT_RUN_SCORE_WEIGHTS,
   DEFAULT_TRACE_ANALYST_KINDS,
   FAILURE_MODE_KIND_SPEC,
   IMPROVEMENT_KIND_SPEC,
@@ -142,7 +138,9 @@ import {
   POLICY_EDIT_TARGET_SURFACES,
   PolicyEditValidationError,
   admitPolicyEdit,
+  aggregateRunScore,
   applyPolicyEditToSurface,
+  clamp01,
   computeFindingId,
   computePolicyEditId,
   createTraceAnalystKind,
@@ -156,7 +154,7 @@ import {
   scorePolicyEditReadiness,
   validatePolicyEdit,
   validatePolicyEditCandidateRecord
-} from "./chunk-CIUOICJT.js";
+} from "./chunk-MGEHEHSN.js";
 import {
   allCriticalPassed,
   controlFailureClassFromVerification,
@@ -187,7 +185,7 @@ import {
 } from "./chunk-MOXWMGPC.js";
 import {
   runEvalCampaign
-} from "./chunk-ONM6PEAE.js";
+} from "./chunk-GQCZRZ7L.js";
 import "./chunk-ARU2PZFM.js";
 import {
   evaluateInterimReleaseConfidence,
@@ -270,13 +268,14 @@ import {
   scoreTraceInsightReadiness,
   tokenizeDomainWords,
   traceAnalystOnRunComplete
-} from "./chunk-TLDB7WRY.js";
+} from "./chunk-YZPO4UHR.js";
 import {
   FAILURE_CLASSES,
   TRACE_SCHEMA_VERSION,
   aggregateLlm,
   argHash,
   groupBy,
+  hasCapturedToolArgs,
   isJudgeSpan,
   isLlmSpan,
   isRetrievalSpan,
@@ -287,13 +286,13 @@ import {
   runFailureClass,
   runsForScenario,
   toolSpans
-} from "./chunk-MHNQWM4I.js";
+} from "./chunk-LQUTGLOZ.js";
 import {
   TRACE_ANALYST_ACTOR_DESCRIPTION,
   TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION,
   TRACE_ANALYST_SUBAGENT_DESCRIPTION,
   analyzeTraces
-} from "./chunk-RPDDVKI7.js";
+} from "./chunk-4JLWXDYA.js";
 import {
   DEFAULT_REDACTION_RULES,
   REDACTION_VERSION,
@@ -302,6 +301,7 @@ import {
 } from "./chunk-GGE4NNQT.js";
 import {
   DEFAULT_TRACE_ANALYST_BUDGETS,
+  INPUT_VALUE,
   LLM_CACHED_TOKENS,
   LLM_CACHED_TOKEN_ATTR_KEYS,
   LLM_COST_ATTR_KEYS,
@@ -313,14 +313,18 @@ import {
   LLM_OUTPUT_TOKENS,
   LLM_OUTPUT_TOKEN_ATTR_KEYS,
   OPENINFERENCE_SPAN_KIND,
+  OUTPUT_VALUE,
   OtlpFileTraceStore,
   SPAN_KIND_ATTR_KEYS,
   SpanNotFoundError,
+  TOOL_ARGS_CAPTURED,
+  TOOL_LATENCY_MS,
   TOOL_NAME,
   TOOL_NAME_ATTR_KEYS,
   TRACE_ANALYST_TRUNCATION_MARKER_PREFIX,
   TraceFileMissingError,
   TraceNotFoundError,
+  applyToolSpanOtlpAttributes,
   asNumber,
   asString,
   buildTraceAnalystTools,
@@ -333,7 +337,7 @@ import {
   stringField,
   traceAnalystFunctionGroup,
   traceSpanKindToOpenInferenceKind
-} from "./chunk-LNQEP766.js";
+} from "./chunk-S2F4J57L.js";
 import {
   RunIntegrityError,
   assertRunCaptured,
@@ -376,15 +380,39 @@ import {
 import {
   LlmCallError,
   LlmClient,
+  LlmResponseError,
   LlmRouteAssertionError,
   assertLlmRoute,
   backoffMs,
   callLlm,
   callLlmJson,
+  costReceiptFromLlm,
+  costReceiptFromLlmError,
   isTransientLlmError,
+  maximumChargeForLlmRequest,
   probeLlm,
   stripFencedJson
-} from "./chunk-GY4SYVPJ.js";
+} from "./chunk-NJC7U437.js";
+import {
+  CostAccountingIncompleteError,
+  CostCallConflictError,
+  CostCeilingReachedError,
+  CostLedger,
+  CostLedgerPersistenceError,
+  CostReceiptCaptureError,
+  CostReservationExceededError,
+  costForUsage,
+  modelPriceKey
+} from "./chunk-VCTY3W6J.js";
+import {
+  MODEL_PRICING,
+  MetricsCollector,
+  TokenCounter,
+  estimateCost,
+  estimateTokens,
+  isModelPriced,
+  resolveModelPricing
+} from "./chunk-VI2UW6B6.js";
 import {
   FileSystemRawProviderSink,
   InMemoryRawProviderSink,
@@ -710,6 +738,12 @@ function errMessage(err) {
 async function executeScenario(tc, scenario, config) {
   const startTime = Date.now();
   const model = config.model ?? "gpt-4o";
+  const costLedger = config.costLedger ?? new CostLedger();
+  const costTags = {
+    ...config.costTags,
+    scenarioId: scenario.id,
+    executionId: globalThis.crypto.randomUUID()
+  };
   const systemPrompt = [config.systemPrompt, scenario.systemPromptAppend ?? ""].filter(Boolean).join("\n\n");
   const messages = [{ role: "system", content: systemPrompt }];
   const turns = [];
@@ -721,12 +755,25 @@ async function executeScenario(tc, scenario, config) {
     const turn = scenario.turns[i];
     const turnStart = Date.now();
     messages.push({ role: "user", content: turn.user });
-    const resp = await tc.chat({
+    const request = {
       model,
       messages,
       temperature: 0.4,
       maxTokens: 3e3
+    };
+    const paid = await costLedger.runPaidCall({
+      channel: "agent",
+      phase: config.costPhase ?? "benchmark.agent",
+      actor: "scenario-agent",
+      model,
+      maximumCharge: maximumChargeForTCloudRequest(request, config.tcloudMaximumAttempts),
+      tags: costTags,
+      signal: config.signal,
+      execute: () => tc.chat(request),
+      receipt: (response) => costReceiptFromTCloud(response, model)
     });
+    if (!paid.succeeded) throw paid.error;
+    const resp = paid.value;
     const message = resp.choices?.[0]?.message;
     const rawContent = message?.content;
     if (message === void 0 || message === null || typeof rawContent !== "string") {
@@ -812,7 +859,16 @@ async function executeScenario(tc, scenario, config) {
         };
     }
   });
-  const judgeInput = { scenario, turns, artifacts };
+  const judgeInput = {
+    scenario,
+    turns,
+    artifacts,
+    costLedger,
+    costPhase: config.costPhase ?? "benchmark.judge",
+    costTags,
+    signal: config.signal,
+    tcloudMaximumAttempts: config.tcloudMaximumAttempts
+  };
   const judgeResults = [];
   let failedJudges = 0;
   const judgeFailures = [];
@@ -881,7 +937,8 @@ async function executeScenario(tc, scenario, config) {
     judgeErrors: errorScores.length + failedJudges,
     overallScore,
     totalDurationMs: Date.now() - startTime,
-    artifacts
+    artifacts,
+    cost: costLedger.summary({ tags: costTags })
   };
   if (judgeFailures.length > 0) result.judgeFailures = judgeFailures;
   return result;
@@ -898,6 +955,8 @@ var BenchmarkRunner = class {
   async run(scenarios) {
     const toRun = scenarios ?? this.config.scenarios;
     const passThreshold = this.config.passThreshold ?? 6;
+    const costLedger = this.config.costLedger ?? new CostLedger();
+    const costTags = { benchmarkRunId: globalThis.crypto.randomUUID() };
     console.log("=".repeat(70));
     console.log(" AGENT EVAL \u2014 BENCHMARK");
     console.log(" Multi-turn scenarios x Multi-judge panel");
@@ -915,7 +974,10 @@ var BenchmarkRunner = class {
       const result = await executeScenario(this.tc, scenario, {
         systemPrompt: this.config.systemPrompt,
         model: this.config.model,
-        judges: this.config.judges
+        judges: this.config.judges,
+        costLedger,
+        costTags,
+        tcloudMaximumAttempts: this.config.tcloudMaximumAttempts
       });
       results.push(result);
       for (const turn of result.turns) {
@@ -1015,6 +1077,7 @@ var BenchmarkRunner = class {
       promptVersion: this.config.promptVersion ?? "v1",
       scenarioCount: toRun.length,
       results,
+      cost: costLedger.summary({ tags: costTags }),
       summary: { overallAvg, byPersona, byDimension, weakest, strongest }
     };
   }
@@ -1621,11 +1684,15 @@ var AgentDriver = class {
   client;
   driverModel;
   productContext;
+  costLedger;
+  tcloudMaximumAttempts;
   constructor(tc, config) {
     this.tc = tc;
     this.client = config.client;
     this.driverModel = config.driverModel ?? "claude-sonnet-4-6";
     this.productContext = config.productContext ?? "";
+    this.costLedger = config.costLedger ?? new CostLedger();
+    this.tcloudMaximumAttempts = config.tcloudMaximumAttempts;
   }
   /**
    * Run a persona through the product.
@@ -1634,6 +1701,7 @@ var AgentDriver = class {
    * quality curve, and convergence curve.
    */
   async run(persona) {
+    const costTags = { driverRunId: globalThis.crypto.randomUUID() };
     const email = `eval-driver-${Date.now()}@test.agent-eval.local`;
     await this.client.signup(`Driver ${persona.role}`, email, "eval-driver-pass");
     await this.client.login(email, "eval-driver-pass");
@@ -1648,7 +1716,12 @@ var AgentDriver = class {
     let criteriaMetAtTurn = null;
     for (let turn = 1; turn <= persona.maxTurns; turn++) {
       const state = await metrics.getState();
-      const userMessage = await this.decideNextMessage(persona, state, conversationHistory);
+      const userMessage = await this.decideNextMessage(
+        persona,
+        state,
+        conversationHistory,
+        costTags
+      );
       if (userMessage === "DONE") {
         completed = true;
         turnsToCompletion = turn - 1;
@@ -1696,18 +1769,21 @@ var AgentDriver = class {
       metrics: turnMetrics,
       finalState,
       convergenceCurve: convergence.getCurve(),
-      totalCostUsd: 0,
+      totalCostUsd: this.costLedger.summary({ tags: costTags }).totalCostUsd,
       finalQualityScore: null
     };
   }
   /** Use the driver LLM to decide what the "user" says next */
-  async decideNextMessage(persona, state, history) {
+  async decideNextMessage(persona, state, history, costTags) {
     return decideNextUserTurn(this.tc, {
       persona,
       state,
       history,
       productContext: this.productContext,
-      model: this.driverModel
+      model: this.driverModel,
+      costLedger: this.costLedger,
+      costTags,
+      tcloudMaximumAttempts: this.tcloudMaximumAttempts
     });
   }
   /** Handle pending approvals based on persona feedback patterns */
@@ -1816,7 +1892,7 @@ async function decideNextUserTurn(tc, opts) {
   const { persona, state, history, productContext = "", model = "claude-sonnet-4-6" } = opts;
   const lastResponse = history.length > 0 ? history[history.length - 1].content.slice(0, 2e3) : "(no conversation yet \u2014 this is the first message)";
   const recentHistory = history.slice(-6).map((h) => `${h.role}: ${h.content.slice(0, 500)}`).join("\n\n");
-  const resp = await tc.chat({
+  const request = {
     model,
     messages: [
       { role: "system", content: buildDriverSystemPrompt(persona, state, productContext) },
@@ -1831,7 +1907,19 @@ ${lastResponse}` : "No conversation yet. Send your opening message \u2014 in cha
     ],
     temperature: 0.5,
     maxTokens: 700
+  };
+  const paid = await (opts.costLedger ?? new CostLedger()).runPaidCall({
+    channel: "driver",
+    phase: "driver-turn",
+    actor: "decideNextUserTurn",
+    model,
+    tags: opts.costTags,
+    maximumCharge: maximumChargeForTCloudRequest(request, opts.tcloudMaximumAttempts),
+    execute: () => tc.chat(request),
+    receipt: (response) => costReceiptFromTCloud(response, model)
   });
+  if (!paid.succeeded) throw paid.error;
+  const resp = paid.value;
   const content = resp.choices?.[0]?.message?.content ?? "";
   return content.trim();
 }
@@ -3794,7 +3882,6 @@ function aggregateJudgeVerdicts(verdicts, dimensionKeys, weights) {
   const dimAcc = {};
   for (const d of dimensionKeys) dimAcc[d] = [];
   let rationale = "";
-  let costUsd = 0;
   const seenCount = /* @__PURE__ */ new Map();
   const keyFor = (model) => {
     const n = (seenCount.get(model) ?? 0) + 1;
@@ -3802,7 +3889,6 @@ function aggregateJudgeVerdicts(verdicts, dimensionKeys, weights) {
     return n === 1 ? model : `${model}#${n}`;
   };
   for (const v of verdicts) {
-    costUsd += v.costUsd ?? 0;
     const key = keyFor(v.model);
     if (!v.perDimension) {
       failedJudges.push(key);
@@ -3844,7 +3930,6 @@ function aggregateJudgeVerdicts(verdicts, dimensionKeys, weights) {
     perJudge,
     maxDisagreement,
     failedJudges,
-    costUsd,
     rationale: rationale || "llm-judge",
     verdicts: [...verdicts]
   };
@@ -3924,37 +4009,92 @@ function ensembleJudge(opts) {
   if (opts.crossFamily !== false) {
     assertCrossFamily(opts.models);
   }
-  const scoreOne = async (model, input) => {
-    if (opts.retry) {
-      const outcome = await withJudgeRetry((m) => opts.scoreWith(m, input), {
-        ...opts.retry,
-        models: [model]
-      });
-      if (!outcome.succeeded || outcome.value === null) {
-        return {
+  const declaredJudgeVersion = opts.judgeVersion?.trim();
+  if (opts.judgeVersion !== void 0 && !declaredJudgeVersion) {
+    throw new Error(`ensembleJudge '${opts.name}': judgeVersion must be non-empty when provided`);
+  }
+  const judgeVersion = declaredJudgeVersion ?? contentHash({
+    kind: "ensembleJudge",
+    models: opts.models,
+    dimensions: opts.dimensions,
+    weights: opts.weights ?? null,
+    crossFamily: opts.crossFamily ?? true,
+    maximumCharge: typeof opts.maximumCharge === "function" ? opts.maximumCharge.toString() : opts.maximumCharge ?? null,
+    retry: opts.retry ? {
+      maxAttempts: opts.retry.maxAttempts ?? null,
+      timeoutMs: opts.retry.timeoutMs ?? null,
+      models: opts.retry.models ?? null,
+      backoffMs: opts.retry.backoffMs?.toString() ?? null,
+      isRetryable: opts.retry.isRetryable?.toString() ?? null
+    } : null,
+    scoreWith: opts.scoreWith.toString()
+  });
+  const directCostLedger = opts.costLedger ?? new CostLedger();
+  const scoreOne = async (args) => {
+    const outcome = await withJudgeRetry(
+      async (model, retrySignal) => {
+        const paid = await args.costLedger.runPaidCall({
+          channel: "judge",
+          phase: args.costPhase,
+          actor: `${opts.name}.${model}`,
           model,
-          perDimension: null,
-          rationale: outcome.error?.message ?? "judge failed after retries"
-        };
-      }
-      return outcome.value;
-    }
-    try {
-      return await opts.scoreWith(model, input);
-    } catch (err) {
+          maximumCharge: typeof opts.maximumCharge === "function" ? opts.maximumCharge(model) : opts.maximumCharge,
+          tags: args.costTags,
+          signal: AbortSignal.any([args.signal, retrySignal]),
+          execute: (signal) => opts.scoreWith(model, { artifact: args.artifact, scenario: args.scenario, signal }),
+          receipt: (verdict) => {
+            const cachedTokens = verdict.usage?.cachedPromptTokens ?? 0;
+            const usageUnknown = !verdict.usage || verdict.usage.captured === false;
+            return {
+              model: verdict.model,
+              inputTokens: Math.max(0, (verdict.usage?.promptTokens ?? 0) - cachedTokens),
+              outputTokens: verdict.usage?.completionTokens ?? 0,
+              cachedTokens: cachedTokens > 0 ? cachedTokens : void 0,
+              usageUnknown,
+              ...verdict.costUsd === void 0 ? {} : { actualCostUsd: verdict.costUsd }
+            };
+          },
+          receiptFromError: (error) => opts.receiptFromError?.(error, model)
+        });
+        if (!paid.succeeded) throw paid.error;
+        return paid.value;
+      },
+      opts.retry ? { ...opts.retry, models: [args.model] } : { maxAttempts: 1, models: [args.model], isRetryable: () => false }
+    );
+    if (!outcome.succeeded || outcome.value === null) {
       return {
-        model,
+        model: args.model,
         perDimension: null,
-        rationale: err instanceof Error ? err.message : String(err)
+        rationale: outcome.error?.message ?? "judge failed"
       };
     }
+    return outcome.value;
   };
   return {
     name: opts.name,
     dimensions: opts.dimensions.map((d) => ({ key: d, description: d })),
-    async score({ artifact, scenario }) {
-      const input = { artifact, scenario };
-      const verdicts = await Promise.all(opts.models.map((model) => scoreOne(model, input)));
+    judgeVersion,
+    async score({
+      artifact,
+      scenario,
+      signal,
+      costLedger,
+      costPhase,
+      costTags
+    }) {
+      const verdicts = await Promise.all(
+        opts.models.map(
+          (model) => scoreOne({
+            model,
+            artifact,
+            scenario,
+            signal,
+            costLedger: costLedger ?? directCostLedger,
+            costPhase: costPhase ?? "judge",
+            costTags
+          })
+        )
+      );
       const agg = aggregateJudgeVerdicts(verdicts, opts.dimensions, opts.weights);
       const score = {
         dimensions: agg.perDimension,
@@ -4570,118 +4710,13 @@ function clampUnit(value) {
   return Math.max(0, Math.min(1, value));
 }
 
-// src/cost-ledger.ts
-function modelPriceKey(model) {
-  return isModelPriced(model) ? model : null;
-}
-function costForUsage(model, usage) {
-  assertNonNegative(usage.inputTokens, "inputTokens");
-  assertNonNegative(usage.outputTokens, "outputTokens");
-  if (usage.cachedTokens !== void 0) assertNonNegative(usage.cachedTokens, "cachedTokens");
-  const pricing = resolveModelPricing(model);
-  if (!pricing) return { costUsd: 0, costUnknown: true };
-  const billedInput = usage.inputTokens + (usage.cachedTokens ?? 0);
-  return { costUsd: estimateCost(billedInput, usage.outputTokens, model), costUnknown: false };
-}
-var CostLedger = class {
-  entries = [];
-  completedTasks = 0;
-  /**
-   * Record one LLM call. The cost is computed from pricing unless
-   * `actualCostUsd` is supplied (a finite observed cost from the provider
-   * response), in which case `costUnknown` is false regardless of pricing.
-   */
-  record(input) {
-    const { costUsd, costUnknown } = costForUsage(input.model, input.usage);
-    const hasActual = typeof input.actualCostUsd === "number" && Number.isFinite(input.actualCostUsd);
-    if (hasActual) assertNonNegative(input.actualCostUsd, "actualCostUsd");
-    const entry = {
-      model: input.model,
-      channel: input.channel,
-      inputTokens: input.usage.inputTokens,
-      outputTokens: input.usage.outputTokens,
-      cachedTokens: input.usage.cachedTokens,
-      costUsd: hasActual ? input.actualCostUsd : costUsd,
-      costUnknown: hasActual ? false : costUnknown,
-      actualCostUsd: hasActual ? input.actualCostUsd : void 0,
-      tags: input.tags,
-      timestamp: input.timestamp ?? Date.now()
-    };
-    this.entries.push(entry);
-    return entry;
-  }
-  /** Increment the completed-task counter (used for cost-per-completed-task). */
-  markCompleted(count = 1) {
-    if (!Number.isInteger(count) || count < 0) {
-      throw new ValidationError(
-        `CostLedger.markCompleted: count must be a non-negative integer, got ${count}`
-      );
-    }
-    this.completedTasks += count;
-  }
-  list() {
-    return [...this.entries];
-  }
-  summary() {
-    const byChannel = /* @__PURE__ */ new Map();
-    const unpriced = /* @__PURE__ */ new Set();
-    let totalCost = 0;
-    let inputTokens = 0;
-    let outputTokens = 0;
-    let cachedTokens = 0;
-    for (const e of this.entries) {
-      totalCost += e.costUsd;
-      inputTokens += e.inputTokens;
-      outputTokens += e.outputTokens;
-      cachedTokens += e.cachedTokens ?? 0;
-      if (e.costUnknown) unpriced.add(e.model);
-      const roll = byChannel.get(e.channel) ?? {
-        channel: e.channel,
-        calls: 0,
-        inputTokens: 0,
-        outputTokens: 0,
-        cachedTokens: 0,
-        costUsd: 0,
-        unpricedCalls: 0
-      };
-      roll.calls += 1;
-      roll.inputTokens += e.inputTokens;
-      roll.outputTokens += e.outputTokens;
-      roll.cachedTokens += e.cachedTokens ?? 0;
-      roll.costUsd += e.costUsd;
-      if (e.costUnknown) roll.unpricedCalls += 1;
-      byChannel.set(e.channel, roll);
-    }
-    return {
-      totalCalls: this.entries.length,
-      inputTokens,
-      outputTokens,
-      cachedTokens,
-      totalCostUsd: totalCost,
-      byChannel: [...byChannel.values()].sort((a, b) => a.channel.localeCompare(b.channel)),
-      unpricedModels: [...unpriced].sort(),
-      fullyPriced: unpriced.size === 0
-    };
-  }
-  /** Total spend divided by completed tasks; null when nothing completed. */
-  costPerCompletedTask() {
-    if (this.completedTasks === 0) return null;
-    return this.summary().totalCostUsd / this.completedTasks;
-  }
-};
-function assertNonNegative(n, name) {
-  if (!Number.isFinite(n) || n < 0) {
-    throw new ValidationError(`CostLedger: ${name} must be a non-negative finite number, got ${n}`);
-  }
-}
-
 // src/cost-tracker.ts
 var CostTracker = class {
   byScenario = /* @__PURE__ */ new Map();
   record(entry) {
     const full = { timestamp: entry.timestamp ?? Date.now(), ...entry };
-    assertNonNegative2(full.inputTokens, "inputTokens");
-    assertNonNegative2(full.outputTokens, "outputTokens");
+    assertNonNegative(full.inputTokens, "inputTokens");
+    assertNonNegative(full.outputTokens, "outputTokens");
     let bucket = this.byScenario.get(full.scenarioId);
     if (!bucket) {
       bucket = {
@@ -4760,7 +4795,7 @@ function costFor(entry) {
   }
   return estimateCost(entry.inputTokens, entry.outputTokens, entry.model);
 }
-function assertNonNegative2(n, name) {
+function assertNonNegative(n, name) {
   if (!Number.isFinite(n) || n < 0) {
     throw new Error(`CostTracker: ${name} must be a non-negative finite number, got ${n}`);
   }
@@ -7986,6 +8021,7 @@ function flowLayer(input) {
 var INTENT_MATCH_JUDGE_VERSION = "intent-match-judge-v1-2026-04-24";
 var DEFAULT_MODEL = "claude-sonnet-4-6";
 var DEFAULT_TIMEOUT = 3e5;
+var DEFAULT_MAX_TOKENS = 800;
 var DEFAULT_MAX_SOURCE = 25e3;
 var DEFAULT_MAX_PER_FILE = 12e3;
 var DEFAULT_MAX_HTML = 2e4;
@@ -8055,10 +8091,14 @@ async function runIntentMatchJudge(input, options = {}) {
   const opts = {
     model: options.model ?? DEFAULT_MODEL,
     timeoutMs: options.timeoutMs ?? DEFAULT_TIMEOUT,
+    maxTokens: options.maxTokens ?? DEFAULT_MAX_TOKENS,
     maxSourceChars: options.maxSourceChars ?? DEFAULT_MAX_SOURCE,
     maxPerFileChars: options.maxPerFileChars ?? DEFAULT_MAX_PER_FILE,
     maxHtmlChars: options.maxHtmlChars ?? DEFAULT_MAX_HTML,
-    llm: options.llm ?? {}
+    llm: options.llm ?? {},
+    costLedger: options.costLedger ?? new CostLedger(),
+    costPhase: options.costPhase ?? "judge.intent-match",
+    signal: options.signal ?? new AbortController().signal
   };
   if (input.sourceFiles.length === 0 && !input.servedHtml) {
     return {
@@ -8072,23 +8112,40 @@ async function runIntentMatchJudge(input, options = {}) {
       error: "no input artifact"
     };
   }
+  let receipt;
   try {
-    const { value, result } = await callLlmJson(
-      {
-        model: opts.model,
-        messages: [
-          {
-            role: "system",
-            content: "You are a holistic code reviewer answering one question: did the agent build the right app for the user. Return strict JSON. No prose outside."
-          },
-          { role: "user", content: buildPrompt(input, opts) }
-        ],
-        jsonSchema: { name: "intent_match_judge", schema: INTENT_SCHEMA },
-        temperature: 0,
-        timeoutMs: opts.timeoutMs
-      },
-      opts.llm
-    );
+    const request = {
+      model: opts.model,
+      messages: [
+        {
+          role: "system",
+          content: "You are a holistic code reviewer answering one question: did the agent build the right app for the user. Return strict JSON. No prose outside."
+        },
+        { role: "user", content: buildPrompt(input, opts) }
+      ],
+      jsonSchema: { name: "intent_match_judge", schema: INTENT_SCHEMA },
+      temperature: 0,
+      maxTokens: opts.maxTokens,
+      timeoutMs: opts.timeoutMs
+    };
+    const paid = await opts.costLedger.runPaidCall({
+      channel: "judge",
+      phase: opts.costPhase,
+      actor: "intent-match",
+      model: opts.model,
+      maximumCharge: maximumChargeForLlmRequest(request, opts.llm),
+      signal: opts.signal,
+      execute: (signal, callId) => callLlmJson(request, {
+        ...opts.llm,
+        signal,
+        idempotencyKey: callId
+      }),
+      receipt: ({ result }) => costReceiptFromLlm(result),
+      receiptFromError: costReceiptFromLlmError
+    });
+    receipt = paid.receipt;
+    if (!paid.succeeded) throw paid.error;
+    const { value } = paid.value;
     const score = Math.max(0, Math.min(1, Number(value?.score ?? 0)));
     return {
       kind: "intent-match",
@@ -8096,7 +8153,7 @@ async function runIntentMatchJudge(input, options = {}) {
       score: Number(score.toFixed(3)),
       evidence: String(value?.evidence ?? "").slice(0, 400),
       durationMs: Date.now() - start,
-      costUsd: result.costUsd ?? null,
+      costUsd: paid.receipt.costUnknown ? null : paid.receipt.costUsd,
       available: true
     };
   } catch (err) {
@@ -8106,7 +8163,7 @@ async function runIntentMatchJudge(input, options = {}) {
       score: 0,
       evidence: "",
       durationMs: Date.now() - start,
-      costUsd: null,
+      costUsd: receipt && !receipt.costUnknown ? receipt.costUsd : null,
       available: false,
       error: err instanceof Error ? err.message : String(err)
     };
@@ -9146,23 +9203,42 @@ function createDefaultReviewer(options) {
   };
   const promptBuilder = options.promptBuilder ?? buildReviewerPrompt;
   const timeoutMs = options.timeoutMs ?? 3e5;
+  const maxTokens = options.maxTokens ?? 4e3;
+  const costLedger = options.costLedger ?? new CostLedger();
   return async (input) => {
     const start = Date.now();
     const { system, user } = promptBuilder(input);
+    let receipt;
     try {
-      const { value, result } = await callLlmJson(
-        {
-          model: options.model,
-          messages: [
-            { role: "system", content: system },
-            { role: "user", content: user }
-          ],
-          jsonSchema: { name: "reviewer_output", schema: REVIEWER_SCHEMA },
-          temperature: 0,
-          timeoutMs
-        },
-        options.llm ?? {}
-      );
+      const request = {
+        model: options.model,
+        messages: [
+          { role: "system", content: system },
+          { role: "user", content: user }
+        ],
+        jsonSchema: { name: "reviewer_output", schema: REVIEWER_SCHEMA },
+        temperature: 0,
+        maxTokens,
+        timeoutMs
+      };
+      const paid = await costLedger.runPaidCall({
+        channel: "analyst",
+        phase: options.costPhase ?? "review",
+        actor: "default-reviewer",
+        model: options.model,
+        maximumCharge: maximumChargeForLlmRequest(request, options.llm),
+        signal: options.signal,
+        execute: (signal, callId) => callLlmJson(request, {
+          ...options.llm,
+          signal,
+          idempotencyKey: callId
+        }),
+        receipt: ({ result }) => costReceiptFromLlm(result),
+        receiptFromError: costReceiptFromLlmError
+      });
+      receipt = paid.receipt;
+      if (!paid.succeeded) throw paid.error;
+      const { value } = paid.value;
       return {
         shot: input.shot,
         observations: String(value.observations ?? softFail2.observations),
@@ -9170,7 +9246,7 @@ function createDefaultReviewer(options) {
         nextShotInstruction: String(value.nextShotInstruction ?? softFail2.nextShotInstruction),
         shouldContinue: Boolean(value.shouldContinue),
         confidence: Math.max(0, Math.min(1, Number(value.confidence ?? softFail2.confidence))),
-        costUsd: result.costUsd ?? null,
+        costUsd: paid.receipt.costUnknown ? null : paid.receipt.costUsd,
         durationMs: Date.now() - start,
         available: true
       };
@@ -9182,7 +9258,7 @@ function createDefaultReviewer(options) {
         nextShotInstruction: softFail2.nextShotInstruction,
         shouldContinue: softFail2.shouldContinue,
         confidence: softFail2.confidence,
-        costUsd: null,
+        costUsd: receipt && !receipt.costUnknown ? receipt.costUsd : null,
         durationMs: Date.now() - start,
         available: false,
         error: err instanceof Error ? err.message : String(err)
@@ -10262,18 +10338,23 @@ async function runDistillation(opts) {
         input: scenario.input,
         scenarioId: scenario.id
       });
-      const response = await chat.chat(
-        {
-          model: opts.studentModel,
-          messages: prompt,
-          jsonMode: true,
-          temperature: studentTemperature,
-          maxTokens: studentMaxTokens
-        },
-        { signal: ctx.signal }
-      );
-      reportUsage(ctx.cost, response);
-      return parse(response.content, scenario.id);
+      const request = {
+        model: opts.studentModel,
+        messages: prompt,
+        jsonMode: true,
+        temperature: studentTemperature,
+        maxTokens: studentMaxTokens
+      };
+      const paid = await ctx.cost.runPaidCall({
+        actor: "distillation-student",
+        model: opts.studentModel,
+        maximumCharge: chat.maximumAttempts === void 0 ? void 0 : maximumChargeForLlmRequest(request, { maxRetries: chat.maximumAttempts }),
+        execute: (signal, callId) => chat.chat(request, { signal, idempotencyKey: callId }),
+        receipt: costReceiptFromLlm,
+        receiptFromError: costReceiptFromLlmError
+      });
+      if (!paid.succeeded) throw paid.error;
+      return parse(paid.value.content, scenario.id);
     }
   });
   const winnerPrompt = typeof loop.winnerSurface === "string" ? loop.winnerSurface : opts.baselinePrompt;
@@ -10285,14 +10366,6 @@ async function runDistillation(opts) {
     holdoutAgreement: { baseline, winner, delta: winner - baseline }
   };
 }
-function reportUsage(cost, response) {
-  if (typeof response.costUsd === "number") cost.observe(response.costUsd, "distillation-student");
-  cost.observeTokens({
-    input: response.usage.promptTokens,
-    output: response.usage.completionTokens,
-    cached: response.usage.cachedPromptTokens
-  });
-}
 var DEFAULT_MUTATION_PRIMITIVES2 = [
   "Add an explicit output-schema instruction so the model emits exactly the gold label fields as JSON.",
   "Add a one-line decision rule for each verdict field the student keeps getting wrong.",
@@ -11455,7 +11528,13 @@ export {
   CaptureIntegrityError,
   ConfigError,
   ConvergenceTracker,
+  CostAccountingIncompleteError,
+  CostCallConflictError,
+  CostCeilingReachedError,
   CostLedger,
+  CostLedgerPersistenceError,
+  CostReceiptCaptureError,
+  CostReservationExceededError,
   CostTracker,
   CrossFamilyError,
   DEFAULT_AGENT_SLOS,
@@ -11490,6 +11569,7 @@ export {
   HoldoutAuditor,
   HoldoutLockedError,
   IMPROVEMENT_KIND_SPEC,
+  INPUT_VALUE,
   INTENT_MATCH_JUDGE_VERSION,
   InMemoryFeedbackTrajectoryStore,
   InMemoryRawProviderSink,
@@ -11512,6 +11592,7 @@ export {
   LLM_OUTPUT_TOKEN_ATTR_KEYS,
   LlmCallError,
   LlmClient,
+  LlmResponseError,
   LlmRouteAssertionError,
   LockedJsonlAppender,
   MODEL_PRICING,
@@ -11524,6 +11605,7 @@ export {
   NotFoundError,
   OPENINFERENCE_SPAN_KIND,
   OTEL_AGENT_EVAL_SCOPE,
+  OUTPUT_VALUE,
   OtlpFileTraceStore,
   POLICY_EDIT_AXES,
   POLICY_EDIT_CANDIDATE_RECORD_SCHEMA,
@@ -11533,6 +11615,8 @@ export {
   ProductClient,
   PromptRegistry,
   REDACTION_VERSION,
+  REFERENCE_EQUIVALENCE_INPUT_LIMITS,
+  REFERENCE_EQUIVALENCE_JUDGE_VERSION,
   RESEARCH_REPORT_HARD_PAIR_FLOOR,
   ReplayCache,
   ReplayCacheMissError,
@@ -11550,6 +11634,8 @@ export {
   SkillUsageAnalyst,
   SpanNotFoundError,
   SubprocessSandboxDriver,
+  TOOL_ARGS_CAPTURED,
+  TOOL_LATENCY_MS,
   TOOL_NAME,
   TOOL_NAME_ATTR_KEYS,
   TRACE_ANALYST_ACTOR_DESCRIPTION,
@@ -11586,6 +11672,7 @@ export {
   analyzeTraces,
   appendScorecard,
   applyPolicyEditToSurface,
+  applyToolSpanOtlpAttributes,
   argHash,
   asNumber,
   asString,
@@ -11678,6 +11765,8 @@ export {
   corpusInterRaterAgreement,
   corpusInterRaterAgreementFromJudgeScores,
   costForUsage,
+  costReceiptFromLlm,
+  costReceiptFromLlmError,
   costReport,
   createAnalystAi,
   createAntiSlopJudge,
@@ -11691,6 +11780,7 @@ export {
   createLlmReviewer,
   createOtelExporter,
   createOtelTracingStore,
+  createReferenceEquivalenceJudge,
   createReplayFetch,
   createSandboxPool,
   createSemanticConceptJudge,
@@ -11781,6 +11871,7 @@ export {
   groupBy,
   groupRunsByAgentProfileCell,
   harnessAxisOf,
+  hasCapturedToolArgs,
   hashContent,
   hashJson,
   hashScenarios,
@@ -11840,6 +11931,7 @@ export {
   mannWhitneyU,
   matchGoldens,
   matchSpan,
+  maximumChargeForLlmRequest,
   mcnemar,
   mcnemarPower,
   mcnemarRequiredN,
@@ -11955,6 +12047,7 @@ export {
   runProposeReview,
   runProposeReviewAsControlLoop,
   runRecordToProductBenchmarkRecord,
+  runReferenceEquivalenceJudge,
   runReferenceReplay,
   runScore,
   runSelfPlay,