@tangle-network/agent-eval 0.117.0 → 0.118.0

package/dist/campaign/index.js

@@ -66,7 +66,7 @@ import {
   userStoryScoreboard,
   validateSearchLedgerEvent,
   verifyCodeSurface
-} from "../chunk-ZUXV7UWZ.js";
+} from "../chunk-Q442S5AS.js";
 import {
   assertCodeSurfaceIdentity,
   buildEvidenceVector,
@@ -103,7 +103,7 @@ import {
   runOptimization,
   surfaceContentHash,
   surfaceHash
-} from "../chunk-HQPHZGL6.js";
+} from "../chunk-YLMUS4MM.js";
 import {
   SearchLedgerConflictError,
   SearchLedgerError,
@@ -120,20 +120,22 @@ import "../chunk-3YYRZDON.js";
 import {
   POLICY_EDIT_CANDIDATE_RECORD_SCHEMA,
   validatePolicyEditCandidateRecord
-} from "../chunk-MGEHEHSN.js";
+} from "../chunk-OIIMMLRB.js";
 import "../chunk-ARU2PZFM.js";
-import "../chunk-PJQFMIOX.js";
-import "../chunk-4JLWXDYA.js";
-import "../chunk-GGE4NNQT.js";
-import "../chunk-S2F4J57L.js";
-import "../chunk-5UF54T55.js";
-import "../chunk-XJYR7XFV.js";
-import "../chunk-VSMTAMNK.js";
 import "../chunk-NJC7U437.js";
+import "../chunk-PJQFMIOX.js";
 import "../chunk-VCTY3W6J.js";
 import "../chunk-VI2UW6B6.js";
+import "../chunk-3EHHMC6E.js";
+import "../chunk-GGE4NNQT.js";
+import "../chunk-QKEGNI5B.js";
 import "../chunk-PC4UYEBM.js";
+import "../chunk-S3UZOQ5Y.js";
+import "../chunk-MA6HLL3S.js";
+import "../chunk-XJYR7XFV.js";
+import "../chunk-VSMTAMNK.js";
 import "../chunk-ONWEPEDO.js";
+import "../chunk-WDHBCA3M.js";
 import "../chunk-PZ5AY32C.js";
 export {
   DEFAULT_POLICY_EDIT_HISTORY_LIMITS,

package/dist/{chunk-VF3XSYTI.js → chunk-33JA4TFA.js}

@@ -6,16 +6,16 @@ import {
   DEFAULT_TRACE_ANALYST_KINDS,
   createTraceAnalystKind,
   makeFinding
-} from "./chunk-MGEHEHSN.js";
+} from "./chunk-OIIMMLRB.js";
+import {
+  LlmClient
+} from "./chunk-NJC7U437.js";
 import {
   LLM_INPUT_TOKEN_ATTR_KEYS,
   LLM_OUTPUT_TOKEN_ATTR_KEYS,
   TOOL_NAME_ATTR_KEYS,
   spanEpochMillis
-} from "./chunk-S2F4J57L.js";
-import {
-  LlmClient
-} from "./chunk-NJC7U437.js";
+} from "./chunk-QKEGNI5B.js";
 
 // src/analyst/chat-client.ts
 function createChatClient(opts) {
@@ -542,4 +542,4 @@ export {
   behavioralAnalyst,
   buildDefaultAnalystRegistry
 };
-//# sourceMappingURL=chunk-VF3XSYTI.js.map
+//# sourceMappingURL=chunk-33JA4TFA.js.map

package/dist/{chunk-4JLWXDYA.js → chunk-3EHHMC6E.js}

@@ -2,7 +2,7 @@ import {
   OtlpFileTraceStore,
   TraceFileMissingError,
   buildTraceAnalystTools
-} from "./chunk-S2F4J57L.js";
+} from "./chunk-QKEGNI5B.js";
 
 // src/trace-analyst/prompts.ts
 var TRACE_ANALYST_ACTOR_DESCRIPTION = `You answer questions about an OTLP-shaped JSONL trace dataset using the trace tools provided in the \`traces\` namespace.
@@ -198,4 +198,4 @@ export {
   TRACE_ANALYST_SUBAGENT_DESCRIPTION,
   analyzeTraces
 };
-//# sourceMappingURL=chunk-4JLWXDYA.js.map
+//# sourceMappingURL=chunk-3EHHMC6E.js.map

package/dist/{chunk-CCZIVI3F.js → chunk-BFW56GTT.js}

@@ -5,7 +5,7 @@ import {
   aggregateRunScore,
   clamp01,
   computeFindingId
-} from "./chunk-MGEHEHSN.js";
+} from "./chunk-OIIMMLRB.js";
 import {
   callLlmJson,
   costReceiptFromLlm,
@@ -759,4 +759,4 @@ export {
   runSemanticConceptJudge,
   createSemanticConceptJudge
 };
-//# sourceMappingURL=chunk-CCZIVI3F.js.map
+//# sourceMappingURL=chunk-BFW56GTT.js.map

package/dist/{chunk-YZPO4UHR.js → chunk-DP3BBMYJ.js}

@@ -1,40 +1,48 @@
+import {
+  recordAggregateMeasurements,
+  summarizeExecutionMeasurements
+} from "./chunk-KSDQVPLR.js";
+import {
+  extractUsage,
+  extractUsageFromSse
+} from "./chunk-PXE2VKMX.js";
 import {
   analyzeTraces
-} from "./chunk-4JLWXDYA.js";
+} from "./chunk-3EHHMC6E.js";
 import {
-  LLM_CACHED_TOKEN_ATTR_KEYS,
-  LLM_COST_ATTR_KEYS,
-  LLM_COST_USD,
-  LLM_INPUT_TOKENS,
-  LLM_INPUT_TOKEN_ATTR_KEYS,
   LLM_MODEL_ATTR_KEYS,
-  LLM_MODEL_NAME,
-  LLM_OUTPUT_TOKENS,
-  LLM_OUTPUT_TOKEN_ATTR_KEYS,
-  OPENINFERENCE_SPAN_KIND,
-  TOOL_NAME,
   applyToolSpanOtlpAttributes,
   compareSpanTime,
-  firstNumberAttr,
   firstStringAttr,
   projectOtlpFlatLine,
   spanEpochMillis,
   traceSpanKindToOpenInferenceKind
-} from "./chunk-S2F4J57L.js";
+} from "./chunk-QKEGNI5B.js";
+import {
+  defaultProviderRedactor,
+  providerFromBaseUrl
+} from "./chunk-PC4UYEBM.js";
 import {
   validateRunRecord
-} from "./chunk-5UF54T55.js";
+} from "./chunk-S3UZOQ5Y.js";
 import {
   canonicalize,
   hashJson
 } from "./chunk-VSMTAMNK.js";
-import {
-  defaultProviderRedactor,
-  providerFromBaseUrl
-} from "./chunk-PC4UYEBM.js";
 import {
   ReplayError
 } from "./chunk-ONWEPEDO.js";
+import {
+  LLM_CACHED_TOKENS,
+  LLM_CACHE_WRITE_TOKENS,
+  LLM_COST_USD,
+  LLM_INPUT_TOKENS,
+  LLM_MODEL_NAME,
+  LLM_OUTPUT_TOKENS,
+  LLM_REASONING_TOKENS,
+  OPENINFERENCE_SPAN_KIND,
+  TOOL_NAME
+} from "./chunk-WDHBCA3M.js";
 
 // src/trace-analyst/hook.ts
 var DEFAULT_QUESTION = "Summarise what happened in this run. Surface any failure modes, surprising findings, or evidence that the run's verdict is wrong.";
@@ -447,7 +455,7 @@ function otlpToTraceRunRecords(otlpJsonl, opts) {
     const spans = byTrace.get(traceId);
     const agg = aggregateTrace(traceId, spans, fallbackModel);
     const score = resolveScore(opts, traceId, agg);
-    const { costUsd, costUnpriced } = resolveCost(opts, agg, spans);
+    const { costUsd, costProvenance } = resolveCost(opts, agg);
     const raw = {
       span_count: agg.spanCount,
       llm_span_count: agg.llmSpanCount,
@@ -457,10 +465,18 @@ function otlpToTraceRunRecords(otlpJsonl, opts) {
       prompt_tokens: agg.tokenUsage.input,
       completion_tokens: agg.tokenUsage.output
     };
+    if (agg.tokenUsage.reasoning !== void 0) raw.reasoning_tokens = agg.tokenUsage.reasoning;
     if (agg.tokenUsage.cached !== void 0) raw.cached_tokens = agg.tokenUsage.cached;
-    if (costUnpriced) raw.cost_unpriced = 1;
+    if (agg.tokenUsage.cacheWrite !== void 0) {
+      raw.cache_write_tokens = agg.tokenUsage.cacheWrite;
+    }
+    if (agg.costMeasurement.value !== void 0 && !agg.costMeasurement.complete) {
+      raw.partial_observed_cost_usd = agg.costMeasurement.value;
+    }
+    recordAggregateMeasurements(raw, agg.aggregateMeasurement);
+    if (costProvenance.kind === "uncaptured") raw.cost_unpriced = 1;
     const outcome = splitTag === "holdout" ? { holdoutScore: score, raw } : { searchScore: score, raw };
-    const { promptText, completionText } = extractPromptCompletion(spans);
+    const { promptText, completionText } = extractPromptCompletion(spans, agg.callSpanIds);
     const judgeMetadata = opts.judgeMetadataForTrace?.(traceId);
     const record = validateRunRecord({
       runId: `otlp:${opts.experimentId}:${opts.candidateId}:${traceId}`,
@@ -473,6 +489,7 @@ function otlpToTraceRunRecords(otlpJsonl, opts) {
       commitSha,
       wallMs: agg.wallMs,
       costUsd,
+      costProvenance,
       tokenUsage: agg.tokenUsage,
       ...judgeMetadata ? { judgeMetadata } : {},
       outcome,
@@ -512,11 +529,15 @@ function aggregateTrace(traceId, spans, fallbackModel) {
   const ordered = [...spans].sort(
     (a2, b2) => compareSpanTime(a2.start_time, b2.start_time) || a2.span_id.localeCompare(b2.span_id)
   );
-  let input = 0;
-  let output = 0;
-  let cached = 0;
-  let sawCached = false;
-  let llmSpanCount = 0;
+  const measurements = summarizeExecutionMeasurements(
+    ordered.map((span) => ({
+      id: span.span_id,
+      ...span.parent_span_id ? { parentId: span.parent_span_id } : {},
+      attributes: span.attributes,
+      modelCall: span.kind === "LLM" || span.kind === "UNKNOWN" && (span.model_name !== null || typeof span.attributes["gen_ai.operation.name"] === "string"),
+      aggregate: span.kind !== "LLM" && span.kind !== "UNKNOWN"
+    }))
+  );
   let toolSpanCount = 0;
   let agentSpanCount = 0;
   let errorSpanCount = 0;
@@ -528,20 +549,7 @@ function aggregateTrace(traceId, spans, fallbackModel) {
     if (s.start_time && (!earliest || compareSpanTime(s.start_time, earliest) < 0))
       earliest = s.start_time;
     if (s.end_time && (!latest || compareSpanTime(s.end_time, latest) > 0)) latest = s.end_time;
-    if (s.kind === "LLM") {
-      llmSpanCount += 1;
-      const i = firstNumberAttr(s.attributes, LLM_INPUT_TOKEN_ATTR_KEYS);
-      const o = firstNumberAttr(s.attributes, LLM_OUTPUT_TOKEN_ATTR_KEYS);
-      const c = firstNumberAttr(s.attributes, LLM_CACHED_TOKEN_ATTR_KEYS);
-      if (i !== null) input += i;
-      if (o !== null) output += o;
-      if (c !== null) {
-        cached += c;
-        sawCached = true;
-      }
-      const m = firstStringAttr(s.attributes, LLM_MODEL_ATTR_KEYS) ?? s.model_name;
-      if (m) modelVotes.set(m, (modelVotes.get(m) ?? 0) + 1);
-    } else if (s.kind === "TOOL") {
+    if (s.kind === "TOOL") {
       toolSpanCount += 1;
     } else if (s.kind === "AGENT") {
       agentSpanCount += 1;
@@ -553,25 +561,33 @@ function aggregateTrace(traceId, spans, fallbackModel) {
       }
     }
   }
+  const callSpanIds = new Set(measurements.callSpanIds);
+  for (const span of ordered) {
+    if (!callSpanIds.has(span.span_id)) continue;
+    const model2 = firstStringAttr(span.attributes, LLM_MODEL_ATTR_KEYS) ?? span.model_name;
+    if (model2) modelVotes.set(model2, (modelVotes.get(model2) ?? 0) + 1);
+  }
   const model = topVote(modelVotes) ?? firstModelAttr(ordered) ?? fallbackModel;
   let wallMs = 0;
   const a = spanEpochMillis(earliest);
   const b = spanEpochMillis(latest);
   if (a !== null && b !== null) wallMs = Math.max(0, b - a);
-  const tokenUsage = sawCached ? { input, output, cached } : { input, output };
   return {
     traceId,
     spanCount: spans.length,
-    llmSpanCount,
+    llmSpanCount: measurements.modelCallCount,
     toolSpanCount,
     agentSpanCount,
     errorSpanCount,
-    tokenUsage,
+    tokenUsage: measurements.tokenUsage,
     firstErrorMessage,
     model,
     startTime: earliest,
     endTime: latest,
-    wallMs
+    wallMs,
+    callSpanIds: measurements.callSpanIds,
+    costMeasurement: measurements.cost,
+    ...measurements.aggregate ? { aggregateMeasurement: measurements.aggregate } : {}
   };
 }
 function resolveScore(opts, traceId, agg) {
@@ -586,26 +602,31 @@ function resolveScore(opts, traceId, agg) {
   }
   return agg.errorSpanCount > 0 ? 0 : 1;
 }
-function resolveCost(opts, agg, spans) {
-  let perSpanCost = 0;
-  let sawCost = false;
-  for (const s of spans) {
-    if (s.kind !== "LLM") continue;
-    const c = firstNumberAttr(s.attributes, LLM_COST_ATTR_KEYS);
-    if (c !== null) {
-      perSpanCost += c;
-      sawCost = true;
-    }
+function resolveCost(opts, agg) {
+  const observedCost = agg.costMeasurement;
+  if (observedCost.complete && observedCost.value !== void 0) {
+    return {
+      costUsd: observedCost.value,
+      costProvenance: { kind: "observed", usd: observedCost.value }
+    };
+  }
+  if (agg.aggregateMeasurement?.costUsd !== void 0) {
+    return {
+      costUsd: agg.aggregateMeasurement.costUsd,
+      costProvenance: { kind: "observed", usd: agg.aggregateMeasurement.costUsd }
+    };
   }
-  if (sawCost) return { costUsd: perSpanCost, costUnpriced: false };
   if (opts.priceUsdPerToken !== void 0) {
     const totalTokens = agg.tokenUsage.input + agg.tokenUsage.output;
-    return { costUsd: totalTokens * opts.priceUsdPerToken, costUnpriced: false };
+    const costUsd = totalTokens * opts.priceUsdPerToken;
+    return { costUsd, costProvenance: { kind: "estimated", usd: costUsd } };
   }
-  return { costUsd: 0, costUnpriced: true };
+  return { costUsd: 0, costProvenance: { kind: "uncaptured", usd: null } };
 }
-function extractPromptCompletion(spans) {
-  const llm = spans.filter((s) => s.kind === "LLM").sort(
+function extractPromptCompletion(spans, callSpanIds) {
+  const callIds = new Set(callSpanIds);
+  const measuredCalls = spans.filter((span) => callIds.has(span.span_id));
+  const llm = (measuredCalls.length > 0 ? measuredCalls : spans.filter((s) => s.kind === "LLM")).sort(
     (a, b) => compareSpanTime(a.start_time, b.start_time) || a.span_id.localeCompare(b.span_id)
   );
   if (llm.length === 0) return {};
@@ -945,85 +966,6 @@ var FileSystemTraceStore = class {
   }
 };
 
-// src/trace/extract-usage.ts
-var INPUT_KEYS = ["prompt_tokens", "input_tokens", "promptTokens", "inputTokens"];
-var OUTPUT_KEYS = [
-  "completion_tokens",
-  "output_tokens",
-  "completionTokens",
-  "outputTokens"
-];
-var CACHED_KEYS = [
-  "cached_tokens",
-  "cache_read_input_tokens",
-  "cachedTokens",
-  "cacheReadInputTokens"
-];
-function nonNegNumber(source, keys) {
-  for (const key of keys) {
-    const value = source[key];
-    if (typeof value === "number" && Number.isFinite(value) && value >= 0) return value;
-  }
-  return null;
-}
-function extractUsage(body) {
-  if (!body || typeof body !== "object") return null;
-  const obj = body;
-  const usage = obj.usage && typeof obj.usage === "object" ? obj.usage : obj;
-  const input = nonNegNumber(usage, INPUT_KEYS);
-  const output = nonNegNumber(usage, OUTPUT_KEYS);
-  if (input === null && output === null) return null;
-  const cached = nonNegNumber(usage, CACHED_KEYS);
-  const result = { input: input ?? 0, output: output ?? 0 };
-  if (cached !== null) result.cached = cached;
-  return result;
-}
-function extractUsageFromSse(text) {
-  let input = 0;
-  let output = 0;
-  let cached = 0;
-  let sawCached = false;
-  let found = false;
-  for (const line of text.split(/\r?\n/)) {
-    const trimmed = line.trim();
-    if (!trimmed.startsWith("data:")) continue;
-    const payload = trimmed.slice("data:".length).trim();
-    if (!payload || payload === "[DONE]") continue;
-    let parsed;
-    try {
-      parsed = JSON.parse(payload);
-    } catch {
-      continue;
-    }
-    const usage = extractUsage(parsed);
-    if (!usage) continue;
-    input += usage.input;
-    output += usage.output;
-    if (usage.cached !== void 0) {
-      cached += usage.cached;
-      sawCached = true;
-    }
-    found = true;
-  }
-  if (!found) return null;
-  return sawCached ? { input, output, cached } : { input, output };
-}
-async function extractUsageFromResponse(response) {
-  let text;
-  try {
-    text = await response.clone().text();
-  } catch {
-    return null;
-  }
-  let json;
-  try {
-    json = JSON.parse(text);
-  } catch {
-    json = void 0;
-  }
-  return extractUsage(json) ?? extractUsageFromSse(text);
-}
-
 // src/trace/capture-fetch.ts
 var DEFAULT_BODY_CAP = 2 * 1024 * 1024;
 function headersToRecord(headers) {
@@ -1136,7 +1078,7 @@ function captureFetchToRawSink(fetch2, sink, ctx, opts = {}) {
     if (opts.onUsage && rawText !== void 0) {
       try {
         const parsedForUsage = parseMaybeJson(rawText);
-        const usage = extractUsage(parsedForUsage) ?? (typeof parsedForUsage === "string" ? extractUsageFromSse(rawText) : null);
+        const usage = extractUsage(parsedForUsage) ?? (typeof parsedForUsage === "string" ? extractUsageFromSse(rawText, { mode: opts.sseUsageMode }) : null);
         if (usage) opts.onUsage(usage, ctx);
       } catch (err) {
         if (opts.failClosed) throw err;
@@ -1224,6 +1166,11 @@ function flattenSpanAttributes(span) {
     base[LLM_MODEL_NAME] = span.model;
     if (span.inputTokens !== void 0) base[LLM_INPUT_TOKENS] = span.inputTokens;
     if (span.outputTokens !== void 0) base[LLM_OUTPUT_TOKENS] = span.outputTokens;
+    if (span.reasoningTokens !== void 0) base[LLM_REASONING_TOKENS] = span.reasoningTokens;
+    if (span.cachedTokens !== void 0) base[LLM_CACHED_TOKENS] = span.cachedTokens;
+    if (span.cacheWriteTokens !== void 0) {
+      base[LLM_CACHE_WRITE_TOKENS] = span.cacheWriteTokens;
+    }
     if (span.costUsd !== void 0) base[LLM_COST_USD] = span.costUsd;
     if (span.finishReason) base["llm.finish_reason"] = span.finishReason;
   } else if (span.kind === "tool") {
@@ -1339,6 +1286,9 @@ function storeSpanToExportable(span, traceId) {
     model: llm?.model,
     inputTokens: llm?.inputTokens,
     outputTokens: llm?.outputTokens,
+    reasoningTokens: llm?.reasoningTokens,
+    cachedTokens: llm?.cachedTokens,
+    cacheWriteTokens: llm?.cacheWriteTokens,
     costUsd: llm?.costUsd,
     tool: tool ? {
       toolName: tool.toolName,
@@ -1448,6 +1398,11 @@ function toOtlpSpan(span) {
   if (span.model) attrs[LLM_MODEL_NAME] = span.model;
   if (span.inputTokens !== void 0) attrs[LLM_INPUT_TOKENS] = span.inputTokens;
   if (span.outputTokens !== void 0) attrs[LLM_OUTPUT_TOKENS] = span.outputTokens;
+  if (span.reasoningTokens !== void 0) attrs[LLM_REASONING_TOKENS] = span.reasoningTokens;
+  if (span.cachedTokens !== void 0) attrs[LLM_CACHED_TOKENS] = span.cachedTokens;
+  if (span.cacheWriteTokens !== void 0) {
+    attrs[LLM_CACHE_WRITE_TOKENS] = span.cacheWriteTokens;
+  }
   if (span.costUsd !== void 0) attrs[LLM_COST_USD] = span.costUsd;
   if (span.tool) applyToolSpanOtlpAttributes(attrs, span.tool);
   return {
@@ -1699,6 +1654,11 @@ function spanToAttributes(span, events) {
     attrs[LLM_MODEL_NAME] = span.model;
     if (span.inputTokens !== void 0) attrs[LLM_INPUT_TOKENS] = span.inputTokens;
     if (span.outputTokens !== void 0) attrs[LLM_OUTPUT_TOKENS] = span.outputTokens;
+    if (span.reasoningTokens !== void 0) attrs[LLM_REASONING_TOKENS] = span.reasoningTokens;
+    if (span.cachedTokens !== void 0) attrs[LLM_CACHED_TOKENS] = span.cachedTokens;
+    if (span.cacheWriteTokens !== void 0) {
+      attrs[LLM_CACHE_WRITE_TOKENS] = span.cacheWriteTokens;
+    }
     if (span.costUsd !== void 0) attrs[LLM_COST_USD] = span.costUsd;
     if (span.finishReason) attrs["llm.finish_reason"] = span.finishReason;
     if (Array.isArray(span.messages)) {
@@ -1936,9 +1896,6 @@ export {
   otlpToTraceRunRecords,
   InMemoryTraceStore,
   FileSystemTraceStore,
-  extractUsage,
-  extractUsageFromSse,
-  extractUsageFromResponse,
   captureFetchToRawSink,
   OTEL_AGENT_EVAL_SCOPE,
   exportRunAsOtlp,
@@ -1951,4 +1908,4 @@ export {
   createReplayFetch,
   iterateRawCalls
 };
-//# sourceMappingURL=chunk-YZPO4UHR.js.map
+//# sourceMappingURL=chunk-DP3BBMYJ.js.map