@tangle-network/agent-eval 0.117.1 → 0.118.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. package/CHANGELOG.md +30 -0
  2. package/dist/analyst/index.d.ts +2772 -21
  3. package/dist/analyst/index.js +7 -6
  4. package/dist/analyst/index.js.map +1 -1
  5. package/dist/belief-state/index.d.ts +706 -10
  6. package/dist/belief-state/index.js +2 -1
  7. package/dist/belief-state/index.js.map +1 -1
  8. package/dist/benchmarks/index.d.ts +958 -14
  9. package/dist/benchmarks/index.js +12 -10
  10. package/dist/builder-eval/index.d.ts +449 -4
  11. package/dist/builder-eval/index.js +4 -3
  12. package/dist/builder-eval/index.js.map +1 -1
  13. package/dist/campaign/index.d.ts +4275 -73
  14. package/dist/campaign/index.js +12 -10
  15. package/dist/{chunk-VF3XSYTI.js → chunk-33JA4TFA.js} +6 -6
  16. package/dist/{chunk-4JLWXDYA.js → chunk-3EHHMC6E.js} +2 -2
  17. package/dist/{chunk-CCZIVI3F.js → chunk-BFW56GTT.js} +2 -2
  18. package/dist/{chunk-YZPO4UHR.js → chunk-FTUMG2U7.js} +124 -149
  19. package/dist/chunk-FTUMG2U7.js.map +1 -0
  20. package/dist/{chunk-E4BUPP7Z.js → chunk-HKUCJ437.js} +38 -66
  21. package/dist/chunk-HKUCJ437.js.map +1 -0
  22. package/dist/{chunk-HZHNRYHK.js → chunk-K6N6XJJX.js} +2 -2
  23. package/dist/chunk-KSDQVPLR.js +286 -0
  24. package/dist/chunk-KSDQVPLR.js.map +1 -0
  25. package/dist/chunk-MA6HLL3S.js +65 -0
  26. package/dist/chunk-MA6HLL3S.js.map +1 -0
  27. package/dist/{chunk-MGEHEHSN.js → chunk-OIIMMLRB.js} +11 -11
  28. package/dist/{chunk-DXZRATT5.js → chunk-OYZAPX5G.js} +3 -3
  29. package/dist/chunk-PXE2VKMX.js +140 -0
  30. package/dist/chunk-PXE2VKMX.js.map +1 -0
  31. package/dist/{chunk-JSJZ4PJ6.js → chunk-Q442S5AS.js} +17 -17
  32. package/dist/{chunk-ODVOOEWQ.js → chunk-QBRSJK47.js} +2 -2
  33. package/dist/{chunk-S2F4J57L.js → chunk-QKEGNI5B.js} +77 -32
  34. package/dist/chunk-QKEGNI5B.js.map +1 -0
  35. package/dist/{chunk-5UF54T55.js → chunk-S3UZOQ5Y.js} +34 -7
  36. package/dist/chunk-S3UZOQ5Y.js.map +1 -0
  37. package/dist/{chunk-FQNLDL4D.js → chunk-SVH2ANFD.js} +136 -4
  38. package/dist/chunk-SVH2ANFD.js.map +1 -0
  39. package/dist/{chunk-GQCZRZ7L.js → chunk-U5CHZ5M3.js} +9 -9
  40. package/dist/{chunk-TVVP3ZZQ.js → chunk-VQMK5FMP.js} +2 -1
  41. package/dist/{chunk-TVVP3ZZQ.js.map → chunk-VQMK5FMP.js.map} +1 -1
  42. package/dist/chunk-WDHBCA3M.js +31 -0
  43. package/dist/chunk-WDHBCA3M.js.map +1 -0
  44. package/dist/{chunk-HQPHZGL6.js → chunk-YLMUS4MM.js} +9 -9
  45. package/dist/{chunk-LQUTGLOZ.js → chunk-ZET2UAYW.js} +15 -65
  46. package/dist/chunk-ZET2UAYW.js.map +1 -0
  47. package/dist/contract/index.d.ts +4012 -38
  48. package/dist/contract/index.js +56 -29
  49. package/dist/contract/index.js.map +1 -1
  50. package/dist/control.d.ts +1013 -9
  51. package/dist/control.js +4 -3
  52. package/dist/fuzz.d.ts +194 -4
  53. package/dist/fuzz.js +3 -3
  54. package/dist/hosted/index.d.ts +498 -17
  55. package/dist/index.d.ts +10992 -1288
  56. package/dist/index.js +101 -80
  57. package/dist/index.js.map +1 -1
  58. package/dist/matrix/index.d.ts +139 -4
  59. package/dist/meta-eval/index.d.ts +862 -15
  60. package/dist/meta-eval/index.js +2 -1
  61. package/dist/meta-eval/index.js.map +1 -1
  62. package/dist/multishot/index.d.ts +214 -14
  63. package/dist/openapi.json +1 -1
  64. package/dist/pipelines/index.d.ts +392 -7
  65. package/dist/pipelines/index.js +5 -3
  66. package/dist/pipelines/index.js.map +1 -1
  67. package/dist/reporting.d.ts +1277 -17
  68. package/dist/rl.d.ts +2359 -28
  69. package/dist/rl.js +6 -5
  70. package/dist/rl.js.map +1 -1
  71. package/dist/storyboard/index.d.ts +86 -1
  72. package/dist/trace-attributes.d.ts +16 -0
  73. package/dist/trace-attributes.js +32 -0
  74. package/dist/trace-attributes.js.map +1 -0
  75. package/dist/traces.d.ts +1978 -697
  76. package/dist/traces.js +53 -32
  77. package/dist/wire/index.d.ts +655 -9
  78. package/docs/insight-report.md +44 -0
  79. package/package.json +9 -3
  80. package/dist/adversarial-B7loGVVX.d.ts +0 -19
  81. package/dist/analyst-C8HHvfJp.d.ts +0 -88
  82. package/dist/analyze-runs--2x39HZ7.d.ts +0 -81
  83. package/dist/baseline-DKq3gJpP.d.ts +0 -141
  84. package/dist/calibration-C8MTS7cw.d.ts +0 -101
  85. package/dist/chunk-5UF54T55.js.map +0 -1
  86. package/dist/chunk-E4BUPP7Z.js.map +0 -1
  87. package/dist/chunk-FQNLDL4D.js.map +0 -1
  88. package/dist/chunk-LQUTGLOZ.js.map +0 -1
  89. package/dist/chunk-S2F4J57L.js.map +0 -1
  90. package/dist/chunk-YZPO4UHR.js.map +0 -1
  91. package/dist/code-agent-session-CjZsVd19.d.ts +0 -87
  92. package/dist/control-6vuGfmDH.d.ts +0 -258
  93. package/dist/cost-ledger-DWy3XdJc.d.ts +0 -183
  94. package/dist/dataset-NENEzRgk.d.ts +0 -115
  95. package/dist/default-registry-DaK8b3fv.d.ts +0 -155
  96. package/dist/emitter-CjD7vUwv.d.ts +0 -122
  97. package/dist/errors-oeQrLqXC.d.ts +0 -74
  98. package/dist/failure-cluster-DOAcSJ87.d.ts +0 -76
  99. package/dist/feedback-trajectory-BUnM58xL.d.ts +0 -348
  100. package/dist/gepa-eESocoDi.d.ts +0 -642
  101. package/dist/index-PdX4VnPA.d.ts +0 -423
  102. package/dist/insight-report-DY4nDW9Q.d.ts +0 -310
  103. package/dist/integrity-DqlBiLyK.d.ts +0 -81
  104. package/dist/judge-calibration-7C-IDmKr.d.ts +0 -145
  105. package/dist/kind-factory-ClZmO25A.d.ts +0 -171
  106. package/dist/llm-client-qoDd18Qz.d.ts +0 -289
  107. package/dist/multi-layer-verifier-BsqKuLyN.d.ts +0 -150
  108. package/dist/off-policy-DiwuKKg7.d.ts +0 -132
  109. package/dist/outcome-store-rnXLEqSn.d.ts +0 -63
  110. package/dist/policy-edit-wG9uFEFm.d.ts +0 -455
  111. package/dist/pre-registration-BWQhJ3vz.d.ts +0 -761
  112. package/dist/provenance-DpjwyseI.d.ts +0 -541
  113. package/dist/query-CF7PG61p.d.ts +0 -35
  114. package/dist/raw-provider-sink-C46HDghv.d.ts +0 -132
  115. package/dist/release-report-C8G2i5Xi.d.ts +0 -236
  116. package/dist/researcher-C8XyxQsu.d.ts +0 -387
  117. package/dist/rubric-predictive-validity-p49lLVrE.d.ts +0 -105
  118. package/dist/run-record-BDH49H2E.d.ts +0 -360
  119. package/dist/runtime-trajectory-DGBIUt4B.d.ts +0 -49
  120. package/dist/schema-B3Q3l9Z_.d.ts +0 -201
  121. package/dist/semantic-concept-judge-CXnPEJbf.d.ts +0 -723
  122. package/dist/sequential-5iSVfzl2.d.ts +0 -139
  123. package/dist/series-convergence-D5OWMBg6.d.ts +0 -33
  124. package/dist/statistics-KUnG73jH.d.ts +0 -494
  125. package/dist/storage-DrX3v_5B.d.ts +0 -50
  126. package/dist/store-C1YxJDEK.d.ts +0 -248
  127. package/dist/store-DGqD0Pyo.d.ts +0 -116
  128. package/dist/summary-report-C5bKFfm-.d.ts +0 -445
  129. package/dist/test-graded-scenario-B0ybnPY7.d.ts +0 -166
  130. package/dist/types-BSw1rOUB.d.ts +0 -634
  131. package/dist/types-BUxNaJ8c.d.ts +0 -108
  132. package/dist/types-BkfcQnxV.d.ts +0 -313
  133. package/dist/verdict-C9MlYujm.d.ts +0 -35
  134. /package/dist/{chunk-VF3XSYTI.js.map → chunk-33JA4TFA.js.map} +0 -0
  135. /package/dist/{chunk-4JLWXDYA.js.map → chunk-3EHHMC6E.js.map} +0 -0
  136. /package/dist/{chunk-CCZIVI3F.js.map → chunk-BFW56GTT.js.map} +0 -0
  137. /package/dist/{chunk-HZHNRYHK.js.map → chunk-K6N6XJJX.js.map} +0 -0
  138. /package/dist/{chunk-MGEHEHSN.js.map → chunk-OIIMMLRB.js.map} +0 -0
  139. /package/dist/{chunk-DXZRATT5.js.map → chunk-OYZAPX5G.js.map} +0 -0
  140. /package/dist/{chunk-JSJZ4PJ6.js.map → chunk-Q442S5AS.js.map} +0 -0
  141. /package/dist/{chunk-ODVOOEWQ.js.map → chunk-QBRSJK47.js.map} +0 -0
  142. /package/dist/{chunk-GQCZRZ7L.js.map → chunk-U5CHZ5M3.js.map} +0 -0
  143. /package/dist/{chunk-HQPHZGL6.js.map → chunk-YLMUS4MM.js.map} +0 -0
@@ -66,7 +66,7 @@ import {
66
66
  userStoryScoreboard,
67
67
  validateSearchLedgerEvent,
68
68
  verifyCodeSurface
69
- } from "../chunk-JSJZ4PJ6.js";
69
+ } from "../chunk-Q442S5AS.js";
70
70
  import {
71
71
  assertCodeSurfaceIdentity,
72
72
  buildEvidenceVector,
@@ -103,7 +103,7 @@ import {
103
103
  runOptimization,
104
104
  surfaceContentHash,
105
105
  surfaceHash
106
- } from "../chunk-HQPHZGL6.js";
106
+ } from "../chunk-YLMUS4MM.js";
107
107
  import {
108
108
  SearchLedgerConflictError,
109
109
  SearchLedgerError,
@@ -120,20 +120,22 @@ import "../chunk-3YYRZDON.js";
120
120
  import {
121
121
  POLICY_EDIT_CANDIDATE_RECORD_SCHEMA,
122
122
  validatePolicyEditCandidateRecord
123
- } from "../chunk-MGEHEHSN.js";
123
+ } from "../chunk-OIIMMLRB.js";
124
124
  import "../chunk-ARU2PZFM.js";
125
- import "../chunk-PJQFMIOX.js";
126
- import "../chunk-4JLWXDYA.js";
127
- import "../chunk-GGE4NNQT.js";
128
- import "../chunk-S2F4J57L.js";
129
- import "../chunk-5UF54T55.js";
130
- import "../chunk-XJYR7XFV.js";
131
- import "../chunk-VSMTAMNK.js";
132
125
  import "../chunk-NJC7U437.js";
126
+ import "../chunk-PJQFMIOX.js";
133
127
  import "../chunk-VCTY3W6J.js";
134
128
  import "../chunk-VI2UW6B6.js";
129
+ import "../chunk-3EHHMC6E.js";
130
+ import "../chunk-GGE4NNQT.js";
131
+ import "../chunk-QKEGNI5B.js";
135
132
  import "../chunk-PC4UYEBM.js";
133
+ import "../chunk-S3UZOQ5Y.js";
134
+ import "../chunk-MA6HLL3S.js";
135
+ import "../chunk-XJYR7XFV.js";
136
+ import "../chunk-VSMTAMNK.js";
136
137
  import "../chunk-ONWEPEDO.js";
138
+ import "../chunk-WDHBCA3M.js";
137
139
  import "../chunk-PZ5AY32C.js";
138
140
  export {
139
141
  DEFAULT_POLICY_EDIT_HISTORY_LIMITS,
@@ -6,16 +6,16 @@ import {
6
6
  DEFAULT_TRACE_ANALYST_KINDS,
7
7
  createTraceAnalystKind,
8
8
  makeFinding
9
- } from "./chunk-MGEHEHSN.js";
9
+ } from "./chunk-OIIMMLRB.js";
10
+ import {
11
+ LlmClient
12
+ } from "./chunk-NJC7U437.js";
10
13
  import {
11
14
  LLM_INPUT_TOKEN_ATTR_KEYS,
12
15
  LLM_OUTPUT_TOKEN_ATTR_KEYS,
13
16
  TOOL_NAME_ATTR_KEYS,
14
17
  spanEpochMillis
15
- } from "./chunk-S2F4J57L.js";
16
- import {
17
- LlmClient
18
- } from "./chunk-NJC7U437.js";
18
+ } from "./chunk-QKEGNI5B.js";
19
19
 
20
20
  // src/analyst/chat-client.ts
21
21
  function createChatClient(opts) {
@@ -542,4 +542,4 @@ export {
542
542
  behavioralAnalyst,
543
543
  buildDefaultAnalystRegistry
544
544
  };
545
- //# sourceMappingURL=chunk-VF3XSYTI.js.map
545
+ //# sourceMappingURL=chunk-33JA4TFA.js.map
@@ -2,7 +2,7 @@ import {
2
2
  OtlpFileTraceStore,
3
3
  TraceFileMissingError,
4
4
  buildTraceAnalystTools
5
- } from "./chunk-S2F4J57L.js";
5
+ } from "./chunk-QKEGNI5B.js";
6
6
 
7
7
  // src/trace-analyst/prompts.ts
8
8
  var TRACE_ANALYST_ACTOR_DESCRIPTION = `You answer questions about an OTLP-shaped JSONL trace dataset using the trace tools provided in the \`traces\` namespace.
@@ -198,4 +198,4 @@ export {
198
198
  TRACE_ANALYST_SUBAGENT_DESCRIPTION,
199
199
  analyzeTraces
200
200
  };
201
- //# sourceMappingURL=chunk-4JLWXDYA.js.map
201
+ //# sourceMappingURL=chunk-3EHHMC6E.js.map
@@ -5,7 +5,7 @@ import {
5
5
  aggregateRunScore,
6
6
  clamp01,
7
7
  computeFindingId
8
- } from "./chunk-MGEHEHSN.js";
8
+ } from "./chunk-OIIMMLRB.js";
9
9
  import {
10
10
  callLlmJson,
11
11
  costReceiptFromLlm,
@@ -759,4 +759,4 @@ export {
759
759
  runSemanticConceptJudge,
760
760
  createSemanticConceptJudge
761
761
  };
762
- //# sourceMappingURL=chunk-CCZIVI3F.js.map
762
+ //# sourceMappingURL=chunk-BFW56GTT.js.map
@@ -1,40 +1,48 @@
1
+ import {
2
+ recordAggregateMeasurements,
3
+ summarizeExecutionMeasurements
4
+ } from "./chunk-KSDQVPLR.js";
5
+ import {
6
+ extractUsage,
7
+ extractUsageFromSse
8
+ } from "./chunk-PXE2VKMX.js";
1
9
  import {
2
10
  analyzeTraces
3
- } from "./chunk-4JLWXDYA.js";
11
+ } from "./chunk-3EHHMC6E.js";
4
12
  import {
5
- LLM_CACHED_TOKEN_ATTR_KEYS,
6
- LLM_COST_ATTR_KEYS,
7
- LLM_COST_USD,
8
- LLM_INPUT_TOKENS,
9
- LLM_INPUT_TOKEN_ATTR_KEYS,
10
13
  LLM_MODEL_ATTR_KEYS,
11
- LLM_MODEL_NAME,
12
- LLM_OUTPUT_TOKENS,
13
- LLM_OUTPUT_TOKEN_ATTR_KEYS,
14
- OPENINFERENCE_SPAN_KIND,
15
- TOOL_NAME,
16
14
  applyToolSpanOtlpAttributes,
17
15
  compareSpanTime,
18
- firstNumberAttr,
19
16
  firstStringAttr,
20
17
  projectOtlpFlatLine,
21
18
  spanEpochMillis,
22
19
  traceSpanKindToOpenInferenceKind
23
- } from "./chunk-S2F4J57L.js";
20
+ } from "./chunk-QKEGNI5B.js";
21
+ import {
22
+ defaultProviderRedactor,
23
+ providerFromBaseUrl
24
+ } from "./chunk-PC4UYEBM.js";
24
25
  import {
25
26
  validateRunRecord
26
- } from "./chunk-5UF54T55.js";
27
+ } from "./chunk-S3UZOQ5Y.js";
27
28
  import {
28
29
  canonicalize,
29
30
  hashJson
30
31
  } from "./chunk-VSMTAMNK.js";
31
- import {
32
- defaultProviderRedactor,
33
- providerFromBaseUrl
34
- } from "./chunk-PC4UYEBM.js";
35
32
  import {
36
33
  ReplayError
37
34
  } from "./chunk-ONWEPEDO.js";
35
+ import {
36
+ LLM_CACHED_TOKENS,
37
+ LLM_CACHE_WRITE_TOKENS,
38
+ LLM_COST_USD,
39
+ LLM_INPUT_TOKENS,
40
+ LLM_MODEL_NAME,
41
+ LLM_OUTPUT_TOKENS,
42
+ LLM_REASONING_TOKENS,
43
+ OPENINFERENCE_SPAN_KIND,
44
+ TOOL_NAME
45
+ } from "./chunk-WDHBCA3M.js";
38
46
 
39
47
  // src/trace-analyst/hook.ts
40
48
  var DEFAULT_QUESTION = "Summarise what happened in this run. Surface any failure modes, surprising findings, or evidence that the run's verdict is wrong.";
@@ -428,17 +436,25 @@ function flattenOtlpExportToNdjson(otlpExport, opts = {}) {
428
436
  function otlpToRunRecords(otlpJsonl, opts) {
429
437
  return otlpToTraceRunRecords(otlpJsonl, opts).map((r) => r.record);
430
438
  }
439
+ function otlpRowsToRunRecords(rows, opts) {
440
+ return otlpRowsToTraceRunRecords(rows, opts).map((row) => row.record);
441
+ }
431
442
  function otlpToTraceRunRecords(otlpJsonl, opts) {
443
+ return traceRunRecordsFromSpans(groupJsonlSpansByTrace(otlpJsonl), opts);
444
+ }
445
+ function otlpRowsToTraceRunRecords(rows, opts) {
446
+ return traceRunRecordsFromSpans(groupRowsByTrace(rows), opts);
447
+ }
448
+ function traceRunRecordsFromSpans(byTrace, opts) {
432
449
  const splitTag = opts.splitTag ?? "holdout";
433
450
  const commitSha = opts.commitSha ?? "unknown";
434
451
  const promptHash = opts.promptHash ?? "unknown";
435
452
  const configHash = opts.configHash ?? "unknown";
436
453
  const seed = opts.seed ?? 0;
437
454
  const fallbackModel = opts.fallbackModel ?? "unknown@otlp";
438
- const byTrace = groupSpansByTrace(otlpJsonl);
439
455
  if (byTrace.size === 0) {
440
456
  throw new Error(
441
- "otlpToRunRecords: OTLP input produced zero valid spans \u2014 every line was empty, malformed, or missing trace_id/span_id"
457
+ "otlpToRunRecords: OTLP input produced zero valid spans \u2014 every row was empty, malformed, or missing trace_id/span_id"
442
458
  );
443
459
  }
444
460
  const traceIds = [...byTrace.keys()].sort();
@@ -447,7 +463,7 @@ function otlpToTraceRunRecords(otlpJsonl, opts) {
447
463
  const spans = byTrace.get(traceId);
448
464
  const agg = aggregateTrace(traceId, spans, fallbackModel);
449
465
  const score = resolveScore(opts, traceId, agg);
450
- const { costUsd, costUnpriced } = resolveCost(opts, agg, spans);
466
+ const { costUsd, costProvenance } = resolveCost(opts, agg);
451
467
  const raw = {
452
468
  span_count: agg.spanCount,
453
469
  llm_span_count: agg.llmSpanCount,
@@ -457,10 +473,18 @@ function otlpToTraceRunRecords(otlpJsonl, opts) {
457
473
  prompt_tokens: agg.tokenUsage.input,
458
474
  completion_tokens: agg.tokenUsage.output
459
475
  };
476
+ if (agg.tokenUsage.reasoning !== void 0) raw.reasoning_tokens = agg.tokenUsage.reasoning;
460
477
  if (agg.tokenUsage.cached !== void 0) raw.cached_tokens = agg.tokenUsage.cached;
461
- if (costUnpriced) raw.cost_unpriced = 1;
478
+ if (agg.tokenUsage.cacheWrite !== void 0) {
479
+ raw.cache_write_tokens = agg.tokenUsage.cacheWrite;
480
+ }
481
+ if (agg.costMeasurement.value !== void 0 && !agg.costMeasurement.complete) {
482
+ raw.partial_observed_cost_usd = agg.costMeasurement.value;
483
+ }
484
+ recordAggregateMeasurements(raw, agg.aggregateMeasurement);
485
+ if (costProvenance.kind === "uncaptured") raw.cost_unpriced = 1;
462
486
  const outcome = splitTag === "holdout" ? { holdoutScore: score, raw } : { searchScore: score, raw };
463
- const { promptText, completionText } = extractPromptCompletion(spans);
487
+ const { promptText, completionText } = extractPromptCompletion(spans, agg.callSpanIds);
464
488
  const judgeMetadata = opts.judgeMetadataForTrace?.(traceId);
465
489
  const record = validateRunRecord({
466
490
  runId: `otlp:${opts.experimentId}:${opts.candidateId}:${traceId}`,
@@ -473,6 +497,7 @@ function otlpToTraceRunRecords(otlpJsonl, opts) {
473
497
  commitSha,
474
498
  wallMs: agg.wallMs,
475
499
  costUsd,
500
+ costProvenance,
476
501
  tokenUsage: agg.tokenUsage,
477
502
  ...judgeMetadata ? { judgeMetadata } : {},
478
503
  outcome,
@@ -488,8 +513,7 @@ function otlpToTraceRunRecords(otlpJsonl, opts) {
488
513
  }
489
514
  return out;
490
515
  }
491
- function groupSpansByTrace(otlpJsonl) {
492
- const byTrace = /* @__PURE__ */ new Map();
516
+ function* yieldJsonlRows(otlpJsonl) {
493
517
  for (const line of otlpJsonl.split("\n")) {
494
518
  const trimmed = line.trim();
495
519
  if (trimmed.length === 0) continue;
@@ -499,8 +523,17 @@ function groupSpansByTrace(otlpJsonl) {
499
523
  } catch {
500
524
  continue;
501
525
  }
502
- if (!parsed || typeof parsed !== "object") continue;
503
- const span = projectOtlpFlatLine(parsed);
526
+ if (parsed && typeof parsed === "object") yield parsed;
527
+ }
528
+ }
529
+ function groupJsonlSpansByTrace(otlpJsonl) {
530
+ return groupRowsByTrace(yieldJsonlRows(otlpJsonl));
531
+ }
532
+ function groupRowsByTrace(rows) {
533
+ const byTrace = /* @__PURE__ */ new Map();
534
+ for (const row of rows) {
535
+ if (!row || typeof row !== "object") continue;
536
+ const span = projectOtlpFlatLine(row);
504
537
  if (!span) continue;
505
538
  const arr = byTrace.get(span.trace_id);
506
539
  if (arr) arr.push(span);
@@ -512,11 +545,15 @@ function aggregateTrace(traceId, spans, fallbackModel) {
512
545
  const ordered = [...spans].sort(
513
546
  (a2, b2) => compareSpanTime(a2.start_time, b2.start_time) || a2.span_id.localeCompare(b2.span_id)
514
547
  );
515
- let input = 0;
516
- let output = 0;
517
- let cached = 0;
518
- let sawCached = false;
519
- let llmSpanCount = 0;
548
+ const measurements = summarizeExecutionMeasurements(
549
+ ordered.map((span) => ({
550
+ id: span.span_id,
551
+ ...span.parent_span_id ? { parentId: span.parent_span_id } : {},
552
+ attributes: span.attributes,
553
+ modelCall: span.kind === "LLM" || span.kind === "UNKNOWN" && (span.model_name !== null || typeof span.attributes["gen_ai.operation.name"] === "string"),
554
+ aggregate: span.kind !== "LLM" && span.kind !== "UNKNOWN"
555
+ }))
556
+ );
520
557
  let toolSpanCount = 0;
521
558
  let agentSpanCount = 0;
522
559
  let errorSpanCount = 0;
@@ -528,20 +565,7 @@ function aggregateTrace(traceId, spans, fallbackModel) {
528
565
  if (s.start_time && (!earliest || compareSpanTime(s.start_time, earliest) < 0))
529
566
  earliest = s.start_time;
530
567
  if (s.end_time && (!latest || compareSpanTime(s.end_time, latest) > 0)) latest = s.end_time;
531
- if (s.kind === "LLM") {
532
- llmSpanCount += 1;
533
- const i = firstNumberAttr(s.attributes, LLM_INPUT_TOKEN_ATTR_KEYS);
534
- const o = firstNumberAttr(s.attributes, LLM_OUTPUT_TOKEN_ATTR_KEYS);
535
- const c = firstNumberAttr(s.attributes, LLM_CACHED_TOKEN_ATTR_KEYS);
536
- if (i !== null) input += i;
537
- if (o !== null) output += o;
538
- if (c !== null) {
539
- cached += c;
540
- sawCached = true;
541
- }
542
- const m = firstStringAttr(s.attributes, LLM_MODEL_ATTR_KEYS) ?? s.model_name;
543
- if (m) modelVotes.set(m, (modelVotes.get(m) ?? 0) + 1);
544
- } else if (s.kind === "TOOL") {
568
+ if (s.kind === "TOOL") {
545
569
  toolSpanCount += 1;
546
570
  } else if (s.kind === "AGENT") {
547
571
  agentSpanCount += 1;
@@ -553,25 +577,33 @@ function aggregateTrace(traceId, spans, fallbackModel) {
553
577
  }
554
578
  }
555
579
  }
580
+ const callSpanIds = new Set(measurements.callSpanIds);
581
+ for (const span of ordered) {
582
+ if (!callSpanIds.has(span.span_id)) continue;
583
+ const model2 = firstStringAttr(span.attributes, LLM_MODEL_ATTR_KEYS) ?? span.model_name;
584
+ if (model2) modelVotes.set(model2, (modelVotes.get(model2) ?? 0) + 1);
585
+ }
556
586
  const model = topVote(modelVotes) ?? firstModelAttr(ordered) ?? fallbackModel;
557
587
  let wallMs = 0;
558
588
  const a = spanEpochMillis(earliest);
559
589
  const b = spanEpochMillis(latest);
560
590
  if (a !== null && b !== null) wallMs = Math.max(0, b - a);
561
- const tokenUsage = sawCached ? { input, output, cached } : { input, output };
562
591
  return {
563
592
  traceId,
564
593
  spanCount: spans.length,
565
- llmSpanCount,
594
+ llmSpanCount: measurements.modelCallCount,
566
595
  toolSpanCount,
567
596
  agentSpanCount,
568
597
  errorSpanCount,
569
- tokenUsage,
598
+ tokenUsage: measurements.tokenUsage,
570
599
  firstErrorMessage,
571
600
  model,
572
601
  startTime: earliest,
573
602
  endTime: latest,
574
- wallMs
603
+ wallMs,
604
+ callSpanIds: measurements.callSpanIds,
605
+ costMeasurement: measurements.cost,
606
+ ...measurements.aggregate ? { aggregateMeasurement: measurements.aggregate } : {}
575
607
  };
576
608
  }
577
609
  function resolveScore(opts, traceId, agg) {
@@ -586,26 +618,31 @@ function resolveScore(opts, traceId, agg) {
586
618
  }
587
619
  return agg.errorSpanCount > 0 ? 0 : 1;
588
620
  }
589
- function resolveCost(opts, agg, spans) {
590
- let perSpanCost = 0;
591
- let sawCost = false;
592
- for (const s of spans) {
593
- if (s.kind !== "LLM") continue;
594
- const c = firstNumberAttr(s.attributes, LLM_COST_ATTR_KEYS);
595
- if (c !== null) {
596
- perSpanCost += c;
597
- sawCost = true;
598
- }
621
+ function resolveCost(opts, agg) {
622
+ const observedCost = agg.costMeasurement;
623
+ if (observedCost.complete && observedCost.value !== void 0) {
624
+ return {
625
+ costUsd: observedCost.value,
626
+ costProvenance: { kind: "observed", usd: observedCost.value }
627
+ };
628
+ }
629
+ if (agg.aggregateMeasurement?.costUsd !== void 0) {
630
+ return {
631
+ costUsd: agg.aggregateMeasurement.costUsd,
632
+ costProvenance: { kind: "observed", usd: agg.aggregateMeasurement.costUsd }
633
+ };
599
634
  }
600
- if (sawCost) return { costUsd: perSpanCost, costUnpriced: false };
601
635
  if (opts.priceUsdPerToken !== void 0) {
602
636
  const totalTokens = agg.tokenUsage.input + agg.tokenUsage.output;
603
- return { costUsd: totalTokens * opts.priceUsdPerToken, costUnpriced: false };
637
+ const costUsd = totalTokens * opts.priceUsdPerToken;
638
+ return { costUsd, costProvenance: { kind: "estimated", usd: costUsd } };
604
639
  }
605
- return { costUsd: 0, costUnpriced: true };
640
+ return { costUsd: 0, costProvenance: { kind: "uncaptured", usd: null } };
606
641
  }
607
- function extractPromptCompletion(spans) {
608
- const llm = spans.filter((s) => s.kind === "LLM").sort(
642
+ function extractPromptCompletion(spans, callSpanIds) {
643
+ const callIds = new Set(callSpanIds);
644
+ const measuredCalls = spans.filter((span) => callIds.has(span.span_id));
645
+ const llm = (measuredCalls.length > 0 ? measuredCalls : spans.filter((s) => s.kind === "LLM")).sort(
609
646
  (a, b) => compareSpanTime(a.start_time, b.start_time) || a.span_id.localeCompare(b.span_id)
610
647
  );
611
648
  if (llm.length === 0) return {};
@@ -945,85 +982,6 @@ var FileSystemTraceStore = class {
945
982
  }
946
983
  };
947
984
 
948
- // src/trace/extract-usage.ts
949
- var INPUT_KEYS = ["prompt_tokens", "input_tokens", "promptTokens", "inputTokens"];
950
- var OUTPUT_KEYS = [
951
- "completion_tokens",
952
- "output_tokens",
953
- "completionTokens",
954
- "outputTokens"
955
- ];
956
- var CACHED_KEYS = [
957
- "cached_tokens",
958
- "cache_read_input_tokens",
959
- "cachedTokens",
960
- "cacheReadInputTokens"
961
- ];
962
- function nonNegNumber(source, keys) {
963
- for (const key of keys) {
964
- const value = source[key];
965
- if (typeof value === "number" && Number.isFinite(value) && value >= 0) return value;
966
- }
967
- return null;
968
- }
969
- function extractUsage(body) {
970
- if (!body || typeof body !== "object") return null;
971
- const obj = body;
972
- const usage = obj.usage && typeof obj.usage === "object" ? obj.usage : obj;
973
- const input = nonNegNumber(usage, INPUT_KEYS);
974
- const output = nonNegNumber(usage, OUTPUT_KEYS);
975
- if (input === null && output === null) return null;
976
- const cached = nonNegNumber(usage, CACHED_KEYS);
977
- const result = { input: input ?? 0, output: output ?? 0 };
978
- if (cached !== null) result.cached = cached;
979
- return result;
980
- }
981
- function extractUsageFromSse(text) {
982
- let input = 0;
983
- let output = 0;
984
- let cached = 0;
985
- let sawCached = false;
986
- let found = false;
987
- for (const line of text.split(/\r?\n/)) {
988
- const trimmed = line.trim();
989
- if (!trimmed.startsWith("data:")) continue;
990
- const payload = trimmed.slice("data:".length).trim();
991
- if (!payload || payload === "[DONE]") continue;
992
- let parsed;
993
- try {
994
- parsed = JSON.parse(payload);
995
- } catch {
996
- continue;
997
- }
998
- const usage = extractUsage(parsed);
999
- if (!usage) continue;
1000
- input += usage.input;
1001
- output += usage.output;
1002
- if (usage.cached !== void 0) {
1003
- cached += usage.cached;
1004
- sawCached = true;
1005
- }
1006
- found = true;
1007
- }
1008
- if (!found) return null;
1009
- return sawCached ? { input, output, cached } : { input, output };
1010
- }
1011
- async function extractUsageFromResponse(response) {
1012
- let text;
1013
- try {
1014
- text = await response.clone().text();
1015
- } catch {
1016
- return null;
1017
- }
1018
- let json;
1019
- try {
1020
- json = JSON.parse(text);
1021
- } catch {
1022
- json = void 0;
1023
- }
1024
- return extractUsage(json) ?? extractUsageFromSse(text);
1025
- }
1026
-
1027
985
  // src/trace/capture-fetch.ts
1028
986
  var DEFAULT_BODY_CAP = 2 * 1024 * 1024;
1029
987
  function headersToRecord(headers) {
@@ -1136,7 +1094,7 @@ function captureFetchToRawSink(fetch2, sink, ctx, opts = {}) {
1136
1094
  if (opts.onUsage && rawText !== void 0) {
1137
1095
  try {
1138
1096
  const parsedForUsage = parseMaybeJson(rawText);
1139
- const usage = extractUsage(parsedForUsage) ?? (typeof parsedForUsage === "string" ? extractUsageFromSse(rawText) : null);
1097
+ const usage = extractUsage(parsedForUsage) ?? (typeof parsedForUsage === "string" ? extractUsageFromSse(rawText, { mode: opts.sseUsageMode }) : null);
1140
1098
  if (usage) opts.onUsage(usage, ctx);
1141
1099
  } catch (err) {
1142
1100
  if (opts.failClosed) throw err;
@@ -1224,6 +1182,11 @@ function flattenSpanAttributes(span) {
1224
1182
  base[LLM_MODEL_NAME] = span.model;
1225
1183
  if (span.inputTokens !== void 0) base[LLM_INPUT_TOKENS] = span.inputTokens;
1226
1184
  if (span.outputTokens !== void 0) base[LLM_OUTPUT_TOKENS] = span.outputTokens;
1185
+ if (span.reasoningTokens !== void 0) base[LLM_REASONING_TOKENS] = span.reasoningTokens;
1186
+ if (span.cachedTokens !== void 0) base[LLM_CACHED_TOKENS] = span.cachedTokens;
1187
+ if (span.cacheWriteTokens !== void 0) {
1188
+ base[LLM_CACHE_WRITE_TOKENS] = span.cacheWriteTokens;
1189
+ }
1227
1190
  if (span.costUsd !== void 0) base[LLM_COST_USD] = span.costUsd;
1228
1191
  if (span.finishReason) base["llm.finish_reason"] = span.finishReason;
1229
1192
  } else if (span.kind === "tool") {
@@ -1339,6 +1302,9 @@ function storeSpanToExportable(span, traceId) {
1339
1302
  model: llm?.model,
1340
1303
  inputTokens: llm?.inputTokens,
1341
1304
  outputTokens: llm?.outputTokens,
1305
+ reasoningTokens: llm?.reasoningTokens,
1306
+ cachedTokens: llm?.cachedTokens,
1307
+ cacheWriteTokens: llm?.cacheWriteTokens,
1342
1308
  costUsd: llm?.costUsd,
1343
1309
  tool: tool ? {
1344
1310
  toolName: tool.toolName,
@@ -1448,6 +1414,11 @@ function toOtlpSpan(span) {
1448
1414
  if (span.model) attrs[LLM_MODEL_NAME] = span.model;
1449
1415
  if (span.inputTokens !== void 0) attrs[LLM_INPUT_TOKENS] = span.inputTokens;
1450
1416
  if (span.outputTokens !== void 0) attrs[LLM_OUTPUT_TOKENS] = span.outputTokens;
1417
+ if (span.reasoningTokens !== void 0) attrs[LLM_REASONING_TOKENS] = span.reasoningTokens;
1418
+ if (span.cachedTokens !== void 0) attrs[LLM_CACHED_TOKENS] = span.cachedTokens;
1419
+ if (span.cacheWriteTokens !== void 0) {
1420
+ attrs[LLM_CACHE_WRITE_TOKENS] = span.cacheWriteTokens;
1421
+ }
1451
1422
  if (span.costUsd !== void 0) attrs[LLM_COST_USD] = span.costUsd;
1452
1423
  if (span.tool) applyToolSpanOtlpAttributes(attrs, span.tool);
1453
1424
  return {
@@ -1699,6 +1670,11 @@ function spanToAttributes(span, events) {
1699
1670
  attrs[LLM_MODEL_NAME] = span.model;
1700
1671
  if (span.inputTokens !== void 0) attrs[LLM_INPUT_TOKENS] = span.inputTokens;
1701
1672
  if (span.outputTokens !== void 0) attrs[LLM_OUTPUT_TOKENS] = span.outputTokens;
1673
+ if (span.reasoningTokens !== void 0) attrs[LLM_REASONING_TOKENS] = span.reasoningTokens;
1674
+ if (span.cachedTokens !== void 0) attrs[LLM_CACHED_TOKENS] = span.cachedTokens;
1675
+ if (span.cacheWriteTokens !== void 0) {
1676
+ attrs[LLM_CACHE_WRITE_TOKENS] = span.cacheWriteTokens;
1677
+ }
1702
1678
  if (span.costUsd !== void 0) attrs[LLM_COST_USD] = span.costUsd;
1703
1679
  if (span.finishReason) attrs["llm.finish_reason"] = span.finishReason;
1704
1680
  if (Array.isArray(span.messages)) {
@@ -1933,12 +1909,11 @@ export {
1933
1909
  buildTraceInsightPrompt,
1934
1910
  flattenOtlpExportToNdjson,
1935
1911
  otlpToRunRecords,
1912
+ otlpRowsToRunRecords,
1936
1913
  otlpToTraceRunRecords,
1914
+ otlpRowsToTraceRunRecords,
1937
1915
  InMemoryTraceStore,
1938
1916
  FileSystemTraceStore,
1939
- extractUsage,
1940
- extractUsageFromSse,
1941
- extractUsageFromResponse,
1942
1917
  captureFetchToRawSink,
1943
1918
  OTEL_AGENT_EVAL_SCOPE,
1944
1919
  exportRunAsOtlp,
@@ -1951,4 +1926,4 @@ export {
1951
1926
  createReplayFetch,
1952
1927
  iterateRawCalls
1953
1928
  };
1954
- //# sourceMappingURL=chunk-YZPO4UHR.js.map
1929
+ //# sourceMappingURL=chunk-FTUMG2U7.js.map