@tangle-network/agent-eval 0.117.0 → 0.118.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. package/CHANGELOG.md +30 -0
  2. package/dist/analyst/index.d.ts +2772 -21
  3. package/dist/analyst/index.js +7 -6
  4. package/dist/analyst/index.js.map +1 -1
  5. package/dist/belief-state/index.d.ts +706 -10
  6. package/dist/belief-state/index.js +2 -1
  7. package/dist/belief-state/index.js.map +1 -1
  8. package/dist/benchmarks/index.d.ts +958 -14
  9. package/dist/benchmarks/index.js +12 -10
  10. package/dist/builder-eval/index.d.ts +449 -4
  11. package/dist/builder-eval/index.js +4 -3
  12. package/dist/builder-eval/index.js.map +1 -1
  13. package/dist/campaign/index.d.ts +4276 -74
  14. package/dist/campaign/index.js +12 -10
  15. package/dist/{chunk-VF3XSYTI.js → chunk-33JA4TFA.js} +6 -6
  16. package/dist/{chunk-4JLWXDYA.js → chunk-3EHHMC6E.js} +2 -2
  17. package/dist/{chunk-CCZIVI3F.js → chunk-BFW56GTT.js} +2 -2
  18. package/dist/{chunk-YZPO4UHR.js → chunk-DP3BBMYJ.js} +100 -143
  19. package/dist/chunk-DP3BBMYJ.js.map +1 -0
  20. package/dist/{chunk-E4BUPP7Z.js → chunk-HKUCJ437.js} +38 -66
  21. package/dist/chunk-HKUCJ437.js.map +1 -0
  22. package/dist/{chunk-HZHNRYHK.js → chunk-K6N6XJJX.js} +2 -2
  23. package/dist/chunk-KSDQVPLR.js +286 -0
  24. package/dist/chunk-KSDQVPLR.js.map +1 -0
  25. package/dist/chunk-MA6HLL3S.js +65 -0
  26. package/dist/chunk-MA6HLL3S.js.map +1 -0
  27. package/dist/{chunk-MGEHEHSN.js → chunk-OIIMMLRB.js} +11 -11
  28. package/dist/{chunk-DXZRATT5.js → chunk-OYZAPX5G.js} +3 -3
  29. package/dist/chunk-PXE2VKMX.js +140 -0
  30. package/dist/chunk-PXE2VKMX.js.map +1 -0
  31. package/dist/{chunk-ZUXV7UWZ.js → chunk-Q442S5AS.js} +73 -19
  32. package/dist/{chunk-ZUXV7UWZ.js.map → chunk-Q442S5AS.js.map} +1 -1
  33. package/dist/{chunk-ODVOOEWQ.js → chunk-QBRSJK47.js} +2 -2
  34. package/dist/{chunk-S2F4J57L.js → chunk-QKEGNI5B.js} +77 -32
  35. package/dist/chunk-QKEGNI5B.js.map +1 -0
  36. package/dist/{chunk-5UF54T55.js → chunk-S3UZOQ5Y.js} +34 -7
  37. package/dist/chunk-S3UZOQ5Y.js.map +1 -0
  38. package/dist/{chunk-FQNLDL4D.js → chunk-SVH2ANFD.js} +136 -4
  39. package/dist/chunk-SVH2ANFD.js.map +1 -0
  40. package/dist/{chunk-GQCZRZ7L.js → chunk-U5CHZ5M3.js} +9 -9
  41. package/dist/{chunk-TVVP3ZZQ.js → chunk-VQMK5FMP.js} +2 -1
  42. package/dist/{chunk-TVVP3ZZQ.js.map → chunk-VQMK5FMP.js.map} +1 -1
  43. package/dist/chunk-WDHBCA3M.js +31 -0
  44. package/dist/chunk-WDHBCA3M.js.map +1 -0
  45. package/dist/{chunk-HQPHZGL6.js → chunk-YLMUS4MM.js} +9 -9
  46. package/dist/{chunk-LQUTGLOZ.js → chunk-ZET2UAYW.js} +15 -65
  47. package/dist/chunk-ZET2UAYW.js.map +1 -0
  48. package/dist/contract/index.d.ts +4012 -38
  49. package/dist/contract/index.js +56 -29
  50. package/dist/contract/index.js.map +1 -1
  51. package/dist/control.d.ts +1013 -9
  52. package/dist/control.js +4 -3
  53. package/dist/fuzz.d.ts +194 -4
  54. package/dist/fuzz.js +3 -3
  55. package/dist/hosted/index.d.ts +498 -17
  56. package/dist/index.d.ts +10982 -1286
  57. package/dist/index.js +97 -80
  58. package/dist/index.js.map +1 -1
  59. package/dist/matrix/index.d.ts +139 -4
  60. package/dist/meta-eval/index.d.ts +862 -15
  61. package/dist/meta-eval/index.js +2 -1
  62. package/dist/meta-eval/index.js.map +1 -1
  63. package/dist/multishot/index.d.ts +214 -14
  64. package/dist/openapi.json +1 -1
  65. package/dist/pipelines/index.d.ts +392 -7
  66. package/dist/pipelines/index.js +5 -3
  67. package/dist/pipelines/index.js.map +1 -1
  68. package/dist/reporting.d.ts +1277 -17
  69. package/dist/rl.d.ts +2359 -28
  70. package/dist/rl.js +6 -5
  71. package/dist/rl.js.map +1 -1
  72. package/dist/storyboard/index.d.ts +86 -1
  73. package/dist/trace-attributes.d.ts +16 -0
  74. package/dist/trace-attributes.js +32 -0
  75. package/dist/trace-attributes.js.map +1 -0
  76. package/dist/traces.d.ts +1970 -697
  77. package/dist/traces.js +49 -32
  78. package/dist/wire/index.d.ts +655 -9
  79. package/docs/insight-report.md +44 -0
  80. package/package.json +9 -3
  81. package/dist/adversarial-B7loGVVX.d.ts +0 -19
  82. package/dist/analyst-C8HHvfJp.d.ts +0 -88
  83. package/dist/analyze-runs--2x39HZ7.d.ts +0 -81
  84. package/dist/baseline-DKq3gJpP.d.ts +0 -141
  85. package/dist/calibration-C8MTS7cw.d.ts +0 -101
  86. package/dist/chunk-5UF54T55.js.map +0 -1
  87. package/dist/chunk-E4BUPP7Z.js.map +0 -1
  88. package/dist/chunk-FQNLDL4D.js.map +0 -1
  89. package/dist/chunk-LQUTGLOZ.js.map +0 -1
  90. package/dist/chunk-S2F4J57L.js.map +0 -1
  91. package/dist/chunk-YZPO4UHR.js.map +0 -1
  92. package/dist/code-agent-session-CjZsVd19.d.ts +0 -87
  93. package/dist/control-6vuGfmDH.d.ts +0 -258
  94. package/dist/cost-ledger-DWy3XdJc.d.ts +0 -183
  95. package/dist/dataset-NENEzRgk.d.ts +0 -115
  96. package/dist/default-registry-DaK8b3fv.d.ts +0 -155
  97. package/dist/emitter-CjD7vUwv.d.ts +0 -122
  98. package/dist/errors-oeQrLqXC.d.ts +0 -74
  99. package/dist/failure-cluster-DOAcSJ87.d.ts +0 -76
  100. package/dist/feedback-trajectory-BUnM58xL.d.ts +0 -348
  101. package/dist/gepa-eESocoDi.d.ts +0 -642
  102. package/dist/index-PdX4VnPA.d.ts +0 -423
  103. package/dist/insight-report-DY4nDW9Q.d.ts +0 -310
  104. package/dist/integrity-DqlBiLyK.d.ts +0 -81
  105. package/dist/judge-calibration-7C-IDmKr.d.ts +0 -145
  106. package/dist/kind-factory-ClZmO25A.d.ts +0 -171
  107. package/dist/llm-client-qoDd18Qz.d.ts +0 -289
  108. package/dist/multi-layer-verifier-BsqKuLyN.d.ts +0 -150
  109. package/dist/off-policy-DiwuKKg7.d.ts +0 -132
  110. package/dist/outcome-store-rnXLEqSn.d.ts +0 -63
  111. package/dist/policy-edit-wG9uFEFm.d.ts +0 -455
  112. package/dist/pre-registration-BWQhJ3vz.d.ts +0 -761
  113. package/dist/provenance-DpjwyseI.d.ts +0 -541
  114. package/dist/query-CF7PG61p.d.ts +0 -35
  115. package/dist/raw-provider-sink-C46HDghv.d.ts +0 -132
  116. package/dist/release-report-C8G2i5Xi.d.ts +0 -236
  117. package/dist/researcher-C8XyxQsu.d.ts +0 -387
  118. package/dist/rubric-predictive-validity-p49lLVrE.d.ts +0 -105
  119. package/dist/run-record-BDH49H2E.d.ts +0 -360
  120. package/dist/runtime-trajectory-DGBIUt4B.d.ts +0 -49
  121. package/dist/schema-B3Q3l9Z_.d.ts +0 -201
  122. package/dist/semantic-concept-judge-CXnPEJbf.d.ts +0 -723
  123. package/dist/sequential-5iSVfzl2.d.ts +0 -139
  124. package/dist/series-convergence-D5OWMBg6.d.ts +0 -33
  125. package/dist/statistics-KUnG73jH.d.ts +0 -494
  126. package/dist/storage-DrX3v_5B.d.ts +0 -50
  127. package/dist/store-C1YxJDEK.d.ts +0 -248
  128. package/dist/store-DGqD0Pyo.d.ts +0 -116
  129. package/dist/summary-report-C5bKFfm-.d.ts +0 -445
  130. package/dist/test-graded-scenario-B0ybnPY7.d.ts +0 -166
  131. package/dist/types-BSw1rOUB.d.ts +0 -634
  132. package/dist/types-BUxNaJ8c.d.ts +0 -108
  133. package/dist/types-BkfcQnxV.d.ts +0 -313
  134. package/dist/verdict-C9MlYujm.d.ts +0 -35
  135. /package/dist/{chunk-VF3XSYTI.js.map → chunk-33JA4TFA.js.map} +0 -0
  136. /package/dist/{chunk-4JLWXDYA.js.map → chunk-3EHHMC6E.js.map} +0 -0
  137. /package/dist/{chunk-CCZIVI3F.js.map → chunk-BFW56GTT.js.map} +0 -0
  138. /package/dist/{chunk-HZHNRYHK.js.map → chunk-K6N6XJJX.js.map} +0 -0
  139. /package/dist/{chunk-MGEHEHSN.js.map → chunk-OIIMMLRB.js.map} +0 -0
  140. /package/dist/{chunk-DXZRATT5.js.map → chunk-OYZAPX5G.js.map} +0 -0
  141. /package/dist/{chunk-ODVOOEWQ.js.map → chunk-QBRSJK47.js.map} +0 -0
  142. /package/dist/{chunk-GQCZRZ7L.js.map → chunk-U5CHZ5M3.js.map} +0 -0
  143. /package/dist/{chunk-HQPHZGL6.js.map → chunk-YLMUS4MM.js.map} +0 -0
package/dist/traces.js CHANGED
@@ -15,9 +15,6 @@ import {
15
15
  describeTraceInsightScope,
16
16
  domainEvidencePattern,
17
17
  exportRunAsOtlp,
18
- extractUsage,
19
- extractUsageFromResponse,
20
- extractUsageFromSse,
21
18
  flattenOtlpExportToNdjson,
22
19
  inferDomainKeywords,
23
20
  iterateRawCalls,
@@ -28,31 +25,30 @@ import {
28
25
  scoreTraceInsightReadiness,
29
26
  tokenizeDomainWords,
30
27
  traceAnalystOnRunComplete
31
- } from "./chunk-YZPO4UHR.js";
28
+ } from "./chunk-DP3BBMYJ.js";
29
+ import "./chunk-KSDQVPLR.js";
30
+ import {
31
+ extractUsage,
32
+ extractUsageFromResponse,
33
+ extractUsageFromSse
34
+ } from "./chunk-PXE2VKMX.js";
32
35
  import {
33
- FAILURE_CLASSES,
34
- TRACE_SCHEMA_VERSION,
35
36
  aggregateLlm,
36
37
  argHash,
37
38
  groupBy,
38
39
  hasCapturedToolArgs,
39
- isJudgeSpan,
40
- isLlmSpan,
41
- isRetrievalSpan,
42
- isSandboxSpan,
43
- isToolSpan,
44
40
  judgeSpans,
45
41
  llmSpans,
46
42
  runFailureClass,
47
43
  runsForScenario,
48
44
  toolSpans
49
- } from "./chunk-LQUTGLOZ.js";
45
+ } from "./chunk-ZET2UAYW.js";
50
46
  import {
51
47
  TRACE_ANALYST_ACTOR_DESCRIPTION,
52
48
  TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION,
53
49
  TRACE_ANALYST_SUBAGENT_DESCRIPTION,
54
50
  analyzeTraces
55
- } from "./chunk-4JLWXDYA.js";
51
+ } from "./chunk-3EHHMC6E.js";
56
52
  import {
57
53
  DEFAULT_REDACTION_RULES,
58
54
  REDACTION_VERSION,
@@ -61,25 +57,17 @@ import {
61
57
  } from "./chunk-GGE4NNQT.js";
62
58
  import {
63
59
  DEFAULT_TRACE_ANALYST_BUDGETS,
64
- INPUT_VALUE,
65
- LLM_CACHED_TOKENS,
66
60
  LLM_CACHED_TOKEN_ATTR_KEYS,
61
+ LLM_CACHE_WRITE_TOKEN_ATTR_KEYS,
67
62
  LLM_COST_ATTR_KEYS,
68
- LLM_COST_USD,
69
- LLM_INPUT_TOKENS,
70
63
  LLM_INPUT_TOKEN_ATTR_KEYS,
71
64
  LLM_MODEL_ATTR_KEYS,
72
- LLM_MODEL_NAME,
73
- LLM_OUTPUT_TOKENS,
74
65
  LLM_OUTPUT_TOKEN_ATTR_KEYS,
75
- OPENINFERENCE_SPAN_KIND,
76
- OUTPUT_VALUE,
66
+ LLM_REASONING_TOKEN_ATTR_KEYS,
77
67
  OtlpFileTraceStore,
68
+ RUN_COST_ATTR_KEYS,
78
69
  SPAN_KIND_ATTR_KEYS,
79
70
  SpanNotFoundError,
80
- TOOL_ARGS_CAPTURED,
81
- TOOL_LATENCY_MS,
82
- TOOL_NAME,
83
71
  TOOL_NAME_ATTR_KEYS,
84
72
  TRACE_ANALYST_TRUNCATION_MARKER_PREFIX,
85
73
  TraceFileMissingError,
@@ -97,19 +85,12 @@ import {
97
85
  stringField,
98
86
  traceAnalystFunctionGroup,
99
87
  traceSpanKindToOpenInferenceKind
100
- } from "./chunk-S2F4J57L.js";
88
+ } from "./chunk-QKEGNI5B.js";
101
89
  import {
102
90
  RunIntegrityError,
103
91
  assertRunCaptured,
104
92
  throwIfRunIncomplete
105
93
  } from "./chunk-TT4KNT67.js";
106
- import {
107
- TraceEmitter,
108
- llmSpanFromProvider
109
- } from "./chunk-TVVP3ZZQ.js";
110
- import "./chunk-5UF54T55.js";
111
- import "./chunk-XJYR7XFV.js";
112
- import "./chunk-VSMTAMNK.js";
113
94
  import {
114
95
  FileSystemRawProviderSink,
115
96
  InMemoryRawProviderSink,
@@ -117,7 +98,38 @@ import {
117
98
  defaultProviderRedactor,
118
99
  providerFromBaseUrl
119
100
  } from "./chunk-PC4UYEBM.js";
101
+ import {
102
+ TraceEmitter,
103
+ llmSpanFromProvider
104
+ } from "./chunk-VQMK5FMP.js";
105
+ import "./chunk-S3UZOQ5Y.js";
106
+ import {
107
+ FAILURE_CLASSES,
108
+ TRACE_SCHEMA_VERSION,
109
+ isJudgeSpan,
110
+ isLlmSpan,
111
+ isRetrievalSpan,
112
+ isSandboxSpan,
113
+ isToolSpan
114
+ } from "./chunk-MA6HLL3S.js";
115
+ import "./chunk-XJYR7XFV.js";
116
+ import "./chunk-VSMTAMNK.js";
120
117
  import "./chunk-ONWEPEDO.js";
118
+ import {
119
+ INPUT_VALUE,
120
+ LLM_CACHED_TOKENS,
121
+ LLM_CACHE_WRITE_TOKENS,
122
+ LLM_COST_USD,
123
+ LLM_INPUT_TOKENS,
124
+ LLM_MODEL_NAME,
125
+ LLM_OUTPUT_TOKENS,
126
+ LLM_REASONING_TOKENS,
127
+ OPENINFERENCE_SPAN_KIND,
128
+ OUTPUT_VALUE,
129
+ TOOL_ARGS_CAPTURED,
130
+ TOOL_LATENCY_MS,
131
+ TOOL_NAME
132
+ } from "./chunk-WDHBCA3M.js";
121
133
  import "./chunk-PZ5AY32C.js";
122
134
  export {
123
135
  DEFAULT_REDACTION_RULES,
@@ -130,6 +142,8 @@ export {
130
142
  InMemoryTraceStore,
131
143
  LLM_CACHED_TOKENS,
132
144
  LLM_CACHED_TOKEN_ATTR_KEYS,
145
+ LLM_CACHE_WRITE_TOKENS,
146
+ LLM_CACHE_WRITE_TOKEN_ATTR_KEYS,
133
147
  LLM_COST_ATTR_KEYS,
134
148
  LLM_COST_USD,
135
149
  LLM_INPUT_TOKENS,
@@ -138,12 +152,15 @@ export {
138
152
  LLM_MODEL_NAME,
139
153
  LLM_OUTPUT_TOKENS,
140
154
  LLM_OUTPUT_TOKEN_ATTR_KEYS,
155
+ LLM_REASONING_TOKENS,
156
+ LLM_REASONING_TOKEN_ATTR_KEYS,
141
157
  NoopRawProviderSink,
142
158
  OPENINFERENCE_SPAN_KIND,
143
159
  OTEL_AGENT_EVAL_SCOPE,
144
160
  OUTPUT_VALUE,
145
161
  OtlpFileTraceStore,
146
162
  REDACTION_VERSION,
163
+ RUN_COST_ATTR_KEYS,
147
164
  ReplayCache,
148
165
  ReplayCacheMissError,
149
166
  RunIntegrityError,