@tangle-network/agent-eval 0.117.1 → 0.118.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. package/CHANGELOG.md +30 -0
  2. package/dist/analyst/index.d.ts +2772 -21
  3. package/dist/analyst/index.js +7 -6
  4. package/dist/analyst/index.js.map +1 -1
  5. package/dist/belief-state/index.d.ts +706 -10
  6. package/dist/belief-state/index.js +2 -1
  7. package/dist/belief-state/index.js.map +1 -1
  8. package/dist/benchmarks/index.d.ts +958 -14
  9. package/dist/benchmarks/index.js +12 -10
  10. package/dist/builder-eval/index.d.ts +449 -4
  11. package/dist/builder-eval/index.js +4 -3
  12. package/dist/builder-eval/index.js.map +1 -1
  13. package/dist/campaign/index.d.ts +4275 -73
  14. package/dist/campaign/index.js +12 -10
  15. package/dist/{chunk-VF3XSYTI.js → chunk-33JA4TFA.js} +6 -6
  16. package/dist/{chunk-4JLWXDYA.js → chunk-3EHHMC6E.js} +2 -2
  17. package/dist/{chunk-CCZIVI3F.js → chunk-BFW56GTT.js} +2 -2
  18. package/dist/{chunk-YZPO4UHR.js → chunk-FTUMG2U7.js} +124 -149
  19. package/dist/chunk-FTUMG2U7.js.map +1 -0
  20. package/dist/{chunk-E4BUPP7Z.js → chunk-HKUCJ437.js} +38 -66
  21. package/dist/chunk-HKUCJ437.js.map +1 -0
  22. package/dist/{chunk-HZHNRYHK.js → chunk-K6N6XJJX.js} +2 -2
  23. package/dist/chunk-KSDQVPLR.js +286 -0
  24. package/dist/chunk-KSDQVPLR.js.map +1 -0
  25. package/dist/chunk-MA6HLL3S.js +65 -0
  26. package/dist/chunk-MA6HLL3S.js.map +1 -0
  27. package/dist/{chunk-MGEHEHSN.js → chunk-OIIMMLRB.js} +11 -11
  28. package/dist/{chunk-DXZRATT5.js → chunk-OYZAPX5G.js} +3 -3
  29. package/dist/chunk-PXE2VKMX.js +140 -0
  30. package/dist/chunk-PXE2VKMX.js.map +1 -0
  31. package/dist/{chunk-JSJZ4PJ6.js → chunk-Q442S5AS.js} +17 -17
  32. package/dist/{chunk-ODVOOEWQ.js → chunk-QBRSJK47.js} +2 -2
  33. package/dist/{chunk-S2F4J57L.js → chunk-QKEGNI5B.js} +77 -32
  34. package/dist/chunk-QKEGNI5B.js.map +1 -0
  35. package/dist/{chunk-5UF54T55.js → chunk-S3UZOQ5Y.js} +34 -7
  36. package/dist/chunk-S3UZOQ5Y.js.map +1 -0
  37. package/dist/{chunk-FQNLDL4D.js → chunk-SVH2ANFD.js} +136 -4
  38. package/dist/chunk-SVH2ANFD.js.map +1 -0
  39. package/dist/{chunk-GQCZRZ7L.js → chunk-U5CHZ5M3.js} +9 -9
  40. package/dist/{chunk-TVVP3ZZQ.js → chunk-VQMK5FMP.js} +2 -1
  41. package/dist/{chunk-TVVP3ZZQ.js.map → chunk-VQMK5FMP.js.map} +1 -1
  42. package/dist/chunk-WDHBCA3M.js +31 -0
  43. package/dist/chunk-WDHBCA3M.js.map +1 -0
  44. package/dist/{chunk-HQPHZGL6.js → chunk-YLMUS4MM.js} +9 -9
  45. package/dist/{chunk-LQUTGLOZ.js → chunk-ZET2UAYW.js} +15 -65
  46. package/dist/chunk-ZET2UAYW.js.map +1 -0
  47. package/dist/contract/index.d.ts +4012 -38
  48. package/dist/contract/index.js +56 -29
  49. package/dist/contract/index.js.map +1 -1
  50. package/dist/control.d.ts +1013 -9
  51. package/dist/control.js +4 -3
  52. package/dist/fuzz.d.ts +194 -4
  53. package/dist/fuzz.js +3 -3
  54. package/dist/hosted/index.d.ts +498 -17
  55. package/dist/index.d.ts +10992 -1288
  56. package/dist/index.js +101 -80
  57. package/dist/index.js.map +1 -1
  58. package/dist/matrix/index.d.ts +139 -4
  59. package/dist/meta-eval/index.d.ts +862 -15
  60. package/dist/meta-eval/index.js +2 -1
  61. package/dist/meta-eval/index.js.map +1 -1
  62. package/dist/multishot/index.d.ts +214 -14
  63. package/dist/openapi.json +1 -1
  64. package/dist/pipelines/index.d.ts +392 -7
  65. package/dist/pipelines/index.js +5 -3
  66. package/dist/pipelines/index.js.map +1 -1
  67. package/dist/reporting.d.ts +1277 -17
  68. package/dist/rl.d.ts +2359 -28
  69. package/dist/rl.js +6 -5
  70. package/dist/rl.js.map +1 -1
  71. package/dist/storyboard/index.d.ts +86 -1
  72. package/dist/trace-attributes.d.ts +16 -0
  73. package/dist/trace-attributes.js +32 -0
  74. package/dist/trace-attributes.js.map +1 -0
  75. package/dist/traces.d.ts +1978 -697
  76. package/dist/traces.js +53 -32
  77. package/dist/wire/index.d.ts +655 -9
  78. package/docs/insight-report.md +44 -0
  79. package/package.json +9 -3
  80. package/dist/adversarial-B7loGVVX.d.ts +0 -19
  81. package/dist/analyst-C8HHvfJp.d.ts +0 -88
  82. package/dist/analyze-runs--2x39HZ7.d.ts +0 -81
  83. package/dist/baseline-DKq3gJpP.d.ts +0 -141
  84. package/dist/calibration-C8MTS7cw.d.ts +0 -101
  85. package/dist/chunk-5UF54T55.js.map +0 -1
  86. package/dist/chunk-E4BUPP7Z.js.map +0 -1
  87. package/dist/chunk-FQNLDL4D.js.map +0 -1
  88. package/dist/chunk-LQUTGLOZ.js.map +0 -1
  89. package/dist/chunk-S2F4J57L.js.map +0 -1
  90. package/dist/chunk-YZPO4UHR.js.map +0 -1
  91. package/dist/code-agent-session-CjZsVd19.d.ts +0 -87
  92. package/dist/control-6vuGfmDH.d.ts +0 -258
  93. package/dist/cost-ledger-DWy3XdJc.d.ts +0 -183
  94. package/dist/dataset-NENEzRgk.d.ts +0 -115
  95. package/dist/default-registry-DaK8b3fv.d.ts +0 -155
  96. package/dist/emitter-CjD7vUwv.d.ts +0 -122
  97. package/dist/errors-oeQrLqXC.d.ts +0 -74
  98. package/dist/failure-cluster-DOAcSJ87.d.ts +0 -76
  99. package/dist/feedback-trajectory-BUnM58xL.d.ts +0 -348
  100. package/dist/gepa-eESocoDi.d.ts +0 -642
  101. package/dist/index-PdX4VnPA.d.ts +0 -423
  102. package/dist/insight-report-DY4nDW9Q.d.ts +0 -310
  103. package/dist/integrity-DqlBiLyK.d.ts +0 -81
  104. package/dist/judge-calibration-7C-IDmKr.d.ts +0 -145
  105. package/dist/kind-factory-ClZmO25A.d.ts +0 -171
  106. package/dist/llm-client-qoDd18Qz.d.ts +0 -289
  107. package/dist/multi-layer-verifier-BsqKuLyN.d.ts +0 -150
  108. package/dist/off-policy-DiwuKKg7.d.ts +0 -132
  109. package/dist/outcome-store-rnXLEqSn.d.ts +0 -63
  110. package/dist/policy-edit-wG9uFEFm.d.ts +0 -455
  111. package/dist/pre-registration-BWQhJ3vz.d.ts +0 -761
  112. package/dist/provenance-DpjwyseI.d.ts +0 -541
  113. package/dist/query-CF7PG61p.d.ts +0 -35
  114. package/dist/raw-provider-sink-C46HDghv.d.ts +0 -132
  115. package/dist/release-report-C8G2i5Xi.d.ts +0 -236
  116. package/dist/researcher-C8XyxQsu.d.ts +0 -387
  117. package/dist/rubric-predictive-validity-p49lLVrE.d.ts +0 -105
  118. package/dist/run-record-BDH49H2E.d.ts +0 -360
  119. package/dist/runtime-trajectory-DGBIUt4B.d.ts +0 -49
  120. package/dist/schema-B3Q3l9Z_.d.ts +0 -201
  121. package/dist/semantic-concept-judge-CXnPEJbf.d.ts +0 -723
  122. package/dist/sequential-5iSVfzl2.d.ts +0 -139
  123. package/dist/series-convergence-D5OWMBg6.d.ts +0 -33
  124. package/dist/statistics-KUnG73jH.d.ts +0 -494
  125. package/dist/storage-DrX3v_5B.d.ts +0 -50
  126. package/dist/store-C1YxJDEK.d.ts +0 -248
  127. package/dist/store-DGqD0Pyo.d.ts +0 -116
  128. package/dist/summary-report-C5bKFfm-.d.ts +0 -445
  129. package/dist/test-graded-scenario-B0ybnPY7.d.ts +0 -166
  130. package/dist/types-BSw1rOUB.d.ts +0 -634
  131. package/dist/types-BUxNaJ8c.d.ts +0 -108
  132. package/dist/types-BkfcQnxV.d.ts +0 -313
  133. package/dist/verdict-C9MlYujm.d.ts +0 -35
  134. /package/dist/{chunk-VF3XSYTI.js.map → chunk-33JA4TFA.js.map} +0 -0
  135. /package/dist/{chunk-4JLWXDYA.js.map → chunk-3EHHMC6E.js.map} +0 -0
  136. /package/dist/{chunk-CCZIVI3F.js.map → chunk-BFW56GTT.js.map} +0 -0
  137. /package/dist/{chunk-HZHNRYHK.js.map → chunk-K6N6XJJX.js.map} +0 -0
  138. /package/dist/{chunk-MGEHEHSN.js.map → chunk-OIIMMLRB.js.map} +0 -0
  139. /package/dist/{chunk-DXZRATT5.js.map → chunk-OYZAPX5G.js.map} +0 -0
  140. /package/dist/{chunk-JSJZ4PJ6.js.map → chunk-Q442S5AS.js.map} +0 -0
  141. /package/dist/{chunk-ODVOOEWQ.js.map → chunk-QBRSJK47.js.map} +0 -0
  142. /package/dist/{chunk-GQCZRZ7L.js.map → chunk-U5CHZ5M3.js.map} +0 -0
  143. /package/dist/{chunk-HQPHZGL6.js.map → chunk-YLMUS4MM.js.map} +0 -0
package/dist/traces.js CHANGED
@@ -15,44 +15,42 @@ import {
15
15
  describeTraceInsightScope,
16
16
  domainEvidencePattern,
17
17
  exportRunAsOtlp,
18
- extractUsage,
19
- extractUsageFromResponse,
20
- extractUsageFromSse,
21
18
  flattenOtlpExportToNdjson,
22
19
  inferDomainKeywords,
23
20
  iterateRawCalls,
24
21
  otelRunCompleteHook,
22
+ otlpRowsToRunRecords,
23
+ otlpRowsToTraceRunRecords,
25
24
  otlpToRunRecords,
26
25
  otlpToTraceRunRecords,
27
26
  planTraceInsightQuestions,
28
27
  scoreTraceInsightReadiness,
29
28
  tokenizeDomainWords,
30
29
  traceAnalystOnRunComplete
31
- } from "./chunk-YZPO4UHR.js";
30
+ } from "./chunk-FTUMG2U7.js";
31
+ import "./chunk-KSDQVPLR.js";
32
+ import {
33
+ extractUsage,
34
+ extractUsageFromResponse,
35
+ extractUsageFromSse
36
+ } from "./chunk-PXE2VKMX.js";
32
37
  import {
33
- FAILURE_CLASSES,
34
- TRACE_SCHEMA_VERSION,
35
38
  aggregateLlm,
36
39
  argHash,
37
40
  groupBy,
38
41
  hasCapturedToolArgs,
39
- isJudgeSpan,
40
- isLlmSpan,
41
- isRetrievalSpan,
42
- isSandboxSpan,
43
- isToolSpan,
44
42
  judgeSpans,
45
43
  llmSpans,
46
44
  runFailureClass,
47
45
  runsForScenario,
48
46
  toolSpans
49
- } from "./chunk-LQUTGLOZ.js";
47
+ } from "./chunk-ZET2UAYW.js";
50
48
  import {
51
49
  TRACE_ANALYST_ACTOR_DESCRIPTION,
52
50
  TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION,
53
51
  TRACE_ANALYST_SUBAGENT_DESCRIPTION,
54
52
  analyzeTraces
55
- } from "./chunk-4JLWXDYA.js";
53
+ } from "./chunk-3EHHMC6E.js";
56
54
  import {
57
55
  DEFAULT_REDACTION_RULES,
58
56
  REDACTION_VERSION,
@@ -61,25 +59,17 @@ import {
61
59
  } from "./chunk-GGE4NNQT.js";
62
60
  import {
63
61
  DEFAULT_TRACE_ANALYST_BUDGETS,
64
- INPUT_VALUE,
65
- LLM_CACHED_TOKENS,
66
62
  LLM_CACHED_TOKEN_ATTR_KEYS,
63
+ LLM_CACHE_WRITE_TOKEN_ATTR_KEYS,
67
64
  LLM_COST_ATTR_KEYS,
68
- LLM_COST_USD,
69
- LLM_INPUT_TOKENS,
70
65
  LLM_INPUT_TOKEN_ATTR_KEYS,
71
66
  LLM_MODEL_ATTR_KEYS,
72
- LLM_MODEL_NAME,
73
- LLM_OUTPUT_TOKENS,
74
67
  LLM_OUTPUT_TOKEN_ATTR_KEYS,
75
- OPENINFERENCE_SPAN_KIND,
76
- OUTPUT_VALUE,
68
+ LLM_REASONING_TOKEN_ATTR_KEYS,
77
69
  OtlpFileTraceStore,
70
+ RUN_COST_ATTR_KEYS,
78
71
  SPAN_KIND_ATTR_KEYS,
79
72
  SpanNotFoundError,
80
- TOOL_ARGS_CAPTURED,
81
- TOOL_LATENCY_MS,
82
- TOOL_NAME,
83
73
  TOOL_NAME_ATTR_KEYS,
84
74
  TRACE_ANALYST_TRUNCATION_MARKER_PREFIX,
85
75
  TraceFileMissingError,
@@ -97,19 +87,12 @@ import {
97
87
  stringField,
98
88
  traceAnalystFunctionGroup,
99
89
  traceSpanKindToOpenInferenceKind
100
- } from "./chunk-S2F4J57L.js";
90
+ } from "./chunk-QKEGNI5B.js";
101
91
  import {
102
92
  RunIntegrityError,
103
93
  assertRunCaptured,
104
94
  throwIfRunIncomplete
105
95
  } from "./chunk-TT4KNT67.js";
106
- import {
107
- TraceEmitter,
108
- llmSpanFromProvider
109
- } from "./chunk-TVVP3ZZQ.js";
110
- import "./chunk-5UF54T55.js";
111
- import "./chunk-XJYR7XFV.js";
112
- import "./chunk-VSMTAMNK.js";
113
96
  import {
114
97
  FileSystemRawProviderSink,
115
98
  InMemoryRawProviderSink,
@@ -117,7 +100,38 @@ import {
117
100
  defaultProviderRedactor,
118
101
  providerFromBaseUrl
119
102
  } from "./chunk-PC4UYEBM.js";
103
+ import {
104
+ TraceEmitter,
105
+ llmSpanFromProvider
106
+ } from "./chunk-VQMK5FMP.js";
107
+ import "./chunk-S3UZOQ5Y.js";
108
+ import {
109
+ FAILURE_CLASSES,
110
+ TRACE_SCHEMA_VERSION,
111
+ isJudgeSpan,
112
+ isLlmSpan,
113
+ isRetrievalSpan,
114
+ isSandboxSpan,
115
+ isToolSpan
116
+ } from "./chunk-MA6HLL3S.js";
117
+ import "./chunk-XJYR7XFV.js";
118
+ import "./chunk-VSMTAMNK.js";
120
119
  import "./chunk-ONWEPEDO.js";
120
+ import {
121
+ INPUT_VALUE,
122
+ LLM_CACHED_TOKENS,
123
+ LLM_CACHE_WRITE_TOKENS,
124
+ LLM_COST_USD,
125
+ LLM_INPUT_TOKENS,
126
+ LLM_MODEL_NAME,
127
+ LLM_OUTPUT_TOKENS,
128
+ LLM_REASONING_TOKENS,
129
+ OPENINFERENCE_SPAN_KIND,
130
+ OUTPUT_VALUE,
131
+ TOOL_ARGS_CAPTURED,
132
+ TOOL_LATENCY_MS,
133
+ TOOL_NAME
134
+ } from "./chunk-WDHBCA3M.js";
121
135
  import "./chunk-PZ5AY32C.js";
122
136
  export {
123
137
  DEFAULT_REDACTION_RULES,
@@ -130,6 +144,8 @@ export {
130
144
  InMemoryTraceStore,
131
145
  LLM_CACHED_TOKENS,
132
146
  LLM_CACHED_TOKEN_ATTR_KEYS,
147
+ LLM_CACHE_WRITE_TOKENS,
148
+ LLM_CACHE_WRITE_TOKEN_ATTR_KEYS,
133
149
  LLM_COST_ATTR_KEYS,
134
150
  LLM_COST_USD,
135
151
  LLM_INPUT_TOKENS,
@@ -138,12 +154,15 @@ export {
138
154
  LLM_MODEL_NAME,
139
155
  LLM_OUTPUT_TOKENS,
140
156
  LLM_OUTPUT_TOKEN_ATTR_KEYS,
157
+ LLM_REASONING_TOKENS,
158
+ LLM_REASONING_TOKEN_ATTR_KEYS,
141
159
  NoopRawProviderSink,
142
160
  OPENINFERENCE_SPAN_KIND,
143
161
  OTEL_AGENT_EVAL_SCOPE,
144
162
  OUTPUT_VALUE,
145
163
  OtlpFileTraceStore,
146
164
  REDACTION_VERSION,
165
+ RUN_COST_ATTR_KEYS,
147
166
  ReplayCache,
148
167
  ReplayCacheMissError,
149
168
  RunIntegrityError,
@@ -202,6 +221,8 @@ export {
202
221
  llmSpanFromProvider,
203
222
  llmSpans,
204
223
  otelRunCompleteHook,
224
+ otlpRowsToRunRecords,
225
+ otlpRowsToTraceRunRecords,
205
226
  otlpToRunRecords,
206
227
  otlpToTraceRunRecords,
207
228
  planTraceInsightQuestions,