@tangle-network/agent-eval 0.117.1 → 0.118.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. package/CHANGELOG.md +23 -0
  2. package/dist/analyst/index.d.ts +2772 -21
  3. package/dist/analyst/index.js +7 -6
  4. package/dist/analyst/index.js.map +1 -1
  5. package/dist/belief-state/index.d.ts +706 -10
  6. package/dist/belief-state/index.js +2 -1
  7. package/dist/belief-state/index.js.map +1 -1
  8. package/dist/benchmarks/index.d.ts +958 -14
  9. package/dist/benchmarks/index.js +12 -10
  10. package/dist/builder-eval/index.d.ts +449 -4
  11. package/dist/builder-eval/index.js +4 -3
  12. package/dist/builder-eval/index.js.map +1 -1
  13. package/dist/campaign/index.d.ts +4275 -73
  14. package/dist/campaign/index.js +12 -10
  15. package/dist/{chunk-VF3XSYTI.js → chunk-33JA4TFA.js} +6 -6
  16. package/dist/{chunk-4JLWXDYA.js → chunk-3EHHMC6E.js} +2 -2
  17. package/dist/{chunk-CCZIVI3F.js → chunk-BFW56GTT.js} +2 -2
  18. package/dist/{chunk-YZPO4UHR.js → chunk-DP3BBMYJ.js} +100 -143
  19. package/dist/chunk-DP3BBMYJ.js.map +1 -0
  20. package/dist/{chunk-E4BUPP7Z.js → chunk-HKUCJ437.js} +38 -66
  21. package/dist/chunk-HKUCJ437.js.map +1 -0
  22. package/dist/{chunk-HZHNRYHK.js → chunk-K6N6XJJX.js} +2 -2
  23. package/dist/chunk-KSDQVPLR.js +286 -0
  24. package/dist/chunk-KSDQVPLR.js.map +1 -0
  25. package/dist/chunk-MA6HLL3S.js +65 -0
  26. package/dist/chunk-MA6HLL3S.js.map +1 -0
  27. package/dist/{chunk-MGEHEHSN.js → chunk-OIIMMLRB.js} +11 -11
  28. package/dist/{chunk-DXZRATT5.js → chunk-OYZAPX5G.js} +3 -3
  29. package/dist/chunk-PXE2VKMX.js +140 -0
  30. package/dist/chunk-PXE2VKMX.js.map +1 -0
  31. package/dist/{chunk-JSJZ4PJ6.js → chunk-Q442S5AS.js} +17 -17
  32. package/dist/{chunk-ODVOOEWQ.js → chunk-QBRSJK47.js} +2 -2
  33. package/dist/{chunk-S2F4J57L.js → chunk-QKEGNI5B.js} +77 -32
  34. package/dist/chunk-QKEGNI5B.js.map +1 -0
  35. package/dist/{chunk-5UF54T55.js → chunk-S3UZOQ5Y.js} +34 -7
  36. package/dist/chunk-S3UZOQ5Y.js.map +1 -0
  37. package/dist/{chunk-FQNLDL4D.js → chunk-SVH2ANFD.js} +136 -4
  38. package/dist/chunk-SVH2ANFD.js.map +1 -0
  39. package/dist/{chunk-GQCZRZ7L.js → chunk-U5CHZ5M3.js} +9 -9
  40. package/dist/{chunk-TVVP3ZZQ.js → chunk-VQMK5FMP.js} +2 -1
  41. package/dist/{chunk-TVVP3ZZQ.js.map → chunk-VQMK5FMP.js.map} +1 -1
  42. package/dist/chunk-WDHBCA3M.js +31 -0
  43. package/dist/chunk-WDHBCA3M.js.map +1 -0
  44. package/dist/{chunk-HQPHZGL6.js → chunk-YLMUS4MM.js} +9 -9
  45. package/dist/{chunk-LQUTGLOZ.js → chunk-ZET2UAYW.js} +15 -65
  46. package/dist/chunk-ZET2UAYW.js.map +1 -0
  47. package/dist/contract/index.d.ts +4012 -38
  48. package/dist/contract/index.js +56 -29
  49. package/dist/contract/index.js.map +1 -1
  50. package/dist/control.d.ts +1013 -9
  51. package/dist/control.js +4 -3
  52. package/dist/fuzz.d.ts +194 -4
  53. package/dist/fuzz.js +3 -3
  54. package/dist/hosted/index.d.ts +498 -17
  55. package/dist/index.d.ts +10982 -1286
  56. package/dist/index.js +97 -80
  57. package/dist/index.js.map +1 -1
  58. package/dist/matrix/index.d.ts +139 -4
  59. package/dist/meta-eval/index.d.ts +862 -15
  60. package/dist/meta-eval/index.js +2 -1
  61. package/dist/meta-eval/index.js.map +1 -1
  62. package/dist/multishot/index.d.ts +214 -14
  63. package/dist/openapi.json +1 -1
  64. package/dist/pipelines/index.d.ts +392 -7
  65. package/dist/pipelines/index.js +5 -3
  66. package/dist/pipelines/index.js.map +1 -1
  67. package/dist/reporting.d.ts +1277 -17
  68. package/dist/rl.d.ts +2359 -28
  69. package/dist/rl.js +6 -5
  70. package/dist/rl.js.map +1 -1
  71. package/dist/storyboard/index.d.ts +86 -1
  72. package/dist/trace-attributes.d.ts +16 -0
  73. package/dist/trace-attributes.js +32 -0
  74. package/dist/trace-attributes.js.map +1 -0
  75. package/dist/traces.d.ts +1970 -697
  76. package/dist/traces.js +49 -32
  77. package/dist/wire/index.d.ts +655 -9
  78. package/docs/insight-report.md +44 -0
  79. package/package.json +9 -3
  80. package/dist/adversarial-B7loGVVX.d.ts +0 -19
  81. package/dist/analyst-C8HHvfJp.d.ts +0 -88
  82. package/dist/analyze-runs--2x39HZ7.d.ts +0 -81
  83. package/dist/baseline-DKq3gJpP.d.ts +0 -141
  84. package/dist/calibration-C8MTS7cw.d.ts +0 -101
  85. package/dist/chunk-5UF54T55.js.map +0 -1
  86. package/dist/chunk-E4BUPP7Z.js.map +0 -1
  87. package/dist/chunk-FQNLDL4D.js.map +0 -1
  88. package/dist/chunk-LQUTGLOZ.js.map +0 -1
  89. package/dist/chunk-S2F4J57L.js.map +0 -1
  90. package/dist/chunk-YZPO4UHR.js.map +0 -1
  91. package/dist/code-agent-session-CjZsVd19.d.ts +0 -87
  92. package/dist/control-6vuGfmDH.d.ts +0 -258
  93. package/dist/cost-ledger-DWy3XdJc.d.ts +0 -183
  94. package/dist/dataset-NENEzRgk.d.ts +0 -115
  95. package/dist/default-registry-DaK8b3fv.d.ts +0 -155
  96. package/dist/emitter-CjD7vUwv.d.ts +0 -122
  97. package/dist/errors-oeQrLqXC.d.ts +0 -74
  98. package/dist/failure-cluster-DOAcSJ87.d.ts +0 -76
  99. package/dist/feedback-trajectory-BUnM58xL.d.ts +0 -348
  100. package/dist/gepa-eESocoDi.d.ts +0 -642
  101. package/dist/index-PdX4VnPA.d.ts +0 -423
  102. package/dist/insight-report-DY4nDW9Q.d.ts +0 -310
  103. package/dist/integrity-DqlBiLyK.d.ts +0 -81
  104. package/dist/judge-calibration-7C-IDmKr.d.ts +0 -145
  105. package/dist/kind-factory-ClZmO25A.d.ts +0 -171
  106. package/dist/llm-client-qoDd18Qz.d.ts +0 -289
  107. package/dist/multi-layer-verifier-BsqKuLyN.d.ts +0 -150
  108. package/dist/off-policy-DiwuKKg7.d.ts +0 -132
  109. package/dist/outcome-store-rnXLEqSn.d.ts +0 -63
  110. package/dist/policy-edit-wG9uFEFm.d.ts +0 -455
  111. package/dist/pre-registration-BWQhJ3vz.d.ts +0 -761
  112. package/dist/provenance-DpjwyseI.d.ts +0 -541
  113. package/dist/query-CF7PG61p.d.ts +0 -35
  114. package/dist/raw-provider-sink-C46HDghv.d.ts +0 -132
  115. package/dist/release-report-C8G2i5Xi.d.ts +0 -236
  116. package/dist/researcher-C8XyxQsu.d.ts +0 -387
  117. package/dist/rubric-predictive-validity-p49lLVrE.d.ts +0 -105
  118. package/dist/run-record-BDH49H2E.d.ts +0 -360
  119. package/dist/runtime-trajectory-DGBIUt4B.d.ts +0 -49
  120. package/dist/schema-B3Q3l9Z_.d.ts +0 -201
  121. package/dist/semantic-concept-judge-CXnPEJbf.d.ts +0 -723
  122. package/dist/sequential-5iSVfzl2.d.ts +0 -139
  123. package/dist/series-convergence-D5OWMBg6.d.ts +0 -33
  124. package/dist/statistics-KUnG73jH.d.ts +0 -494
  125. package/dist/storage-DrX3v_5B.d.ts +0 -50
  126. package/dist/store-C1YxJDEK.d.ts +0 -248
  127. package/dist/store-DGqD0Pyo.d.ts +0 -116
  128. package/dist/summary-report-C5bKFfm-.d.ts +0 -445
  129. package/dist/test-graded-scenario-B0ybnPY7.d.ts +0 -166
  130. package/dist/types-BSw1rOUB.d.ts +0 -634
  131. package/dist/types-BUxNaJ8c.d.ts +0 -108
  132. package/dist/types-BkfcQnxV.d.ts +0 -313
  133. package/dist/verdict-C9MlYujm.d.ts +0 -35
  134. /package/dist/{chunk-VF3XSYTI.js.map → chunk-33JA4TFA.js.map} +0 -0
  135. /package/dist/{chunk-4JLWXDYA.js.map → chunk-3EHHMC6E.js.map} +0 -0
  136. /package/dist/{chunk-CCZIVI3F.js.map → chunk-BFW56GTT.js.map} +0 -0
  137. /package/dist/{chunk-HZHNRYHK.js.map → chunk-K6N6XJJX.js.map} +0 -0
  138. /package/dist/{chunk-MGEHEHSN.js.map → chunk-OIIMMLRB.js.map} +0 -0
  139. /package/dist/{chunk-DXZRATT5.js.map → chunk-OYZAPX5G.js.map} +0 -0
  140. /package/dist/{chunk-JSJZ4PJ6.js.map → chunk-Q442S5AS.js.map} +0 -0
  141. /package/dist/{chunk-ODVOOEWQ.js.map → chunk-QBRSJK47.js.map} +0 -0
  142. /package/dist/{chunk-GQCZRZ7L.js.map → chunk-U5CHZ5M3.js.map} +0 -0
  143. /package/dist/{chunk-HQPHZGL6.js.map → chunk-YLMUS4MM.js.map} +0 -0
package/dist/traces.js CHANGED
@@ -15,9 +15,6 @@ import {
15
15
  describeTraceInsightScope,
16
16
  domainEvidencePattern,
17
17
  exportRunAsOtlp,
18
- extractUsage,
19
- extractUsageFromResponse,
20
- extractUsageFromSse,
21
18
  flattenOtlpExportToNdjson,
22
19
  inferDomainKeywords,
23
20
  iterateRawCalls,
@@ -28,31 +25,30 @@ import {
28
25
  scoreTraceInsightReadiness,
29
26
  tokenizeDomainWords,
30
27
  traceAnalystOnRunComplete
31
- } from "./chunk-YZPO4UHR.js";
28
+ } from "./chunk-DP3BBMYJ.js";
29
+ import "./chunk-KSDQVPLR.js";
30
+ import {
31
+ extractUsage,
32
+ extractUsageFromResponse,
33
+ extractUsageFromSse
34
+ } from "./chunk-PXE2VKMX.js";
32
35
  import {
33
- FAILURE_CLASSES,
34
- TRACE_SCHEMA_VERSION,
35
36
  aggregateLlm,
36
37
  argHash,
37
38
  groupBy,
38
39
  hasCapturedToolArgs,
39
- isJudgeSpan,
40
- isLlmSpan,
41
- isRetrievalSpan,
42
- isSandboxSpan,
43
- isToolSpan,
44
40
  judgeSpans,
45
41
  llmSpans,
46
42
  runFailureClass,
47
43
  runsForScenario,
48
44
  toolSpans
49
- } from "./chunk-LQUTGLOZ.js";
45
+ } from "./chunk-ZET2UAYW.js";
50
46
  import {
51
47
  TRACE_ANALYST_ACTOR_DESCRIPTION,
52
48
  TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION,
53
49
  TRACE_ANALYST_SUBAGENT_DESCRIPTION,
54
50
  analyzeTraces
55
- } from "./chunk-4JLWXDYA.js";
51
+ } from "./chunk-3EHHMC6E.js";
56
52
  import {
57
53
  DEFAULT_REDACTION_RULES,
58
54
  REDACTION_VERSION,
@@ -61,25 +57,17 @@ import {
61
57
  } from "./chunk-GGE4NNQT.js";
62
58
  import {
63
59
  DEFAULT_TRACE_ANALYST_BUDGETS,
64
- INPUT_VALUE,
65
- LLM_CACHED_TOKENS,
66
60
  LLM_CACHED_TOKEN_ATTR_KEYS,
61
+ LLM_CACHE_WRITE_TOKEN_ATTR_KEYS,
67
62
  LLM_COST_ATTR_KEYS,
68
- LLM_COST_USD,
69
- LLM_INPUT_TOKENS,
70
63
  LLM_INPUT_TOKEN_ATTR_KEYS,
71
64
  LLM_MODEL_ATTR_KEYS,
72
- LLM_MODEL_NAME,
73
- LLM_OUTPUT_TOKENS,
74
65
  LLM_OUTPUT_TOKEN_ATTR_KEYS,
75
- OPENINFERENCE_SPAN_KIND,
76
- OUTPUT_VALUE,
66
+ LLM_REASONING_TOKEN_ATTR_KEYS,
77
67
  OtlpFileTraceStore,
68
+ RUN_COST_ATTR_KEYS,
78
69
  SPAN_KIND_ATTR_KEYS,
79
70
  SpanNotFoundError,
80
- TOOL_ARGS_CAPTURED,
81
- TOOL_LATENCY_MS,
82
- TOOL_NAME,
83
71
  TOOL_NAME_ATTR_KEYS,
84
72
  TRACE_ANALYST_TRUNCATION_MARKER_PREFIX,
85
73
  TraceFileMissingError,
@@ -97,19 +85,12 @@ import {
97
85
  stringField,
98
86
  traceAnalystFunctionGroup,
99
87
  traceSpanKindToOpenInferenceKind
100
- } from "./chunk-S2F4J57L.js";
88
+ } from "./chunk-QKEGNI5B.js";
101
89
  import {
102
90
  RunIntegrityError,
103
91
  assertRunCaptured,
104
92
  throwIfRunIncomplete
105
93
  } from "./chunk-TT4KNT67.js";
106
- import {
107
- TraceEmitter,
108
- llmSpanFromProvider
109
- } from "./chunk-TVVP3ZZQ.js";
110
- import "./chunk-5UF54T55.js";
111
- import "./chunk-XJYR7XFV.js";
112
- import "./chunk-VSMTAMNK.js";
113
94
  import {
114
95
  FileSystemRawProviderSink,
115
96
  InMemoryRawProviderSink,
@@ -117,7 +98,38 @@ import {
117
98
  defaultProviderRedactor,
118
99
  providerFromBaseUrl
119
100
  } from "./chunk-PC4UYEBM.js";
101
+ import {
102
+ TraceEmitter,
103
+ llmSpanFromProvider
104
+ } from "./chunk-VQMK5FMP.js";
105
+ import "./chunk-S3UZOQ5Y.js";
106
+ import {
107
+ FAILURE_CLASSES,
108
+ TRACE_SCHEMA_VERSION,
109
+ isJudgeSpan,
110
+ isLlmSpan,
111
+ isRetrievalSpan,
112
+ isSandboxSpan,
113
+ isToolSpan
114
+ } from "./chunk-MA6HLL3S.js";
115
+ import "./chunk-XJYR7XFV.js";
116
+ import "./chunk-VSMTAMNK.js";
120
117
  import "./chunk-ONWEPEDO.js";
118
+ import {
119
+ INPUT_VALUE,
120
+ LLM_CACHED_TOKENS,
121
+ LLM_CACHE_WRITE_TOKENS,
122
+ LLM_COST_USD,
123
+ LLM_INPUT_TOKENS,
124
+ LLM_MODEL_NAME,
125
+ LLM_OUTPUT_TOKENS,
126
+ LLM_REASONING_TOKENS,
127
+ OPENINFERENCE_SPAN_KIND,
128
+ OUTPUT_VALUE,
129
+ TOOL_ARGS_CAPTURED,
130
+ TOOL_LATENCY_MS,
131
+ TOOL_NAME
132
+ } from "./chunk-WDHBCA3M.js";
121
133
  import "./chunk-PZ5AY32C.js";
122
134
  export {
123
135
  DEFAULT_REDACTION_RULES,
@@ -130,6 +142,8 @@ export {
130
142
  InMemoryTraceStore,
131
143
  LLM_CACHED_TOKENS,
132
144
  LLM_CACHED_TOKEN_ATTR_KEYS,
145
+ LLM_CACHE_WRITE_TOKENS,
146
+ LLM_CACHE_WRITE_TOKEN_ATTR_KEYS,
133
147
  LLM_COST_ATTR_KEYS,
134
148
  LLM_COST_USD,
135
149
  LLM_INPUT_TOKENS,
@@ -138,12 +152,15 @@ export {
138
152
  LLM_MODEL_NAME,
139
153
  LLM_OUTPUT_TOKENS,
140
154
  LLM_OUTPUT_TOKEN_ATTR_KEYS,
155
+ LLM_REASONING_TOKENS,
156
+ LLM_REASONING_TOKEN_ATTR_KEYS,
141
157
  NoopRawProviderSink,
142
158
  OPENINFERENCE_SPAN_KIND,
143
159
  OTEL_AGENT_EVAL_SCOPE,
144
160
  OUTPUT_VALUE,
145
161
  OtlpFileTraceStore,
146
162
  REDACTION_VERSION,
163
+ RUN_COST_ATTR_KEYS,
147
164
  ReplayCache,
148
165
  ReplayCacheMissError,
149
166
  RunIntegrityError,