@tangle-network/agent-eval 0.117.0 → 0.118.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +30 -0
- package/dist/analyst/index.d.ts +2772 -21
- package/dist/analyst/index.js +7 -6
- package/dist/analyst/index.js.map +1 -1
- package/dist/belief-state/index.d.ts +706 -10
- package/dist/belief-state/index.js +2 -1
- package/dist/belief-state/index.js.map +1 -1
- package/dist/benchmarks/index.d.ts +958 -14
- package/dist/benchmarks/index.js +12 -10
- package/dist/builder-eval/index.d.ts +449 -4
- package/dist/builder-eval/index.js +4 -3
- package/dist/builder-eval/index.js.map +1 -1
- package/dist/campaign/index.d.ts +4276 -74
- package/dist/campaign/index.js +12 -10
- package/dist/{chunk-VF3XSYTI.js → chunk-33JA4TFA.js} +6 -6
- package/dist/{chunk-4JLWXDYA.js → chunk-3EHHMC6E.js} +2 -2
- package/dist/{chunk-CCZIVI3F.js → chunk-BFW56GTT.js} +2 -2
- package/dist/{chunk-YZPO4UHR.js → chunk-DP3BBMYJ.js} +100 -143
- package/dist/chunk-DP3BBMYJ.js.map +1 -0
- package/dist/{chunk-E4BUPP7Z.js → chunk-HKUCJ437.js} +38 -66
- package/dist/chunk-HKUCJ437.js.map +1 -0
- package/dist/{chunk-HZHNRYHK.js → chunk-K6N6XJJX.js} +2 -2
- package/dist/chunk-KSDQVPLR.js +286 -0
- package/dist/chunk-KSDQVPLR.js.map +1 -0
- package/dist/chunk-MA6HLL3S.js +65 -0
- package/dist/chunk-MA6HLL3S.js.map +1 -0
- package/dist/{chunk-MGEHEHSN.js → chunk-OIIMMLRB.js} +11 -11
- package/dist/{chunk-DXZRATT5.js → chunk-OYZAPX5G.js} +3 -3
- package/dist/chunk-PXE2VKMX.js +140 -0
- package/dist/chunk-PXE2VKMX.js.map +1 -0
- package/dist/{chunk-ZUXV7UWZ.js → chunk-Q442S5AS.js} +73 -19
- package/dist/{chunk-ZUXV7UWZ.js.map → chunk-Q442S5AS.js.map} +1 -1
- package/dist/{chunk-ODVOOEWQ.js → chunk-QBRSJK47.js} +2 -2
- package/dist/{chunk-S2F4J57L.js → chunk-QKEGNI5B.js} +77 -32
- package/dist/chunk-QKEGNI5B.js.map +1 -0
- package/dist/{chunk-5UF54T55.js → chunk-S3UZOQ5Y.js} +34 -7
- package/dist/chunk-S3UZOQ5Y.js.map +1 -0
- package/dist/{chunk-FQNLDL4D.js → chunk-SVH2ANFD.js} +136 -4
- package/dist/chunk-SVH2ANFD.js.map +1 -0
- package/dist/{chunk-GQCZRZ7L.js → chunk-U5CHZ5M3.js} +9 -9
- package/dist/{chunk-TVVP3ZZQ.js → chunk-VQMK5FMP.js} +2 -1
- package/dist/{chunk-TVVP3ZZQ.js.map → chunk-VQMK5FMP.js.map} +1 -1
- package/dist/chunk-WDHBCA3M.js +31 -0
- package/dist/chunk-WDHBCA3M.js.map +1 -0
- package/dist/{chunk-HQPHZGL6.js → chunk-YLMUS4MM.js} +9 -9
- package/dist/{chunk-LQUTGLOZ.js → chunk-ZET2UAYW.js} +15 -65
- package/dist/chunk-ZET2UAYW.js.map +1 -0
- package/dist/contract/index.d.ts +4012 -38
- package/dist/contract/index.js +56 -29
- package/dist/contract/index.js.map +1 -1
- package/dist/control.d.ts +1013 -9
- package/dist/control.js +4 -3
- package/dist/fuzz.d.ts +194 -4
- package/dist/fuzz.js +3 -3
- package/dist/hosted/index.d.ts +498 -17
- package/dist/index.d.ts +10982 -1286
- package/dist/index.js +97 -80
- package/dist/index.js.map +1 -1
- package/dist/matrix/index.d.ts +139 -4
- package/dist/meta-eval/index.d.ts +862 -15
- package/dist/meta-eval/index.js +2 -1
- package/dist/meta-eval/index.js.map +1 -1
- package/dist/multishot/index.d.ts +214 -14
- package/dist/openapi.json +1 -1
- package/dist/pipelines/index.d.ts +392 -7
- package/dist/pipelines/index.js +5 -3
- package/dist/pipelines/index.js.map +1 -1
- package/dist/reporting.d.ts +1277 -17
- package/dist/rl.d.ts +2359 -28
- package/dist/rl.js +6 -5
- package/dist/rl.js.map +1 -1
- package/dist/storyboard/index.d.ts +86 -1
- package/dist/trace-attributes.d.ts +16 -0
- package/dist/trace-attributes.js +32 -0
- package/dist/trace-attributes.js.map +1 -0
- package/dist/traces.d.ts +1970 -697
- package/dist/traces.js +49 -32
- package/dist/wire/index.d.ts +655 -9
- package/docs/insight-report.md +44 -0
- package/package.json +9 -3
- package/dist/adversarial-B7loGVVX.d.ts +0 -19
- package/dist/analyst-C8HHvfJp.d.ts +0 -88
- package/dist/analyze-runs--2x39HZ7.d.ts +0 -81
- package/dist/baseline-DKq3gJpP.d.ts +0 -141
- package/dist/calibration-C8MTS7cw.d.ts +0 -101
- package/dist/chunk-5UF54T55.js.map +0 -1
- package/dist/chunk-E4BUPP7Z.js.map +0 -1
- package/dist/chunk-FQNLDL4D.js.map +0 -1
- package/dist/chunk-LQUTGLOZ.js.map +0 -1
- package/dist/chunk-S2F4J57L.js.map +0 -1
- package/dist/chunk-YZPO4UHR.js.map +0 -1
- package/dist/code-agent-session-CjZsVd19.d.ts +0 -87
- package/dist/control-6vuGfmDH.d.ts +0 -258
- package/dist/cost-ledger-DWy3XdJc.d.ts +0 -183
- package/dist/dataset-NENEzRgk.d.ts +0 -115
- package/dist/default-registry-DaK8b3fv.d.ts +0 -155
- package/dist/emitter-CjD7vUwv.d.ts +0 -122
- package/dist/errors-oeQrLqXC.d.ts +0 -74
- package/dist/failure-cluster-DOAcSJ87.d.ts +0 -76
- package/dist/feedback-trajectory-BUnM58xL.d.ts +0 -348
- package/dist/gepa-eESocoDi.d.ts +0 -642
- package/dist/index-PdX4VnPA.d.ts +0 -423
- package/dist/insight-report-DY4nDW9Q.d.ts +0 -310
- package/dist/integrity-DqlBiLyK.d.ts +0 -81
- package/dist/judge-calibration-7C-IDmKr.d.ts +0 -145
- package/dist/kind-factory-ClZmO25A.d.ts +0 -171
- package/dist/llm-client-qoDd18Qz.d.ts +0 -289
- package/dist/multi-layer-verifier-BsqKuLyN.d.ts +0 -150
- package/dist/off-policy-DiwuKKg7.d.ts +0 -132
- package/dist/outcome-store-rnXLEqSn.d.ts +0 -63
- package/dist/policy-edit-wG9uFEFm.d.ts +0 -455
- package/dist/pre-registration-BWQhJ3vz.d.ts +0 -761
- package/dist/provenance-DpjwyseI.d.ts +0 -541
- package/dist/query-CF7PG61p.d.ts +0 -35
- package/dist/raw-provider-sink-C46HDghv.d.ts +0 -132
- package/dist/release-report-C8G2i5Xi.d.ts +0 -236
- package/dist/researcher-C8XyxQsu.d.ts +0 -387
- package/dist/rubric-predictive-validity-p49lLVrE.d.ts +0 -105
- package/dist/run-record-BDH49H2E.d.ts +0 -360
- package/dist/runtime-trajectory-DGBIUt4B.d.ts +0 -49
- package/dist/schema-B3Q3l9Z_.d.ts +0 -201
- package/dist/semantic-concept-judge-CXnPEJbf.d.ts +0 -723
- package/dist/sequential-5iSVfzl2.d.ts +0 -139
- package/dist/series-convergence-D5OWMBg6.d.ts +0 -33
- package/dist/statistics-KUnG73jH.d.ts +0 -494
- package/dist/storage-DrX3v_5B.d.ts +0 -50
- package/dist/store-C1YxJDEK.d.ts +0 -248
- package/dist/store-DGqD0Pyo.d.ts +0 -116
- package/dist/summary-report-C5bKFfm-.d.ts +0 -445
- package/dist/test-graded-scenario-B0ybnPY7.d.ts +0 -166
- package/dist/types-BSw1rOUB.d.ts +0 -634
- package/dist/types-BUxNaJ8c.d.ts +0 -108
- package/dist/types-BkfcQnxV.d.ts +0 -313
- package/dist/verdict-C9MlYujm.d.ts +0 -35
- /package/dist/{chunk-VF3XSYTI.js.map → chunk-33JA4TFA.js.map} +0 -0
- /package/dist/{chunk-4JLWXDYA.js.map → chunk-3EHHMC6E.js.map} +0 -0
- /package/dist/{chunk-CCZIVI3F.js.map → chunk-BFW56GTT.js.map} +0 -0
- /package/dist/{chunk-HZHNRYHK.js.map → chunk-K6N6XJJX.js.map} +0 -0
- /package/dist/{chunk-MGEHEHSN.js.map → chunk-OIIMMLRB.js.map} +0 -0
- /package/dist/{chunk-DXZRATT5.js.map → chunk-OYZAPX5G.js.map} +0 -0
- /package/dist/{chunk-ODVOOEWQ.js.map → chunk-QBRSJK47.js.map} +0 -0
- /package/dist/{chunk-GQCZRZ7L.js.map → chunk-U5CHZ5M3.js.map} +0 -0
- /package/dist/{chunk-HQPHZGL6.js.map → chunk-YLMUS4MM.js.map} +0 -0
package/dist/traces.js
CHANGED
|
@@ -15,9 +15,6 @@ import {
|
|
|
15
15
|
describeTraceInsightScope,
|
|
16
16
|
domainEvidencePattern,
|
|
17
17
|
exportRunAsOtlp,
|
|
18
|
-
extractUsage,
|
|
19
|
-
extractUsageFromResponse,
|
|
20
|
-
extractUsageFromSse,
|
|
21
18
|
flattenOtlpExportToNdjson,
|
|
22
19
|
inferDomainKeywords,
|
|
23
20
|
iterateRawCalls,
|
|
@@ -28,31 +25,30 @@ import {
|
|
|
28
25
|
scoreTraceInsightReadiness,
|
|
29
26
|
tokenizeDomainWords,
|
|
30
27
|
traceAnalystOnRunComplete
|
|
31
|
-
} from "./chunk-
|
|
28
|
+
} from "./chunk-DP3BBMYJ.js";
|
|
29
|
+
import "./chunk-KSDQVPLR.js";
|
|
30
|
+
import {
|
|
31
|
+
extractUsage,
|
|
32
|
+
extractUsageFromResponse,
|
|
33
|
+
extractUsageFromSse
|
|
34
|
+
} from "./chunk-PXE2VKMX.js";
|
|
32
35
|
import {
|
|
33
|
-
FAILURE_CLASSES,
|
|
34
|
-
TRACE_SCHEMA_VERSION,
|
|
35
36
|
aggregateLlm,
|
|
36
37
|
argHash,
|
|
37
38
|
groupBy,
|
|
38
39
|
hasCapturedToolArgs,
|
|
39
|
-
isJudgeSpan,
|
|
40
|
-
isLlmSpan,
|
|
41
|
-
isRetrievalSpan,
|
|
42
|
-
isSandboxSpan,
|
|
43
|
-
isToolSpan,
|
|
44
40
|
judgeSpans,
|
|
45
41
|
llmSpans,
|
|
46
42
|
runFailureClass,
|
|
47
43
|
runsForScenario,
|
|
48
44
|
toolSpans
|
|
49
|
-
} from "./chunk-
|
|
45
|
+
} from "./chunk-ZET2UAYW.js";
|
|
50
46
|
import {
|
|
51
47
|
TRACE_ANALYST_ACTOR_DESCRIPTION,
|
|
52
48
|
TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION,
|
|
53
49
|
TRACE_ANALYST_SUBAGENT_DESCRIPTION,
|
|
54
50
|
analyzeTraces
|
|
55
|
-
} from "./chunk-
|
|
51
|
+
} from "./chunk-3EHHMC6E.js";
|
|
56
52
|
import {
|
|
57
53
|
DEFAULT_REDACTION_RULES,
|
|
58
54
|
REDACTION_VERSION,
|
|
@@ -61,25 +57,17 @@ import {
|
|
|
61
57
|
} from "./chunk-GGE4NNQT.js";
|
|
62
58
|
import {
|
|
63
59
|
DEFAULT_TRACE_ANALYST_BUDGETS,
|
|
64
|
-
INPUT_VALUE,
|
|
65
|
-
LLM_CACHED_TOKENS,
|
|
66
60
|
LLM_CACHED_TOKEN_ATTR_KEYS,
|
|
61
|
+
LLM_CACHE_WRITE_TOKEN_ATTR_KEYS,
|
|
67
62
|
LLM_COST_ATTR_KEYS,
|
|
68
|
-
LLM_COST_USD,
|
|
69
|
-
LLM_INPUT_TOKENS,
|
|
70
63
|
LLM_INPUT_TOKEN_ATTR_KEYS,
|
|
71
64
|
LLM_MODEL_ATTR_KEYS,
|
|
72
|
-
LLM_MODEL_NAME,
|
|
73
|
-
LLM_OUTPUT_TOKENS,
|
|
74
65
|
LLM_OUTPUT_TOKEN_ATTR_KEYS,
|
|
75
|
-
|
|
76
|
-
OUTPUT_VALUE,
|
|
66
|
+
LLM_REASONING_TOKEN_ATTR_KEYS,
|
|
77
67
|
OtlpFileTraceStore,
|
|
68
|
+
RUN_COST_ATTR_KEYS,
|
|
78
69
|
SPAN_KIND_ATTR_KEYS,
|
|
79
70
|
SpanNotFoundError,
|
|
80
|
-
TOOL_ARGS_CAPTURED,
|
|
81
|
-
TOOL_LATENCY_MS,
|
|
82
|
-
TOOL_NAME,
|
|
83
71
|
TOOL_NAME_ATTR_KEYS,
|
|
84
72
|
TRACE_ANALYST_TRUNCATION_MARKER_PREFIX,
|
|
85
73
|
TraceFileMissingError,
|
|
@@ -97,19 +85,12 @@ import {
|
|
|
97
85
|
stringField,
|
|
98
86
|
traceAnalystFunctionGroup,
|
|
99
87
|
traceSpanKindToOpenInferenceKind
|
|
100
|
-
} from "./chunk-
|
|
88
|
+
} from "./chunk-QKEGNI5B.js";
|
|
101
89
|
import {
|
|
102
90
|
RunIntegrityError,
|
|
103
91
|
assertRunCaptured,
|
|
104
92
|
throwIfRunIncomplete
|
|
105
93
|
} from "./chunk-TT4KNT67.js";
|
|
106
|
-
import {
|
|
107
|
-
TraceEmitter,
|
|
108
|
-
llmSpanFromProvider
|
|
109
|
-
} from "./chunk-TVVP3ZZQ.js";
|
|
110
|
-
import "./chunk-5UF54T55.js";
|
|
111
|
-
import "./chunk-XJYR7XFV.js";
|
|
112
|
-
import "./chunk-VSMTAMNK.js";
|
|
113
94
|
import {
|
|
114
95
|
FileSystemRawProviderSink,
|
|
115
96
|
InMemoryRawProviderSink,
|
|
@@ -117,7 +98,38 @@ import {
|
|
|
117
98
|
defaultProviderRedactor,
|
|
118
99
|
providerFromBaseUrl
|
|
119
100
|
} from "./chunk-PC4UYEBM.js";
|
|
101
|
+
import {
|
|
102
|
+
TraceEmitter,
|
|
103
|
+
llmSpanFromProvider
|
|
104
|
+
} from "./chunk-VQMK5FMP.js";
|
|
105
|
+
import "./chunk-S3UZOQ5Y.js";
|
|
106
|
+
import {
|
|
107
|
+
FAILURE_CLASSES,
|
|
108
|
+
TRACE_SCHEMA_VERSION,
|
|
109
|
+
isJudgeSpan,
|
|
110
|
+
isLlmSpan,
|
|
111
|
+
isRetrievalSpan,
|
|
112
|
+
isSandboxSpan,
|
|
113
|
+
isToolSpan
|
|
114
|
+
} from "./chunk-MA6HLL3S.js";
|
|
115
|
+
import "./chunk-XJYR7XFV.js";
|
|
116
|
+
import "./chunk-VSMTAMNK.js";
|
|
120
117
|
import "./chunk-ONWEPEDO.js";
|
|
118
|
+
import {
|
|
119
|
+
INPUT_VALUE,
|
|
120
|
+
LLM_CACHED_TOKENS,
|
|
121
|
+
LLM_CACHE_WRITE_TOKENS,
|
|
122
|
+
LLM_COST_USD,
|
|
123
|
+
LLM_INPUT_TOKENS,
|
|
124
|
+
LLM_MODEL_NAME,
|
|
125
|
+
LLM_OUTPUT_TOKENS,
|
|
126
|
+
LLM_REASONING_TOKENS,
|
|
127
|
+
OPENINFERENCE_SPAN_KIND,
|
|
128
|
+
OUTPUT_VALUE,
|
|
129
|
+
TOOL_ARGS_CAPTURED,
|
|
130
|
+
TOOL_LATENCY_MS,
|
|
131
|
+
TOOL_NAME
|
|
132
|
+
} from "./chunk-WDHBCA3M.js";
|
|
121
133
|
import "./chunk-PZ5AY32C.js";
|
|
122
134
|
export {
|
|
123
135
|
DEFAULT_REDACTION_RULES,
|
|
@@ -130,6 +142,8 @@ export {
|
|
|
130
142
|
InMemoryTraceStore,
|
|
131
143
|
LLM_CACHED_TOKENS,
|
|
132
144
|
LLM_CACHED_TOKEN_ATTR_KEYS,
|
|
145
|
+
LLM_CACHE_WRITE_TOKENS,
|
|
146
|
+
LLM_CACHE_WRITE_TOKEN_ATTR_KEYS,
|
|
133
147
|
LLM_COST_ATTR_KEYS,
|
|
134
148
|
LLM_COST_USD,
|
|
135
149
|
LLM_INPUT_TOKENS,
|
|
@@ -138,12 +152,15 @@ export {
|
|
|
138
152
|
LLM_MODEL_NAME,
|
|
139
153
|
LLM_OUTPUT_TOKENS,
|
|
140
154
|
LLM_OUTPUT_TOKEN_ATTR_KEYS,
|
|
155
|
+
LLM_REASONING_TOKENS,
|
|
156
|
+
LLM_REASONING_TOKEN_ATTR_KEYS,
|
|
141
157
|
NoopRawProviderSink,
|
|
142
158
|
OPENINFERENCE_SPAN_KIND,
|
|
143
159
|
OTEL_AGENT_EVAL_SCOPE,
|
|
144
160
|
OUTPUT_VALUE,
|
|
145
161
|
OtlpFileTraceStore,
|
|
146
162
|
REDACTION_VERSION,
|
|
163
|
+
RUN_COST_ATTR_KEYS,
|
|
147
164
|
ReplayCache,
|
|
148
165
|
ReplayCacheMissError,
|
|
149
166
|
RunIntegrityError,
|