@tangle-network/agent-eval 0.117.1 → 0.118.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +30 -0
- package/dist/analyst/index.d.ts +2772 -21
- package/dist/analyst/index.js +7 -6
- package/dist/analyst/index.js.map +1 -1
- package/dist/belief-state/index.d.ts +706 -10
- package/dist/belief-state/index.js +2 -1
- package/dist/belief-state/index.js.map +1 -1
- package/dist/benchmarks/index.d.ts +958 -14
- package/dist/benchmarks/index.js +12 -10
- package/dist/builder-eval/index.d.ts +449 -4
- package/dist/builder-eval/index.js +4 -3
- package/dist/builder-eval/index.js.map +1 -1
- package/dist/campaign/index.d.ts +4275 -73
- package/dist/campaign/index.js +12 -10
- package/dist/{chunk-VF3XSYTI.js → chunk-33JA4TFA.js} +6 -6
- package/dist/{chunk-4JLWXDYA.js → chunk-3EHHMC6E.js} +2 -2
- package/dist/{chunk-CCZIVI3F.js → chunk-BFW56GTT.js} +2 -2
- package/dist/{chunk-YZPO4UHR.js → chunk-FTUMG2U7.js} +124 -149
- package/dist/chunk-FTUMG2U7.js.map +1 -0
- package/dist/{chunk-E4BUPP7Z.js → chunk-HKUCJ437.js} +38 -66
- package/dist/chunk-HKUCJ437.js.map +1 -0
- package/dist/{chunk-HZHNRYHK.js → chunk-K6N6XJJX.js} +2 -2
- package/dist/chunk-KSDQVPLR.js +286 -0
- package/dist/chunk-KSDQVPLR.js.map +1 -0
- package/dist/chunk-MA6HLL3S.js +65 -0
- package/dist/chunk-MA6HLL3S.js.map +1 -0
- package/dist/{chunk-MGEHEHSN.js → chunk-OIIMMLRB.js} +11 -11
- package/dist/{chunk-DXZRATT5.js → chunk-OYZAPX5G.js} +3 -3
- package/dist/chunk-PXE2VKMX.js +140 -0
- package/dist/chunk-PXE2VKMX.js.map +1 -0
- package/dist/{chunk-JSJZ4PJ6.js → chunk-Q442S5AS.js} +17 -17
- package/dist/{chunk-ODVOOEWQ.js → chunk-QBRSJK47.js} +2 -2
- package/dist/{chunk-S2F4J57L.js → chunk-QKEGNI5B.js} +77 -32
- package/dist/chunk-QKEGNI5B.js.map +1 -0
- package/dist/{chunk-5UF54T55.js → chunk-S3UZOQ5Y.js} +34 -7
- package/dist/chunk-S3UZOQ5Y.js.map +1 -0
- package/dist/{chunk-FQNLDL4D.js → chunk-SVH2ANFD.js} +136 -4
- package/dist/chunk-SVH2ANFD.js.map +1 -0
- package/dist/{chunk-GQCZRZ7L.js → chunk-U5CHZ5M3.js} +9 -9
- package/dist/{chunk-TVVP3ZZQ.js → chunk-VQMK5FMP.js} +2 -1
- package/dist/{chunk-TVVP3ZZQ.js.map → chunk-VQMK5FMP.js.map} +1 -1
- package/dist/chunk-WDHBCA3M.js +31 -0
- package/dist/chunk-WDHBCA3M.js.map +1 -0
- package/dist/{chunk-HQPHZGL6.js → chunk-YLMUS4MM.js} +9 -9
- package/dist/{chunk-LQUTGLOZ.js → chunk-ZET2UAYW.js} +15 -65
- package/dist/chunk-ZET2UAYW.js.map +1 -0
- package/dist/contract/index.d.ts +4012 -38
- package/dist/contract/index.js +56 -29
- package/dist/contract/index.js.map +1 -1
- package/dist/control.d.ts +1013 -9
- package/dist/control.js +4 -3
- package/dist/fuzz.d.ts +194 -4
- package/dist/fuzz.js +3 -3
- package/dist/hosted/index.d.ts +498 -17
- package/dist/index.d.ts +10992 -1288
- package/dist/index.js +101 -80
- package/dist/index.js.map +1 -1
- package/dist/matrix/index.d.ts +139 -4
- package/dist/meta-eval/index.d.ts +862 -15
- package/dist/meta-eval/index.js +2 -1
- package/dist/meta-eval/index.js.map +1 -1
- package/dist/multishot/index.d.ts +214 -14
- package/dist/openapi.json +1 -1
- package/dist/pipelines/index.d.ts +392 -7
- package/dist/pipelines/index.js +5 -3
- package/dist/pipelines/index.js.map +1 -1
- package/dist/reporting.d.ts +1277 -17
- package/dist/rl.d.ts +2359 -28
- package/dist/rl.js +6 -5
- package/dist/rl.js.map +1 -1
- package/dist/storyboard/index.d.ts +86 -1
- package/dist/trace-attributes.d.ts +16 -0
- package/dist/trace-attributes.js +32 -0
- package/dist/trace-attributes.js.map +1 -0
- package/dist/traces.d.ts +1978 -697
- package/dist/traces.js +53 -32
- package/dist/wire/index.d.ts +655 -9
- package/docs/insight-report.md +44 -0
- package/package.json +9 -3
- package/dist/adversarial-B7loGVVX.d.ts +0 -19
- package/dist/analyst-C8HHvfJp.d.ts +0 -88
- package/dist/analyze-runs--2x39HZ7.d.ts +0 -81
- package/dist/baseline-DKq3gJpP.d.ts +0 -141
- package/dist/calibration-C8MTS7cw.d.ts +0 -101
- package/dist/chunk-5UF54T55.js.map +0 -1
- package/dist/chunk-E4BUPP7Z.js.map +0 -1
- package/dist/chunk-FQNLDL4D.js.map +0 -1
- package/dist/chunk-LQUTGLOZ.js.map +0 -1
- package/dist/chunk-S2F4J57L.js.map +0 -1
- package/dist/chunk-YZPO4UHR.js.map +0 -1
- package/dist/code-agent-session-CjZsVd19.d.ts +0 -87
- package/dist/control-6vuGfmDH.d.ts +0 -258
- package/dist/cost-ledger-DWy3XdJc.d.ts +0 -183
- package/dist/dataset-NENEzRgk.d.ts +0 -115
- package/dist/default-registry-DaK8b3fv.d.ts +0 -155
- package/dist/emitter-CjD7vUwv.d.ts +0 -122
- package/dist/errors-oeQrLqXC.d.ts +0 -74
- package/dist/failure-cluster-DOAcSJ87.d.ts +0 -76
- package/dist/feedback-trajectory-BUnM58xL.d.ts +0 -348
- package/dist/gepa-eESocoDi.d.ts +0 -642
- package/dist/index-PdX4VnPA.d.ts +0 -423
- package/dist/insight-report-DY4nDW9Q.d.ts +0 -310
- package/dist/integrity-DqlBiLyK.d.ts +0 -81
- package/dist/judge-calibration-7C-IDmKr.d.ts +0 -145
- package/dist/kind-factory-ClZmO25A.d.ts +0 -171
- package/dist/llm-client-qoDd18Qz.d.ts +0 -289
- package/dist/multi-layer-verifier-BsqKuLyN.d.ts +0 -150
- package/dist/off-policy-DiwuKKg7.d.ts +0 -132
- package/dist/outcome-store-rnXLEqSn.d.ts +0 -63
- package/dist/policy-edit-wG9uFEFm.d.ts +0 -455
- package/dist/pre-registration-BWQhJ3vz.d.ts +0 -761
- package/dist/provenance-DpjwyseI.d.ts +0 -541
- package/dist/query-CF7PG61p.d.ts +0 -35
- package/dist/raw-provider-sink-C46HDghv.d.ts +0 -132
- package/dist/release-report-C8G2i5Xi.d.ts +0 -236
- package/dist/researcher-C8XyxQsu.d.ts +0 -387
- package/dist/rubric-predictive-validity-p49lLVrE.d.ts +0 -105
- package/dist/run-record-BDH49H2E.d.ts +0 -360
- package/dist/runtime-trajectory-DGBIUt4B.d.ts +0 -49
- package/dist/schema-B3Q3l9Z_.d.ts +0 -201
- package/dist/semantic-concept-judge-CXnPEJbf.d.ts +0 -723
- package/dist/sequential-5iSVfzl2.d.ts +0 -139
- package/dist/series-convergence-D5OWMBg6.d.ts +0 -33
- package/dist/statistics-KUnG73jH.d.ts +0 -494
- package/dist/storage-DrX3v_5B.d.ts +0 -50
- package/dist/store-C1YxJDEK.d.ts +0 -248
- package/dist/store-DGqD0Pyo.d.ts +0 -116
- package/dist/summary-report-C5bKFfm-.d.ts +0 -445
- package/dist/test-graded-scenario-B0ybnPY7.d.ts +0 -166
- package/dist/types-BSw1rOUB.d.ts +0 -634
- package/dist/types-BUxNaJ8c.d.ts +0 -108
- package/dist/types-BkfcQnxV.d.ts +0 -313
- package/dist/verdict-C9MlYujm.d.ts +0 -35
- /package/dist/{chunk-VF3XSYTI.js.map → chunk-33JA4TFA.js.map} +0 -0
- /package/dist/{chunk-4JLWXDYA.js.map → chunk-3EHHMC6E.js.map} +0 -0
- /package/dist/{chunk-CCZIVI3F.js.map → chunk-BFW56GTT.js.map} +0 -0
- /package/dist/{chunk-HZHNRYHK.js.map → chunk-K6N6XJJX.js.map} +0 -0
- /package/dist/{chunk-MGEHEHSN.js.map → chunk-OIIMMLRB.js.map} +0 -0
- /package/dist/{chunk-DXZRATT5.js.map → chunk-OYZAPX5G.js.map} +0 -0
- /package/dist/{chunk-JSJZ4PJ6.js.map → chunk-Q442S5AS.js.map} +0 -0
- /package/dist/{chunk-ODVOOEWQ.js.map → chunk-QBRSJK47.js.map} +0 -0
- /package/dist/{chunk-GQCZRZ7L.js.map → chunk-U5CHZ5M3.js.map} +0 -0
- /package/dist/{chunk-HQPHZGL6.js.map → chunk-YLMUS4MM.js.map} +0 -0
package/dist/campaign/index.js
CHANGED
|
@@ -66,7 +66,7 @@ import {
|
|
|
66
66
|
userStoryScoreboard,
|
|
67
67
|
validateSearchLedgerEvent,
|
|
68
68
|
verifyCodeSurface
|
|
69
|
-
} from "../chunk-
|
|
69
|
+
} from "../chunk-Q442S5AS.js";
|
|
70
70
|
import {
|
|
71
71
|
assertCodeSurfaceIdentity,
|
|
72
72
|
buildEvidenceVector,
|
|
@@ -103,7 +103,7 @@ import {
|
|
|
103
103
|
runOptimization,
|
|
104
104
|
surfaceContentHash,
|
|
105
105
|
surfaceHash
|
|
106
|
-
} from "../chunk-
|
|
106
|
+
} from "../chunk-YLMUS4MM.js";
|
|
107
107
|
import {
|
|
108
108
|
SearchLedgerConflictError,
|
|
109
109
|
SearchLedgerError,
|
|
@@ -120,20 +120,22 @@ import "../chunk-3YYRZDON.js";
|
|
|
120
120
|
import {
|
|
121
121
|
POLICY_EDIT_CANDIDATE_RECORD_SCHEMA,
|
|
122
122
|
validatePolicyEditCandidateRecord
|
|
123
|
-
} from "../chunk-
|
|
123
|
+
} from "../chunk-OIIMMLRB.js";
|
|
124
124
|
import "../chunk-ARU2PZFM.js";
|
|
125
|
-
import "../chunk-PJQFMIOX.js";
|
|
126
|
-
import "../chunk-4JLWXDYA.js";
|
|
127
|
-
import "../chunk-GGE4NNQT.js";
|
|
128
|
-
import "../chunk-S2F4J57L.js";
|
|
129
|
-
import "../chunk-5UF54T55.js";
|
|
130
|
-
import "../chunk-XJYR7XFV.js";
|
|
131
|
-
import "../chunk-VSMTAMNK.js";
|
|
132
125
|
import "../chunk-NJC7U437.js";
|
|
126
|
+
import "../chunk-PJQFMIOX.js";
|
|
133
127
|
import "../chunk-VCTY3W6J.js";
|
|
134
128
|
import "../chunk-VI2UW6B6.js";
|
|
129
|
+
import "../chunk-3EHHMC6E.js";
|
|
130
|
+
import "../chunk-GGE4NNQT.js";
|
|
131
|
+
import "../chunk-QKEGNI5B.js";
|
|
135
132
|
import "../chunk-PC4UYEBM.js";
|
|
133
|
+
import "../chunk-S3UZOQ5Y.js";
|
|
134
|
+
import "../chunk-MA6HLL3S.js";
|
|
135
|
+
import "../chunk-XJYR7XFV.js";
|
|
136
|
+
import "../chunk-VSMTAMNK.js";
|
|
136
137
|
import "../chunk-ONWEPEDO.js";
|
|
138
|
+
import "../chunk-WDHBCA3M.js";
|
|
137
139
|
import "../chunk-PZ5AY32C.js";
|
|
138
140
|
export {
|
|
139
141
|
DEFAULT_POLICY_EDIT_HISTORY_LIMITS,
|
|
@@ -6,16 +6,16 @@ import {
|
|
|
6
6
|
DEFAULT_TRACE_ANALYST_KINDS,
|
|
7
7
|
createTraceAnalystKind,
|
|
8
8
|
makeFinding
|
|
9
|
-
} from "./chunk-
|
|
9
|
+
} from "./chunk-OIIMMLRB.js";
|
|
10
|
+
import {
|
|
11
|
+
LlmClient
|
|
12
|
+
} from "./chunk-NJC7U437.js";
|
|
10
13
|
import {
|
|
11
14
|
LLM_INPUT_TOKEN_ATTR_KEYS,
|
|
12
15
|
LLM_OUTPUT_TOKEN_ATTR_KEYS,
|
|
13
16
|
TOOL_NAME_ATTR_KEYS,
|
|
14
17
|
spanEpochMillis
|
|
15
|
-
} from "./chunk-
|
|
16
|
-
import {
|
|
17
|
-
LlmClient
|
|
18
|
-
} from "./chunk-NJC7U437.js";
|
|
18
|
+
} from "./chunk-QKEGNI5B.js";
|
|
19
19
|
|
|
20
20
|
// src/analyst/chat-client.ts
|
|
21
21
|
function createChatClient(opts) {
|
|
@@ -542,4 +542,4 @@ export {
|
|
|
542
542
|
behavioralAnalyst,
|
|
543
543
|
buildDefaultAnalystRegistry
|
|
544
544
|
};
|
|
545
|
-
//# sourceMappingURL=chunk-
|
|
545
|
+
//# sourceMappingURL=chunk-33JA4TFA.js.map
|
|
@@ -2,7 +2,7 @@ import {
|
|
|
2
2
|
OtlpFileTraceStore,
|
|
3
3
|
TraceFileMissingError,
|
|
4
4
|
buildTraceAnalystTools
|
|
5
|
-
} from "./chunk-
|
|
5
|
+
} from "./chunk-QKEGNI5B.js";
|
|
6
6
|
|
|
7
7
|
// src/trace-analyst/prompts.ts
|
|
8
8
|
var TRACE_ANALYST_ACTOR_DESCRIPTION = `You answer questions about an OTLP-shaped JSONL trace dataset using the trace tools provided in the \`traces\` namespace.
|
|
@@ -198,4 +198,4 @@ export {
|
|
|
198
198
|
TRACE_ANALYST_SUBAGENT_DESCRIPTION,
|
|
199
199
|
analyzeTraces
|
|
200
200
|
};
|
|
201
|
-
//# sourceMappingURL=chunk-
|
|
201
|
+
//# sourceMappingURL=chunk-3EHHMC6E.js.map
|
|
@@ -5,7 +5,7 @@ import {
|
|
|
5
5
|
aggregateRunScore,
|
|
6
6
|
clamp01,
|
|
7
7
|
computeFindingId
|
|
8
|
-
} from "./chunk-
|
|
8
|
+
} from "./chunk-OIIMMLRB.js";
|
|
9
9
|
import {
|
|
10
10
|
callLlmJson,
|
|
11
11
|
costReceiptFromLlm,
|
|
@@ -759,4 +759,4 @@ export {
|
|
|
759
759
|
runSemanticConceptJudge,
|
|
760
760
|
createSemanticConceptJudge
|
|
761
761
|
};
|
|
762
|
-
//# sourceMappingURL=chunk-
|
|
762
|
+
//# sourceMappingURL=chunk-BFW56GTT.js.map
|
|
@@ -1,40 +1,48 @@
|
|
|
1
|
+
import {
|
|
2
|
+
recordAggregateMeasurements,
|
|
3
|
+
summarizeExecutionMeasurements
|
|
4
|
+
} from "./chunk-KSDQVPLR.js";
|
|
5
|
+
import {
|
|
6
|
+
extractUsage,
|
|
7
|
+
extractUsageFromSse
|
|
8
|
+
} from "./chunk-PXE2VKMX.js";
|
|
1
9
|
import {
|
|
2
10
|
analyzeTraces
|
|
3
|
-
} from "./chunk-
|
|
11
|
+
} from "./chunk-3EHHMC6E.js";
|
|
4
12
|
import {
|
|
5
|
-
LLM_CACHED_TOKEN_ATTR_KEYS,
|
|
6
|
-
LLM_COST_ATTR_KEYS,
|
|
7
|
-
LLM_COST_USD,
|
|
8
|
-
LLM_INPUT_TOKENS,
|
|
9
|
-
LLM_INPUT_TOKEN_ATTR_KEYS,
|
|
10
13
|
LLM_MODEL_ATTR_KEYS,
|
|
11
|
-
LLM_MODEL_NAME,
|
|
12
|
-
LLM_OUTPUT_TOKENS,
|
|
13
|
-
LLM_OUTPUT_TOKEN_ATTR_KEYS,
|
|
14
|
-
OPENINFERENCE_SPAN_KIND,
|
|
15
|
-
TOOL_NAME,
|
|
16
14
|
applyToolSpanOtlpAttributes,
|
|
17
15
|
compareSpanTime,
|
|
18
|
-
firstNumberAttr,
|
|
19
16
|
firstStringAttr,
|
|
20
17
|
projectOtlpFlatLine,
|
|
21
18
|
spanEpochMillis,
|
|
22
19
|
traceSpanKindToOpenInferenceKind
|
|
23
|
-
} from "./chunk-
|
|
20
|
+
} from "./chunk-QKEGNI5B.js";
|
|
21
|
+
import {
|
|
22
|
+
defaultProviderRedactor,
|
|
23
|
+
providerFromBaseUrl
|
|
24
|
+
} from "./chunk-PC4UYEBM.js";
|
|
24
25
|
import {
|
|
25
26
|
validateRunRecord
|
|
26
|
-
} from "./chunk-
|
|
27
|
+
} from "./chunk-S3UZOQ5Y.js";
|
|
27
28
|
import {
|
|
28
29
|
canonicalize,
|
|
29
30
|
hashJson
|
|
30
31
|
} from "./chunk-VSMTAMNK.js";
|
|
31
|
-
import {
|
|
32
|
-
defaultProviderRedactor,
|
|
33
|
-
providerFromBaseUrl
|
|
34
|
-
} from "./chunk-PC4UYEBM.js";
|
|
35
32
|
import {
|
|
36
33
|
ReplayError
|
|
37
34
|
} from "./chunk-ONWEPEDO.js";
|
|
35
|
+
import {
|
|
36
|
+
LLM_CACHED_TOKENS,
|
|
37
|
+
LLM_CACHE_WRITE_TOKENS,
|
|
38
|
+
LLM_COST_USD,
|
|
39
|
+
LLM_INPUT_TOKENS,
|
|
40
|
+
LLM_MODEL_NAME,
|
|
41
|
+
LLM_OUTPUT_TOKENS,
|
|
42
|
+
LLM_REASONING_TOKENS,
|
|
43
|
+
OPENINFERENCE_SPAN_KIND,
|
|
44
|
+
TOOL_NAME
|
|
45
|
+
} from "./chunk-WDHBCA3M.js";
|
|
38
46
|
|
|
39
47
|
// src/trace-analyst/hook.ts
|
|
40
48
|
var DEFAULT_QUESTION = "Summarise what happened in this run. Surface any failure modes, surprising findings, or evidence that the run's verdict is wrong.";
|
|
@@ -428,17 +436,25 @@ function flattenOtlpExportToNdjson(otlpExport, opts = {}) {
|
|
|
428
436
|
function otlpToRunRecords(otlpJsonl, opts) {
|
|
429
437
|
return otlpToTraceRunRecords(otlpJsonl, opts).map((r) => r.record);
|
|
430
438
|
}
|
|
439
|
+
function otlpRowsToRunRecords(rows, opts) {
|
|
440
|
+
return otlpRowsToTraceRunRecords(rows, opts).map((row) => row.record);
|
|
441
|
+
}
|
|
431
442
|
function otlpToTraceRunRecords(otlpJsonl, opts) {
|
|
443
|
+
return traceRunRecordsFromSpans(groupJsonlSpansByTrace(otlpJsonl), opts);
|
|
444
|
+
}
|
|
445
|
+
function otlpRowsToTraceRunRecords(rows, opts) {
|
|
446
|
+
return traceRunRecordsFromSpans(groupRowsByTrace(rows), opts);
|
|
447
|
+
}
|
|
448
|
+
function traceRunRecordsFromSpans(byTrace, opts) {
|
|
432
449
|
const splitTag = opts.splitTag ?? "holdout";
|
|
433
450
|
const commitSha = opts.commitSha ?? "unknown";
|
|
434
451
|
const promptHash = opts.promptHash ?? "unknown";
|
|
435
452
|
const configHash = opts.configHash ?? "unknown";
|
|
436
453
|
const seed = opts.seed ?? 0;
|
|
437
454
|
const fallbackModel = opts.fallbackModel ?? "unknown@otlp";
|
|
438
|
-
const byTrace = groupSpansByTrace(otlpJsonl);
|
|
439
455
|
if (byTrace.size === 0) {
|
|
440
456
|
throw new Error(
|
|
441
|
-
"otlpToRunRecords: OTLP input produced zero valid spans \u2014 every
|
|
457
|
+
"otlpToRunRecords: OTLP input produced zero valid spans \u2014 every row was empty, malformed, or missing trace_id/span_id"
|
|
442
458
|
);
|
|
443
459
|
}
|
|
444
460
|
const traceIds = [...byTrace.keys()].sort();
|
|
@@ -447,7 +463,7 @@ function otlpToTraceRunRecords(otlpJsonl, opts) {
|
|
|
447
463
|
const spans = byTrace.get(traceId);
|
|
448
464
|
const agg = aggregateTrace(traceId, spans, fallbackModel);
|
|
449
465
|
const score = resolveScore(opts, traceId, agg);
|
|
450
|
-
const { costUsd,
|
|
466
|
+
const { costUsd, costProvenance } = resolveCost(opts, agg);
|
|
451
467
|
const raw = {
|
|
452
468
|
span_count: agg.spanCount,
|
|
453
469
|
llm_span_count: agg.llmSpanCount,
|
|
@@ -457,10 +473,18 @@ function otlpToTraceRunRecords(otlpJsonl, opts) {
|
|
|
457
473
|
prompt_tokens: agg.tokenUsage.input,
|
|
458
474
|
completion_tokens: agg.tokenUsage.output
|
|
459
475
|
};
|
|
476
|
+
if (agg.tokenUsage.reasoning !== void 0) raw.reasoning_tokens = agg.tokenUsage.reasoning;
|
|
460
477
|
if (agg.tokenUsage.cached !== void 0) raw.cached_tokens = agg.tokenUsage.cached;
|
|
461
|
-
if (
|
|
478
|
+
if (agg.tokenUsage.cacheWrite !== void 0) {
|
|
479
|
+
raw.cache_write_tokens = agg.tokenUsage.cacheWrite;
|
|
480
|
+
}
|
|
481
|
+
if (agg.costMeasurement.value !== void 0 && !agg.costMeasurement.complete) {
|
|
482
|
+
raw.partial_observed_cost_usd = agg.costMeasurement.value;
|
|
483
|
+
}
|
|
484
|
+
recordAggregateMeasurements(raw, agg.aggregateMeasurement);
|
|
485
|
+
if (costProvenance.kind === "uncaptured") raw.cost_unpriced = 1;
|
|
462
486
|
const outcome = splitTag === "holdout" ? { holdoutScore: score, raw } : { searchScore: score, raw };
|
|
463
|
-
const { promptText, completionText } = extractPromptCompletion(spans);
|
|
487
|
+
const { promptText, completionText } = extractPromptCompletion(spans, agg.callSpanIds);
|
|
464
488
|
const judgeMetadata = opts.judgeMetadataForTrace?.(traceId);
|
|
465
489
|
const record = validateRunRecord({
|
|
466
490
|
runId: `otlp:${opts.experimentId}:${opts.candidateId}:${traceId}`,
|
|
@@ -473,6 +497,7 @@ function otlpToTraceRunRecords(otlpJsonl, opts) {
|
|
|
473
497
|
commitSha,
|
|
474
498
|
wallMs: agg.wallMs,
|
|
475
499
|
costUsd,
|
|
500
|
+
costProvenance,
|
|
476
501
|
tokenUsage: agg.tokenUsage,
|
|
477
502
|
...judgeMetadata ? { judgeMetadata } : {},
|
|
478
503
|
outcome,
|
|
@@ -488,8 +513,7 @@ function otlpToTraceRunRecords(otlpJsonl, opts) {
|
|
|
488
513
|
}
|
|
489
514
|
return out;
|
|
490
515
|
}
|
|
491
|
-
function
|
|
492
|
-
const byTrace = /* @__PURE__ */ new Map();
|
|
516
|
+
function* yieldJsonlRows(otlpJsonl) {
|
|
493
517
|
for (const line of otlpJsonl.split("\n")) {
|
|
494
518
|
const trimmed = line.trim();
|
|
495
519
|
if (trimmed.length === 0) continue;
|
|
@@ -499,8 +523,17 @@ function groupSpansByTrace(otlpJsonl) {
|
|
|
499
523
|
} catch {
|
|
500
524
|
continue;
|
|
501
525
|
}
|
|
502
|
-
if (
|
|
503
|
-
|
|
526
|
+
if (parsed && typeof parsed === "object") yield parsed;
|
|
527
|
+
}
|
|
528
|
+
}
|
|
529
|
+
function groupJsonlSpansByTrace(otlpJsonl) {
|
|
530
|
+
return groupRowsByTrace(yieldJsonlRows(otlpJsonl));
|
|
531
|
+
}
|
|
532
|
+
function groupRowsByTrace(rows) {
|
|
533
|
+
const byTrace = /* @__PURE__ */ new Map();
|
|
534
|
+
for (const row of rows) {
|
|
535
|
+
if (!row || typeof row !== "object") continue;
|
|
536
|
+
const span = projectOtlpFlatLine(row);
|
|
504
537
|
if (!span) continue;
|
|
505
538
|
const arr = byTrace.get(span.trace_id);
|
|
506
539
|
if (arr) arr.push(span);
|
|
@@ -512,11 +545,15 @@ function aggregateTrace(traceId, spans, fallbackModel) {
|
|
|
512
545
|
const ordered = [...spans].sort(
|
|
513
546
|
(a2, b2) => compareSpanTime(a2.start_time, b2.start_time) || a2.span_id.localeCompare(b2.span_id)
|
|
514
547
|
);
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
548
|
+
const measurements = summarizeExecutionMeasurements(
|
|
549
|
+
ordered.map((span) => ({
|
|
550
|
+
id: span.span_id,
|
|
551
|
+
...span.parent_span_id ? { parentId: span.parent_span_id } : {},
|
|
552
|
+
attributes: span.attributes,
|
|
553
|
+
modelCall: span.kind === "LLM" || span.kind === "UNKNOWN" && (span.model_name !== null || typeof span.attributes["gen_ai.operation.name"] === "string"),
|
|
554
|
+
aggregate: span.kind !== "LLM" && span.kind !== "UNKNOWN"
|
|
555
|
+
}))
|
|
556
|
+
);
|
|
520
557
|
let toolSpanCount = 0;
|
|
521
558
|
let agentSpanCount = 0;
|
|
522
559
|
let errorSpanCount = 0;
|
|
@@ -528,20 +565,7 @@ function aggregateTrace(traceId, spans, fallbackModel) {
|
|
|
528
565
|
if (s.start_time && (!earliest || compareSpanTime(s.start_time, earliest) < 0))
|
|
529
566
|
earliest = s.start_time;
|
|
530
567
|
if (s.end_time && (!latest || compareSpanTime(s.end_time, latest) > 0)) latest = s.end_time;
|
|
531
|
-
if (s.kind === "
|
|
532
|
-
llmSpanCount += 1;
|
|
533
|
-
const i = firstNumberAttr(s.attributes, LLM_INPUT_TOKEN_ATTR_KEYS);
|
|
534
|
-
const o = firstNumberAttr(s.attributes, LLM_OUTPUT_TOKEN_ATTR_KEYS);
|
|
535
|
-
const c = firstNumberAttr(s.attributes, LLM_CACHED_TOKEN_ATTR_KEYS);
|
|
536
|
-
if (i !== null) input += i;
|
|
537
|
-
if (o !== null) output += o;
|
|
538
|
-
if (c !== null) {
|
|
539
|
-
cached += c;
|
|
540
|
-
sawCached = true;
|
|
541
|
-
}
|
|
542
|
-
const m = firstStringAttr(s.attributes, LLM_MODEL_ATTR_KEYS) ?? s.model_name;
|
|
543
|
-
if (m) modelVotes.set(m, (modelVotes.get(m) ?? 0) + 1);
|
|
544
|
-
} else if (s.kind === "TOOL") {
|
|
568
|
+
if (s.kind === "TOOL") {
|
|
545
569
|
toolSpanCount += 1;
|
|
546
570
|
} else if (s.kind === "AGENT") {
|
|
547
571
|
agentSpanCount += 1;
|
|
@@ -553,25 +577,33 @@ function aggregateTrace(traceId, spans, fallbackModel) {
|
|
|
553
577
|
}
|
|
554
578
|
}
|
|
555
579
|
}
|
|
580
|
+
const callSpanIds = new Set(measurements.callSpanIds);
|
|
581
|
+
for (const span of ordered) {
|
|
582
|
+
if (!callSpanIds.has(span.span_id)) continue;
|
|
583
|
+
const model2 = firstStringAttr(span.attributes, LLM_MODEL_ATTR_KEYS) ?? span.model_name;
|
|
584
|
+
if (model2) modelVotes.set(model2, (modelVotes.get(model2) ?? 0) + 1);
|
|
585
|
+
}
|
|
556
586
|
const model = topVote(modelVotes) ?? firstModelAttr(ordered) ?? fallbackModel;
|
|
557
587
|
let wallMs = 0;
|
|
558
588
|
const a = spanEpochMillis(earliest);
|
|
559
589
|
const b = spanEpochMillis(latest);
|
|
560
590
|
if (a !== null && b !== null) wallMs = Math.max(0, b - a);
|
|
561
|
-
const tokenUsage = sawCached ? { input, output, cached } : { input, output };
|
|
562
591
|
return {
|
|
563
592
|
traceId,
|
|
564
593
|
spanCount: spans.length,
|
|
565
|
-
llmSpanCount,
|
|
594
|
+
llmSpanCount: measurements.modelCallCount,
|
|
566
595
|
toolSpanCount,
|
|
567
596
|
agentSpanCount,
|
|
568
597
|
errorSpanCount,
|
|
569
|
-
tokenUsage,
|
|
598
|
+
tokenUsage: measurements.tokenUsage,
|
|
570
599
|
firstErrorMessage,
|
|
571
600
|
model,
|
|
572
601
|
startTime: earliest,
|
|
573
602
|
endTime: latest,
|
|
574
|
-
wallMs
|
|
603
|
+
wallMs,
|
|
604
|
+
callSpanIds: measurements.callSpanIds,
|
|
605
|
+
costMeasurement: measurements.cost,
|
|
606
|
+
...measurements.aggregate ? { aggregateMeasurement: measurements.aggregate } : {}
|
|
575
607
|
};
|
|
576
608
|
}
|
|
577
609
|
function resolveScore(opts, traceId, agg) {
|
|
@@ -586,26 +618,31 @@ function resolveScore(opts, traceId, agg) {
|
|
|
586
618
|
}
|
|
587
619
|
return agg.errorSpanCount > 0 ? 0 : 1;
|
|
588
620
|
}
|
|
589
|
-
function resolveCost(opts, agg
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
621
|
+
function resolveCost(opts, agg) {
|
|
622
|
+
const observedCost = agg.costMeasurement;
|
|
623
|
+
if (observedCost.complete && observedCost.value !== void 0) {
|
|
624
|
+
return {
|
|
625
|
+
costUsd: observedCost.value,
|
|
626
|
+
costProvenance: { kind: "observed", usd: observedCost.value }
|
|
627
|
+
};
|
|
628
|
+
}
|
|
629
|
+
if (agg.aggregateMeasurement?.costUsd !== void 0) {
|
|
630
|
+
return {
|
|
631
|
+
costUsd: agg.aggregateMeasurement.costUsd,
|
|
632
|
+
costProvenance: { kind: "observed", usd: agg.aggregateMeasurement.costUsd }
|
|
633
|
+
};
|
|
599
634
|
}
|
|
600
|
-
if (sawCost) return { costUsd: perSpanCost, costUnpriced: false };
|
|
601
635
|
if (opts.priceUsdPerToken !== void 0) {
|
|
602
636
|
const totalTokens = agg.tokenUsage.input + agg.tokenUsage.output;
|
|
603
|
-
|
|
637
|
+
const costUsd = totalTokens * opts.priceUsdPerToken;
|
|
638
|
+
return { costUsd, costProvenance: { kind: "estimated", usd: costUsd } };
|
|
604
639
|
}
|
|
605
|
-
return { costUsd: 0,
|
|
640
|
+
return { costUsd: 0, costProvenance: { kind: "uncaptured", usd: null } };
|
|
606
641
|
}
|
|
607
|
-
function extractPromptCompletion(spans) {
|
|
608
|
-
const
|
|
642
|
+
function extractPromptCompletion(spans, callSpanIds) {
|
|
643
|
+
const callIds = new Set(callSpanIds);
|
|
644
|
+
const measuredCalls = spans.filter((span) => callIds.has(span.span_id));
|
|
645
|
+
const llm = (measuredCalls.length > 0 ? measuredCalls : spans.filter((s) => s.kind === "LLM")).sort(
|
|
609
646
|
(a, b) => compareSpanTime(a.start_time, b.start_time) || a.span_id.localeCompare(b.span_id)
|
|
610
647
|
);
|
|
611
648
|
if (llm.length === 0) return {};
|
|
@@ -945,85 +982,6 @@ var FileSystemTraceStore = class {
|
|
|
945
982
|
}
|
|
946
983
|
};
|
|
947
984
|
|
|
948
|
-
// src/trace/extract-usage.ts
|
|
949
|
-
var INPUT_KEYS = ["prompt_tokens", "input_tokens", "promptTokens", "inputTokens"];
|
|
950
|
-
var OUTPUT_KEYS = [
|
|
951
|
-
"completion_tokens",
|
|
952
|
-
"output_tokens",
|
|
953
|
-
"completionTokens",
|
|
954
|
-
"outputTokens"
|
|
955
|
-
];
|
|
956
|
-
var CACHED_KEYS = [
|
|
957
|
-
"cached_tokens",
|
|
958
|
-
"cache_read_input_tokens",
|
|
959
|
-
"cachedTokens",
|
|
960
|
-
"cacheReadInputTokens"
|
|
961
|
-
];
|
|
962
|
-
function nonNegNumber(source, keys) {
|
|
963
|
-
for (const key of keys) {
|
|
964
|
-
const value = source[key];
|
|
965
|
-
if (typeof value === "number" && Number.isFinite(value) && value >= 0) return value;
|
|
966
|
-
}
|
|
967
|
-
return null;
|
|
968
|
-
}
|
|
969
|
-
function extractUsage(body) {
|
|
970
|
-
if (!body || typeof body !== "object") return null;
|
|
971
|
-
const obj = body;
|
|
972
|
-
const usage = obj.usage && typeof obj.usage === "object" ? obj.usage : obj;
|
|
973
|
-
const input = nonNegNumber(usage, INPUT_KEYS);
|
|
974
|
-
const output = nonNegNumber(usage, OUTPUT_KEYS);
|
|
975
|
-
if (input === null && output === null) return null;
|
|
976
|
-
const cached = nonNegNumber(usage, CACHED_KEYS);
|
|
977
|
-
const result = { input: input ?? 0, output: output ?? 0 };
|
|
978
|
-
if (cached !== null) result.cached = cached;
|
|
979
|
-
return result;
|
|
980
|
-
}
|
|
981
|
-
function extractUsageFromSse(text) {
|
|
982
|
-
let input = 0;
|
|
983
|
-
let output = 0;
|
|
984
|
-
let cached = 0;
|
|
985
|
-
let sawCached = false;
|
|
986
|
-
let found = false;
|
|
987
|
-
for (const line of text.split(/\r?\n/)) {
|
|
988
|
-
const trimmed = line.trim();
|
|
989
|
-
if (!trimmed.startsWith("data:")) continue;
|
|
990
|
-
const payload = trimmed.slice("data:".length).trim();
|
|
991
|
-
if (!payload || payload === "[DONE]") continue;
|
|
992
|
-
let parsed;
|
|
993
|
-
try {
|
|
994
|
-
parsed = JSON.parse(payload);
|
|
995
|
-
} catch {
|
|
996
|
-
continue;
|
|
997
|
-
}
|
|
998
|
-
const usage = extractUsage(parsed);
|
|
999
|
-
if (!usage) continue;
|
|
1000
|
-
input += usage.input;
|
|
1001
|
-
output += usage.output;
|
|
1002
|
-
if (usage.cached !== void 0) {
|
|
1003
|
-
cached += usage.cached;
|
|
1004
|
-
sawCached = true;
|
|
1005
|
-
}
|
|
1006
|
-
found = true;
|
|
1007
|
-
}
|
|
1008
|
-
if (!found) return null;
|
|
1009
|
-
return sawCached ? { input, output, cached } : { input, output };
|
|
1010
|
-
}
|
|
1011
|
-
async function extractUsageFromResponse(response) {
|
|
1012
|
-
let text;
|
|
1013
|
-
try {
|
|
1014
|
-
text = await response.clone().text();
|
|
1015
|
-
} catch {
|
|
1016
|
-
return null;
|
|
1017
|
-
}
|
|
1018
|
-
let json;
|
|
1019
|
-
try {
|
|
1020
|
-
json = JSON.parse(text);
|
|
1021
|
-
} catch {
|
|
1022
|
-
json = void 0;
|
|
1023
|
-
}
|
|
1024
|
-
return extractUsage(json) ?? extractUsageFromSse(text);
|
|
1025
|
-
}
|
|
1026
|
-
|
|
1027
985
|
// src/trace/capture-fetch.ts
|
|
1028
986
|
var DEFAULT_BODY_CAP = 2 * 1024 * 1024;
|
|
1029
987
|
function headersToRecord(headers) {
|
|
@@ -1136,7 +1094,7 @@ function captureFetchToRawSink(fetch2, sink, ctx, opts = {}) {
|
|
|
1136
1094
|
if (opts.onUsage && rawText !== void 0) {
|
|
1137
1095
|
try {
|
|
1138
1096
|
const parsedForUsage = parseMaybeJson(rawText);
|
|
1139
|
-
const usage = extractUsage(parsedForUsage) ?? (typeof parsedForUsage === "string" ? extractUsageFromSse(rawText) : null);
|
|
1097
|
+
const usage = extractUsage(parsedForUsage) ?? (typeof parsedForUsage === "string" ? extractUsageFromSse(rawText, { mode: opts.sseUsageMode }) : null);
|
|
1140
1098
|
if (usage) opts.onUsage(usage, ctx);
|
|
1141
1099
|
} catch (err) {
|
|
1142
1100
|
if (opts.failClosed) throw err;
|
|
@@ -1224,6 +1182,11 @@ function flattenSpanAttributes(span) {
|
|
|
1224
1182
|
base[LLM_MODEL_NAME] = span.model;
|
|
1225
1183
|
if (span.inputTokens !== void 0) base[LLM_INPUT_TOKENS] = span.inputTokens;
|
|
1226
1184
|
if (span.outputTokens !== void 0) base[LLM_OUTPUT_TOKENS] = span.outputTokens;
|
|
1185
|
+
if (span.reasoningTokens !== void 0) base[LLM_REASONING_TOKENS] = span.reasoningTokens;
|
|
1186
|
+
if (span.cachedTokens !== void 0) base[LLM_CACHED_TOKENS] = span.cachedTokens;
|
|
1187
|
+
if (span.cacheWriteTokens !== void 0) {
|
|
1188
|
+
base[LLM_CACHE_WRITE_TOKENS] = span.cacheWriteTokens;
|
|
1189
|
+
}
|
|
1227
1190
|
if (span.costUsd !== void 0) base[LLM_COST_USD] = span.costUsd;
|
|
1228
1191
|
if (span.finishReason) base["llm.finish_reason"] = span.finishReason;
|
|
1229
1192
|
} else if (span.kind === "tool") {
|
|
@@ -1339,6 +1302,9 @@ function storeSpanToExportable(span, traceId) {
|
|
|
1339
1302
|
model: llm?.model,
|
|
1340
1303
|
inputTokens: llm?.inputTokens,
|
|
1341
1304
|
outputTokens: llm?.outputTokens,
|
|
1305
|
+
reasoningTokens: llm?.reasoningTokens,
|
|
1306
|
+
cachedTokens: llm?.cachedTokens,
|
|
1307
|
+
cacheWriteTokens: llm?.cacheWriteTokens,
|
|
1342
1308
|
costUsd: llm?.costUsd,
|
|
1343
1309
|
tool: tool ? {
|
|
1344
1310
|
toolName: tool.toolName,
|
|
@@ -1448,6 +1414,11 @@ function toOtlpSpan(span) {
|
|
|
1448
1414
|
if (span.model) attrs[LLM_MODEL_NAME] = span.model;
|
|
1449
1415
|
if (span.inputTokens !== void 0) attrs[LLM_INPUT_TOKENS] = span.inputTokens;
|
|
1450
1416
|
if (span.outputTokens !== void 0) attrs[LLM_OUTPUT_TOKENS] = span.outputTokens;
|
|
1417
|
+
if (span.reasoningTokens !== void 0) attrs[LLM_REASONING_TOKENS] = span.reasoningTokens;
|
|
1418
|
+
if (span.cachedTokens !== void 0) attrs[LLM_CACHED_TOKENS] = span.cachedTokens;
|
|
1419
|
+
if (span.cacheWriteTokens !== void 0) {
|
|
1420
|
+
attrs[LLM_CACHE_WRITE_TOKENS] = span.cacheWriteTokens;
|
|
1421
|
+
}
|
|
1451
1422
|
if (span.costUsd !== void 0) attrs[LLM_COST_USD] = span.costUsd;
|
|
1452
1423
|
if (span.tool) applyToolSpanOtlpAttributes(attrs, span.tool);
|
|
1453
1424
|
return {
|
|
@@ -1699,6 +1670,11 @@ function spanToAttributes(span, events) {
|
|
|
1699
1670
|
attrs[LLM_MODEL_NAME] = span.model;
|
|
1700
1671
|
if (span.inputTokens !== void 0) attrs[LLM_INPUT_TOKENS] = span.inputTokens;
|
|
1701
1672
|
if (span.outputTokens !== void 0) attrs[LLM_OUTPUT_TOKENS] = span.outputTokens;
|
|
1673
|
+
if (span.reasoningTokens !== void 0) attrs[LLM_REASONING_TOKENS] = span.reasoningTokens;
|
|
1674
|
+
if (span.cachedTokens !== void 0) attrs[LLM_CACHED_TOKENS] = span.cachedTokens;
|
|
1675
|
+
if (span.cacheWriteTokens !== void 0) {
|
|
1676
|
+
attrs[LLM_CACHE_WRITE_TOKENS] = span.cacheWriteTokens;
|
|
1677
|
+
}
|
|
1702
1678
|
if (span.costUsd !== void 0) attrs[LLM_COST_USD] = span.costUsd;
|
|
1703
1679
|
if (span.finishReason) attrs["llm.finish_reason"] = span.finishReason;
|
|
1704
1680
|
if (Array.isArray(span.messages)) {
|
|
@@ -1933,12 +1909,11 @@ export {
|
|
|
1933
1909
|
buildTraceInsightPrompt,
|
|
1934
1910
|
flattenOtlpExportToNdjson,
|
|
1935
1911
|
otlpToRunRecords,
|
|
1912
|
+
otlpRowsToRunRecords,
|
|
1936
1913
|
otlpToTraceRunRecords,
|
|
1914
|
+
otlpRowsToTraceRunRecords,
|
|
1937
1915
|
InMemoryTraceStore,
|
|
1938
1916
|
FileSystemTraceStore,
|
|
1939
|
-
extractUsage,
|
|
1940
|
-
extractUsageFromSse,
|
|
1941
|
-
extractUsageFromResponse,
|
|
1942
1917
|
captureFetchToRawSink,
|
|
1943
1918
|
OTEL_AGENT_EVAL_SCOPE,
|
|
1944
1919
|
exportRunAsOtlp,
|
|
@@ -1951,4 +1926,4 @@ export {
|
|
|
1951
1926
|
createReplayFetch,
|
|
1952
1927
|
iterateRawCalls
|
|
1953
1928
|
};
|
|
1954
|
-
//# sourceMappingURL=chunk-
|
|
1929
|
+
//# sourceMappingURL=chunk-FTUMG2U7.js.map
|