braintrust 3.20.0 → 3.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +20 -0
- package/README.md +6 -0
- package/dev/dist/index.d.mts +1338 -199
- package/dev/dist/index.d.ts +1338 -199
- package/dev/dist/index.js +2904 -997
- package/dev/dist/index.mjs +2418 -511
- package/dist/apply-auto-instrumentation.js +781 -199
- package/dist/apply-auto-instrumentation.mjs +588 -6
- package/dist/auto-instrumentations/bundler/esbuild.cjs +594 -2
- package/dist/auto-instrumentations/bundler/esbuild.d.mts +2 -2
- package/dist/auto-instrumentations/bundler/esbuild.d.ts +2 -2
- package/dist/auto-instrumentations/bundler/esbuild.mjs +3 -3
- package/dist/auto-instrumentations/bundler/next.cjs +594 -2
- package/dist/auto-instrumentations/bundler/next.mjs +4 -4
- package/dist/auto-instrumentations/bundler/rollup.cjs +594 -2
- package/dist/auto-instrumentations/bundler/rollup.d.mts +2 -2
- package/dist/auto-instrumentations/bundler/rollup.d.ts +2 -2
- package/dist/auto-instrumentations/bundler/rollup.mjs +3 -3
- package/dist/auto-instrumentations/bundler/vite.cjs +594 -2
- package/dist/auto-instrumentations/bundler/vite.d.mts +2 -2
- package/dist/auto-instrumentations/bundler/vite.d.ts +2 -2
- package/dist/auto-instrumentations/bundler/vite.mjs +3 -3
- package/dist/auto-instrumentations/bundler/webpack-loader.cjs +593 -9
- package/dist/auto-instrumentations/bundler/webpack-loader.d.ts +68 -1
- package/dist/auto-instrumentations/bundler/webpack.cjs +594 -2
- package/dist/auto-instrumentations/bundler/webpack.d.mts +2 -2
- package/dist/auto-instrumentations/bundler/webpack.d.ts +2 -2
- package/dist/auto-instrumentations/bundler/webpack.mjs +4 -4
- package/dist/auto-instrumentations/{chunk-W4E36GIW.mjs → chunk-BRQX23KL.mjs} +16 -0
- package/dist/auto-instrumentations/chunk-M6DLIJ2Z.mjs +784 -0
- package/dist/auto-instrumentations/{chunk-K74TZGGM.mjs → chunk-T6J4C7LX.mjs} +1 -1
- package/dist/auto-instrumentations/{chunk-S4374IP6.mjs → chunk-TKRPRPGD.mjs} +4 -6
- package/dist/auto-instrumentations/chunk-VT6DDNKM.mjs +856 -0
- package/dist/auto-instrumentations/hook.mjs +824 -56
- package/dist/auto-instrumentations/index.cjs +16 -0
- package/dist/auto-instrumentations/index.d.mts +2 -3
- package/dist/auto-instrumentations/index.d.ts +2 -3
- package/dist/auto-instrumentations/index.mjs +1 -1
- package/dist/auto-instrumentations/loader/cjs-patch.cjs +578 -4
- package/dist/auto-instrumentations/loader/cjs-patch.d.mts +1 -1
- package/dist/auto-instrumentations/loader/cjs-patch.d.ts +1 -1
- package/dist/auto-instrumentations/loader/cjs-patch.mjs +3 -7
- package/dist/auto-instrumentations/loader/esm-hook.mjs +2 -6
- package/dist/auto-instrumentations/{plugin-D7nDswtC.d.mts → plugin-D0KHwSJv.d.mts} +1 -1
- package/dist/auto-instrumentations/{plugin-D7nDswtC.d.ts → plugin-G6fgxk1b.d.ts} +1 -1
- package/dist/auto-instrumentations/types-RNPaKi9o.d.mts +70 -0
- package/dist/auto-instrumentations/types-RNPaKi9o.d.ts +70 -0
- package/dist/browser.d.mts +4411 -719
- package/dist/browser.d.ts +4411 -719
- package/dist/browser.js +4274 -480
- package/dist/browser.mjs +4274 -480
- package/dist/{chunk-LPC4W2WX.js → chunk-BFGIH2ZJ.js} +4 -0
- package/dist/{chunk-Y7W7WP6H.mjs → chunk-KMGUTPB7.mjs} +4 -0
- package/dist/{chunk-POCCIJAL.js → chunk-MWVVR5LR.js} +3278 -1334
- package/dist/{chunk-F73OMF66.mjs → chunk-ZG2O3XVF.mjs} +2542 -598
- package/dist/cli.js +2338 -419
- package/dist/edge-light.d.mts +1 -1
- package/dist/edge-light.d.ts +1 -1
- package/dist/edge-light.js +4274 -480
- package/dist/edge-light.mjs +4274 -480
- package/dist/index.d.mts +4926 -1234
- package/dist/index.d.ts +4926 -1234
- package/dist/index.js +2303 -435
- package/dist/index.mjs +1914 -46
- package/dist/instrumentation/index.d.mts +1026 -20
- package/dist/instrumentation/index.d.ts +1026 -20
- package/dist/instrumentation/index.js +3864 -483
- package/dist/instrumentation/index.mjs +3862 -483
- package/dist/vitest-evals-reporter.js +16 -16
- package/dist/vitest-evals-reporter.mjs +2 -2
- package/dist/workerd.d.mts +1 -1
- package/dist/workerd.d.ts +1 -1
- package/dist/workerd.js +4274 -480
- package/dist/workerd.mjs +4274 -480
- package/licenses/import-in-the-middle/LICENSE +201 -0
- package/licenses/import-in-the-middle/LICENSE-3rdparty.csv +4 -0
- package/licenses/import-in-the-middle/NOTICE +34 -0
- package/licenses/orchestrion-js/LICENSE +202 -0
- package/licenses/orchestrion-js/LICENSE-3rdparty.csv +8 -0
- package/licenses/orchestrion-js/NOTICE +3 -0
- package/licenses/require-in-the-middle/LICENSE +23 -0
- package/package.json +16 -4
- package/util/dist/index.d.mts +1 -0
- package/util/dist/index.d.ts +1 -0
- package/util/dist/index.js +4 -4
- package/util/dist/index.mjs +2 -2
- package/dist/auto-instrumentations/chunk-J57YF7WS.mjs +0 -208
- package/dist/auto-instrumentations/chunk-QFMACSOL.mjs +0 -280
package/dist/index.mjs
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
import {
|
|
2
2
|
Attachment,
|
|
3
3
|
AttachmentReference,
|
|
4
|
+
BAGGAGE_HEADER,
|
|
4
5
|
BRAINTRUST_CURRENT_SPAN_STORE,
|
|
5
6
|
BRAINTRUST_LANGCHAIN_CALLBACK_HANDLER_NAME,
|
|
7
|
+
BRAINTRUST_PARENT_KEY,
|
|
6
8
|
BaseAttachment,
|
|
7
9
|
BraintrustLangChainCallbackHandler,
|
|
8
10
|
BraintrustObservabilityExporter,
|
|
@@ -25,6 +27,7 @@ import {
|
|
|
25
27
|
JSONAttachment,
|
|
26
28
|
LEGACY_CACHED_HEADER,
|
|
27
29
|
LOGS3_OVERFLOW_REFERENCE_TYPE,
|
|
30
|
+
LRUCache,
|
|
28
31
|
LazyValue,
|
|
29
32
|
Logger,
|
|
30
33
|
LoginInvalidOrgError,
|
|
@@ -32,6 +35,7 @@ import {
|
|
|
32
35
|
NOOP_SPAN,
|
|
33
36
|
NOOP_SPAN_PERMALINK,
|
|
34
37
|
NoopSpan,
|
|
38
|
+
OTELIDGenerator,
|
|
35
39
|
ObjectFetcher,
|
|
36
40
|
ObjectReference,
|
|
37
41
|
Project,
|
|
@@ -41,8 +45,10 @@ import {
|
|
|
41
45
|
ReadonlyExperiment,
|
|
42
46
|
RemoteEvalParameters,
|
|
43
47
|
RunEval,
|
|
44
|
-
|
|
48
|
+
SpanComponentsV4,
|
|
45
49
|
SpanImpl,
|
|
50
|
+
TRACEPARENT_HEADER,
|
|
51
|
+
TRACESTATE_HEADER,
|
|
46
52
|
TestBackgroundLogger,
|
|
47
53
|
ToolFunctionDefinition,
|
|
48
54
|
UUIDGenerator,
|
|
@@ -50,6 +56,7 @@ import {
|
|
|
50
56
|
_exportsForTestingOnly,
|
|
51
57
|
_internalGetGlobalState,
|
|
52
58
|
_internalSetInitialState,
|
|
59
|
+
_internalStartSpanWithInitialMerge,
|
|
53
60
|
addAzureBlobHeaders,
|
|
54
61
|
braintrustAISDKTelemetry,
|
|
55
62
|
braintrustFlueInstrumentation,
|
|
@@ -63,11 +70,13 @@ import {
|
|
|
63
70
|
currentExperiment,
|
|
64
71
|
currentLogger,
|
|
65
72
|
currentSpan,
|
|
73
|
+
currentWorkflowAgentWrapperSpan,
|
|
66
74
|
debugLogger,
|
|
67
75
|
deepCopyEvent,
|
|
68
76
|
deserializePlainStringAsJSON,
|
|
69
77
|
devNullWritableStream,
|
|
70
78
|
extractAnthropicCacheTokens,
|
|
79
|
+
extractTraceContextFromHeaders,
|
|
71
80
|
finalizeAnthropicTokens,
|
|
72
81
|
flush,
|
|
73
82
|
getBedrockRuntimeOperation,
|
|
@@ -81,6 +90,7 @@ import {
|
|
|
81
90
|
initDataset,
|
|
82
91
|
initExperiment,
|
|
83
92
|
initLogger,
|
|
93
|
+
injectTraceContext,
|
|
84
94
|
isEmpty,
|
|
85
95
|
isObject,
|
|
86
96
|
isTemplateFormat,
|
|
@@ -129,7 +139,7 @@ import {
|
|
|
129
139
|
wrapMastraAgent,
|
|
130
140
|
wrapTraced,
|
|
131
141
|
zodToJsonSchema
|
|
132
|
-
} from "./chunk-
|
|
142
|
+
} from "./chunk-ZG2O3XVF.mjs";
|
|
133
143
|
import {
|
|
134
144
|
__export,
|
|
135
145
|
aiSDKChannels,
|
|
@@ -152,15 +162,17 @@ import {
|
|
|
152
162
|
openRouterChannels,
|
|
153
163
|
piCodingAgentChannels,
|
|
154
164
|
strandsAgentSDKChannels
|
|
155
|
-
} from "./chunk-
|
|
165
|
+
} from "./chunk-KMGUTPB7.mjs";
|
|
156
166
|
|
|
157
167
|
// src/exports.ts
|
|
158
168
|
var exports_exports = {};
|
|
159
169
|
__export(exports_exports, {
|
|
160
170
|
Attachment: () => Attachment,
|
|
161
171
|
AttachmentReference: () => AttachmentReference,
|
|
172
|
+
BAGGAGE_HEADER: () => BAGGAGE_HEADER,
|
|
162
173
|
BRAINTRUST_CURRENT_SPAN_STORE: () => BRAINTRUST_CURRENT_SPAN_STORE,
|
|
163
174
|
BRAINTRUST_LANGCHAIN_CALLBACK_HANDLER_NAME: () => BRAINTRUST_LANGCHAIN_CALLBACK_HANDLER_NAME,
|
|
175
|
+
BRAINTRUST_PARENT_KEY: () => BRAINTRUST_PARENT_KEY,
|
|
164
176
|
BaseAttachment: () => BaseAttachment,
|
|
165
177
|
BaseExperiment: () => BaseExperiment,
|
|
166
178
|
BraintrustLangChainCallbackHandler: () => BraintrustLangChainCallbackHandler,
|
|
@@ -193,6 +205,7 @@ __export(exports_exports, {
|
|
|
193
205
|
NOOP_SPAN: () => NOOP_SPAN,
|
|
194
206
|
NOOP_SPAN_PERMALINK: () => NOOP_SPAN_PERMALINK,
|
|
195
207
|
NoopSpan: () => NoopSpan,
|
|
208
|
+
OTELIDGenerator: () => OTELIDGenerator,
|
|
196
209
|
ObjectFetcher: () => ObjectFetcher,
|
|
197
210
|
Project: () => Project2,
|
|
198
211
|
ProjectNameIdMap: () => ProjectNameIdMap,
|
|
@@ -204,6 +217,8 @@ __export(exports_exports, {
|
|
|
204
217
|
ScorerBuilder: () => ScorerBuilder,
|
|
205
218
|
SpanFetcher: () => SpanFetcher,
|
|
206
219
|
SpanImpl: () => SpanImpl,
|
|
220
|
+
TRACEPARENT_HEADER: () => TRACEPARENT_HEADER,
|
|
221
|
+
TRACESTATE_HEADER: () => TRACESTATE_HEADER,
|
|
207
222
|
TestBackgroundLogger: () => TestBackgroundLogger,
|
|
208
223
|
ToolBuilder: () => ToolBuilder,
|
|
209
224
|
UUIDGenerator: () => UUIDGenerator,
|
|
@@ -213,7 +228,10 @@ __export(exports_exports, {
|
|
|
213
228
|
_internalIso: () => isomorph_default,
|
|
214
229
|
_internalSetInitialState: () => _internalSetInitialState,
|
|
215
230
|
addAzureBlobHeaders: () => addAzureBlobHeaders,
|
|
231
|
+
agentAssertionScorer: () => agentAssertionScorer,
|
|
216
232
|
braintrustAISDKTelemetry: () => braintrustAISDKTelemetry,
|
|
233
|
+
braintrustEveHook: () => braintrustEveHook,
|
|
234
|
+
braintrustEveInstrumentation: () => braintrustEveInstrumentation,
|
|
217
235
|
braintrustFlueInstrumentation: () => braintrustFlueInstrumentation,
|
|
218
236
|
braintrustFlueObserver: () => braintrustFlueObserver,
|
|
219
237
|
braintrustStreamChunkSchema: () => braintrustStreamChunkSchema,
|
|
@@ -230,6 +248,7 @@ __export(exports_exports, {
|
|
|
230
248
|
devNullWritableStream: () => devNullWritableStream,
|
|
231
249
|
evaluatorDefinitionSchema: () => evaluatorDefinitionSchema,
|
|
232
250
|
evaluatorDefinitionsSchema: () => evaluatorDefinitionsSchema,
|
|
251
|
+
extractTraceContextFromHeaders: () => extractTraceContextFromHeaders,
|
|
233
252
|
flush: () => flush,
|
|
234
253
|
getContextManager: () => getContextManager,
|
|
235
254
|
getIdGenerator: () => getIdGenerator,
|
|
@@ -243,6 +262,7 @@ __export(exports_exports, {
|
|
|
243
262
|
initFunction: () => initFunction,
|
|
244
263
|
initLogger: () => initLogger,
|
|
245
264
|
initNodeTestSuite: () => initNodeTestSuite,
|
|
265
|
+
injectTraceContext: () => injectTraceContext,
|
|
246
266
|
invoke: () => invoke,
|
|
247
267
|
isTemplateFormat: () => isTemplateFormat,
|
|
248
268
|
loadParameters: () => loadParameters,
|
|
@@ -874,6 +894,7 @@ function wrapAISDK(aiSDK, options = {}) {
|
|
|
874
894
|
case "Agent":
|
|
875
895
|
case "Experimental_Agent":
|
|
876
896
|
case "ToolLoopAgent":
|
|
897
|
+
case "WorkflowAgent":
|
|
877
898
|
return original ? wrapAgentClass(original, options) : original;
|
|
878
899
|
}
|
|
879
900
|
return original;
|
|
@@ -892,10 +913,10 @@ var wrapAgentClass = (AgentClass, options = {}) => {
|
|
|
892
913
|
return new Proxy(instance, {
|
|
893
914
|
get(instanceTarget, prop, instanceReceiver) {
|
|
894
915
|
const original = Reflect.get(instanceTarget, prop, instanceTarget);
|
|
895
|
-
if (prop === "generate") {
|
|
916
|
+
if (prop === "generate" && typeof original === "function" && instanceTarget.constructor.name !== "WorkflowAgent") {
|
|
896
917
|
return wrapAgentGenerate(original, instanceTarget, options);
|
|
897
918
|
}
|
|
898
|
-
if (prop === "stream") {
|
|
919
|
+
if (prop === "stream" && typeof original === "function") {
|
|
899
920
|
return wrapAgentStream(original, instanceTarget, options);
|
|
900
921
|
}
|
|
901
922
|
if (typeof original === "function") {
|
|
@@ -910,7 +931,7 @@ var wrapAgentClass = (AgentClass, options = {}) => {
|
|
|
910
931
|
var wrapAgentGenerate = (generate, instance, options = {}) => {
|
|
911
932
|
const defaultName = `${instance.constructor.name}.generate`;
|
|
912
933
|
return async (params) => makeGenerateTextWrapper(
|
|
913
|
-
|
|
934
|
+
generateChannelForAgent(instance.constructor.name),
|
|
914
935
|
defaultName,
|
|
915
936
|
generate.bind(instance),
|
|
916
937
|
{
|
|
@@ -920,19 +941,42 @@ var wrapAgentGenerate = (generate, instance, options = {}) => {
|
|
|
920
941
|
options
|
|
921
942
|
)({ ...instance.settings, ...params });
|
|
922
943
|
};
|
|
944
|
+
function generateChannelForAgent(agentName) {
|
|
945
|
+
if (agentName === "ToolLoopAgent") {
|
|
946
|
+
return aiSDKChannels.toolLoopAgentGenerate;
|
|
947
|
+
}
|
|
948
|
+
return aiSDKChannels.agentGenerate;
|
|
949
|
+
}
|
|
923
950
|
var wrapAgentStream = (stream, instance, options = {}) => {
|
|
924
951
|
const defaultName = `${instance.constructor.name}.stream`;
|
|
925
|
-
return (params) =>
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
952
|
+
return (params) => {
|
|
953
|
+
const workflowAgent = instance.constructor.name === "WorkflowAgent";
|
|
954
|
+
if (workflowAgent && currentWorkflowAgentWrapperSpan()) {
|
|
955
|
+
const { span_info: _spanInfo, ...cleanParams } = params;
|
|
956
|
+
return stream.call(instance, { ...instance.settings, ...cleanParams });
|
|
957
|
+
}
|
|
958
|
+
const trace = () => makeStreamWrapper(
|
|
959
|
+
streamChannelForAgent(instance.constructor.name),
|
|
960
|
+
defaultName,
|
|
961
|
+
stream.bind(instance),
|
|
962
|
+
{
|
|
963
|
+
self: instance,
|
|
964
|
+
spanType: "function" /* FUNCTION */
|
|
965
|
+
},
|
|
966
|
+
options
|
|
967
|
+
)({ ...instance.settings, ...params });
|
|
968
|
+
return trace();
|
|
969
|
+
};
|
|
935
970
|
};
|
|
971
|
+
function streamChannelForAgent(agentName) {
|
|
972
|
+
if (agentName === "ToolLoopAgent") {
|
|
973
|
+
return aiSDKChannels.toolLoopAgentStream;
|
|
974
|
+
}
|
|
975
|
+
if (agentName === "WorkflowAgent") {
|
|
976
|
+
return aiSDKChannels.workflowAgentStream;
|
|
977
|
+
}
|
|
978
|
+
return aiSDKChannels.agentStream;
|
|
979
|
+
}
|
|
936
980
|
var makeGenerateTextWrapper = (channel, name, generateText, contextOptions = {}, options = {}) => {
|
|
937
981
|
const wrapper = async function(allParams) {
|
|
938
982
|
const { span_info, ...params } = allParams;
|
|
@@ -1194,7 +1238,7 @@ var BraintrustLanguageModelWrapper = class {
|
|
|
1194
1238
|
let time_to_first_token = void 0;
|
|
1195
1239
|
let usage = void 0;
|
|
1196
1240
|
let fullText = void 0;
|
|
1197
|
-
const
|
|
1241
|
+
const toolCalls2 = {};
|
|
1198
1242
|
let finishReason = void 0;
|
|
1199
1243
|
return {
|
|
1200
1244
|
...ret,
|
|
@@ -1213,7 +1257,7 @@ var BraintrustLanguageModelWrapper = class {
|
|
|
1213
1257
|
fullText += chunk.textDelta;
|
|
1214
1258
|
break;
|
|
1215
1259
|
case "tool-call":
|
|
1216
|
-
|
|
1260
|
+
toolCalls2[chunk.toolCallId] = {
|
|
1217
1261
|
toolCallType: chunk.toolCallType,
|
|
1218
1262
|
toolCallId: chunk.toolCallId,
|
|
1219
1263
|
toolName: chunk.toolName,
|
|
@@ -1221,15 +1265,15 @@ var BraintrustLanguageModelWrapper = class {
|
|
|
1221
1265
|
};
|
|
1222
1266
|
break;
|
|
1223
1267
|
case "tool-call-delta":
|
|
1224
|
-
if (
|
|
1225
|
-
|
|
1268
|
+
if (toolCalls2[chunk.toolCallId] === void 0) {
|
|
1269
|
+
toolCalls2[chunk.toolCallId] = {
|
|
1226
1270
|
toolCallType: chunk.toolCallType,
|
|
1227
1271
|
toolCallId: chunk.toolCallId,
|
|
1228
1272
|
toolName: chunk.toolName,
|
|
1229
1273
|
args: ""
|
|
1230
1274
|
};
|
|
1231
1275
|
}
|
|
1232
|
-
|
|
1276
|
+
toolCalls2[chunk.toolCallId].args += chunk.argsTextDelta;
|
|
1233
1277
|
break;
|
|
1234
1278
|
case "finish":
|
|
1235
1279
|
usage = chunk.usage;
|
|
@@ -1242,7 +1286,7 @@ var BraintrustLanguageModelWrapper = class {
|
|
|
1242
1286
|
span.log({
|
|
1243
1287
|
output: postProcessOutput(
|
|
1244
1288
|
fullText,
|
|
1245
|
-
Object.keys(
|
|
1289
|
+
Object.keys(toolCalls2).length > 0 ? Object.values(toolCalls2) : void 0,
|
|
1246
1290
|
finishReason
|
|
1247
1291
|
),
|
|
1248
1292
|
metrics: {
|
|
@@ -1343,15 +1387,15 @@ function postProcessPrompt(prompt) {
|
|
|
1343
1387
|
}
|
|
1344
1388
|
});
|
|
1345
1389
|
}
|
|
1346
|
-
function postProcessOutput(text,
|
|
1390
|
+
function postProcessOutput(text, toolCalls2, finishReason) {
|
|
1347
1391
|
return [
|
|
1348
1392
|
{
|
|
1349
1393
|
index: 0,
|
|
1350
1394
|
message: {
|
|
1351
1395
|
role: "assistant",
|
|
1352
1396
|
content: text ?? "",
|
|
1353
|
-
...
|
|
1354
|
-
tool_calls:
|
|
1397
|
+
...toolCalls2 && toolCalls2.length > 0 ? {
|
|
1398
|
+
tool_calls: toolCalls2.map((toolCall) => ({
|
|
1355
1399
|
id: toolCall.toolCallId,
|
|
1356
1400
|
function: {
|
|
1357
1401
|
name: toolCall.toolName,
|
|
@@ -1450,29 +1494,29 @@ function normalizeFinishReason(reason) {
|
|
|
1450
1494
|
if (typeof reason !== "string") return void 0;
|
|
1451
1495
|
return reason.replace(/-/g, "_");
|
|
1452
1496
|
}
|
|
1453
|
-
function buildAssistantOutputWithToolCalls(result,
|
|
1497
|
+
function buildAssistantOutputWithToolCalls(result, toolCalls2) {
|
|
1454
1498
|
return [
|
|
1455
1499
|
{
|
|
1456
1500
|
index: 0,
|
|
1457
1501
|
logprobs: null,
|
|
1458
|
-
finish_reason: normalizeFinishReason(result?.finishReason) ?? (
|
|
1502
|
+
finish_reason: normalizeFinishReason(result?.finishReason) ?? (toolCalls2.length ? "tool_calls" : void 0),
|
|
1459
1503
|
message: {
|
|
1460
1504
|
role: "assistant",
|
|
1461
|
-
tool_calls:
|
|
1505
|
+
tool_calls: toolCalls2.length > 0 ? toolCalls2 : void 0
|
|
1462
1506
|
}
|
|
1463
1507
|
}
|
|
1464
1508
|
];
|
|
1465
1509
|
}
|
|
1466
1510
|
function extractToolCallsFromSteps(steps) {
|
|
1467
|
-
const
|
|
1468
|
-
if (!Array.isArray(steps)) return
|
|
1511
|
+
const toolCalls2 = [];
|
|
1512
|
+
if (!Array.isArray(steps)) return toolCalls2;
|
|
1469
1513
|
let idx = 0;
|
|
1470
1514
|
for (const step of steps) {
|
|
1471
1515
|
const blocks = step?.content;
|
|
1472
1516
|
if (!Array.isArray(blocks)) continue;
|
|
1473
1517
|
for (const block of blocks) {
|
|
1474
1518
|
if (block && typeof block === "object" && block.type === "tool-call") {
|
|
1475
|
-
|
|
1519
|
+
toolCalls2.push({
|
|
1476
1520
|
id: block.toolCallId,
|
|
1477
1521
|
type: "function",
|
|
1478
1522
|
index: idx++,
|
|
@@ -1484,7 +1528,7 @@ function extractToolCallsFromSteps(steps) {
|
|
|
1484
1528
|
}
|
|
1485
1529
|
}
|
|
1486
1530
|
}
|
|
1487
|
-
return
|
|
1531
|
+
return toolCalls2;
|
|
1488
1532
|
}
|
|
1489
1533
|
function extractToolCallsFromBlocks(blocks) {
|
|
1490
1534
|
if (!Array.isArray(blocks)) return [];
|
|
@@ -1550,12 +1594,12 @@ function BraintrustMiddleware(config = {}) {
|
|
|
1550
1594
|
metadata.model = modelId;
|
|
1551
1595
|
}
|
|
1552
1596
|
}
|
|
1553
|
-
let
|
|
1554
|
-
if (!
|
|
1555
|
-
|
|
1597
|
+
let toolCalls2 = extractToolCallsFromSteps(result?.steps);
|
|
1598
|
+
if (!toolCalls2 || toolCalls2.length === 0) {
|
|
1599
|
+
toolCalls2 = extractToolCallsFromBlocks(result?.content);
|
|
1556
1600
|
}
|
|
1557
1601
|
span.log({
|
|
1558
|
-
output:
|
|
1602
|
+
output: toolCalls2.length > 0 ? buildAssistantOutputWithToolCalls(result, toolCalls2) : result?.content,
|
|
1559
1603
|
metadata,
|
|
1560
1604
|
metrics: normalizeUsageMetrics(
|
|
1561
1605
|
result.usage,
|
|
@@ -1644,13 +1688,13 @@ function BraintrustMiddleware(config = {}) {
|
|
|
1644
1688
|
metadata.model = model;
|
|
1645
1689
|
}
|
|
1646
1690
|
if (toolBlocks.length > 0) {
|
|
1647
|
-
const
|
|
1691
|
+
const toolCalls2 = extractToolCallsFromSteps([
|
|
1648
1692
|
{ content: toolBlocks }
|
|
1649
1693
|
]);
|
|
1650
|
-
if (
|
|
1694
|
+
if (toolCalls2.length > 0) {
|
|
1651
1695
|
output = buildAssistantOutputWithToolCalls(
|
|
1652
1696
|
resultForDetection,
|
|
1653
|
-
|
|
1697
|
+
toolCalls2
|
|
1654
1698
|
);
|
|
1655
1699
|
}
|
|
1656
1700
|
}
|
|
@@ -1688,6 +1732,1469 @@ function BraintrustMiddleware(config = {}) {
|
|
|
1688
1732
|
};
|
|
1689
1733
|
}
|
|
1690
1734
|
|
|
1735
|
+
// src/instrumentation/plugins/eve-plugin.ts
|
|
1736
|
+
var EVE_TRACE_STATE_KEY = "braintrust.eve.tracing";
|
|
1737
|
+
var MAX_EVE_CACHE_ENTRIES = 1e4;
|
|
1738
|
+
var MAX_STORED_LLM_INPUTS = 100;
|
|
1739
|
+
var MAX_STORED_SPAN_REFERENCES = 1e4;
|
|
1740
|
+
var MAX_STORED_STEP_STARTS = 1e4;
|
|
1741
|
+
function braintrustEveHook(options) {
|
|
1742
|
+
const state = options.defineState(EVE_TRACE_STATE_KEY, emptyEveTraceState);
|
|
1743
|
+
const bridge = new EveBridge(state);
|
|
1744
|
+
return {
|
|
1745
|
+
events: {
|
|
1746
|
+
"*": async (event, ctx) => {
|
|
1747
|
+
await bridge.handle(event, ctx, options.metadata);
|
|
1748
|
+
}
|
|
1749
|
+
}
|
|
1750
|
+
};
|
|
1751
|
+
}
|
|
1752
|
+
function braintrustEveInstrumentation(options) {
|
|
1753
|
+
const state = options.defineState(EVE_TRACE_STATE_KEY, emptyEveTraceState);
|
|
1754
|
+
return {
|
|
1755
|
+
events: {
|
|
1756
|
+
"step.started": (input) => {
|
|
1757
|
+
try {
|
|
1758
|
+
captureEveModelInput(state, input);
|
|
1759
|
+
} catch (error) {
|
|
1760
|
+
debugLogger.warn("Error in Eve LLM input capture:", error);
|
|
1761
|
+
}
|
|
1762
|
+
}
|
|
1763
|
+
},
|
|
1764
|
+
recordInputs: false,
|
|
1765
|
+
recordOutputs: false,
|
|
1766
|
+
setup: options.setup
|
|
1767
|
+
};
|
|
1768
|
+
}
|
|
1769
|
+
function isEveHandleMessageStreamEvent(event) {
|
|
1770
|
+
return isObject(event) && typeof event["type"] === "string";
|
|
1771
|
+
}
|
|
1772
|
+
var ResumedEveSpan = class {
|
|
1773
|
+
constructor(reference) {
|
|
1774
|
+
this.reference = reference;
|
|
1775
|
+
this.endTime = reference.endTime;
|
|
1776
|
+
}
|
|
1777
|
+
endTime;
|
|
1778
|
+
get rootSpanId() {
|
|
1779
|
+
return this.reference.rootSpanId;
|
|
1780
|
+
}
|
|
1781
|
+
get spanId() {
|
|
1782
|
+
return this.reference.spanId;
|
|
1783
|
+
}
|
|
1784
|
+
log(event) {
|
|
1785
|
+
const metrics = {
|
|
1786
|
+
...this.reference.startEvent?.metrics,
|
|
1787
|
+
...this.endTime === void 0 ? {} : { end: this.endTime },
|
|
1788
|
+
...event.metrics
|
|
1789
|
+
};
|
|
1790
|
+
updateSpan({
|
|
1791
|
+
exported: this.reference.exported,
|
|
1792
|
+
...this.reference.startEvent,
|
|
1793
|
+
...event,
|
|
1794
|
+
...Object.keys(metrics).length > 0 ? { metrics } : {}
|
|
1795
|
+
});
|
|
1796
|
+
}
|
|
1797
|
+
end(args) {
|
|
1798
|
+
if (this.endTime === void 0) {
|
|
1799
|
+
this.endTime = args?.endTime ?? getCurrentUnixTimestamp();
|
|
1800
|
+
this.log({ metrics: { end: this.endTime } });
|
|
1801
|
+
}
|
|
1802
|
+
return this.endTime;
|
|
1803
|
+
}
|
|
1804
|
+
};
|
|
1805
|
+
var EveBridge = class {
|
|
1806
|
+
constructor(state) {
|
|
1807
|
+
this.state = state;
|
|
1808
|
+
}
|
|
1809
|
+
eventQueuesBySession = /* @__PURE__ */ new Map();
|
|
1810
|
+
sessionsById = new LRUCache({
|
|
1811
|
+
max: MAX_EVE_CACHE_ENTRIES
|
|
1812
|
+
});
|
|
1813
|
+
completedToolKeys = new LRUCache({
|
|
1814
|
+
max: MAX_EVE_CACHE_ENTRIES
|
|
1815
|
+
});
|
|
1816
|
+
toolsByCallKey = new LRUCache({
|
|
1817
|
+
max: MAX_EVE_CACHE_ENTRIES
|
|
1818
|
+
});
|
|
1819
|
+
turnsByKey = new LRUCache({
|
|
1820
|
+
max: MAX_EVE_CACHE_ENTRIES
|
|
1821
|
+
});
|
|
1822
|
+
async startEveSpan(args) {
|
|
1823
|
+
const rowId = args?.event?.id;
|
|
1824
|
+
const reference = typeof rowId === "string" && readEveTraceState(this.state).spanReferences.find(
|
|
1825
|
+
(candidate) => candidate.rowId === rowId
|
|
1826
|
+
);
|
|
1827
|
+
if (reference) {
|
|
1828
|
+
return new ResumedEveSpan(reference);
|
|
1829
|
+
}
|
|
1830
|
+
const startTime = args?.startTime ?? getCurrentUnixTimestamp();
|
|
1831
|
+
const parentSpanIds = args?.parentSpanIds;
|
|
1832
|
+
const startEvent = {
|
|
1833
|
+
created: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1834
|
+
metrics: { start: startTime },
|
|
1835
|
+
span_attributes: {
|
|
1836
|
+
...args?.name ? { name: args.name } : {},
|
|
1837
|
+
...args?.type ? { type: args.type } : {},
|
|
1838
|
+
...args?.spanAttributes
|
|
1839
|
+
},
|
|
1840
|
+
span_parents: parentSpanIds ? "spanId" in parentSpanIds ? [parentSpanIds.spanId] : parentSpanIds.parentSpanIds : []
|
|
1841
|
+
};
|
|
1842
|
+
const span = withCurrent(
|
|
1843
|
+
NOOP_SPAN,
|
|
1844
|
+
() => _internalStartSpanWithInitialMerge({ ...args, startTime })
|
|
1845
|
+
);
|
|
1846
|
+
if (typeof rowId !== "string") {
|
|
1847
|
+
return span;
|
|
1848
|
+
}
|
|
1849
|
+
try {
|
|
1850
|
+
const exported = await span.export();
|
|
1851
|
+
const reference2 = {
|
|
1852
|
+
exported,
|
|
1853
|
+
rootSpanId: span.rootSpanId,
|
|
1854
|
+
rowId,
|
|
1855
|
+
spanId: span.spanId,
|
|
1856
|
+
startEvent
|
|
1857
|
+
};
|
|
1858
|
+
this.state.update((current) => {
|
|
1859
|
+
const normalized = normalizeEveTraceState(current);
|
|
1860
|
+
return normalized.spanReferences.some(
|
|
1861
|
+
(candidate) => candidate.rowId === rowId
|
|
1862
|
+
) ? normalized : {
|
|
1863
|
+
...normalized,
|
|
1864
|
+
spanReferences: [...normalized.spanReferences, reference2].slice(
|
|
1865
|
+
-MAX_STORED_SPAN_REFERENCES
|
|
1866
|
+
)
|
|
1867
|
+
};
|
|
1868
|
+
});
|
|
1869
|
+
} catch (error) {
|
|
1870
|
+
debugLogger.warn("Error exporting Eve span for resumption:", error);
|
|
1871
|
+
}
|
|
1872
|
+
return span;
|
|
1873
|
+
}
|
|
1874
|
+
async startEveChildSpan(parent, args) {
|
|
1875
|
+
return await this.startEveSpan({
|
|
1876
|
+
...args,
|
|
1877
|
+
parentSpanIds: {
|
|
1878
|
+
rootSpanId: parent.rootSpanId,
|
|
1879
|
+
spanId: parent.spanId
|
|
1880
|
+
}
|
|
1881
|
+
});
|
|
1882
|
+
}
|
|
1883
|
+
stepOrdinal(event) {
|
|
1884
|
+
let ordinal = 0;
|
|
1885
|
+
this.state.update((current) => {
|
|
1886
|
+
const state = normalizeEveTraceState(current);
|
|
1887
|
+
const previous = state.stepStarts.filter(
|
|
1888
|
+
(entry) => entry.turnId === event.data.turnId && entry.stepIndex === event.data.stepIndex
|
|
1889
|
+
).at(-1);
|
|
1890
|
+
if (previous?.open) {
|
|
1891
|
+
ordinal = previous.ordinal;
|
|
1892
|
+
return state;
|
|
1893
|
+
}
|
|
1894
|
+
ordinal = state.stepStarts.filter(
|
|
1895
|
+
(entry) => entry.turnId === event.data.turnId
|
|
1896
|
+
).length;
|
|
1897
|
+
return {
|
|
1898
|
+
...state,
|
|
1899
|
+
stepStarts: [
|
|
1900
|
+
...state.stepStarts,
|
|
1901
|
+
{
|
|
1902
|
+
open: true,
|
|
1903
|
+
ordinal,
|
|
1904
|
+
stepIndex: event.data.stepIndex,
|
|
1905
|
+
turnId: event.data.turnId
|
|
1906
|
+
}
|
|
1907
|
+
].slice(-MAX_STORED_STEP_STARTS)
|
|
1908
|
+
};
|
|
1909
|
+
});
|
|
1910
|
+
return ordinal;
|
|
1911
|
+
}
|
|
1912
|
+
markStepEnded(turnId, stepIndex) {
|
|
1913
|
+
this.state.update((current) => {
|
|
1914
|
+
const state = normalizeEveTraceState(current);
|
|
1915
|
+
let index = -1;
|
|
1916
|
+
for (let i = state.stepStarts.length - 1; i >= 0; i--) {
|
|
1917
|
+
const entry = state.stepStarts[i];
|
|
1918
|
+
if (entry?.turnId === turnId && entry.stepIndex === stepIndex) {
|
|
1919
|
+
index = i;
|
|
1920
|
+
break;
|
|
1921
|
+
}
|
|
1922
|
+
}
|
|
1923
|
+
if (index < 0 || !state.stepStarts[index]?.open) {
|
|
1924
|
+
return state;
|
|
1925
|
+
}
|
|
1926
|
+
return {
|
|
1927
|
+
...state,
|
|
1928
|
+
stepStarts: state.stepStarts.map(
|
|
1929
|
+
(entry, entryIndex) => entryIndex === index ? { ...entry, open: false } : entry
|
|
1930
|
+
)
|
|
1931
|
+
};
|
|
1932
|
+
});
|
|
1933
|
+
}
|
|
1934
|
+
async handle(event, ctx, hookMetadata) {
|
|
1935
|
+
if (!isEveHandleMessageStreamEvent(event)) {
|
|
1936
|
+
return;
|
|
1937
|
+
}
|
|
1938
|
+
const run = async () => {
|
|
1939
|
+
try {
|
|
1940
|
+
if (!await this.handleEvent(event, ctx, hookMetadata)) {
|
|
1941
|
+
return;
|
|
1942
|
+
}
|
|
1943
|
+
if (event.type === "session.failed") {
|
|
1944
|
+
const sessionId2 = event.data.sessionId || sessionIdFromContext(ctx);
|
|
1945
|
+
await this.flushInstrumentation();
|
|
1946
|
+
if (sessionId2) {
|
|
1947
|
+
this.cleanupSession(sessionId2);
|
|
1948
|
+
}
|
|
1949
|
+
} else if (event.type === "session.completed") {
|
|
1950
|
+
const sessionId2 = sessionIdFromContext(ctx);
|
|
1951
|
+
await this.flushInstrumentation();
|
|
1952
|
+
if (sessionId2) {
|
|
1953
|
+
this.cleanupSession(sessionId2);
|
|
1954
|
+
}
|
|
1955
|
+
}
|
|
1956
|
+
} catch (error) {
|
|
1957
|
+
debugLogger.warn("Error in Eve hook instrumentation:", error);
|
|
1958
|
+
}
|
|
1959
|
+
};
|
|
1960
|
+
const sessionId = event.type === "session.failed" ? event.data.sessionId || sessionIdFromContext(ctx) : sessionIdFromContext(ctx);
|
|
1961
|
+
if (!sessionId) {
|
|
1962
|
+
await run();
|
|
1963
|
+
return;
|
|
1964
|
+
}
|
|
1965
|
+
const previous = this.eventQueuesBySession.get(sessionId);
|
|
1966
|
+
const queued = previous ? previous.then(run) : run();
|
|
1967
|
+
this.eventQueuesBySession.set(sessionId, queued);
|
|
1968
|
+
try {
|
|
1969
|
+
await queued;
|
|
1970
|
+
} finally {
|
|
1971
|
+
if (this.eventQueuesBySession.get(sessionId) === queued) {
|
|
1972
|
+
this.eventQueuesBySession.delete(sessionId);
|
|
1973
|
+
}
|
|
1974
|
+
}
|
|
1975
|
+
}
|
|
1976
|
+
async handleEvent(event, ctx, hookMetadata) {
|
|
1977
|
+
switch (event.type) {
|
|
1978
|
+
case "session.started":
|
|
1979
|
+
await this.handleSessionStarted(event, ctx, hookMetadata);
|
|
1980
|
+
return true;
|
|
1981
|
+
case "turn.started":
|
|
1982
|
+
await this.handleTurnStarted(event, ctx, hookMetadata);
|
|
1983
|
+
return true;
|
|
1984
|
+
case "message.received":
|
|
1985
|
+
await this.handleMessageReceived(event, ctx, hookMetadata);
|
|
1986
|
+
return true;
|
|
1987
|
+
case "step.started":
|
|
1988
|
+
await this.handleStepStarted(event, ctx, hookMetadata);
|
|
1989
|
+
return true;
|
|
1990
|
+
case "message.completed":
|
|
1991
|
+
this.handleMessageCompleted(event, ctx);
|
|
1992
|
+
return true;
|
|
1993
|
+
case "result.completed":
|
|
1994
|
+
this.handleResultCompleted(event, ctx);
|
|
1995
|
+
return true;
|
|
1996
|
+
case "actions.requested":
|
|
1997
|
+
await this.handleActionsRequested(event, ctx, hookMetadata);
|
|
1998
|
+
return true;
|
|
1999
|
+
case "action.result":
|
|
2000
|
+
await this.handleActionResult(event, ctx, hookMetadata);
|
|
2001
|
+
return true;
|
|
2002
|
+
case "subagent.called":
|
|
2003
|
+
await this.handleSubagentCalled(event, ctx, hookMetadata);
|
|
2004
|
+
return true;
|
|
2005
|
+
case "subagent.completed":
|
|
2006
|
+
await this.handleSubagentCompleted(event, ctx, hookMetadata);
|
|
2007
|
+
return true;
|
|
2008
|
+
case "step.completed":
|
|
2009
|
+
this.handleStepCompleted(event, ctx);
|
|
2010
|
+
return true;
|
|
2011
|
+
case "step.failed":
|
|
2012
|
+
this.handleStepFailed(event, ctx);
|
|
2013
|
+
return true;
|
|
2014
|
+
case "turn.completed":
|
|
2015
|
+
await this.handleTurnCompleted(event, ctx);
|
|
2016
|
+
return true;
|
|
2017
|
+
case "turn.failed":
|
|
2018
|
+
await this.handleTurnFailed(event, ctx);
|
|
2019
|
+
return true;
|
|
2020
|
+
case "session.failed":
|
|
2021
|
+
await this.handleSessionFailed(event, ctx);
|
|
2022
|
+
return true;
|
|
2023
|
+
case "session.completed":
|
|
2024
|
+
await this.handleSessionCompleted(event, ctx);
|
|
2025
|
+
return true;
|
|
2026
|
+
default:
|
|
2027
|
+
return false;
|
|
2028
|
+
}
|
|
2029
|
+
}
|
|
2030
|
+
async handleSessionStarted(event, ctx, hookMetadata) {
|
|
2031
|
+
const sessionId = sessionIdFromContext(ctx);
|
|
2032
|
+
if (!sessionId) {
|
|
2033
|
+
return;
|
|
2034
|
+
}
|
|
2035
|
+
const metadata = {
|
|
2036
|
+
...hookMetadata ?? {},
|
|
2037
|
+
...modelMetadataFromRuntime(event.data.runtime)
|
|
2038
|
+
};
|
|
2039
|
+
this.state.update((current) => {
|
|
2040
|
+
const normalized = normalizeEveTraceState(current);
|
|
2041
|
+
return {
|
|
2042
|
+
...normalized,
|
|
2043
|
+
metadata: { ...normalized.metadata, ...metadata }
|
|
2044
|
+
};
|
|
2045
|
+
});
|
|
2046
|
+
await this.ensureSession(sessionId, ctx, metadata, eventTime(event));
|
|
2047
|
+
for (const [key, turn] of this.turnsByKey) {
|
|
2048
|
+
if (!key.startsWith(`${sessionId}:`)) {
|
|
2049
|
+
continue;
|
|
2050
|
+
}
|
|
2051
|
+
turn.metadata = { ...turn.metadata, ...metadata };
|
|
2052
|
+
turn.span.log({ metadata: turn.metadata });
|
|
2053
|
+
for (const step of turn.stepsByIndex.values()) {
|
|
2054
|
+
step.metadata = { ...step.metadata, ...metadata };
|
|
2055
|
+
step.span.log({ metadata: step.metadata });
|
|
2056
|
+
}
|
|
2057
|
+
}
|
|
2058
|
+
}
|
|
2059
|
+
async handleTurnStarted(event, ctx, hookMetadata) {
|
|
2060
|
+
const sessionId = sessionIdFromContext(ctx);
|
|
2061
|
+
if (!sessionId) {
|
|
2062
|
+
return;
|
|
2063
|
+
}
|
|
2064
|
+
const session = await this.ensureSession(
|
|
2065
|
+
sessionId,
|
|
2066
|
+
ctx,
|
|
2067
|
+
hookMetadata ?? {},
|
|
2068
|
+
eventTime(event)
|
|
2069
|
+
);
|
|
2070
|
+
const key = turnKey(sessionId, event.data.turnId);
|
|
2071
|
+
const metadata = { ...session.metadata };
|
|
2072
|
+
const existing = this.turnsByKey.get(key);
|
|
2073
|
+
if (existing) {
|
|
2074
|
+
existing.metadata = { ...existing.metadata, ...metadata };
|
|
2075
|
+
existing.span.log({ metadata: existing.metadata });
|
|
2076
|
+
return;
|
|
2077
|
+
}
|
|
2078
|
+
const span = await this.startTurnSpan(session, event, metadata);
|
|
2079
|
+
span.log({ metadata });
|
|
2080
|
+
this.turnsByKey.set(key, {
|
|
2081
|
+
key,
|
|
2082
|
+
metadata,
|
|
2083
|
+
metrics: {},
|
|
2084
|
+
span,
|
|
2085
|
+
stepsByIndex: /* @__PURE__ */ new Map(),
|
|
2086
|
+
turnId: event.data.turnId
|
|
2087
|
+
});
|
|
2088
|
+
}
|
|
2089
|
+
async handleMessageReceived(event, ctx, hookMetadata) {
|
|
2090
|
+
const turn = await this.ensureTurn(event, ctx, hookMetadata);
|
|
2091
|
+
if (!turn) {
|
|
2092
|
+
return;
|
|
2093
|
+
}
|
|
2094
|
+
const input = [{ content: event.data.message, role: "user" }];
|
|
2095
|
+
turn.span.log({ input });
|
|
2096
|
+
}
|
|
2097
|
+
async handleStepStarted(event, ctx, hookMetadata) {
|
|
2098
|
+
const turn = await this.ensureTurn(event, ctx, hookMetadata);
|
|
2099
|
+
const sessionId = sessionIdFromContext(ctx);
|
|
2100
|
+
if (!turn || !sessionId) {
|
|
2101
|
+
return;
|
|
2102
|
+
}
|
|
2103
|
+
const existing = turn.stepsByIndex.get(event.data.stepIndex);
|
|
2104
|
+
if (existing) {
|
|
2105
|
+
existing.span.log({
|
|
2106
|
+
...existing.input !== void 0 ? { input: existing.input } : {},
|
|
2107
|
+
metadata: existing.metadata,
|
|
2108
|
+
metrics: existing.metrics,
|
|
2109
|
+
output: existing.output
|
|
2110
|
+
});
|
|
2111
|
+
const endTime = eventTime(event);
|
|
2112
|
+
existing.span.end(endTime === void 0 ? void 0 : { endTime });
|
|
2113
|
+
this.markStepEnded(event.data.turnId, event.data.stepIndex);
|
|
2114
|
+
}
|
|
2115
|
+
const stepOrdinal = this.stepOrdinal(event);
|
|
2116
|
+
const metadata = {
|
|
2117
|
+
...turn.metadata,
|
|
2118
|
+
...this.sessionsById.get(sessionId)?.metadata ?? {}
|
|
2119
|
+
};
|
|
2120
|
+
const input = consumeCapturedEveModelInput(
|
|
2121
|
+
this.state,
|
|
2122
|
+
sessionId,
|
|
2123
|
+
event.data.turnId,
|
|
2124
|
+
event.data.stepIndex
|
|
2125
|
+
);
|
|
2126
|
+
const { rowId: eventId, spanId } = await generateEveIds(
|
|
2127
|
+
"step",
|
|
2128
|
+
sessionId,
|
|
2129
|
+
event.data.turnId,
|
|
2130
|
+
String(stepOrdinal)
|
|
2131
|
+
);
|
|
2132
|
+
const span = await this.startEveChildSpan(turn.span, {
|
|
2133
|
+
event: {
|
|
2134
|
+
id: eventId,
|
|
2135
|
+
...input !== void 0 ? { input } : {},
|
|
2136
|
+
metadata
|
|
2137
|
+
},
|
|
2138
|
+
name: "eve.step",
|
|
2139
|
+
spanAttributes: { type: "llm" /* LLM */ },
|
|
2140
|
+
spanId,
|
|
2141
|
+
startTime: eventTime(event)
|
|
2142
|
+
});
|
|
2143
|
+
span.log({ ...input !== void 0 ? { input } : {}, metadata });
|
|
2144
|
+
turn.stepsByIndex.set(event.data.stepIndex, {
|
|
2145
|
+
...input !== void 0 ? { input } : {},
|
|
2146
|
+
metadata,
|
|
2147
|
+
metrics: {},
|
|
2148
|
+
span
|
|
2149
|
+
});
|
|
2150
|
+
}
|
|
2151
|
+
handleMessageCompleted(event, ctx) {
|
|
2152
|
+
const step = this.stepForEvent(event, ctx);
|
|
2153
|
+
if (!step) {
|
|
2154
|
+
return;
|
|
2155
|
+
}
|
|
2156
|
+
const existingMessage = Array.isArray(step.output) && isObject(step.output[0]) ? step.output[0].message : void 0;
|
|
2157
|
+
const existingToolCalls = isObject(existingMessage) ? existingMessage.tool_calls : void 0;
|
|
2158
|
+
step.output = [
|
|
2159
|
+
{
|
|
2160
|
+
finish_reason: normalizedFinishReason(event.data.finishReason),
|
|
2161
|
+
index: 0,
|
|
2162
|
+
message: {
|
|
2163
|
+
content: event.data.message,
|
|
2164
|
+
role: "assistant",
|
|
2165
|
+
...Array.isArray(existingToolCalls) ? { tool_calls: existingToolCalls } : {}
|
|
2166
|
+
}
|
|
2167
|
+
}
|
|
2168
|
+
];
|
|
2169
|
+
const turn = this.turnForEvent(event, ctx);
|
|
2170
|
+
if (turn && event.data.finishReason !== "tool-calls") {
|
|
2171
|
+
turn.output = event.data.message;
|
|
2172
|
+
}
|
|
2173
|
+
}
|
|
2174
|
+
handleResultCompleted(event, ctx) {
|
|
2175
|
+
const step = this.stepForEvent(event, ctx);
|
|
2176
|
+
if (step) {
|
|
2177
|
+
step.output = [
|
|
2178
|
+
{
|
|
2179
|
+
finish_reason: "stop",
|
|
2180
|
+
index: 0,
|
|
2181
|
+
message: {
|
|
2182
|
+
content: event.data.result,
|
|
2183
|
+
role: "assistant"
|
|
2184
|
+
}
|
|
2185
|
+
}
|
|
2186
|
+
];
|
|
2187
|
+
}
|
|
2188
|
+
const turn = this.turnForEvent(event, ctx);
|
|
2189
|
+
if (turn) {
|
|
2190
|
+
turn.output = event.data.result;
|
|
2191
|
+
}
|
|
2192
|
+
}
|
|
2193
|
+
async handleActionsRequested(event, ctx, hookMetadata) {
|
|
2194
|
+
const turn = await this.ensureTurn(event, ctx, hookMetadata);
|
|
2195
|
+
const sessionId = sessionIdFromContext(ctx);
|
|
2196
|
+
if (!turn || !sessionId) {
|
|
2197
|
+
return;
|
|
2198
|
+
}
|
|
2199
|
+
const traceActions = event.data.actions.filter(isTraceableActionRequest);
|
|
2200
|
+
if (traceActions.length === 0) {
|
|
2201
|
+
return;
|
|
2202
|
+
}
|
|
2203
|
+
for (const action of traceActions) {
|
|
2204
|
+
if (isToolCallAction(action)) {
|
|
2205
|
+
await this.startRequestedTool(event, turn, sessionId, action);
|
|
2206
|
+
} else if (isLocalSubagentCallAction(action)) {
|
|
2207
|
+
await this.startRequestedSubagent(event, turn, sessionId, action);
|
|
2208
|
+
}
|
|
2209
|
+
}
|
|
2210
|
+
const step = turn.stepsByIndex.get(event.data.stepIndex);
|
|
2211
|
+
if (!step) {
|
|
2212
|
+
return;
|
|
2213
|
+
}
|
|
2214
|
+
step.output = [
|
|
2215
|
+
{
|
|
2216
|
+
finish_reason: "tool_calls",
|
|
2217
|
+
index: 0,
|
|
2218
|
+
message: {
|
|
2219
|
+
content: null,
|
|
2220
|
+
role: "assistant",
|
|
2221
|
+
tool_calls: traceActions.map((action) => {
|
|
2222
|
+
const name = action.kind === "tool-call" ? action.toolName : action.subagentName ?? action.name ?? "agent";
|
|
2223
|
+
return {
|
|
2224
|
+
function: {
|
|
2225
|
+
arguments: JSON.stringify(action.input),
|
|
2226
|
+
name
|
|
2227
|
+
},
|
|
2228
|
+
id: action.callId,
|
|
2229
|
+
type: "function"
|
|
2230
|
+
};
|
|
2231
|
+
})
|
|
2232
|
+
}
|
|
2233
|
+
}
|
|
2234
|
+
];
|
|
2235
|
+
step.span.log({ metadata: step.metadata, output: step.output });
|
|
2236
|
+
}
|
|
2237
|
+
async handleActionResult(event, ctx, hookMetadata) {
|
|
2238
|
+
if (isToolResult(event.data.result)) {
|
|
2239
|
+
await this.handleToolResult(event, ctx, event.data.result, hookMetadata);
|
|
2240
|
+
return;
|
|
2241
|
+
}
|
|
2242
|
+
if (isSubagentResult(event.data.result)) {
|
|
2243
|
+
await this.handleSubagentResult(
|
|
2244
|
+
event,
|
|
2245
|
+
ctx,
|
|
2246
|
+
event.data.result,
|
|
2247
|
+
hookMetadata
|
|
2248
|
+
);
|
|
2249
|
+
}
|
|
2250
|
+
}
|
|
2251
|
+
async handleToolResult(event, ctx, result, hookMetadata) {
|
|
2252
|
+
const sessionId = sessionIdFromContext(ctx);
|
|
2253
|
+
if (!sessionId) {
|
|
2254
|
+
return;
|
|
2255
|
+
}
|
|
2256
|
+
const key = toolKey(sessionId, result.callId);
|
|
2257
|
+
if (this.completedToolKeys.has(key)) {
|
|
2258
|
+
return;
|
|
2259
|
+
}
|
|
2260
|
+
const tool = this.toolsByCallKey.get(key) ?? await this.startSyntheticTool(event, ctx, result, hookMetadata);
|
|
2261
|
+
if (!tool) {
|
|
2262
|
+
return;
|
|
2263
|
+
}
|
|
2264
|
+
const failed = event.data.status === "failed" || result.isError === true || event.data.error !== void 0;
|
|
2265
|
+
tool.span.log({
|
|
2266
|
+
...failed ? {
|
|
2267
|
+
error: actionResultError(event.data.error, result.output)
|
|
2268
|
+
} : {},
|
|
2269
|
+
metadata: tool.metadata,
|
|
2270
|
+
output: result.output
|
|
2271
|
+
});
|
|
2272
|
+
const endTime = eventTime(event);
|
|
2273
|
+
tool.span.end(endTime === void 0 ? void 0 : { endTime });
|
|
2274
|
+
this.toolsByCallKey.delete(key);
|
|
2275
|
+
this.completedToolKeys.set(key, true);
|
|
2276
|
+
}
|
|
2277
|
+
async handleSubagentCalled(event, ctx, hookMetadata) {
|
|
2278
|
+
if (event.data.remote?.url) {
|
|
2279
|
+
return;
|
|
2280
|
+
}
|
|
2281
|
+
const turn = await this.ensureTurn(event, ctx, hookMetadata);
|
|
2282
|
+
const sessionId = sessionIdFromContext(ctx);
|
|
2283
|
+
if (!turn || !sessionId) {
|
|
2284
|
+
return;
|
|
2285
|
+
}
|
|
2286
|
+
const key = toolKey(sessionId, event.data.callId);
|
|
2287
|
+
const metadata = toolMetadataFromTurn(turn);
|
|
2288
|
+
const existing = this.toolsByCallKey.get(key);
|
|
2289
|
+
if (existing) {
|
|
2290
|
+
existing.metadata = { ...existing.metadata, ...metadata };
|
|
2291
|
+
existing.span.log({ metadata: existing.metadata });
|
|
2292
|
+
return;
|
|
2293
|
+
}
|
|
2294
|
+
if (this.completedToolKeys.has(key)) {
|
|
2295
|
+
return;
|
|
2296
|
+
}
|
|
2297
|
+
const { rowId: eventId, spanId } = await generateEveIds(
|
|
2298
|
+
"subagent",
|
|
2299
|
+
sessionId,
|
|
2300
|
+
event.data.callId
|
|
2301
|
+
);
|
|
2302
|
+
const pending = this.toolsByCallKey.get(key);
|
|
2303
|
+
if (pending || this.completedToolKeys.has(key)) {
|
|
2304
|
+
if (pending) {
|
|
2305
|
+
pending.metadata = { ...pending.metadata, ...metadata };
|
|
2306
|
+
pending.span.log({ metadata: pending.metadata });
|
|
2307
|
+
}
|
|
2308
|
+
return;
|
|
2309
|
+
}
|
|
2310
|
+
const span = await this.startEveChildSpan(turn.span, {
|
|
2311
|
+
event: {
|
|
2312
|
+
id: eventId,
|
|
2313
|
+
metadata
|
|
2314
|
+
},
|
|
2315
|
+
name: event.data.toolName ?? event.data.name,
|
|
2316
|
+
spanAttributes: { type: "tool" /* TOOL */ },
|
|
2317
|
+
spanId,
|
|
2318
|
+
startTime: eventTime(event)
|
|
2319
|
+
});
|
|
2320
|
+
span.log({ metadata });
|
|
2321
|
+
this.toolsByCallKey.set(key, {
|
|
2322
|
+
metadata,
|
|
2323
|
+
span,
|
|
2324
|
+
turnKey: turnKey(sessionId, event.data.turnId)
|
|
2325
|
+
});
|
|
2326
|
+
}
|
|
2327
|
+
async handleSubagentCompleted(event, ctx, hookMetadata) {
|
|
2328
|
+
const sessionId = sessionIdFromContext(ctx);
|
|
2329
|
+
if (!sessionId) {
|
|
2330
|
+
return;
|
|
2331
|
+
}
|
|
2332
|
+
const key = toolKey(sessionId, event.data.callId);
|
|
2333
|
+
if (this.completedToolKeys.has(key)) {
|
|
2334
|
+
return;
|
|
2335
|
+
}
|
|
2336
|
+
const subagent = this.toolsByCallKey.get(key) ?? await this.startSyntheticSubagent(event, ctx, hookMetadata);
|
|
2337
|
+
if (!subagent) {
|
|
2338
|
+
return;
|
|
2339
|
+
}
|
|
2340
|
+
subagent.span.log({
|
|
2341
|
+
...event.data.status === "failed" ? {
|
|
2342
|
+
error: actionResultError(event.data.error, event.data.output)
|
|
2343
|
+
} : {},
|
|
2344
|
+
metadata: subagent.metadata,
|
|
2345
|
+
...event.data.output !== void 0 ? { output: event.data.output } : {}
|
|
2346
|
+
});
|
|
2347
|
+
const endTime = eventTime(event);
|
|
2348
|
+
const recordedEndTime = subagent.span.end(
|
|
2349
|
+
endTime === void 0 ? void 0 : { endTime }
|
|
2350
|
+
);
|
|
2351
|
+
this.state.update((current) => {
|
|
2352
|
+
const normalized = normalizeEveTraceState(current);
|
|
2353
|
+
return {
|
|
2354
|
+
...normalized,
|
|
2355
|
+
spanReferences: normalized.spanReferences.map(
|
|
2356
|
+
(reference) => reference.spanId === subagent.span.spanId ? { ...reference, endTime: recordedEndTime } : reference
|
|
2357
|
+
)
|
|
2358
|
+
};
|
|
2359
|
+
});
|
|
2360
|
+
}
|
|
2361
|
+
async handleSubagentResult(event, ctx, result, hookMetadata) {
|
|
2362
|
+
const sessionId = sessionIdFromContext(ctx);
|
|
2363
|
+
if (!sessionId) {
|
|
2364
|
+
return;
|
|
2365
|
+
}
|
|
2366
|
+
const key = toolKey(sessionId, result.callId);
|
|
2367
|
+
if (this.completedToolKeys.has(key)) {
|
|
2368
|
+
return;
|
|
2369
|
+
}
|
|
2370
|
+
const subagent = this.toolsByCallKey.get(key) ?? await this.startSyntheticSubagentResult(
|
|
2371
|
+
event,
|
|
2372
|
+
ctx,
|
|
2373
|
+
result,
|
|
2374
|
+
hookMetadata
|
|
2375
|
+
);
|
|
2376
|
+
if (!subagent) {
|
|
2377
|
+
return;
|
|
2378
|
+
}
|
|
2379
|
+
const isError = event.data.status === "failed" || result.isError === true || event.data.error !== void 0;
|
|
2380
|
+
subagent.span.log({
|
|
2381
|
+
...isError ? {
|
|
2382
|
+
error: actionResultError(event.data.error, result.output)
|
|
2383
|
+
} : {},
|
|
2384
|
+
metadata: subagent.metadata,
|
|
2385
|
+
output: result.output
|
|
2386
|
+
});
|
|
2387
|
+
const endTime = eventTime(event);
|
|
2388
|
+
subagent.span.end(endTime === void 0 ? void 0 : { endTime });
|
|
2389
|
+
this.toolsByCallKey.delete(key);
|
|
2390
|
+
this.completedToolKeys.set(key, true);
|
|
2391
|
+
}
|
|
2392
|
+
handleStepCompleted(event, ctx) {
|
|
2393
|
+
const step = this.stepForEvent(event, ctx);
|
|
2394
|
+
if (!step) {
|
|
2395
|
+
return;
|
|
2396
|
+
}
|
|
2397
|
+
const usage = event.data.usage;
|
|
2398
|
+
const inputTokens = typeof usage?.inputTokens === "number" && Number.isFinite(usage.inputTokens) && usage.inputTokens >= 0 ? usage.inputTokens : void 0;
|
|
2399
|
+
const outputTokens = typeof usage?.outputTokens === "number" && Number.isFinite(usage.outputTokens) && usage.outputTokens >= 0 ? usage.outputTokens : void 0;
|
|
2400
|
+
const cacheReadTokens = typeof usage?.cacheReadTokens === "number" && Number.isFinite(usage.cacheReadTokens) && usage.cacheReadTokens >= 0 ? usage.cacheReadTokens : void 0;
|
|
2401
|
+
const cacheWriteTokens = typeof usage?.cacheWriteTokens === "number" && Number.isFinite(usage.cacheWriteTokens) && usage.cacheWriteTokens >= 0 ? usage.cacheWriteTokens : void 0;
|
|
2402
|
+
const costUsd = typeof usage?.costUsd === "number" && Number.isFinite(usage.costUsd) && usage.costUsd >= 0 ? usage.costUsd : void 0;
|
|
2403
|
+
const total = inputTokens !== void 0 && outputTokens !== void 0 ? inputTokens + outputTokens : void 0;
|
|
2404
|
+
const metrics = {
|
|
2405
|
+
...inputTokens !== void 0 ? { prompt_tokens: inputTokens } : {},
|
|
2406
|
+
...outputTokens !== void 0 ? { completion_tokens: outputTokens } : {},
|
|
2407
|
+
...total !== void 0 ? { tokens: total } : {},
|
|
2408
|
+
...cacheReadTokens !== void 0 ? { prompt_cached_tokens: cacheReadTokens } : {},
|
|
2409
|
+
...cacheWriteTokens !== void 0 ? { prompt_cache_creation_tokens: cacheWriteTokens } : {},
|
|
2410
|
+
...costUsd !== void 0 ? { estimated_cost: costUsd } : {}
|
|
2411
|
+
};
|
|
2412
|
+
step.metrics = { ...step.metrics, ...metrics };
|
|
2413
|
+
if (Array.isArray(step.output) && isObject(step.output[0])) {
|
|
2414
|
+
const finishReason = step.output[0].finish_reason;
|
|
2415
|
+
if (typeof finishReason !== "string") {
|
|
2416
|
+
step.output[0].finish_reason = normalizedFinishReason(
|
|
2417
|
+
event.data.finishReason
|
|
2418
|
+
);
|
|
2419
|
+
}
|
|
2420
|
+
}
|
|
2421
|
+
step.span.log({
|
|
2422
|
+
...step.input !== void 0 ? { input: step.input } : {},
|
|
2423
|
+
metadata: step.metadata,
|
|
2424
|
+
metrics,
|
|
2425
|
+
output: step.output
|
|
2426
|
+
});
|
|
2427
|
+
const endTime = eventTime(event);
|
|
2428
|
+
step.span.end(endTime === void 0 ? void 0 : { endTime });
|
|
2429
|
+
const turn = this.turnForEvent(event, ctx);
|
|
2430
|
+
if (turn) {
|
|
2431
|
+
for (const [key, value] of Object.entries(metrics)) {
|
|
2432
|
+
turn.metrics[key] = (turn.metrics[key] ?? 0) + value;
|
|
2433
|
+
}
|
|
2434
|
+
turn.stepsByIndex.delete(event.data.stepIndex);
|
|
2435
|
+
}
|
|
2436
|
+
this.markStepEnded(event.data.turnId, event.data.stepIndex);
|
|
2437
|
+
}
|
|
2438
|
+
handleStepFailed(event, ctx) {
|
|
2439
|
+
const step = this.stepForEvent(event, ctx);
|
|
2440
|
+
if (step) {
|
|
2441
|
+
step.span.log({
|
|
2442
|
+
error: errorFromMessage(
|
|
2443
|
+
event.data.message,
|
|
2444
|
+
event.data.code,
|
|
2445
|
+
event.data.details
|
|
2446
|
+
)
|
|
2447
|
+
});
|
|
2448
|
+
const endTime = eventTime(event);
|
|
2449
|
+
step.span.end(endTime === void 0 ? void 0 : { endTime });
|
|
2450
|
+
}
|
|
2451
|
+
const turn = this.turnForEvent(event, ctx);
|
|
2452
|
+
turn?.stepsByIndex.delete(event.data.stepIndex);
|
|
2453
|
+
this.markStepEnded(event.data.turnId, event.data.stepIndex);
|
|
2454
|
+
}
|
|
2455
|
+
handleTurnCompleted(event, ctx) {
|
|
2456
|
+
const turn = this.turnForEvent(event, ctx);
|
|
2457
|
+
if (!turn) {
|
|
2458
|
+
return;
|
|
2459
|
+
}
|
|
2460
|
+
this.finalizeTurn(turn, {
|
|
2461
|
+
endTime: eventTime(event)
|
|
2462
|
+
});
|
|
2463
|
+
}
|
|
2464
|
+
handleTurnFailed(event, ctx) {
|
|
2465
|
+
const turn = this.turnForEvent(event, ctx);
|
|
2466
|
+
if (!turn) {
|
|
2467
|
+
return;
|
|
2468
|
+
}
|
|
2469
|
+
this.finalizeTurn(turn, {
|
|
2470
|
+
endTime: eventTime(event),
|
|
2471
|
+
error: errorFromMessage(
|
|
2472
|
+
event.data.message,
|
|
2473
|
+
event.data.code,
|
|
2474
|
+
event.data.details
|
|
2475
|
+
)
|
|
2476
|
+
});
|
|
2477
|
+
}
|
|
2478
|
+
async handleSessionFailed(event, ctx) {
|
|
2479
|
+
const sessionId = event.data.sessionId || sessionIdFromContext(ctx);
|
|
2480
|
+
if (!sessionId) {
|
|
2481
|
+
return;
|
|
2482
|
+
}
|
|
2483
|
+
const session = await this.ensureSession(
|
|
2484
|
+
sessionId,
|
|
2485
|
+
ctx,
|
|
2486
|
+
{},
|
|
2487
|
+
eventTime(event)
|
|
2488
|
+
);
|
|
2489
|
+
const error = errorFromMessage(
|
|
2490
|
+
event.data.message,
|
|
2491
|
+
event.data.code,
|
|
2492
|
+
event.data.details
|
|
2493
|
+
);
|
|
2494
|
+
for (const [key, turn] of this.turnsByKey) {
|
|
2495
|
+
if (!key.startsWith(`${sessionId}:`)) {
|
|
2496
|
+
continue;
|
|
2497
|
+
}
|
|
2498
|
+
this.finalizeTurn(turn, {
|
|
2499
|
+
endTime: eventTime(event),
|
|
2500
|
+
error
|
|
2501
|
+
});
|
|
2502
|
+
}
|
|
2503
|
+
for (const [key, tool] of this.toolsByCallKey) {
|
|
2504
|
+
if (key.startsWith(`${sessionId}:`)) {
|
|
2505
|
+
const endTime2 = eventTime(event);
|
|
2506
|
+
if (!tool.endedByTurn) {
|
|
2507
|
+
tool.span.log({ metadata: tool.metadata });
|
|
2508
|
+
tool.span.end(endTime2 === void 0 ? void 0 : { endTime: endTime2 });
|
|
2509
|
+
tool.endedByTurn = true;
|
|
2510
|
+
}
|
|
2511
|
+
}
|
|
2512
|
+
}
|
|
2513
|
+
session.span.log({ error });
|
|
2514
|
+
const endTime = eventTime(event);
|
|
2515
|
+
session.span.end(endTime === void 0 ? void 0 : { endTime });
|
|
2516
|
+
}
|
|
2517
|
+
async handleSessionCompleted(event, ctx) {
|
|
2518
|
+
const sessionId = sessionIdFromContext(ctx);
|
|
2519
|
+
if (!sessionId) {
|
|
2520
|
+
return;
|
|
2521
|
+
}
|
|
2522
|
+
const session = await this.ensureSession(
|
|
2523
|
+
sessionId,
|
|
2524
|
+
ctx,
|
|
2525
|
+
{},
|
|
2526
|
+
eventTime(event)
|
|
2527
|
+
);
|
|
2528
|
+
for (const [key, turn] of this.turnsByKey) {
|
|
2529
|
+
if (!key.startsWith(`${sessionId}:`)) {
|
|
2530
|
+
continue;
|
|
2531
|
+
}
|
|
2532
|
+
this.finalizeTurn(turn, {
|
|
2533
|
+
endTime: eventTime(event)
|
|
2534
|
+
});
|
|
2535
|
+
}
|
|
2536
|
+
for (const [key, tool] of this.toolsByCallKey) {
|
|
2537
|
+
if (key.startsWith(`${sessionId}:`) && !tool.endedByTurn) {
|
|
2538
|
+
const endTime2 = eventTime(event);
|
|
2539
|
+
tool.span.log({ metadata: tool.metadata });
|
|
2540
|
+
tool.span.end(endTime2 === void 0 ? void 0 : { endTime: endTime2 });
|
|
2541
|
+
tool.endedByTurn = true;
|
|
2542
|
+
}
|
|
2543
|
+
}
|
|
2544
|
+
session.span.log({ metadata: session.metadata });
|
|
2545
|
+
const endTime = eventTime(event);
|
|
2546
|
+
session.span.end(endTime === void 0 ? void 0 : { endTime });
|
|
2547
|
+
}
|
|
2548
|
+
async ensureSession(sessionId, ctx, metadata, startTime) {
|
|
2549
|
+
metadata = {
|
|
2550
|
+
...readEveTraceState(this.state).metadata,
|
|
2551
|
+
...metadata
|
|
2552
|
+
};
|
|
2553
|
+
const existing = this.sessionsById.get(sessionId);
|
|
2554
|
+
if (existing) {
|
|
2555
|
+
existing.metadata = { ...existing.metadata, ...metadata };
|
|
2556
|
+
existing.span.log({ metadata: existing.metadata });
|
|
2557
|
+
return existing;
|
|
2558
|
+
}
|
|
2559
|
+
const lineage = parentLineageFromContext(ctx);
|
|
2560
|
+
const parentSubagent = lineage ? this.toolsByCallKey.get(toolKey(lineage.sessionId, lineage.callId)) : void 0;
|
|
2561
|
+
const activeParent = currentSpan();
|
|
2562
|
+
const [
|
|
2563
|
+
{ rowId: eventId, spanId },
|
|
2564
|
+
parentSubagentSpanId,
|
|
2565
|
+
fallbackRootSpanId
|
|
2566
|
+
] = await Promise.all([
|
|
2567
|
+
generateEveIds("session", sessionId),
|
|
2568
|
+
lineage ? spanIdForSubagent(lineage.sessionId, lineage.callId) : Promise.resolve(void 0),
|
|
2569
|
+
lineage ? rootSpanIdForSession(lineage.rootSessionId) : rootSpanIdForSession(sessionId)
|
|
2570
|
+
]);
|
|
2571
|
+
const span = await this.startEveSpan({
|
|
2572
|
+
event: {
|
|
2573
|
+
id: eventId,
|
|
2574
|
+
metadata
|
|
2575
|
+
},
|
|
2576
|
+
name: "eve.session",
|
|
2577
|
+
parentSpanIds: lineage && parentSubagentSpanId ? {
|
|
2578
|
+
rootSpanId: parentSubagent?.span.rootSpanId ?? fallbackRootSpanId,
|
|
2579
|
+
spanId: parentSubagentSpanId
|
|
2580
|
+
} : !Object.is(activeParent, NOOP_SPAN) ? {
|
|
2581
|
+
rootSpanId: activeParent.rootSpanId,
|
|
2582
|
+
spanId: activeParent.spanId
|
|
2583
|
+
} : {
|
|
2584
|
+
parentSpanIds: [],
|
|
2585
|
+
rootSpanId: fallbackRootSpanId
|
|
2586
|
+
},
|
|
2587
|
+
spanAttributes: { type: "task" /* TASK */ },
|
|
2588
|
+
spanId,
|
|
2589
|
+
startTime
|
|
2590
|
+
});
|
|
2591
|
+
span.log({ metadata });
|
|
2592
|
+
const session = { metadata, sessionId, span };
|
|
2593
|
+
this.sessionsById.set(sessionId, session);
|
|
2594
|
+
return session;
|
|
2595
|
+
}
|
|
2596
|
+
async ensureTurn(event, ctx, hookMetadata) {
|
|
2597
|
+
const sessionId = sessionIdFromContext(ctx);
|
|
2598
|
+
if (!sessionId) {
|
|
2599
|
+
return void 0;
|
|
2600
|
+
}
|
|
2601
|
+
const session = await this.ensureSession(
|
|
2602
|
+
sessionId,
|
|
2603
|
+
ctx,
|
|
2604
|
+
hookMetadata ?? {},
|
|
2605
|
+
eventTime(event)
|
|
2606
|
+
);
|
|
2607
|
+
const key = turnKey(sessionId, event.data.turnId);
|
|
2608
|
+
const existing = this.turnsByKey.get(key);
|
|
2609
|
+
if (existing) {
|
|
2610
|
+
return existing;
|
|
2611
|
+
}
|
|
2612
|
+
const metadata = { ...session.metadata };
|
|
2613
|
+
const span = await this.startTurnSpan(session, event, metadata);
|
|
2614
|
+
span.log({ metadata });
|
|
2615
|
+
const state = {
|
|
2616
|
+
key,
|
|
2617
|
+
metadata,
|
|
2618
|
+
metrics: {},
|
|
2619
|
+
span,
|
|
2620
|
+
stepsByIndex: /* @__PURE__ */ new Map(),
|
|
2621
|
+
turnId: event.data.turnId
|
|
2622
|
+
};
|
|
2623
|
+
this.turnsByKey.set(key, state);
|
|
2624
|
+
return state;
|
|
2625
|
+
}
|
|
2626
|
+
async startRequestedTool(event, turn, sessionId, action) {
|
|
2627
|
+
const key = toolKey(sessionId, action.callId);
|
|
2628
|
+
if (this.toolsByCallKey.has(key) || this.completedToolKeys.has(key)) {
|
|
2629
|
+
return;
|
|
2630
|
+
}
|
|
2631
|
+
const metadata = toolMetadataFromTurn(turn);
|
|
2632
|
+
const { rowId: eventId, spanId } = await generateEveIds(
|
|
2633
|
+
"tool",
|
|
2634
|
+
sessionId,
|
|
2635
|
+
event.data.turnId,
|
|
2636
|
+
action.callId
|
|
2637
|
+
);
|
|
2638
|
+
if (this.toolsByCallKey.has(key) || this.completedToolKeys.has(key)) {
|
|
2639
|
+
return;
|
|
2640
|
+
}
|
|
2641
|
+
const span = await this.startEveChildSpan(turn.span, {
|
|
2642
|
+
event: {
|
|
2643
|
+
id: eventId,
|
|
2644
|
+
input: action.input,
|
|
2645
|
+
metadata
|
|
2646
|
+
},
|
|
2647
|
+
name: action.toolName,
|
|
2648
|
+
spanAttributes: { type: "tool" /* TOOL */ },
|
|
2649
|
+
spanId,
|
|
2650
|
+
startTime: eventTime(event)
|
|
2651
|
+
});
|
|
2652
|
+
span.log({ input: action.input, metadata });
|
|
2653
|
+
this.toolsByCallKey.set(key, {
|
|
2654
|
+
metadata,
|
|
2655
|
+
span,
|
|
2656
|
+
turnKey: turnKey(sessionId, event.data.turnId)
|
|
2657
|
+
});
|
|
2658
|
+
}
|
|
2659
|
+
async startRequestedSubagent(event, turn, sessionId, action) {
|
|
2660
|
+
const key = toolKey(sessionId, action.callId);
|
|
2661
|
+
if (this.toolsByCallKey.has(key) || this.completedToolKeys.has(key)) {
|
|
2662
|
+
return;
|
|
2663
|
+
}
|
|
2664
|
+
const name = action.subagentName ?? action.name ?? "agent";
|
|
2665
|
+
const metadata = toolMetadataFromTurn(turn);
|
|
2666
|
+
const { rowId: eventId, spanId } = await generateEveIds(
|
|
2667
|
+
"subagent",
|
|
2668
|
+
sessionId,
|
|
2669
|
+
action.callId
|
|
2670
|
+
);
|
|
2671
|
+
if (this.toolsByCallKey.has(key) || this.completedToolKeys.has(key)) {
|
|
2672
|
+
return;
|
|
2673
|
+
}
|
|
2674
|
+
const span = await this.startEveChildSpan(turn.span, {
|
|
2675
|
+
event: {
|
|
2676
|
+
id: eventId,
|
|
2677
|
+
input: action.input,
|
|
2678
|
+
metadata
|
|
2679
|
+
},
|
|
2680
|
+
name,
|
|
2681
|
+
spanAttributes: { type: "tool" /* TOOL */ },
|
|
2682
|
+
spanId,
|
|
2683
|
+
startTime: eventTime(event)
|
|
2684
|
+
});
|
|
2685
|
+
span.log({ input: action.input, metadata });
|
|
2686
|
+
this.toolsByCallKey.set(key, {
|
|
2687
|
+
metadata,
|
|
2688
|
+
span,
|
|
2689
|
+
turnKey: turnKey(sessionId, event.data.turnId)
|
|
2690
|
+
});
|
|
2691
|
+
}
|
|
2692
|
+
async startSyntheticTool(event, ctx, result, hookMetadata) {
|
|
2693
|
+
const turn = await this.ensureTurn(event, ctx, hookMetadata);
|
|
2694
|
+
const sessionId = sessionIdFromContext(ctx);
|
|
2695
|
+
if (!turn || !sessionId) {
|
|
2696
|
+
return void 0;
|
|
2697
|
+
}
|
|
2698
|
+
const metadata = toolMetadataFromTurn(turn);
|
|
2699
|
+
const { rowId: eventId, spanId } = await generateEveIds(
|
|
2700
|
+
"tool",
|
|
2701
|
+
sessionId,
|
|
2702
|
+
event.data.turnId,
|
|
2703
|
+
result.callId
|
|
2704
|
+
);
|
|
2705
|
+
const existing = this.toolsByCallKey.get(toolKey(sessionId, result.callId));
|
|
2706
|
+
if (existing) {
|
|
2707
|
+
return existing;
|
|
2708
|
+
}
|
|
2709
|
+
const span = await this.startEveChildSpan(turn.span, {
|
|
2710
|
+
event: {
|
|
2711
|
+
id: eventId,
|
|
2712
|
+
metadata
|
|
2713
|
+
},
|
|
2714
|
+
name: result.toolName,
|
|
2715
|
+
spanAttributes: { type: "tool" /* TOOL */ },
|
|
2716
|
+
spanId,
|
|
2717
|
+
startTime: eventTime(event)
|
|
2718
|
+
});
|
|
2719
|
+
span.log({ metadata });
|
|
2720
|
+
const state = {
|
|
2721
|
+
metadata,
|
|
2722
|
+
span,
|
|
2723
|
+
turnKey: turnKey(sessionId, event.data.turnId)
|
|
2724
|
+
};
|
|
2725
|
+
this.toolsByCallKey.set(toolKey(sessionId, result.callId), state);
|
|
2726
|
+
return state;
|
|
2727
|
+
}
|
|
2728
|
+
async startSyntheticSubagent(event, ctx, hookMetadata) {
|
|
2729
|
+
const turn = await this.ensureTurn(event, ctx, hookMetadata);
|
|
2730
|
+
const sessionId = sessionIdFromContext(ctx);
|
|
2731
|
+
if (!turn || !sessionId) {
|
|
2732
|
+
return void 0;
|
|
2733
|
+
}
|
|
2734
|
+
const metadata = toolMetadataFromTurn(turn);
|
|
2735
|
+
const { rowId: eventId, spanId } = await generateEveIds(
|
|
2736
|
+
"subagent",
|
|
2737
|
+
sessionId,
|
|
2738
|
+
event.data.callId
|
|
2739
|
+
);
|
|
2740
|
+
const existing = this.toolsByCallKey.get(
|
|
2741
|
+
toolKey(sessionId, event.data.callId)
|
|
2742
|
+
);
|
|
2743
|
+
if (existing) {
|
|
2744
|
+
return existing;
|
|
2745
|
+
}
|
|
2746
|
+
const span = await this.startEveChildSpan(turn.span, {
|
|
2747
|
+
event: {
|
|
2748
|
+
id: eventId,
|
|
2749
|
+
metadata
|
|
2750
|
+
},
|
|
2751
|
+
name: event.data.subagentName,
|
|
2752
|
+
spanAttributes: { type: "tool" /* TOOL */ },
|
|
2753
|
+
spanId,
|
|
2754
|
+
startTime: eventTime(event)
|
|
2755
|
+
});
|
|
2756
|
+
span.log({ metadata });
|
|
2757
|
+
const state = {
|
|
2758
|
+
metadata,
|
|
2759
|
+
span,
|
|
2760
|
+
turnKey: turnKey(sessionId, event.data.turnId)
|
|
2761
|
+
};
|
|
2762
|
+
this.toolsByCallKey.set(toolKey(sessionId, event.data.callId), state);
|
|
2763
|
+
return state;
|
|
2764
|
+
}
|
|
2765
|
+
async startSyntheticSubagentResult(event, ctx, result, hookMetadata) {
|
|
2766
|
+
const turn = await this.ensureTurn(event, ctx, hookMetadata);
|
|
2767
|
+
const sessionId = sessionIdFromContext(ctx);
|
|
2768
|
+
if (!turn || !sessionId) {
|
|
2769
|
+
return void 0;
|
|
2770
|
+
}
|
|
2771
|
+
const metadata = toolMetadataFromTurn(turn);
|
|
2772
|
+
const { rowId: eventId, spanId } = await generateEveIds(
|
|
2773
|
+
"subagent",
|
|
2774
|
+
sessionId,
|
|
2775
|
+
result.callId
|
|
2776
|
+
);
|
|
2777
|
+
const existing = this.toolsByCallKey.get(toolKey(sessionId, result.callId));
|
|
2778
|
+
if (existing) {
|
|
2779
|
+
return existing;
|
|
2780
|
+
}
|
|
2781
|
+
const span = await this.startEveChildSpan(turn.span, {
|
|
2782
|
+
event: {
|
|
2783
|
+
id: eventId,
|
|
2784
|
+
metadata
|
|
2785
|
+
},
|
|
2786
|
+
name: result.subagentName,
|
|
2787
|
+
spanAttributes: { type: "tool" /* TOOL */ },
|
|
2788
|
+
spanId,
|
|
2789
|
+
startTime: eventTime(event)
|
|
2790
|
+
});
|
|
2791
|
+
span.log({ metadata });
|
|
2792
|
+
const state = {
|
|
2793
|
+
metadata,
|
|
2794
|
+
span,
|
|
2795
|
+
turnKey: turnKey(sessionId, event.data.turnId)
|
|
2796
|
+
};
|
|
2797
|
+
this.toolsByCallKey.set(toolKey(sessionId, result.callId), state);
|
|
2798
|
+
return state;
|
|
2799
|
+
}
|
|
2800
|
+
async startTurnSpan(session, event, metadata) {
|
|
2801
|
+
const { rowId: eventId, spanId } = await generateEveIds(
|
|
2802
|
+
"turn",
|
|
2803
|
+
session.sessionId,
|
|
2804
|
+
event.data.turnId
|
|
2805
|
+
);
|
|
2806
|
+
return await this.startEveSpan({
|
|
2807
|
+
event: {
|
|
2808
|
+
id: eventId,
|
|
2809
|
+
metadata
|
|
2810
|
+
},
|
|
2811
|
+
name: "eve.turn",
|
|
2812
|
+
parentSpanIds: {
|
|
2813
|
+
rootSpanId: session.span.rootSpanId,
|
|
2814
|
+
spanId: session.span.spanId
|
|
2815
|
+
},
|
|
2816
|
+
spanAttributes: { type: "task" /* TASK */ },
|
|
2817
|
+
spanId,
|
|
2818
|
+
startTime: eventTime(event)
|
|
2819
|
+
});
|
|
2820
|
+
}
|
|
2821
|
+
turnForEvent(event, ctx) {
|
|
2822
|
+
const sessionId = sessionIdFromContext(ctx);
|
|
2823
|
+
return sessionId ? this.turnsByKey.get(turnKey(sessionId, event.data.turnId)) : void 0;
|
|
2824
|
+
}
|
|
2825
|
+
stepForEvent(event, ctx) {
|
|
2826
|
+
return this.turnForEvent(event, ctx)?.stepsByIndex.get(
|
|
2827
|
+
event.data.stepIndex
|
|
2828
|
+
);
|
|
2829
|
+
}
|
|
2830
|
+
finalizeTurn(turn, args) {
|
|
2831
|
+
const { endTime } = args;
|
|
2832
|
+
for (const step of turn.stepsByIndex.values()) {
|
|
2833
|
+
step.span.log({
|
|
2834
|
+
...step.input !== void 0 ? { input: step.input } : {},
|
|
2835
|
+
metadata: step.metadata,
|
|
2836
|
+
metrics: step.metrics,
|
|
2837
|
+
output: step.output
|
|
2838
|
+
});
|
|
2839
|
+
step.span.end(endTime === void 0 ? void 0 : { endTime });
|
|
2840
|
+
}
|
|
2841
|
+
turn.stepsByIndex.clear();
|
|
2842
|
+
for (const tool of this.toolsByCallKey.values()) {
|
|
2843
|
+
if (tool.turnKey !== turn.key) {
|
|
2844
|
+
continue;
|
|
2845
|
+
}
|
|
2846
|
+
if (tool.endedByTurn) {
|
|
2847
|
+
continue;
|
|
2848
|
+
}
|
|
2849
|
+
tool.span.log({ metadata: tool.metadata });
|
|
2850
|
+
tool.span.end(endTime === void 0 ? void 0 : { endTime });
|
|
2851
|
+
tool.endedByTurn = true;
|
|
2852
|
+
}
|
|
2853
|
+
if (args.error) {
|
|
2854
|
+
turn.span.log({ error: args.error });
|
|
2855
|
+
} else {
|
|
2856
|
+
turn.span.log({
|
|
2857
|
+
metadata: turn.metadata,
|
|
2858
|
+
metrics: turn.metrics,
|
|
2859
|
+
output: turn.output
|
|
2860
|
+
});
|
|
2861
|
+
}
|
|
2862
|
+
turn.span.end(endTime === void 0 ? void 0 : { endTime });
|
|
2863
|
+
this.turnsByKey.delete(turn.key);
|
|
2864
|
+
this.state.update((current) => {
|
|
2865
|
+
const normalized = normalizeEveTraceState(current);
|
|
2866
|
+
return {
|
|
2867
|
+
...normalized,
|
|
2868
|
+
stepStarts: normalized.stepStarts.filter(
|
|
2869
|
+
(entry) => entry.turnId !== turn.turnId
|
|
2870
|
+
)
|
|
2871
|
+
};
|
|
2872
|
+
});
|
|
2873
|
+
}
|
|
2874
|
+
cleanupSession(sessionId) {
|
|
2875
|
+
const keyPrefix = `${sessionId}:`;
|
|
2876
|
+
this.sessionsById.delete(sessionId);
|
|
2877
|
+
for (const key of this.turnsByKey.keys()) {
|
|
2878
|
+
if (key.startsWith(keyPrefix)) {
|
|
2879
|
+
this.turnsByKey.delete(key);
|
|
2880
|
+
}
|
|
2881
|
+
}
|
|
2882
|
+
for (const key of this.toolsByCallKey.keys()) {
|
|
2883
|
+
if (key.startsWith(keyPrefix)) {
|
|
2884
|
+
this.toolsByCallKey.delete(key);
|
|
2885
|
+
}
|
|
2886
|
+
}
|
|
2887
|
+
for (const key of this.completedToolKeys.keys()) {
|
|
2888
|
+
if (key.startsWith(keyPrefix)) {
|
|
2889
|
+
this.completedToolKeys.delete(key);
|
|
2890
|
+
}
|
|
2891
|
+
}
|
|
2892
|
+
this.state.update(() => emptyEveTraceState());
|
|
2893
|
+
}
|
|
2894
|
+
async flushInstrumentation() {
|
|
2895
|
+
try {
|
|
2896
|
+
await flush();
|
|
2897
|
+
return true;
|
|
2898
|
+
} catch (error) {
|
|
2899
|
+
debugLogger.warn("Error in Eve flush instrumentation:", error);
|
|
2900
|
+
return false;
|
|
2901
|
+
}
|
|
2902
|
+
}
|
|
2903
|
+
};
|
|
2904
|
+
function emptyEveTraceState() {
|
|
2905
|
+
return {
|
|
2906
|
+
llmInputs: [],
|
|
2907
|
+
metadata: {},
|
|
2908
|
+
spanReferences: [],
|
|
2909
|
+
stepStarts: []
|
|
2910
|
+
};
|
|
2911
|
+
}
|
|
2912
|
+
function normalizeEveTraceState(state) {
|
|
2913
|
+
if (!isObject(state)) {
|
|
2914
|
+
return emptyEveTraceState();
|
|
2915
|
+
}
|
|
2916
|
+
const metadata = isObject(state["metadata"]) ? state["metadata"] : {};
|
|
2917
|
+
const spanReferences = Array.isArray(state["spanReferences"]) ? state["spanReferences"].flatMap((entry) => {
|
|
2918
|
+
if (!isObject(entry)) {
|
|
2919
|
+
return [];
|
|
2920
|
+
}
|
|
2921
|
+
const exported = entry["exported"];
|
|
2922
|
+
const endTime = entry["endTime"];
|
|
2923
|
+
const rootSpanId = entry["rootSpanId"];
|
|
2924
|
+
const rowId = entry["rowId"];
|
|
2925
|
+
const spanId = entry["spanId"];
|
|
2926
|
+
const startEvent = entry["startEvent"];
|
|
2927
|
+
const startEventCreated = isObject(startEvent) ? startEvent["created"] : void 0;
|
|
2928
|
+
const startEventMetrics = isObject(startEvent) ? startEvent["metrics"] : void 0;
|
|
2929
|
+
const startEventSpanAttributes = isObject(startEvent) ? startEvent["span_attributes"] : void 0;
|
|
2930
|
+
const startEventSpanParents = isObject(startEvent) ? startEvent["span_parents"] : void 0;
|
|
2931
|
+
const normalizedStartEvent = typeof startEventCreated === "string" && isObject(startEventMetrics) && typeof startEventMetrics["start"] === "number" && Number.isFinite(startEventMetrics["start"]) && isObject(startEventSpanAttributes) && Array.isArray(startEventSpanParents) && startEventSpanParents.every(
|
|
2932
|
+
(parent) => typeof parent === "string"
|
|
2933
|
+
) ? {
|
|
2934
|
+
created: startEventCreated,
|
|
2935
|
+
metrics: { start: startEventMetrics["start"] },
|
|
2936
|
+
span_attributes: { ...startEventSpanAttributes },
|
|
2937
|
+
span_parents: [...startEventSpanParents]
|
|
2938
|
+
} : void 0;
|
|
2939
|
+
return typeof exported === "string" && typeof rootSpanId === "string" && typeof rowId === "string" && typeof spanId === "string" ? [
|
|
2940
|
+
{
|
|
2941
|
+
...typeof endTime === "number" && Number.isFinite(endTime) ? { endTime } : {},
|
|
2942
|
+
exported,
|
|
2943
|
+
rootSpanId,
|
|
2944
|
+
rowId,
|
|
2945
|
+
spanId,
|
|
2946
|
+
...normalizedStartEvent ? { startEvent: normalizedStartEvent } : {}
|
|
2947
|
+
}
|
|
2948
|
+
] : [];
|
|
2949
|
+
}).slice(-MAX_STORED_SPAN_REFERENCES) : [];
|
|
2950
|
+
const llmInputs = Array.isArray(state["llmInputs"]) ? state["llmInputs"].flatMap((entry) => {
|
|
2951
|
+
if (!isObject(entry)) {
|
|
2952
|
+
return [];
|
|
2953
|
+
}
|
|
2954
|
+
const key = entry["key"];
|
|
2955
|
+
const input = entry["input"];
|
|
2956
|
+
return typeof key === "string" && isCapturedModelInput(input) ? [{ input, key }] : [];
|
|
2957
|
+
}).slice(-MAX_STORED_LLM_INPUTS) : [];
|
|
2958
|
+
const stepStarts = Array.isArray(state["stepStarts"]) ? state["stepStarts"].flatMap((entry) => {
|
|
2959
|
+
if (!isObject(entry)) {
|
|
2960
|
+
return [];
|
|
2961
|
+
}
|
|
2962
|
+
const ordinal = entry["ordinal"];
|
|
2963
|
+
const open = entry["open"];
|
|
2964
|
+
const stepIndex = entry["stepIndex"];
|
|
2965
|
+
const turnId = entry["turnId"];
|
|
2966
|
+
return typeof ordinal === "number" && Number.isInteger(ordinal) && ordinal >= 0 && typeof open === "boolean" && typeof stepIndex === "number" && Number.isInteger(stepIndex) && typeof turnId === "string" ? [{ open, ordinal, stepIndex, turnId }] : [];
|
|
2967
|
+
}).slice(-MAX_STORED_STEP_STARTS) : [];
|
|
2968
|
+
return { llmInputs, metadata: { ...metadata }, spanReferences, stepStarts };
|
|
2969
|
+
}
|
|
2970
|
+
function readEveTraceState(state) {
|
|
2971
|
+
try {
|
|
2972
|
+
return normalizeEveTraceState(state.get());
|
|
2973
|
+
} catch {
|
|
2974
|
+
return emptyEveTraceState();
|
|
2975
|
+
}
|
|
2976
|
+
}
|
|
2977
|
+
function captureEveModelInput(state, input) {
|
|
2978
|
+
if (!isObject(input)) {
|
|
2979
|
+
return;
|
|
2980
|
+
}
|
|
2981
|
+
const session = input["session"];
|
|
2982
|
+
const turn = input["turn"];
|
|
2983
|
+
const step = input["step"];
|
|
2984
|
+
if (!isObject(session) || !isObject(turn) || !isObject(step)) {
|
|
2985
|
+
return;
|
|
2986
|
+
}
|
|
2987
|
+
const sessionId = session["id"];
|
|
2988
|
+
const turnId = turn["id"];
|
|
2989
|
+
const stepIndex = step["index"];
|
|
2990
|
+
if (typeof sessionId !== "string" || typeof turnId !== "string" || typeof stepIndex !== "number" || !Number.isInteger(stepIndex)) {
|
|
2991
|
+
return;
|
|
2992
|
+
}
|
|
2993
|
+
const captured = capturedModelInput(input["modelInput"]);
|
|
2994
|
+
if (!captured) {
|
|
2995
|
+
return;
|
|
2996
|
+
}
|
|
2997
|
+
const key = llmInputKey(sessionId, turnId, stepIndex);
|
|
2998
|
+
state.update((current) => {
|
|
2999
|
+
const normalized = normalizeEveTraceState(current);
|
|
3000
|
+
const llmInputs = [...normalized.llmInputs, { input: captured, key }];
|
|
3001
|
+
return {
|
|
3002
|
+
...normalized,
|
|
3003
|
+
llmInputs: llmInputs.slice(-MAX_STORED_LLM_INPUTS)
|
|
3004
|
+
};
|
|
3005
|
+
});
|
|
3006
|
+
}
|
|
3007
|
+
function consumeCapturedEveModelInput(state, sessionId, turnId, stepIndex) {
|
|
3008
|
+
try {
|
|
3009
|
+
const key = llmInputKey(sessionId, turnId, stepIndex);
|
|
3010
|
+
let input;
|
|
3011
|
+
state.update((current) => {
|
|
3012
|
+
const normalized = normalizeEveTraceState(current);
|
|
3013
|
+
const index = normalized.llmInputs.findIndex(
|
|
3014
|
+
(candidate) => candidate.key === key
|
|
3015
|
+
);
|
|
3016
|
+
if (index < 0) {
|
|
3017
|
+
return normalized;
|
|
3018
|
+
}
|
|
3019
|
+
input = normalized.llmInputs[index]?.input;
|
|
3020
|
+
return {
|
|
3021
|
+
...normalized,
|
|
3022
|
+
llmInputs: normalized.llmInputs.filter(
|
|
3023
|
+
(_, candidateIndex) => candidateIndex !== index
|
|
3024
|
+
)
|
|
3025
|
+
};
|
|
3026
|
+
});
|
|
3027
|
+
return input;
|
|
3028
|
+
} catch (error) {
|
|
3029
|
+
debugLogger.warn("Error in Eve LLM input consumption:", error);
|
|
3030
|
+
return void 0;
|
|
3031
|
+
}
|
|
3032
|
+
}
|
|
3033
|
+
function capturedModelInput(modelInput) {
|
|
3034
|
+
if (!isObject(modelInput)) {
|
|
3035
|
+
return void 0;
|
|
3036
|
+
}
|
|
3037
|
+
const messages = modelInput["messages"];
|
|
3038
|
+
if (!Array.isArray(messages)) {
|
|
3039
|
+
return void 0;
|
|
3040
|
+
}
|
|
3041
|
+
const instructions = modelInput["instructions"];
|
|
3042
|
+
const value = [
|
|
3043
|
+
...instructions !== void 0 ? [{ content: instructions, role: "system" }] : [],
|
|
3044
|
+
...messages
|
|
3045
|
+
];
|
|
3046
|
+
try {
|
|
3047
|
+
const cloned = JSON.parse(JSON.stringify(value));
|
|
3048
|
+
if (!Array.isArray(cloned)) {
|
|
3049
|
+
return void 0;
|
|
3050
|
+
}
|
|
3051
|
+
return cloned;
|
|
3052
|
+
} catch {
|
|
3053
|
+
return void 0;
|
|
3054
|
+
}
|
|
3055
|
+
}
|
|
3056
|
+
function isCapturedModelInput(input) {
|
|
3057
|
+
return Array.isArray(input);
|
|
3058
|
+
}
|
|
3059
|
+
function llmInputKey(sessionId, turnId, stepIndex) {
|
|
3060
|
+
return `${sessionId}\0${turnId}\0${stepIndex}`;
|
|
3061
|
+
}
|
|
3062
|
+
function modelMetadataFromRuntime(runtime) {
|
|
3063
|
+
if (!isObject(runtime)) {
|
|
3064
|
+
return {};
|
|
3065
|
+
}
|
|
3066
|
+
const modelId = runtime["modelId"];
|
|
3067
|
+
return typeof modelId === "string" ? modelMetadataFromModelId(modelId) : {};
|
|
3068
|
+
}
|
|
3069
|
+
function modelMetadataFromModelId(modelId) {
|
|
3070
|
+
const normalized = modelId.trim();
|
|
3071
|
+
if (!normalized) {
|
|
3072
|
+
return {};
|
|
3073
|
+
}
|
|
3074
|
+
const slashIndex = normalized.indexOf("/");
|
|
3075
|
+
if (slashIndex > 0 && slashIndex < normalized.length - 1) {
|
|
3076
|
+
return {
|
|
3077
|
+
model: normalized.slice(slashIndex + 1),
|
|
3078
|
+
provider: normalized.slice(0, slashIndex)
|
|
3079
|
+
};
|
|
3080
|
+
}
|
|
3081
|
+
return {
|
|
3082
|
+
model: normalized
|
|
3083
|
+
};
|
|
3084
|
+
}
|
|
3085
|
+
function sessionIdFromContext(ctx) {
|
|
3086
|
+
if (!isObject(ctx)) {
|
|
3087
|
+
return void 0;
|
|
3088
|
+
}
|
|
3089
|
+
const session = ctx["session"];
|
|
3090
|
+
if (!isObject(session)) {
|
|
3091
|
+
return void 0;
|
|
3092
|
+
}
|
|
3093
|
+
const id = session["id"];
|
|
3094
|
+
return typeof id === "string" ? id : void 0;
|
|
3095
|
+
}
|
|
3096
|
+
function toolMetadataFromTurn(turn) {
|
|
3097
|
+
const { model: _model, provider: _provider, ...metadata } = turn.metadata;
|
|
3098
|
+
return metadata;
|
|
3099
|
+
}
|
|
3100
|
+
function parentLineageFromContext(ctx) {
|
|
3101
|
+
if (!isObject(ctx)) {
|
|
3102
|
+
return void 0;
|
|
3103
|
+
}
|
|
3104
|
+
const session = ctx.session;
|
|
3105
|
+
if (!isObject(session)) {
|
|
3106
|
+
return void 0;
|
|
3107
|
+
}
|
|
3108
|
+
const parent = session["parent"];
|
|
3109
|
+
if (!isObject(parent)) {
|
|
3110
|
+
return void 0;
|
|
3111
|
+
}
|
|
3112
|
+
const callId = parent["callId"];
|
|
3113
|
+
const rootSessionId = parent["rootSessionId"];
|
|
3114
|
+
const sessionId = parent["sessionId"];
|
|
3115
|
+
if (typeof callId !== "string" || typeof rootSessionId !== "string" || typeof sessionId !== "string") {
|
|
3116
|
+
return void 0;
|
|
3117
|
+
}
|
|
3118
|
+
return { callId, rootSessionId, sessionId };
|
|
3119
|
+
}
|
|
3120
|
+
function isToolCallAction(action) {
|
|
3121
|
+
return isObject(action) && action["kind"] === "tool-call" && typeof action["callId"] === "string" && typeof action["toolName"] === "string" && isObject(action["input"]);
|
|
3122
|
+
}
|
|
3123
|
+
function isLocalSubagentCallAction(action) {
|
|
3124
|
+
return isObject(action) && action["kind"] === "subagent-call" && typeof action["callId"] === "string" && isObject(action["input"]);
|
|
3125
|
+
}
|
|
3126
|
+
function isTraceableActionRequest(action) {
|
|
3127
|
+
return isToolCallAction(action) || isLocalSubagentCallAction(action);
|
|
3128
|
+
}
|
|
3129
|
+
function isToolResult(result) {
|
|
3130
|
+
return isObject(result) && result["kind"] === "tool-result" && typeof result["callId"] === "string" && typeof result["toolName"] === "string";
|
|
3131
|
+
}
|
|
3132
|
+
function isSubagentResult(result) {
|
|
3133
|
+
return isObject(result) && result["kind"] === "subagent-result" && typeof result["callId"] === "string" && typeof result["subagentName"] === "string";
|
|
3134
|
+
}
|
|
3135
|
+
function normalizedFinishReason(finishReason) {
|
|
3136
|
+
switch (finishReason) {
|
|
3137
|
+
case "content-filter":
|
|
3138
|
+
return "content_filter";
|
|
3139
|
+
case "tool-calls":
|
|
3140
|
+
return "tool_calls";
|
|
3141
|
+
default:
|
|
3142
|
+
return finishReason;
|
|
3143
|
+
}
|
|
3144
|
+
}
|
|
3145
|
+
function errorFromMessage(message, code, details) {
|
|
3146
|
+
const error = new Error(`${code}: ${message}`);
|
|
3147
|
+
if (details !== void 0) {
|
|
3148
|
+
error.cause = details;
|
|
3149
|
+
}
|
|
3150
|
+
return error;
|
|
3151
|
+
}
|
|
3152
|
+
function actionResultError(error, output) {
|
|
3153
|
+
if (error) {
|
|
3154
|
+
return errorFromMessage(error.message, error.code);
|
|
3155
|
+
}
|
|
3156
|
+
const result = new Error("Eve action failed");
|
|
3157
|
+
if (output !== void 0) {
|
|
3158
|
+
result.cause = output;
|
|
3159
|
+
}
|
|
3160
|
+
return result;
|
|
3161
|
+
}
|
|
3162
|
+
function eventTime(event) {
|
|
3163
|
+
if (!event.meta?.at) {
|
|
3164
|
+
return void 0;
|
|
3165
|
+
}
|
|
3166
|
+
const timestamp = Date.parse(event.meta.at);
|
|
3167
|
+
return Number.isFinite(timestamp) ? timestamp / 1e3 : void 0;
|
|
3168
|
+
}
|
|
3169
|
+
function turnKey(sessionId, turnId) {
|
|
3170
|
+
return `${sessionId}:${turnId}`;
|
|
3171
|
+
}
|
|
3172
|
+
function toolKey(sessionId, callId) {
|
|
3173
|
+
return `${sessionId}:${callId}`;
|
|
3174
|
+
}
|
|
3175
|
+
async function rootSpanIdForSession(sessionId) {
|
|
3176
|
+
return deterministicEveId("eve:root", sessionId);
|
|
3177
|
+
}
|
|
3178
|
+
async function generateEveIds(kind, ...parts) {
|
|
3179
|
+
const [rowId, spanId] = await Promise.all([
|
|
3180
|
+
deterministicEveId(`eve:row:${kind}`, ...parts),
|
|
3181
|
+
deterministicEveId(`eve:${kind}`, ...parts)
|
|
3182
|
+
]);
|
|
3183
|
+
return { rowId, spanId };
|
|
3184
|
+
}
|
|
3185
|
+
async function spanIdForSubagent(sessionId, callId) {
|
|
3186
|
+
return deterministicEveId("eve:subagent", sessionId, callId);
|
|
3187
|
+
}
|
|
3188
|
+
async function deterministicEveId(...parts) {
|
|
3189
|
+
const data = new TextEncoder().encode(
|
|
3190
|
+
parts.map((part) => `${part.length}:${part}`).join("\0")
|
|
3191
|
+
);
|
|
3192
|
+
const digest = await globalThis.crypto.subtle.digest("SHA-256", data);
|
|
3193
|
+
const bytes = Array.from(new Uint8Array(digest, 0, 16));
|
|
3194
|
+
const hex = bytes.map((byte) => byte.toString(16).padStart(2, "0")).join("");
|
|
3195
|
+
return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;
|
|
3196
|
+
}
|
|
3197
|
+
|
|
1691
3198
|
// src/typed-instrumentation-helpers.ts
|
|
1692
3199
|
var TypedApplyProxy = Proxy;
|
|
1693
3200
|
|
|
@@ -5599,15 +7106,21 @@ var waterfall$1 = awaitify(waterfall);
|
|
|
5599
7106
|
|
|
5600
7107
|
// src/trace.ts
|
|
5601
7108
|
var SpanFetcher = class _SpanFetcher extends ObjectFetcher {
|
|
5602
|
-
constructor(objectType, _objectId, rootSpanId, _state, spanTypeFilter, includeScorers = false) {
|
|
7109
|
+
constructor(objectType, _objectId, rootSpanId, _state, spanTypeFilter, includeScorers = false, brainstoreRealtime = true) {
|
|
5603
7110
|
const filterExpr = _SpanFetcher.buildFilter(
|
|
5604
7111
|
rootSpanId,
|
|
5605
7112
|
spanTypeFilter,
|
|
5606
7113
|
includeScorers
|
|
5607
7114
|
);
|
|
5608
|
-
super(
|
|
5609
|
-
|
|
5610
|
-
|
|
7115
|
+
super(
|
|
7116
|
+
objectType,
|
|
7117
|
+
void 0,
|
|
7118
|
+
void 0,
|
|
7119
|
+
{
|
|
7120
|
+
filter: filterExpr
|
|
7121
|
+
},
|
|
7122
|
+
brainstoreRealtime
|
|
7123
|
+
);
|
|
5611
7124
|
this._objectId = _objectId;
|
|
5612
7125
|
this.rootSpanId = rootSpanId;
|
|
5613
7126
|
this._state = _state;
|
|
@@ -5661,7 +7174,7 @@ var CachedSpanFetcher = class {
|
|
|
5661
7174
|
spanCache = /* @__PURE__ */ new Map();
|
|
5662
7175
|
allFetched = false;
|
|
5663
7176
|
fetchFn;
|
|
5664
|
-
constructor(objectTypeOrFetchFn, objectId, rootSpanId, getState) {
|
|
7177
|
+
constructor(objectTypeOrFetchFn, objectId, rootSpanId, getState, brainstoreRealtime = true) {
|
|
5665
7178
|
if (typeof objectTypeOrFetchFn === "function") {
|
|
5666
7179
|
this.fetchFn = (spanType) => objectTypeOrFetchFn(spanType);
|
|
5667
7180
|
} else {
|
|
@@ -5674,7 +7187,8 @@ var CachedSpanFetcher = class {
|
|
|
5674
7187
|
rootSpanId,
|
|
5675
7188
|
state,
|
|
5676
7189
|
spanType,
|
|
5677
|
-
includeScorers
|
|
7190
|
+
includeScorers,
|
|
7191
|
+
brainstoreRealtime
|
|
5678
7192
|
);
|
|
5679
7193
|
const rows = await fetcher.fetchedData();
|
|
5680
7194
|
return rows.map((row) => ({
|
|
@@ -6504,7 +8018,7 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
|
|
|
6504
8018
|
}
|
|
6505
8019
|
};
|
|
6506
8020
|
const parentStr = state.currentParent.getStore();
|
|
6507
|
-
const parentComponents = parentStr ?
|
|
8021
|
+
const parentComponents = typeof parentStr === "string" ? SpanComponentsV4.fromStr(parentStr) : null;
|
|
6508
8022
|
const trace = state ? new LocalTrace({
|
|
6509
8023
|
objectType: parentComponents ? spanObjectTypeV3ToTypedString(
|
|
6510
8024
|
parentComponents.data.object_type
|
|
@@ -7030,6 +8544,350 @@ var defaultReporter = {
|
|
|
7030
8544
|
}
|
|
7031
8545
|
};
|
|
7032
8546
|
|
|
8547
|
+
// src/agent-assertions.ts
|
|
8548
|
+
function agentAssertionScorer(callback, options = {}) {
|
|
8549
|
+
return async (args) => {
|
|
8550
|
+
const { trace: _trace, ...callbackArgs } = args;
|
|
8551
|
+
const callbackMetadata = callbackArgs.metadata ?? {};
|
|
8552
|
+
const assertions = await callback({
|
|
8553
|
+
...callbackArgs,
|
|
8554
|
+
metadata: callbackMetadata,
|
|
8555
|
+
assert: agentAssertionHelpers
|
|
8556
|
+
});
|
|
8557
|
+
const resources = {};
|
|
8558
|
+
if (assertions.some((assertion) => assertion.requiresTrace)) {
|
|
8559
|
+
resources.spans = await args.trace?.getSpans({ spanType: ["tool"] });
|
|
8560
|
+
}
|
|
8561
|
+
const results = await Promise.all(
|
|
8562
|
+
assertions.map(async (assertion) => {
|
|
8563
|
+
const result = await assertion.evaluate(resources);
|
|
8564
|
+
return {
|
|
8565
|
+
name: assertion.name,
|
|
8566
|
+
passed: result.passed,
|
|
8567
|
+
failure: result.failure
|
|
8568
|
+
};
|
|
8569
|
+
})
|
|
8570
|
+
);
|
|
8571
|
+
const passed = results.filter((result) => result.passed).length;
|
|
8572
|
+
const total = results.length;
|
|
8573
|
+
const failed = results.filter((result) => !result.passed).map(
|
|
8574
|
+
(result) => `${result.name}: ${result.failure ?? "assertion did not pass"}`
|
|
8575
|
+
);
|
|
8576
|
+
return {
|
|
8577
|
+
name: options.name ?? "assertions",
|
|
8578
|
+
score: total === 0 ? 1 : passed / total,
|
|
8579
|
+
metadata: {
|
|
8580
|
+
assertions: results.map(({ name, passed: passed2 }) => ({ name, passed: passed2 })),
|
|
8581
|
+
failed
|
|
8582
|
+
}
|
|
8583
|
+
};
|
|
8584
|
+
};
|
|
8585
|
+
}
|
|
8586
|
+
var agentAssertionHelpers = {
|
|
8587
|
+
equals: (actual, expected, name = "equals") => ({
|
|
8588
|
+
name,
|
|
8589
|
+
evaluate: () => {
|
|
8590
|
+
const passed = deepEqual(actual, expected);
|
|
8591
|
+
return {
|
|
8592
|
+
passed,
|
|
8593
|
+
failure: passed ? void 0 : `expected ${formatValue(actual)} to equal ${formatValue(expected)}`
|
|
8594
|
+
};
|
|
8595
|
+
}
|
|
8596
|
+
}),
|
|
8597
|
+
notEquals: (actual, expected, name = "not equals") => ({
|
|
8598
|
+
name,
|
|
8599
|
+
evaluate: () => {
|
|
8600
|
+
const passed = !deepEqual(actual, expected);
|
|
8601
|
+
return {
|
|
8602
|
+
passed,
|
|
8603
|
+
failure: passed ? void 0 : `expected ${formatValue(actual)} not to equal ${formatValue(expected)}`
|
|
8604
|
+
};
|
|
8605
|
+
}
|
|
8606
|
+
}),
|
|
8607
|
+
contains: (value, expected, name = "contains") => ({
|
|
8608
|
+
name,
|
|
8609
|
+
evaluate: () => {
|
|
8610
|
+
const searchedValue = typeof value === "string" ? value : formatValue(value);
|
|
8611
|
+
const passed = expected instanceof RegExp ? testRegex(expected, value) : searchedValue.includes(expected);
|
|
8612
|
+
return {
|
|
8613
|
+
passed,
|
|
8614
|
+
failure: passed ? void 0 : `expected ${formatValue(value)} to contain ${formatValue(expected)}`
|
|
8615
|
+
};
|
|
8616
|
+
}
|
|
8617
|
+
}),
|
|
8618
|
+
matches: (value, schema, name = "matches schema") => ({
|
|
8619
|
+
name,
|
|
8620
|
+
evaluate: async () => {
|
|
8621
|
+
const result = await validateSchema(schema, value);
|
|
8622
|
+
return {
|
|
8623
|
+
passed: result.passed,
|
|
8624
|
+
failure: result.passed ? void 0 : `expected value to match schema: ${result.message}`
|
|
8625
|
+
};
|
|
8626
|
+
}
|
|
8627
|
+
}),
|
|
8628
|
+
calledTool: (toolName, options = {}, name = `called tool ${toolName}`) => ({
|
|
8629
|
+
name,
|
|
8630
|
+
requiresTrace: true,
|
|
8631
|
+
evaluate: ({ spans }) => {
|
|
8632
|
+
const calls = matchingToolCalls(spans ?? [], toolName, options);
|
|
8633
|
+
const passed = options.times === void 0 ? calls.length > 0 : calls.length === options.times;
|
|
8634
|
+
return {
|
|
8635
|
+
passed,
|
|
8636
|
+
failure: passed ? void 0 : options.times === void 0 ? `expected tool "${toolName}" to be called; found ${calls.length} matching call${calls.length === 1 ? "" : "s"}` : `expected tool "${toolName}" to be called ${options.times} time${options.times === 1 ? "" : "s"}; found ${calls.length} matching call${calls.length === 1 ? "" : "s"}`
|
|
8637
|
+
};
|
|
8638
|
+
}
|
|
8639
|
+
}),
|
|
8640
|
+
notCalledTool: (toolName, name = `did not call tool ${toolName}`) => ({
|
|
8641
|
+
name,
|
|
8642
|
+
requiresTrace: true,
|
|
8643
|
+
evaluate: ({ spans }) => {
|
|
8644
|
+
const calls = toolCalls(spans ?? []).filter(
|
|
8645
|
+
(span) => getToolName(span) === toolName
|
|
8646
|
+
);
|
|
8647
|
+
const passed = calls.length === 0;
|
|
8648
|
+
return {
|
|
8649
|
+
passed,
|
|
8650
|
+
failure: passed ? void 0 : `expected tool "${toolName}" not to be called; found ${calls.length} call${calls.length === 1 ? "" : "s"}`
|
|
8651
|
+
};
|
|
8652
|
+
}
|
|
8653
|
+
}),
|
|
8654
|
+
toolOrder: (toolNames, name = "tool order") => ({
|
|
8655
|
+
name,
|
|
8656
|
+
requiresTrace: true,
|
|
8657
|
+
evaluate: ({ spans }) => {
|
|
8658
|
+
const observed = toolCalls(spans ?? []).map(getToolName).filter((toolName) => toolName !== void 0);
|
|
8659
|
+
let fromIndex = 0;
|
|
8660
|
+
const passed = toolNames.every((toolName) => {
|
|
8661
|
+
const index = observed.indexOf(toolName, fromIndex);
|
|
8662
|
+
if (index === -1) return false;
|
|
8663
|
+
fromIndex = index + 1;
|
|
8664
|
+
return true;
|
|
8665
|
+
});
|
|
8666
|
+
return {
|
|
8667
|
+
passed,
|
|
8668
|
+
failure: passed ? void 0 : `expected tool order ${toolNames.join(" -> ")}; observed ${observed.join(" -> ") || "no tools"}`
|
|
8669
|
+
};
|
|
8670
|
+
}
|
|
8671
|
+
}),
|
|
8672
|
+
usedNoTools: (name = "used no tools") => ({
|
|
8673
|
+
name,
|
|
8674
|
+
requiresTrace: true,
|
|
8675
|
+
evaluate: ({ spans }) => {
|
|
8676
|
+
const calls = toolCalls(spans ?? []);
|
|
8677
|
+
const passed = calls.length === 0;
|
|
8678
|
+
return {
|
|
8679
|
+
passed,
|
|
8680
|
+
failure: passed ? void 0 : `expected no tool calls; found ${calls.length}`
|
|
8681
|
+
};
|
|
8682
|
+
}
|
|
8683
|
+
}),
|
|
8684
|
+
maxToolCalls: (max, name = `at most ${max} tool calls`) => ({
|
|
8685
|
+
name,
|
|
8686
|
+
requiresTrace: true,
|
|
8687
|
+
evaluate: ({ spans }) => {
|
|
8688
|
+
const calls = toolCalls(spans ?? []);
|
|
8689
|
+
const passed = calls.length <= max;
|
|
8690
|
+
return {
|
|
8691
|
+
passed,
|
|
8692
|
+
failure: passed ? void 0 : `expected at most ${max} tool call${max === 1 ? "" : "s"}; found ${calls.length}`
|
|
8693
|
+
};
|
|
8694
|
+
}
|
|
8695
|
+
})
|
|
8696
|
+
};
|
|
8697
|
+
async function validateSchema(schema, value) {
|
|
8698
|
+
try {
|
|
8699
|
+
if ("safeParse" in schema) {
|
|
8700
|
+
const result2 = schema.safeParse(value);
|
|
8701
|
+
return result2.success ? { passed: true, message: "" } : { passed: false, message: formatSchemaError(result2.error) };
|
|
8702
|
+
}
|
|
8703
|
+
if ("parse" in schema) {
|
|
8704
|
+
schema.parse(value);
|
|
8705
|
+
return { passed: true, message: "" };
|
|
8706
|
+
}
|
|
8707
|
+
const result = await schema["~standard"].validate(value);
|
|
8708
|
+
if (typeof result === "object" && result !== null && "issues" in result && Array.isArray(result.issues) && result.issues.length > 0) {
|
|
8709
|
+
return { passed: false, message: formatValue(result.issues) };
|
|
8710
|
+
}
|
|
8711
|
+
return { passed: true, message: "" };
|
|
8712
|
+
} catch (e) {
|
|
8713
|
+
return { passed: false, message: formatSchemaError(e) };
|
|
8714
|
+
}
|
|
8715
|
+
}
|
|
8716
|
+
function toolCalls(spans) {
|
|
8717
|
+
return spans.filter((span) => span.span_attributes?.type === "tool");
|
|
8718
|
+
}
|
|
8719
|
+
function matchingToolCalls(spans, toolName, options) {
|
|
8720
|
+
return toolCalls(spans).filter((span) => {
|
|
8721
|
+
if (getToolName(span) !== toolName) return false;
|
|
8722
|
+
if (Object.prototype.hasOwnProperty.call(options, "input") && !matchesValue(span.input, options.input)) {
|
|
8723
|
+
return false;
|
|
8724
|
+
}
|
|
8725
|
+
if (Object.prototype.hasOwnProperty.call(options, "output") && !matchesValue(span.output, options.output)) {
|
|
8726
|
+
return false;
|
|
8727
|
+
}
|
|
8728
|
+
if (options.isError !== void 0 && Boolean(span.error) !== options.isError) {
|
|
8729
|
+
return false;
|
|
8730
|
+
}
|
|
8731
|
+
return true;
|
|
8732
|
+
});
|
|
8733
|
+
}
|
|
8734
|
+
function getToolName(span) {
|
|
8735
|
+
const spanName = [span.span_attributes?.name, span.name].map(normalizeToolName).find((value) => value !== void 0);
|
|
8736
|
+
if (spanName?.includes("/")) {
|
|
8737
|
+
return spanName;
|
|
8738
|
+
}
|
|
8739
|
+
const metadataName = [
|
|
8740
|
+
span.metadata?.tool_name,
|
|
8741
|
+
span.metadata?.["gen_ai.tool.name"]
|
|
8742
|
+
].map(normalizeToolName).find((value) => value !== void 0);
|
|
8743
|
+
const mcpServer = [
|
|
8744
|
+
span.metadata?.["mcp.server"],
|
|
8745
|
+
span.metadata?.["openai_codex.mcp.server"]
|
|
8746
|
+
].find((value) => typeof value === "string" && value !== "");
|
|
8747
|
+
if (metadataName && mcpServer) {
|
|
8748
|
+
return `${mcpServer}/${metadataName}`;
|
|
8749
|
+
}
|
|
8750
|
+
return metadataName ?? spanName;
|
|
8751
|
+
}
|
|
8752
|
+
function normalizeToolName(value) {
|
|
8753
|
+
if (typeof value !== "string" || value === "") {
|
|
8754
|
+
return void 0;
|
|
8755
|
+
}
|
|
8756
|
+
return value.startsWith("tool:") ? value.slice("tool:".length).trim() : value;
|
|
8757
|
+
}
|
|
8758
|
+
function matchesValue(actual, matcher) {
|
|
8759
|
+
if (matcher instanceof RegExp) {
|
|
8760
|
+
return testRegex(matcher, actual);
|
|
8761
|
+
}
|
|
8762
|
+
if (typeof matcher === "function") {
|
|
8763
|
+
return matcher(actual);
|
|
8764
|
+
}
|
|
8765
|
+
if (Array.isArray(matcher)) {
|
|
8766
|
+
return Array.isArray(actual) && actual.length === matcher.length && matcher.every((value, index) => matchesValue(actual[index], value));
|
|
8767
|
+
}
|
|
8768
|
+
if (isPlainObject(matcher) && isPlainObject(actual)) {
|
|
8769
|
+
return Object.entries(matcher).every(
|
|
8770
|
+
([key, value]) => Object.prototype.hasOwnProperty.call(actual, key) && matchesValue(actual[key], value)
|
|
8771
|
+
);
|
|
8772
|
+
}
|
|
8773
|
+
return deepEqual(actual, matcher);
|
|
8774
|
+
}
|
|
8775
|
+
function testRegex(matcher, value) {
|
|
8776
|
+
matcher.lastIndex = 0;
|
|
8777
|
+
return matcher.test(typeof value === "string" ? value : formatValue(value));
|
|
8778
|
+
}
|
|
8779
|
+
function deepEqual(left, right) {
|
|
8780
|
+
if (Object.is(left, right)) return true;
|
|
8781
|
+
if (Array.isArray(left) && Array.isArray(right)) {
|
|
8782
|
+
return left.length === right.length && left.every((item, index) => deepEqual(item, right[index]));
|
|
8783
|
+
}
|
|
8784
|
+
if (isPlainObject(left) && isPlainObject(right)) {
|
|
8785
|
+
const leftKeys = Object.keys(left);
|
|
8786
|
+
const rightKeys = Object.keys(right);
|
|
8787
|
+
return leftKeys.length === rightKeys.length && leftKeys.every(
|
|
8788
|
+
(key) => Object.prototype.hasOwnProperty.call(right, key) && deepEqual(left[key], right[key])
|
|
8789
|
+
);
|
|
8790
|
+
}
|
|
8791
|
+
return false;
|
|
8792
|
+
}
|
|
8793
|
+
function isPlainObject(value) {
|
|
8794
|
+
return typeof value === "object" && value !== null && !Array.isArray(value) && Object.getPrototypeOf(value) === Object.prototype;
|
|
8795
|
+
}
|
|
8796
|
+
function formatSchemaError(error) {
|
|
8797
|
+
if (error instanceof Error) {
|
|
8798
|
+
return error.message;
|
|
8799
|
+
}
|
|
8800
|
+
return formatValue(error);
|
|
8801
|
+
}
|
|
8802
|
+
function formatValue(value) {
|
|
8803
|
+
if (value instanceof RegExp) {
|
|
8804
|
+
return value.toString();
|
|
8805
|
+
}
|
|
8806
|
+
if (typeof value === "string") {
|
|
8807
|
+
return JSON.stringify(value);
|
|
8808
|
+
}
|
|
8809
|
+
try {
|
|
8810
|
+
const serialized = JSON.stringify(value);
|
|
8811
|
+
if (serialized !== void 0 && !hasUndefinedJsonValue(value)) {
|
|
8812
|
+
return serialized;
|
|
8813
|
+
}
|
|
8814
|
+
return formatValueWithUndefined(value, /* @__PURE__ */ new Set());
|
|
8815
|
+
} catch {
|
|
8816
|
+
return String(value);
|
|
8817
|
+
}
|
|
8818
|
+
}
|
|
8819
|
+
function hasUndefinedJsonValue(value, seen = /* @__PURE__ */ new Set()) {
|
|
8820
|
+
if (value === void 0) {
|
|
8821
|
+
return true;
|
|
8822
|
+
}
|
|
8823
|
+
if (typeof value !== "object" || value === null) {
|
|
8824
|
+
return false;
|
|
8825
|
+
}
|
|
8826
|
+
if (seen.has(value)) {
|
|
8827
|
+
return false;
|
|
8828
|
+
}
|
|
8829
|
+
seen.add(value);
|
|
8830
|
+
if (Array.isArray(value)) {
|
|
8831
|
+
return Array.from({ length: value.length }).some(
|
|
8832
|
+
(_, index) => hasUndefinedJsonValue(value[index], seen)
|
|
8833
|
+
);
|
|
8834
|
+
}
|
|
8835
|
+
return Object.keys(value).some(
|
|
8836
|
+
(key) => hasUndefinedJsonValue(value[key], seen)
|
|
8837
|
+
);
|
|
8838
|
+
}
|
|
8839
|
+
function formatValueWithUndefined(value, seen) {
|
|
8840
|
+
if (value === void 0) {
|
|
8841
|
+
return "undefined";
|
|
8842
|
+
}
|
|
8843
|
+
if (value === null) {
|
|
8844
|
+
return "null";
|
|
8845
|
+
}
|
|
8846
|
+
if (typeof value === "string") {
|
|
8847
|
+
return JSON.stringify(value);
|
|
8848
|
+
}
|
|
8849
|
+
if (typeof value === "number" || typeof value === "boolean") {
|
|
8850
|
+
return JSON.stringify(value);
|
|
8851
|
+
}
|
|
8852
|
+
if (typeof value === "bigint" || typeof value === "function") {
|
|
8853
|
+
return String(value);
|
|
8854
|
+
}
|
|
8855
|
+
if (typeof value === "symbol") {
|
|
8856
|
+
return String(value);
|
|
8857
|
+
}
|
|
8858
|
+
if (value instanceof RegExp) {
|
|
8859
|
+
return value.toString();
|
|
8860
|
+
}
|
|
8861
|
+
if (typeof value !== "object") {
|
|
8862
|
+
return String(value);
|
|
8863
|
+
}
|
|
8864
|
+
if (seen.has(value)) {
|
|
8865
|
+
return '"[Circular]"';
|
|
8866
|
+
}
|
|
8867
|
+
seen.add(value);
|
|
8868
|
+
if (typeof value.toJSON === "function") {
|
|
8869
|
+
const jsonValue = value.toJSON();
|
|
8870
|
+
seen.delete(value);
|
|
8871
|
+
return formatValueWithUndefined(jsonValue, seen);
|
|
8872
|
+
}
|
|
8873
|
+
if (Array.isArray(value)) {
|
|
8874
|
+
const formatted2 = Array.from(
|
|
8875
|
+
{ length: value.length },
|
|
8876
|
+
(_, index) => formatValueWithUndefined(value[index], seen)
|
|
8877
|
+
);
|
|
8878
|
+
seen.delete(value);
|
|
8879
|
+
return `[${formatted2.join(",")}]`;
|
|
8880
|
+
}
|
|
8881
|
+
const formatted = Object.keys(value).map(
|
|
8882
|
+
(key) => `${JSON.stringify(key)}:${formatValueWithUndefined(
|
|
8883
|
+
value[key],
|
|
8884
|
+
seen
|
|
8885
|
+
)}`
|
|
8886
|
+
);
|
|
8887
|
+
seen.delete(value);
|
|
8888
|
+
return `{${formatted.join(",")}}`;
|
|
8889
|
+
}
|
|
8890
|
+
|
|
7033
8891
|
// src/dataset-pipeline.ts
|
|
7034
8892
|
function DatasetPipeline(definition) {
|
|
7035
8893
|
if (!globalThis.__braintrust_dataset_pipelines) {
|
|
@@ -7606,8 +9464,10 @@ configureNode();
|
|
|
7606
9464
|
export {
|
|
7607
9465
|
Attachment,
|
|
7608
9466
|
AttachmentReference,
|
|
9467
|
+
BAGGAGE_HEADER,
|
|
7609
9468
|
BRAINTRUST_CURRENT_SPAN_STORE,
|
|
7610
9469
|
BRAINTRUST_LANGCHAIN_CALLBACK_HANDLER_NAME,
|
|
9470
|
+
BRAINTRUST_PARENT_KEY,
|
|
7611
9471
|
BaseAttachment,
|
|
7612
9472
|
BaseExperiment,
|
|
7613
9473
|
BraintrustLangChainCallbackHandler,
|
|
@@ -7640,6 +9500,7 @@ export {
|
|
|
7640
9500
|
NOOP_SPAN,
|
|
7641
9501
|
NOOP_SPAN_PERMALINK,
|
|
7642
9502
|
NoopSpan,
|
|
9503
|
+
OTELIDGenerator,
|
|
7643
9504
|
ObjectFetcher,
|
|
7644
9505
|
Project2 as Project,
|
|
7645
9506
|
ProjectNameIdMap,
|
|
@@ -7651,6 +9512,8 @@ export {
|
|
|
7651
9512
|
ScorerBuilder,
|
|
7652
9513
|
SpanFetcher,
|
|
7653
9514
|
SpanImpl,
|
|
9515
|
+
TRACEPARENT_HEADER,
|
|
9516
|
+
TRACESTATE_HEADER,
|
|
7654
9517
|
TestBackgroundLogger,
|
|
7655
9518
|
ToolBuilder,
|
|
7656
9519
|
UUIDGenerator,
|
|
@@ -7660,7 +9523,10 @@ export {
|
|
|
7660
9523
|
isomorph_default as _internalIso,
|
|
7661
9524
|
_internalSetInitialState,
|
|
7662
9525
|
addAzureBlobHeaders,
|
|
9526
|
+
agentAssertionScorer,
|
|
7663
9527
|
braintrustAISDKTelemetry,
|
|
9528
|
+
braintrustEveHook,
|
|
9529
|
+
braintrustEveInstrumentation,
|
|
7664
9530
|
braintrustFlueInstrumentation,
|
|
7665
9531
|
braintrustFlueObserver,
|
|
7666
9532
|
braintrustStreamChunkSchema,
|
|
@@ -7678,6 +9544,7 @@ export {
|
|
|
7678
9544
|
devNullWritableStream,
|
|
7679
9545
|
evaluatorDefinitionSchema,
|
|
7680
9546
|
evaluatorDefinitionsSchema,
|
|
9547
|
+
extractTraceContextFromHeaders,
|
|
7681
9548
|
flush,
|
|
7682
9549
|
getContextManager,
|
|
7683
9550
|
getIdGenerator,
|
|
@@ -7691,6 +9558,7 @@ export {
|
|
|
7691
9558
|
initFunction,
|
|
7692
9559
|
initLogger,
|
|
7693
9560
|
initNodeTestSuite,
|
|
9561
|
+
injectTraceContext,
|
|
7694
9562
|
invoke,
|
|
7695
9563
|
isTemplateFormat,
|
|
7696
9564
|
loadParameters,
|