braintrust 3.19.0 → 3.21.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. package/NOTICE +20 -0
  2. package/README.md +2 -0
  3. package/dev/dist/index.d.mts +309 -199
  4. package/dev/dist/index.d.ts +309 -199
  5. package/dev/dist/index.js +2833 -987
  6. package/dev/dist/index.mjs +2294 -448
  7. package/dist/apply-auto-instrumentation.js +781 -199
  8. package/dist/apply-auto-instrumentation.mjs +588 -6
  9. package/dist/auto-instrumentations/bundler/esbuild.cjs +594 -2
  10. package/dist/auto-instrumentations/bundler/esbuild.d.mts +2 -2
  11. package/dist/auto-instrumentations/bundler/esbuild.d.ts +2 -2
  12. package/dist/auto-instrumentations/bundler/esbuild.mjs +3 -3
  13. package/dist/auto-instrumentations/bundler/next.cjs +594 -2
  14. package/dist/auto-instrumentations/bundler/next.mjs +4 -4
  15. package/dist/auto-instrumentations/bundler/rollup.cjs +594 -2
  16. package/dist/auto-instrumentations/bundler/rollup.d.mts +2 -2
  17. package/dist/auto-instrumentations/bundler/rollup.d.ts +2 -2
  18. package/dist/auto-instrumentations/bundler/rollup.mjs +3 -3
  19. package/dist/auto-instrumentations/bundler/vite.cjs +594 -2
  20. package/dist/auto-instrumentations/bundler/vite.d.mts +2 -2
  21. package/dist/auto-instrumentations/bundler/vite.d.ts +2 -2
  22. package/dist/auto-instrumentations/bundler/vite.mjs +3 -3
  23. package/dist/auto-instrumentations/bundler/webpack-loader.cjs +593 -9
  24. package/dist/auto-instrumentations/bundler/webpack-loader.d.ts +68 -1
  25. package/dist/auto-instrumentations/bundler/webpack.cjs +594 -2
  26. package/dist/auto-instrumentations/bundler/webpack.d.mts +2 -2
  27. package/dist/auto-instrumentations/bundler/webpack.d.ts +2 -2
  28. package/dist/auto-instrumentations/bundler/webpack.mjs +4 -4
  29. package/dist/auto-instrumentations/{chunk-W4E36GIW.mjs → chunk-BRQX23KL.mjs} +16 -0
  30. package/dist/auto-instrumentations/chunk-M6DLIJ2Z.mjs +784 -0
  31. package/dist/auto-instrumentations/{chunk-K74TZGGM.mjs → chunk-T6J4C7LX.mjs} +1 -1
  32. package/dist/auto-instrumentations/{chunk-S4374IP6.mjs → chunk-TKRPRPGD.mjs} +4 -6
  33. package/dist/auto-instrumentations/chunk-VT6DDNKM.mjs +856 -0
  34. package/dist/auto-instrumentations/hook.mjs +749 -45
  35. package/dist/auto-instrumentations/index.cjs +16 -0
  36. package/dist/auto-instrumentations/index.d.mts +2 -3
  37. package/dist/auto-instrumentations/index.d.ts +2 -3
  38. package/dist/auto-instrumentations/index.mjs +1 -1
  39. package/dist/auto-instrumentations/loader/cjs-patch.cjs +578 -4
  40. package/dist/auto-instrumentations/loader/cjs-patch.d.mts +1 -1
  41. package/dist/auto-instrumentations/loader/cjs-patch.d.ts +1 -1
  42. package/dist/auto-instrumentations/loader/cjs-patch.mjs +3 -7
  43. package/dist/auto-instrumentations/loader/esm-hook.mjs +2 -6
  44. package/dist/auto-instrumentations/{plugin-D7nDswtC.d.mts → plugin-D0KHwSJv.d.mts} +1 -1
  45. package/dist/auto-instrumentations/{plugin-D7nDswtC.d.ts → plugin-G6fgxk1b.d.ts} +1 -1
  46. package/dist/auto-instrumentations/types-RNPaKi9o.d.mts +70 -0
  47. package/dist/auto-instrumentations/types-RNPaKi9o.d.ts +70 -0
  48. package/dist/browser.d.mts +1369 -848
  49. package/dist/browser.d.ts +1369 -848
  50. package/dist/browser.js +2735 -473
  51. package/dist/browser.mjs +2735 -473
  52. package/dist/{chunk-LPC4W2WX.js → chunk-BFGIH2ZJ.js} +4 -0
  53. package/dist/{chunk-37RLJF2U.js → chunk-FY7DAKA5.js} +3149 -1273
  54. package/dist/{chunk-Y7W7WP6H.mjs → chunk-KMGUTPB7.mjs} +4 -0
  55. package/dist/{chunk-VEQ2PCMC.mjs → chunk-O2P765XK.mjs} +2399 -523
  56. package/dist/cli.js +2267 -412
  57. package/dist/edge-light.d.mts +1 -1
  58. package/dist/edge-light.d.ts +1 -1
  59. package/dist/edge-light.js +2735 -473
  60. package/dist/edge-light.mjs +2735 -473
  61. package/dist/index.d.mts +1848 -1327
  62. package/dist/index.d.ts +1848 -1327
  63. package/dist/index.js +754 -352
  64. package/dist/index.mjs +448 -46
  65. package/dist/instrumentation/index.d.mts +119 -21
  66. package/dist/instrumentation/index.d.ts +119 -21
  67. package/dist/instrumentation/index.js +2345 -512
  68. package/dist/instrumentation/index.mjs +2344 -512
  69. package/dist/vitest-evals-reporter.js +16 -16
  70. package/dist/vitest-evals-reporter.mjs +2 -2
  71. package/dist/workerd.d.mts +1 -1
  72. package/dist/workerd.d.ts +1 -1
  73. package/dist/workerd.js +2735 -473
  74. package/dist/workerd.mjs +2735 -473
  75. package/licenses/import-in-the-middle/LICENSE +201 -0
  76. package/licenses/import-in-the-middle/LICENSE-3rdparty.csv +4 -0
  77. package/licenses/import-in-the-middle/NOTICE +34 -0
  78. package/licenses/orchestrion-js/LICENSE +202 -0
  79. package/licenses/orchestrion-js/LICENSE-3rdparty.csv +8 -0
  80. package/licenses/orchestrion-js/NOTICE +3 -0
  81. package/licenses/require-in-the-middle/LICENSE +23 -0
  82. package/package.json +13 -4
  83. package/util/dist/index.d.mts +1 -0
  84. package/util/dist/index.d.ts +1 -0
  85. package/util/dist/index.js +4 -4
  86. package/util/dist/index.mjs +2 -2
  87. package/dist/auto-instrumentations/chunk-J57YF7WS.mjs +0 -208
  88. package/dist/auto-instrumentations/chunk-QFMACSOL.mjs +0 -280
package/dist/index.mjs CHANGED
@@ -1,8 +1,10 @@
1
1
  import {
2
2
  Attachment,
3
3
  AttachmentReference,
4
+ BAGGAGE_HEADER,
4
5
  BRAINTRUST_CURRENT_SPAN_STORE,
5
6
  BRAINTRUST_LANGCHAIN_CALLBACK_HANDLER_NAME,
7
+ BRAINTRUST_PARENT_KEY,
6
8
  BaseAttachment,
7
9
  BraintrustLangChainCallbackHandler,
8
10
  BraintrustObservabilityExporter,
@@ -32,6 +34,7 @@ import {
32
34
  NOOP_SPAN,
33
35
  NOOP_SPAN_PERMALINK,
34
36
  NoopSpan,
37
+ OTELIDGenerator,
35
38
  ObjectFetcher,
36
39
  ObjectReference,
37
40
  Project,
@@ -41,8 +44,10 @@ import {
41
44
  ReadonlyExperiment,
42
45
  RemoteEvalParameters,
43
46
  RunEval,
44
- SpanComponentsV3,
47
+ SpanComponentsV4,
45
48
  SpanImpl,
49
+ TRACEPARENT_HEADER,
50
+ TRACESTATE_HEADER,
46
51
  TestBackgroundLogger,
47
52
  ToolFunctionDefinition,
48
53
  UUIDGenerator,
@@ -52,6 +57,7 @@ import {
52
57
  _internalSetInitialState,
53
58
  addAzureBlobHeaders,
54
59
  braintrustAISDKTelemetry,
60
+ braintrustFlueInstrumentation,
55
61
  braintrustFlueObserver,
56
62
  braintrustStreamChunkSchema,
57
63
  buildBedrockRuntimeSpanInfo,
@@ -62,11 +68,13 @@ import {
62
68
  currentExperiment,
63
69
  currentLogger,
64
70
  currentSpan,
71
+ currentWorkflowAgentWrapperSpan,
65
72
  debugLogger,
66
73
  deepCopyEvent,
67
74
  deserializePlainStringAsJSON,
68
75
  devNullWritableStream,
69
76
  extractAnthropicCacheTokens,
77
+ extractTraceContextFromHeaders,
70
78
  finalizeAnthropicTokens,
71
79
  flush,
72
80
  getBedrockRuntimeOperation,
@@ -80,6 +88,7 @@ import {
80
88
  initDataset,
81
89
  initExperiment,
82
90
  initLogger,
91
+ injectTraceContext,
83
92
  isEmpty,
84
93
  isObject,
85
94
  isTemplateFormat,
@@ -128,7 +137,7 @@ import {
128
137
  wrapMastraAgent,
129
138
  wrapTraced,
130
139
  zodToJsonSchema
131
- } from "./chunk-VEQ2PCMC.mjs";
140
+ } from "./chunk-O2P765XK.mjs";
132
141
  import {
133
142
  __export,
134
143
  aiSDKChannels,
@@ -151,15 +160,17 @@ import {
151
160
  openRouterChannels,
152
161
  piCodingAgentChannels,
153
162
  strandsAgentSDKChannels
154
- } from "./chunk-Y7W7WP6H.mjs";
163
+ } from "./chunk-KMGUTPB7.mjs";
155
164
 
156
165
  // src/exports.ts
157
166
  var exports_exports = {};
158
167
  __export(exports_exports, {
159
168
  Attachment: () => Attachment,
160
169
  AttachmentReference: () => AttachmentReference,
170
+ BAGGAGE_HEADER: () => BAGGAGE_HEADER,
161
171
  BRAINTRUST_CURRENT_SPAN_STORE: () => BRAINTRUST_CURRENT_SPAN_STORE,
162
172
  BRAINTRUST_LANGCHAIN_CALLBACK_HANDLER_NAME: () => BRAINTRUST_LANGCHAIN_CALLBACK_HANDLER_NAME,
173
+ BRAINTRUST_PARENT_KEY: () => BRAINTRUST_PARENT_KEY,
163
174
  BaseAttachment: () => BaseAttachment,
164
175
  BaseExperiment: () => BaseExperiment,
165
176
  BraintrustLangChainCallbackHandler: () => BraintrustLangChainCallbackHandler,
@@ -192,6 +203,7 @@ __export(exports_exports, {
192
203
  NOOP_SPAN: () => NOOP_SPAN,
193
204
  NOOP_SPAN_PERMALINK: () => NOOP_SPAN_PERMALINK,
194
205
  NoopSpan: () => NoopSpan,
206
+ OTELIDGenerator: () => OTELIDGenerator,
195
207
  ObjectFetcher: () => ObjectFetcher,
196
208
  Project: () => Project2,
197
209
  ProjectNameIdMap: () => ProjectNameIdMap,
@@ -203,6 +215,8 @@ __export(exports_exports, {
203
215
  ScorerBuilder: () => ScorerBuilder,
204
216
  SpanFetcher: () => SpanFetcher,
205
217
  SpanImpl: () => SpanImpl,
218
+ TRACEPARENT_HEADER: () => TRACEPARENT_HEADER,
219
+ TRACESTATE_HEADER: () => TRACESTATE_HEADER,
206
220
  TestBackgroundLogger: () => TestBackgroundLogger,
207
221
  ToolBuilder: () => ToolBuilder,
208
222
  UUIDGenerator: () => UUIDGenerator,
@@ -212,7 +226,9 @@ __export(exports_exports, {
212
226
  _internalIso: () => isomorph_default,
213
227
  _internalSetInitialState: () => _internalSetInitialState,
214
228
  addAzureBlobHeaders: () => addAzureBlobHeaders,
229
+ agentAssertionScorer: () => agentAssertionScorer,
215
230
  braintrustAISDKTelemetry: () => braintrustAISDKTelemetry,
231
+ braintrustFlueInstrumentation: () => braintrustFlueInstrumentation,
216
232
  braintrustFlueObserver: () => braintrustFlueObserver,
217
233
  braintrustStreamChunkSchema: () => braintrustStreamChunkSchema,
218
234
  buildLocalSummary: () => buildLocalSummary,
@@ -228,6 +244,7 @@ __export(exports_exports, {
228
244
  devNullWritableStream: () => devNullWritableStream,
229
245
  evaluatorDefinitionSchema: () => evaluatorDefinitionSchema,
230
246
  evaluatorDefinitionsSchema: () => evaluatorDefinitionsSchema,
247
+ extractTraceContextFromHeaders: () => extractTraceContextFromHeaders,
231
248
  flush: () => flush,
232
249
  getContextManager: () => getContextManager,
233
250
  getIdGenerator: () => getIdGenerator,
@@ -241,6 +258,7 @@ __export(exports_exports, {
241
258
  initFunction: () => initFunction,
242
259
  initLogger: () => initLogger,
243
260
  initNodeTestSuite: () => initNodeTestSuite,
261
+ injectTraceContext: () => injectTraceContext,
244
262
  invoke: () => invoke,
245
263
  isTemplateFormat: () => isTemplateFormat,
246
264
  loadParameters: () => loadParameters,
@@ -872,6 +890,7 @@ function wrapAISDK(aiSDK, options = {}) {
872
890
  case "Agent":
873
891
  case "Experimental_Agent":
874
892
  case "ToolLoopAgent":
893
+ case "WorkflowAgent":
875
894
  return original ? wrapAgentClass(original, options) : original;
876
895
  }
877
896
  return original;
@@ -890,10 +909,10 @@ var wrapAgentClass = (AgentClass, options = {}) => {
890
909
  return new Proxy(instance, {
891
910
  get(instanceTarget, prop, instanceReceiver) {
892
911
  const original = Reflect.get(instanceTarget, prop, instanceTarget);
893
- if (prop === "generate") {
912
+ if (prop === "generate" && typeof original === "function" && instanceTarget.constructor.name !== "WorkflowAgent") {
894
913
  return wrapAgentGenerate(original, instanceTarget, options);
895
914
  }
896
- if (prop === "stream") {
915
+ if (prop === "stream" && typeof original === "function") {
897
916
  return wrapAgentStream(original, instanceTarget, options);
898
917
  }
899
918
  if (typeof original === "function") {
@@ -908,7 +927,7 @@ var wrapAgentClass = (AgentClass, options = {}) => {
908
927
  var wrapAgentGenerate = (generate, instance, options = {}) => {
909
928
  const defaultName = `${instance.constructor.name}.generate`;
910
929
  return async (params) => makeGenerateTextWrapper(
911
- aiSDKChannels.generateText,
930
+ generateChannelForAgent(instance.constructor.name),
912
931
  defaultName,
913
932
  generate.bind(instance),
914
933
  {
@@ -918,19 +937,42 @@ var wrapAgentGenerate = (generate, instance, options = {}) => {
918
937
  options
919
938
  )({ ...instance.settings, ...params });
920
939
  };
940
+ function generateChannelForAgent(agentName) {
941
+ if (agentName === "ToolLoopAgent") {
942
+ return aiSDKChannels.toolLoopAgentGenerate;
943
+ }
944
+ return aiSDKChannels.agentGenerate;
945
+ }
921
946
  var wrapAgentStream = (stream, instance, options = {}) => {
922
947
  const defaultName = `${instance.constructor.name}.stream`;
923
- return (params) => makeStreamWrapper(
924
- aiSDKChannels.agentStream,
925
- defaultName,
926
- stream.bind(instance),
927
- {
928
- self: instance,
929
- spanType: "function" /* FUNCTION */
930
- },
931
- options
932
- )({ ...instance.settings, ...params });
948
+ return (params) => {
949
+ const workflowAgent = instance.constructor.name === "WorkflowAgent";
950
+ if (workflowAgent && currentWorkflowAgentWrapperSpan()) {
951
+ const { span_info: _spanInfo, ...cleanParams } = params;
952
+ return stream.call(instance, { ...instance.settings, ...cleanParams });
953
+ }
954
+ const trace = () => makeStreamWrapper(
955
+ streamChannelForAgent(instance.constructor.name),
956
+ defaultName,
957
+ stream.bind(instance),
958
+ {
959
+ self: instance,
960
+ spanType: "function" /* FUNCTION */
961
+ },
962
+ options
963
+ )({ ...instance.settings, ...params });
964
+ return trace();
965
+ };
933
966
  };
967
+ function streamChannelForAgent(agentName) {
968
+ if (agentName === "ToolLoopAgent") {
969
+ return aiSDKChannels.toolLoopAgentStream;
970
+ }
971
+ if (agentName === "WorkflowAgent") {
972
+ return aiSDKChannels.workflowAgentStream;
973
+ }
974
+ return aiSDKChannels.agentStream;
975
+ }
934
976
  var makeGenerateTextWrapper = (channel, name, generateText, contextOptions = {}, options = {}) => {
935
977
  const wrapper = async function(allParams) {
936
978
  const { span_info, ...params } = allParams;
@@ -1192,7 +1234,7 @@ var BraintrustLanguageModelWrapper = class {
1192
1234
  let time_to_first_token = void 0;
1193
1235
  let usage = void 0;
1194
1236
  let fullText = void 0;
1195
- const toolCalls = {};
1237
+ const toolCalls2 = {};
1196
1238
  let finishReason = void 0;
1197
1239
  return {
1198
1240
  ...ret,
@@ -1211,7 +1253,7 @@ var BraintrustLanguageModelWrapper = class {
1211
1253
  fullText += chunk.textDelta;
1212
1254
  break;
1213
1255
  case "tool-call":
1214
- toolCalls[chunk.toolCallId] = {
1256
+ toolCalls2[chunk.toolCallId] = {
1215
1257
  toolCallType: chunk.toolCallType,
1216
1258
  toolCallId: chunk.toolCallId,
1217
1259
  toolName: chunk.toolName,
@@ -1219,15 +1261,15 @@ var BraintrustLanguageModelWrapper = class {
1219
1261
  };
1220
1262
  break;
1221
1263
  case "tool-call-delta":
1222
- if (toolCalls[chunk.toolCallId] === void 0) {
1223
- toolCalls[chunk.toolCallId] = {
1264
+ if (toolCalls2[chunk.toolCallId] === void 0) {
1265
+ toolCalls2[chunk.toolCallId] = {
1224
1266
  toolCallType: chunk.toolCallType,
1225
1267
  toolCallId: chunk.toolCallId,
1226
1268
  toolName: chunk.toolName,
1227
1269
  args: ""
1228
1270
  };
1229
1271
  }
1230
- toolCalls[chunk.toolCallId].args += chunk.argsTextDelta;
1272
+ toolCalls2[chunk.toolCallId].args += chunk.argsTextDelta;
1231
1273
  break;
1232
1274
  case "finish":
1233
1275
  usage = chunk.usage;
@@ -1240,7 +1282,7 @@ var BraintrustLanguageModelWrapper = class {
1240
1282
  span.log({
1241
1283
  output: postProcessOutput(
1242
1284
  fullText,
1243
- Object.keys(toolCalls).length > 0 ? Object.values(toolCalls) : void 0,
1285
+ Object.keys(toolCalls2).length > 0 ? Object.values(toolCalls2) : void 0,
1244
1286
  finishReason
1245
1287
  ),
1246
1288
  metrics: {
@@ -1341,15 +1383,15 @@ function postProcessPrompt(prompt) {
1341
1383
  }
1342
1384
  });
1343
1385
  }
1344
- function postProcessOutput(text, toolCalls, finishReason) {
1386
+ function postProcessOutput(text, toolCalls2, finishReason) {
1345
1387
  return [
1346
1388
  {
1347
1389
  index: 0,
1348
1390
  message: {
1349
1391
  role: "assistant",
1350
1392
  content: text ?? "",
1351
- ...toolCalls && toolCalls.length > 0 ? {
1352
- tool_calls: toolCalls.map((toolCall) => ({
1393
+ ...toolCalls2 && toolCalls2.length > 0 ? {
1394
+ tool_calls: toolCalls2.map((toolCall) => ({
1353
1395
  id: toolCall.toolCallId,
1354
1396
  function: {
1355
1397
  name: toolCall.toolName,
@@ -1448,29 +1490,29 @@ function normalizeFinishReason(reason) {
1448
1490
  if (typeof reason !== "string") return void 0;
1449
1491
  return reason.replace(/-/g, "_");
1450
1492
  }
1451
- function buildAssistantOutputWithToolCalls(result, toolCalls) {
1493
+ function buildAssistantOutputWithToolCalls(result, toolCalls2) {
1452
1494
  return [
1453
1495
  {
1454
1496
  index: 0,
1455
1497
  logprobs: null,
1456
- finish_reason: normalizeFinishReason(result?.finishReason) ?? (toolCalls.length ? "tool_calls" : void 0),
1498
+ finish_reason: normalizeFinishReason(result?.finishReason) ?? (toolCalls2.length ? "tool_calls" : void 0),
1457
1499
  message: {
1458
1500
  role: "assistant",
1459
- tool_calls: toolCalls.length > 0 ? toolCalls : void 0
1501
+ tool_calls: toolCalls2.length > 0 ? toolCalls2 : void 0
1460
1502
  }
1461
1503
  }
1462
1504
  ];
1463
1505
  }
1464
1506
  function extractToolCallsFromSteps(steps) {
1465
- const toolCalls = [];
1466
- if (!Array.isArray(steps)) return toolCalls;
1507
+ const toolCalls2 = [];
1508
+ if (!Array.isArray(steps)) return toolCalls2;
1467
1509
  let idx = 0;
1468
1510
  for (const step of steps) {
1469
1511
  const blocks = step?.content;
1470
1512
  if (!Array.isArray(blocks)) continue;
1471
1513
  for (const block of blocks) {
1472
1514
  if (block && typeof block === "object" && block.type === "tool-call") {
1473
- toolCalls.push({
1515
+ toolCalls2.push({
1474
1516
  id: block.toolCallId,
1475
1517
  type: "function",
1476
1518
  index: idx++,
@@ -1482,7 +1524,7 @@ function extractToolCallsFromSteps(steps) {
1482
1524
  }
1483
1525
  }
1484
1526
  }
1485
- return toolCalls;
1527
+ return toolCalls2;
1486
1528
  }
1487
1529
  function extractToolCallsFromBlocks(blocks) {
1488
1530
  if (!Array.isArray(blocks)) return [];
@@ -1548,12 +1590,12 @@ function BraintrustMiddleware(config = {}) {
1548
1590
  metadata.model = modelId;
1549
1591
  }
1550
1592
  }
1551
- let toolCalls = extractToolCallsFromSteps(result?.steps);
1552
- if (!toolCalls || toolCalls.length === 0) {
1553
- toolCalls = extractToolCallsFromBlocks(result?.content);
1593
+ let toolCalls2 = extractToolCallsFromSteps(result?.steps);
1594
+ if (!toolCalls2 || toolCalls2.length === 0) {
1595
+ toolCalls2 = extractToolCallsFromBlocks(result?.content);
1554
1596
  }
1555
1597
  span.log({
1556
- output: toolCalls.length > 0 ? buildAssistantOutputWithToolCalls(result, toolCalls) : result?.content,
1598
+ output: toolCalls2.length > 0 ? buildAssistantOutputWithToolCalls(result, toolCalls2) : result?.content,
1557
1599
  metadata,
1558
1600
  metrics: normalizeUsageMetrics(
1559
1601
  result.usage,
@@ -1642,13 +1684,13 @@ function BraintrustMiddleware(config = {}) {
1642
1684
  metadata.model = model;
1643
1685
  }
1644
1686
  if (toolBlocks.length > 0) {
1645
- const toolCalls = extractToolCallsFromSteps([
1687
+ const toolCalls2 = extractToolCallsFromSteps([
1646
1688
  { content: toolBlocks }
1647
1689
  ]);
1648
- if (toolCalls.length > 0) {
1690
+ if (toolCalls2.length > 0) {
1649
1691
  output = buildAssistantOutputWithToolCalls(
1650
1692
  resultForDetection,
1651
- toolCalls
1693
+ toolCalls2
1652
1694
  );
1653
1695
  }
1654
1696
  }
@@ -5597,15 +5639,21 @@ var waterfall$1 = awaitify(waterfall);
5597
5639
 
5598
5640
  // src/trace.ts
5599
5641
  var SpanFetcher = class _SpanFetcher extends ObjectFetcher {
5600
- constructor(objectType, _objectId, rootSpanId, _state, spanTypeFilter, includeScorers = false) {
5642
+ constructor(objectType, _objectId, rootSpanId, _state, spanTypeFilter, includeScorers = false, brainstoreRealtime = true) {
5601
5643
  const filterExpr = _SpanFetcher.buildFilter(
5602
5644
  rootSpanId,
5603
5645
  spanTypeFilter,
5604
5646
  includeScorers
5605
5647
  );
5606
- super(objectType, void 0, void 0, {
5607
- filter: filterExpr
5608
- });
5648
+ super(
5649
+ objectType,
5650
+ void 0,
5651
+ void 0,
5652
+ {
5653
+ filter: filterExpr
5654
+ },
5655
+ brainstoreRealtime
5656
+ );
5609
5657
  this._objectId = _objectId;
5610
5658
  this.rootSpanId = rootSpanId;
5611
5659
  this._state = _state;
@@ -5659,7 +5707,7 @@ var CachedSpanFetcher = class {
5659
5707
  spanCache = /* @__PURE__ */ new Map();
5660
5708
  allFetched = false;
5661
5709
  fetchFn;
5662
- constructor(objectTypeOrFetchFn, objectId, rootSpanId, getState) {
5710
+ constructor(objectTypeOrFetchFn, objectId, rootSpanId, getState, brainstoreRealtime = true) {
5663
5711
  if (typeof objectTypeOrFetchFn === "function") {
5664
5712
  this.fetchFn = (spanType) => objectTypeOrFetchFn(spanType);
5665
5713
  } else {
@@ -5672,7 +5720,8 @@ var CachedSpanFetcher = class {
5672
5720
  rootSpanId,
5673
5721
  state,
5674
5722
  spanType,
5675
- includeScorers
5723
+ includeScorers,
5724
+ brainstoreRealtime
5676
5725
  );
5677
5726
  const rows = await fetcher.fetchedData();
5678
5727
  return rows.map((row) => ({
@@ -6502,7 +6551,7 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
6502
6551
  }
6503
6552
  };
6504
6553
  const parentStr = state.currentParent.getStore();
6505
- const parentComponents = parentStr ? SpanComponentsV3.fromStr(parentStr) : null;
6554
+ const parentComponents = typeof parentStr === "string" ? SpanComponentsV4.fromStr(parentStr) : null;
6506
6555
  const trace = state ? new LocalTrace({
6507
6556
  objectType: parentComponents ? spanObjectTypeV3ToTypedString(
6508
6557
  parentComponents.data.object_type
@@ -7028,6 +7077,350 @@ var defaultReporter = {
7028
7077
  }
7029
7078
  };
7030
7079
 
7080
+ // src/agent-assertions.ts
7081
+ function agentAssertionScorer(callback, options = {}) {
7082
+ return async (args) => {
7083
+ const { trace: _trace, ...callbackArgs } = args;
7084
+ const callbackMetadata = callbackArgs.metadata ?? {};
7085
+ const assertions = await callback({
7086
+ ...callbackArgs,
7087
+ metadata: callbackMetadata,
7088
+ assert: agentAssertionHelpers
7089
+ });
7090
+ const resources = {};
7091
+ if (assertions.some((assertion) => assertion.requiresTrace)) {
7092
+ resources.spans = await args.trace?.getSpans({ spanType: ["tool"] });
7093
+ }
7094
+ const results = await Promise.all(
7095
+ assertions.map(async (assertion) => {
7096
+ const result = await assertion.evaluate(resources);
7097
+ return {
7098
+ name: assertion.name,
7099
+ passed: result.passed,
7100
+ failure: result.failure
7101
+ };
7102
+ })
7103
+ );
7104
+ const passed = results.filter((result) => result.passed).length;
7105
+ const total = results.length;
7106
+ const failed = results.filter((result) => !result.passed).map(
7107
+ (result) => `${result.name}: ${result.failure ?? "assertion did not pass"}`
7108
+ );
7109
+ return {
7110
+ name: options.name ?? "assertions",
7111
+ score: total === 0 ? 1 : passed / total,
7112
+ metadata: {
7113
+ assertions: results.map(({ name, passed: passed2 }) => ({ name, passed: passed2 })),
7114
+ failed
7115
+ }
7116
+ };
7117
+ };
7118
+ }
7119
+ var agentAssertionHelpers = {
7120
+ equals: (actual, expected, name = "equals") => ({
7121
+ name,
7122
+ evaluate: () => {
7123
+ const passed = deepEqual(actual, expected);
7124
+ return {
7125
+ passed,
7126
+ failure: passed ? void 0 : `expected ${formatValue(actual)} to equal ${formatValue(expected)}`
7127
+ };
7128
+ }
7129
+ }),
7130
+ notEquals: (actual, expected, name = "not equals") => ({
7131
+ name,
7132
+ evaluate: () => {
7133
+ const passed = !deepEqual(actual, expected);
7134
+ return {
7135
+ passed,
7136
+ failure: passed ? void 0 : `expected ${formatValue(actual)} not to equal ${formatValue(expected)}`
7137
+ };
7138
+ }
7139
+ }),
7140
+ contains: (value, expected, name = "contains") => ({
7141
+ name,
7142
+ evaluate: () => {
7143
+ const searchedValue = typeof value === "string" ? value : formatValue(value);
7144
+ const passed = expected instanceof RegExp ? testRegex(expected, value) : searchedValue.includes(expected);
7145
+ return {
7146
+ passed,
7147
+ failure: passed ? void 0 : `expected ${formatValue(value)} to contain ${formatValue(expected)}`
7148
+ };
7149
+ }
7150
+ }),
7151
+ matches: (value, schema, name = "matches schema") => ({
7152
+ name,
7153
+ evaluate: async () => {
7154
+ const result = await validateSchema(schema, value);
7155
+ return {
7156
+ passed: result.passed,
7157
+ failure: result.passed ? void 0 : `expected value to match schema: ${result.message}`
7158
+ };
7159
+ }
7160
+ }),
7161
+ calledTool: (toolName, options = {}, name = `called tool ${toolName}`) => ({
7162
+ name,
7163
+ requiresTrace: true,
7164
+ evaluate: ({ spans }) => {
7165
+ const calls = matchingToolCalls(spans ?? [], toolName, options);
7166
+ const passed = options.times === void 0 ? calls.length > 0 : calls.length === options.times;
7167
+ return {
7168
+ passed,
7169
+ failure: passed ? void 0 : options.times === void 0 ? `expected tool "${toolName}" to be called; found ${calls.length} matching call${calls.length === 1 ? "" : "s"}` : `expected tool "${toolName}" to be called ${options.times} time${options.times === 1 ? "" : "s"}; found ${calls.length} matching call${calls.length === 1 ? "" : "s"}`
7170
+ };
7171
+ }
7172
+ }),
7173
+ notCalledTool: (toolName, name = `did not call tool ${toolName}`) => ({
7174
+ name,
7175
+ requiresTrace: true,
7176
+ evaluate: ({ spans }) => {
7177
+ const calls = toolCalls(spans ?? []).filter(
7178
+ (span) => getToolName(span) === toolName
7179
+ );
7180
+ const passed = calls.length === 0;
7181
+ return {
7182
+ passed,
7183
+ failure: passed ? void 0 : `expected tool "${toolName}" not to be called; found ${calls.length} call${calls.length === 1 ? "" : "s"}`
7184
+ };
7185
+ }
7186
+ }),
7187
+ toolOrder: (toolNames, name = "tool order") => ({
7188
+ name,
7189
+ requiresTrace: true,
7190
+ evaluate: ({ spans }) => {
7191
+ const observed = toolCalls(spans ?? []).map(getToolName).filter((toolName) => toolName !== void 0);
7192
+ let fromIndex = 0;
7193
+ const passed = toolNames.every((toolName) => {
7194
+ const index = observed.indexOf(toolName, fromIndex);
7195
+ if (index === -1) return false;
7196
+ fromIndex = index + 1;
7197
+ return true;
7198
+ });
7199
+ return {
7200
+ passed,
7201
+ failure: passed ? void 0 : `expected tool order ${toolNames.join(" -> ")}; observed ${observed.join(" -> ") || "no tools"}`
7202
+ };
7203
+ }
7204
+ }),
7205
+ usedNoTools: (name = "used no tools") => ({
7206
+ name,
7207
+ requiresTrace: true,
7208
+ evaluate: ({ spans }) => {
7209
+ const calls = toolCalls(spans ?? []);
7210
+ const passed = calls.length === 0;
7211
+ return {
7212
+ passed,
7213
+ failure: passed ? void 0 : `expected no tool calls; found ${calls.length}`
7214
+ };
7215
+ }
7216
+ }),
7217
+ maxToolCalls: (max, name = `at most ${max} tool calls`) => ({
7218
+ name,
7219
+ requiresTrace: true,
7220
+ evaluate: ({ spans }) => {
7221
+ const calls = toolCalls(spans ?? []);
7222
+ const passed = calls.length <= max;
7223
+ return {
7224
+ passed,
7225
+ failure: passed ? void 0 : `expected at most ${max} tool call${max === 1 ? "" : "s"}; found ${calls.length}`
7226
+ };
7227
+ }
7228
+ })
7229
+ };
7230
+ async function validateSchema(schema, value) {
7231
+ try {
7232
+ if ("safeParse" in schema) {
7233
+ const result2 = schema.safeParse(value);
7234
+ return result2.success ? { passed: true, message: "" } : { passed: false, message: formatSchemaError(result2.error) };
7235
+ }
7236
+ if ("parse" in schema) {
7237
+ schema.parse(value);
7238
+ return { passed: true, message: "" };
7239
+ }
7240
+ const result = await schema["~standard"].validate(value);
7241
+ if (typeof result === "object" && result !== null && "issues" in result && Array.isArray(result.issues) && result.issues.length > 0) {
7242
+ return { passed: false, message: formatValue(result.issues) };
7243
+ }
7244
+ return { passed: true, message: "" };
7245
+ } catch (e) {
7246
+ return { passed: false, message: formatSchemaError(e) };
7247
+ }
7248
+ }
7249
+ function toolCalls(spans) {
7250
+ return spans.filter((span) => span.span_attributes?.type === "tool");
7251
+ }
7252
+ function matchingToolCalls(spans, toolName, options) {
7253
+ return toolCalls(spans).filter((span) => {
7254
+ if (getToolName(span) !== toolName) return false;
7255
+ if (Object.prototype.hasOwnProperty.call(options, "input") && !matchesValue(span.input, options.input)) {
7256
+ return false;
7257
+ }
7258
+ if (Object.prototype.hasOwnProperty.call(options, "output") && !matchesValue(span.output, options.output)) {
7259
+ return false;
7260
+ }
7261
+ if (options.isError !== void 0 && Boolean(span.error) !== options.isError) {
7262
+ return false;
7263
+ }
7264
+ return true;
7265
+ });
7266
+ }
7267
+ function getToolName(span) {
7268
+ const spanName = [span.span_attributes?.name, span.name].map(normalizeToolName).find((value) => value !== void 0);
7269
+ if (spanName?.includes("/")) {
7270
+ return spanName;
7271
+ }
7272
+ const metadataName = [
7273
+ span.metadata?.tool_name,
7274
+ span.metadata?.["gen_ai.tool.name"]
7275
+ ].map(normalizeToolName).find((value) => value !== void 0);
7276
+ const mcpServer = [
7277
+ span.metadata?.["mcp.server"],
7278
+ span.metadata?.["openai_codex.mcp.server"]
7279
+ ].find((value) => typeof value === "string" && value !== "");
7280
+ if (metadataName && mcpServer) {
7281
+ return `${mcpServer}/${metadataName}`;
7282
+ }
7283
+ return metadataName ?? spanName;
7284
+ }
7285
+ function normalizeToolName(value) {
7286
+ if (typeof value !== "string" || value === "") {
7287
+ return void 0;
7288
+ }
7289
+ return value.startsWith("tool:") ? value.slice("tool:".length).trim() : value;
7290
+ }
7291
+ function matchesValue(actual, matcher) {
7292
+ if (matcher instanceof RegExp) {
7293
+ return testRegex(matcher, actual);
7294
+ }
7295
+ if (typeof matcher === "function") {
7296
+ return matcher(actual);
7297
+ }
7298
+ if (Array.isArray(matcher)) {
7299
+ return Array.isArray(actual) && actual.length === matcher.length && matcher.every((value, index) => matchesValue(actual[index], value));
7300
+ }
7301
+ if (isPlainObject(matcher) && isPlainObject(actual)) {
7302
+ return Object.entries(matcher).every(
7303
+ ([key, value]) => Object.prototype.hasOwnProperty.call(actual, key) && matchesValue(actual[key], value)
7304
+ );
7305
+ }
7306
+ return deepEqual(actual, matcher);
7307
+ }
7308
+ function testRegex(matcher, value) {
7309
+ matcher.lastIndex = 0;
7310
+ return matcher.test(typeof value === "string" ? value : formatValue(value));
7311
+ }
7312
+ function deepEqual(left, right) {
7313
+ if (Object.is(left, right)) return true;
7314
+ if (Array.isArray(left) && Array.isArray(right)) {
7315
+ return left.length === right.length && left.every((item, index) => deepEqual(item, right[index]));
7316
+ }
7317
+ if (isPlainObject(left) && isPlainObject(right)) {
7318
+ const leftKeys = Object.keys(left);
7319
+ const rightKeys = Object.keys(right);
7320
+ return leftKeys.length === rightKeys.length && leftKeys.every(
7321
+ (key) => Object.prototype.hasOwnProperty.call(right, key) && deepEqual(left[key], right[key])
7322
+ );
7323
+ }
7324
+ return false;
7325
+ }
7326
+ function isPlainObject(value) {
7327
+ return typeof value === "object" && value !== null && !Array.isArray(value) && Object.getPrototypeOf(value) === Object.prototype;
7328
+ }
7329
+ function formatSchemaError(error) {
7330
+ if (error instanceof Error) {
7331
+ return error.message;
7332
+ }
7333
+ return formatValue(error);
7334
+ }
7335
+ function formatValue(value) {
7336
+ if (value instanceof RegExp) {
7337
+ return value.toString();
7338
+ }
7339
+ if (typeof value === "string") {
7340
+ return JSON.stringify(value);
7341
+ }
7342
+ try {
7343
+ const serialized = JSON.stringify(value);
7344
+ if (serialized !== void 0 && !hasUndefinedJsonValue(value)) {
7345
+ return serialized;
7346
+ }
7347
+ return formatValueWithUndefined(value, /* @__PURE__ */ new Set());
7348
+ } catch {
7349
+ return String(value);
7350
+ }
7351
+ }
7352
+ function hasUndefinedJsonValue(value, seen = /* @__PURE__ */ new Set()) {
7353
+ if (value === void 0) {
7354
+ return true;
7355
+ }
7356
+ if (typeof value !== "object" || value === null) {
7357
+ return false;
7358
+ }
7359
+ if (seen.has(value)) {
7360
+ return false;
7361
+ }
7362
+ seen.add(value);
7363
+ if (Array.isArray(value)) {
7364
+ return Array.from({ length: value.length }).some(
7365
+ (_, index) => hasUndefinedJsonValue(value[index], seen)
7366
+ );
7367
+ }
7368
+ return Object.keys(value).some(
7369
+ (key) => hasUndefinedJsonValue(value[key], seen)
7370
+ );
7371
+ }
7372
+ function formatValueWithUndefined(value, seen) {
7373
+ if (value === void 0) {
7374
+ return "undefined";
7375
+ }
7376
+ if (value === null) {
7377
+ return "null";
7378
+ }
7379
+ if (typeof value === "string") {
7380
+ return JSON.stringify(value);
7381
+ }
7382
+ if (typeof value === "number" || typeof value === "boolean") {
7383
+ return JSON.stringify(value);
7384
+ }
7385
+ if (typeof value === "bigint" || typeof value === "function") {
7386
+ return String(value);
7387
+ }
7388
+ if (typeof value === "symbol") {
7389
+ return String(value);
7390
+ }
7391
+ if (value instanceof RegExp) {
7392
+ return value.toString();
7393
+ }
7394
+ if (typeof value !== "object") {
7395
+ return String(value);
7396
+ }
7397
+ if (seen.has(value)) {
7398
+ return '"[Circular]"';
7399
+ }
7400
+ seen.add(value);
7401
+ if (typeof value.toJSON === "function") {
7402
+ const jsonValue = value.toJSON();
7403
+ seen.delete(value);
7404
+ return formatValueWithUndefined(jsonValue, seen);
7405
+ }
7406
+ if (Array.isArray(value)) {
7407
+ const formatted2 = Array.from(
7408
+ { length: value.length },
7409
+ (_, index) => formatValueWithUndefined(value[index], seen)
7410
+ );
7411
+ seen.delete(value);
7412
+ return `[${formatted2.join(",")}]`;
7413
+ }
7414
+ const formatted = Object.keys(value).map(
7415
+ (key) => `${JSON.stringify(key)}:${formatValueWithUndefined(
7416
+ value[key],
7417
+ seen
7418
+ )}`
7419
+ );
7420
+ seen.delete(value);
7421
+ return `{${formatted.join(",")}}`;
7422
+ }
7423
+
7031
7424
  // src/dataset-pipeline.ts
7032
7425
  function DatasetPipeline(definition) {
7033
7426
  if (!globalThis.__braintrust_dataset_pipelines) {
@@ -7604,8 +7997,10 @@ configureNode();
7604
7997
  export {
7605
7998
  Attachment,
7606
7999
  AttachmentReference,
8000
+ BAGGAGE_HEADER,
7607
8001
  BRAINTRUST_CURRENT_SPAN_STORE,
7608
8002
  BRAINTRUST_LANGCHAIN_CALLBACK_HANDLER_NAME,
8003
+ BRAINTRUST_PARENT_KEY,
7609
8004
  BaseAttachment,
7610
8005
  BaseExperiment,
7611
8006
  BraintrustLangChainCallbackHandler,
@@ -7638,6 +8033,7 @@ export {
7638
8033
  NOOP_SPAN,
7639
8034
  NOOP_SPAN_PERMALINK,
7640
8035
  NoopSpan,
8036
+ OTELIDGenerator,
7641
8037
  ObjectFetcher,
7642
8038
  Project2 as Project,
7643
8039
  ProjectNameIdMap,
@@ -7649,6 +8045,8 @@ export {
7649
8045
  ScorerBuilder,
7650
8046
  SpanFetcher,
7651
8047
  SpanImpl,
8048
+ TRACEPARENT_HEADER,
8049
+ TRACESTATE_HEADER,
7652
8050
  TestBackgroundLogger,
7653
8051
  ToolBuilder,
7654
8052
  UUIDGenerator,
@@ -7658,7 +8056,9 @@ export {
7658
8056
  isomorph_default as _internalIso,
7659
8057
  _internalSetInitialState,
7660
8058
  addAzureBlobHeaders,
8059
+ agentAssertionScorer,
7661
8060
  braintrustAISDKTelemetry,
8061
+ braintrustFlueInstrumentation,
7662
8062
  braintrustFlueObserver,
7663
8063
  braintrustStreamChunkSchema,
7664
8064
  buildLocalSummary,
@@ -7675,6 +8075,7 @@ export {
7675
8075
  devNullWritableStream,
7676
8076
  evaluatorDefinitionSchema,
7677
8077
  evaluatorDefinitionsSchema,
8078
+ extractTraceContextFromHeaders,
7678
8079
  flush,
7679
8080
  getContextManager,
7680
8081
  getIdGenerator,
@@ -7688,6 +8089,7 @@ export {
7688
8089
  initFunction,
7689
8090
  initLogger,
7690
8091
  initNodeTestSuite,
8092
+ injectTraceContext,
7691
8093
  invoke,
7692
8094
  isTemplateFormat,
7693
8095
  loadParameters,