braintrust 3.20.0 → 3.21.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. package/NOTICE +20 -0
  2. package/README.md +2 -0
  3. package/dev/dist/index.d.mts +309 -199
  4. package/dev/dist/index.d.ts +309 -199
  5. package/dev/dist/index.js +2826 -983
  6. package/dev/dist/index.mjs +2286 -443
  7. package/dist/apply-auto-instrumentation.js +781 -199
  8. package/dist/apply-auto-instrumentation.mjs +588 -6
  9. package/dist/auto-instrumentations/bundler/esbuild.cjs +594 -2
  10. package/dist/auto-instrumentations/bundler/esbuild.d.mts +2 -2
  11. package/dist/auto-instrumentations/bundler/esbuild.d.ts +2 -2
  12. package/dist/auto-instrumentations/bundler/esbuild.mjs +3 -3
  13. package/dist/auto-instrumentations/bundler/next.cjs +594 -2
  14. package/dist/auto-instrumentations/bundler/next.mjs +4 -4
  15. package/dist/auto-instrumentations/bundler/rollup.cjs +594 -2
  16. package/dist/auto-instrumentations/bundler/rollup.d.mts +2 -2
  17. package/dist/auto-instrumentations/bundler/rollup.d.ts +2 -2
  18. package/dist/auto-instrumentations/bundler/rollup.mjs +3 -3
  19. package/dist/auto-instrumentations/bundler/vite.cjs +594 -2
  20. package/dist/auto-instrumentations/bundler/vite.d.mts +2 -2
  21. package/dist/auto-instrumentations/bundler/vite.d.ts +2 -2
  22. package/dist/auto-instrumentations/bundler/vite.mjs +3 -3
  23. package/dist/auto-instrumentations/bundler/webpack-loader.cjs +593 -9
  24. package/dist/auto-instrumentations/bundler/webpack-loader.d.ts +68 -1
  25. package/dist/auto-instrumentations/bundler/webpack.cjs +594 -2
  26. package/dist/auto-instrumentations/bundler/webpack.d.mts +2 -2
  27. package/dist/auto-instrumentations/bundler/webpack.d.ts +2 -2
  28. package/dist/auto-instrumentations/bundler/webpack.mjs +4 -4
  29. package/dist/auto-instrumentations/{chunk-W4E36GIW.mjs → chunk-BRQX23KL.mjs} +16 -0
  30. package/dist/auto-instrumentations/chunk-M6DLIJ2Z.mjs +784 -0
  31. package/dist/auto-instrumentations/{chunk-K74TZGGM.mjs → chunk-T6J4C7LX.mjs} +1 -1
  32. package/dist/auto-instrumentations/{chunk-S4374IP6.mjs → chunk-TKRPRPGD.mjs} +4 -6
  33. package/dist/auto-instrumentations/chunk-VT6DDNKM.mjs +856 -0
  34. package/dist/auto-instrumentations/hook.mjs +749 -45
  35. package/dist/auto-instrumentations/index.cjs +16 -0
  36. package/dist/auto-instrumentations/index.d.mts +2 -3
  37. package/dist/auto-instrumentations/index.d.ts +2 -3
  38. package/dist/auto-instrumentations/index.mjs +1 -1
  39. package/dist/auto-instrumentations/loader/cjs-patch.cjs +578 -4
  40. package/dist/auto-instrumentations/loader/cjs-patch.d.mts +1 -1
  41. package/dist/auto-instrumentations/loader/cjs-patch.d.ts +1 -1
  42. package/dist/auto-instrumentations/loader/cjs-patch.mjs +3 -7
  43. package/dist/auto-instrumentations/loader/esm-hook.mjs +2 -6
  44. package/dist/auto-instrumentations/{plugin-D7nDswtC.d.mts → plugin-D0KHwSJv.d.mts} +1 -1
  45. package/dist/auto-instrumentations/{plugin-D7nDswtC.d.ts → plugin-G6fgxk1b.d.ts} +1 -1
  46. package/dist/auto-instrumentations/types-RNPaKi9o.d.mts +70 -0
  47. package/dist/auto-instrumentations/types-RNPaKi9o.d.ts +70 -0
  48. package/dist/browser.d.mts +1357 -847
  49. package/dist/browser.d.ts +1357 -847
  50. package/dist/browser.js +2726 -469
  51. package/dist/browser.mjs +2726 -469
  52. package/dist/{chunk-LPC4W2WX.js → chunk-BFGIH2ZJ.js} +4 -0
  53. package/dist/{chunk-POCCIJAL.js → chunk-FY7DAKA5.js} +3141 -1269
  54. package/dist/{chunk-Y7W7WP6H.mjs → chunk-KMGUTPB7.mjs} +4 -0
  55. package/dist/{chunk-F73OMF66.mjs → chunk-O2P765XK.mjs} +2391 -519
  56. package/dist/cli.js +2260 -408
  57. package/dist/edge-light.d.mts +1 -1
  58. package/dist/edge-light.d.ts +1 -1
  59. package/dist/edge-light.js +2726 -469
  60. package/dist/edge-light.mjs +2726 -469
  61. package/dist/index.d.mts +1837 -1327
  62. package/dist/index.d.ts +1837 -1327
  63. package/dist/index.js +752 -353
  64. package/dist/index.mjs +445 -46
  65. package/dist/instrumentation/index.d.mts +107 -19
  66. package/dist/instrumentation/index.d.ts +107 -19
  67. package/dist/instrumentation/index.js +2337 -509
  68. package/dist/instrumentation/index.mjs +2337 -509
  69. package/dist/vitest-evals-reporter.js +16 -16
  70. package/dist/vitest-evals-reporter.mjs +2 -2
  71. package/dist/workerd.d.mts +1 -1
  72. package/dist/workerd.d.ts +1 -1
  73. package/dist/workerd.js +2726 -469
  74. package/dist/workerd.mjs +2726 -469
  75. package/licenses/import-in-the-middle/LICENSE +201 -0
  76. package/licenses/import-in-the-middle/LICENSE-3rdparty.csv +4 -0
  77. package/licenses/import-in-the-middle/NOTICE +34 -0
  78. package/licenses/orchestrion-js/LICENSE +202 -0
  79. package/licenses/orchestrion-js/LICENSE-3rdparty.csv +8 -0
  80. package/licenses/orchestrion-js/NOTICE +3 -0
  81. package/licenses/require-in-the-middle/LICENSE +23 -0
  82. package/package.json +13 -4
  83. package/util/dist/index.d.mts +1 -0
  84. package/util/dist/index.d.ts +1 -0
  85. package/util/dist/index.js +4 -4
  86. package/util/dist/index.mjs +2 -2
  87. package/dist/auto-instrumentations/chunk-J57YF7WS.mjs +0 -208
  88. package/dist/auto-instrumentations/chunk-QFMACSOL.mjs +0 -280
package/dist/index.mjs CHANGED
@@ -1,8 +1,10 @@
1
1
  import {
2
2
  Attachment,
3
3
  AttachmentReference,
4
+ BAGGAGE_HEADER,
4
5
  BRAINTRUST_CURRENT_SPAN_STORE,
5
6
  BRAINTRUST_LANGCHAIN_CALLBACK_HANDLER_NAME,
7
+ BRAINTRUST_PARENT_KEY,
6
8
  BaseAttachment,
7
9
  BraintrustLangChainCallbackHandler,
8
10
  BraintrustObservabilityExporter,
@@ -32,6 +34,7 @@ import {
32
34
  NOOP_SPAN,
33
35
  NOOP_SPAN_PERMALINK,
34
36
  NoopSpan,
37
+ OTELIDGenerator,
35
38
  ObjectFetcher,
36
39
  ObjectReference,
37
40
  Project,
@@ -41,8 +44,10 @@ import {
41
44
  ReadonlyExperiment,
42
45
  RemoteEvalParameters,
43
46
  RunEval,
44
- SpanComponentsV3,
47
+ SpanComponentsV4,
45
48
  SpanImpl,
49
+ TRACEPARENT_HEADER,
50
+ TRACESTATE_HEADER,
46
51
  TestBackgroundLogger,
47
52
  ToolFunctionDefinition,
48
53
  UUIDGenerator,
@@ -63,11 +68,13 @@ import {
63
68
  currentExperiment,
64
69
  currentLogger,
65
70
  currentSpan,
71
+ currentWorkflowAgentWrapperSpan,
66
72
  debugLogger,
67
73
  deepCopyEvent,
68
74
  deserializePlainStringAsJSON,
69
75
  devNullWritableStream,
70
76
  extractAnthropicCacheTokens,
77
+ extractTraceContextFromHeaders,
71
78
  finalizeAnthropicTokens,
72
79
  flush,
73
80
  getBedrockRuntimeOperation,
@@ -81,6 +88,7 @@ import {
81
88
  initDataset,
82
89
  initExperiment,
83
90
  initLogger,
91
+ injectTraceContext,
84
92
  isEmpty,
85
93
  isObject,
86
94
  isTemplateFormat,
@@ -129,7 +137,7 @@ import {
129
137
  wrapMastraAgent,
130
138
  wrapTraced,
131
139
  zodToJsonSchema
132
- } from "./chunk-F73OMF66.mjs";
140
+ } from "./chunk-O2P765XK.mjs";
133
141
  import {
134
142
  __export,
135
143
  aiSDKChannels,
@@ -152,15 +160,17 @@ import {
152
160
  openRouterChannels,
153
161
  piCodingAgentChannels,
154
162
  strandsAgentSDKChannels
155
- } from "./chunk-Y7W7WP6H.mjs";
163
+ } from "./chunk-KMGUTPB7.mjs";
156
164
 
157
165
  // src/exports.ts
158
166
  var exports_exports = {};
159
167
  __export(exports_exports, {
160
168
  Attachment: () => Attachment,
161
169
  AttachmentReference: () => AttachmentReference,
170
+ BAGGAGE_HEADER: () => BAGGAGE_HEADER,
162
171
  BRAINTRUST_CURRENT_SPAN_STORE: () => BRAINTRUST_CURRENT_SPAN_STORE,
163
172
  BRAINTRUST_LANGCHAIN_CALLBACK_HANDLER_NAME: () => BRAINTRUST_LANGCHAIN_CALLBACK_HANDLER_NAME,
173
+ BRAINTRUST_PARENT_KEY: () => BRAINTRUST_PARENT_KEY,
164
174
  BaseAttachment: () => BaseAttachment,
165
175
  BaseExperiment: () => BaseExperiment,
166
176
  BraintrustLangChainCallbackHandler: () => BraintrustLangChainCallbackHandler,
@@ -193,6 +203,7 @@ __export(exports_exports, {
193
203
  NOOP_SPAN: () => NOOP_SPAN,
194
204
  NOOP_SPAN_PERMALINK: () => NOOP_SPAN_PERMALINK,
195
205
  NoopSpan: () => NoopSpan,
206
+ OTELIDGenerator: () => OTELIDGenerator,
196
207
  ObjectFetcher: () => ObjectFetcher,
197
208
  Project: () => Project2,
198
209
  ProjectNameIdMap: () => ProjectNameIdMap,
@@ -204,6 +215,8 @@ __export(exports_exports, {
204
215
  ScorerBuilder: () => ScorerBuilder,
205
216
  SpanFetcher: () => SpanFetcher,
206
217
  SpanImpl: () => SpanImpl,
218
+ TRACEPARENT_HEADER: () => TRACEPARENT_HEADER,
219
+ TRACESTATE_HEADER: () => TRACESTATE_HEADER,
207
220
  TestBackgroundLogger: () => TestBackgroundLogger,
208
221
  ToolBuilder: () => ToolBuilder,
209
222
  UUIDGenerator: () => UUIDGenerator,
@@ -213,6 +226,7 @@ __export(exports_exports, {
213
226
  _internalIso: () => isomorph_default,
214
227
  _internalSetInitialState: () => _internalSetInitialState,
215
228
  addAzureBlobHeaders: () => addAzureBlobHeaders,
229
+ agentAssertionScorer: () => agentAssertionScorer,
216
230
  braintrustAISDKTelemetry: () => braintrustAISDKTelemetry,
217
231
  braintrustFlueInstrumentation: () => braintrustFlueInstrumentation,
218
232
  braintrustFlueObserver: () => braintrustFlueObserver,
@@ -230,6 +244,7 @@ __export(exports_exports, {
230
244
  devNullWritableStream: () => devNullWritableStream,
231
245
  evaluatorDefinitionSchema: () => evaluatorDefinitionSchema,
232
246
  evaluatorDefinitionsSchema: () => evaluatorDefinitionsSchema,
247
+ extractTraceContextFromHeaders: () => extractTraceContextFromHeaders,
233
248
  flush: () => flush,
234
249
  getContextManager: () => getContextManager,
235
250
  getIdGenerator: () => getIdGenerator,
@@ -243,6 +258,7 @@ __export(exports_exports, {
243
258
  initFunction: () => initFunction,
244
259
  initLogger: () => initLogger,
245
260
  initNodeTestSuite: () => initNodeTestSuite,
261
+ injectTraceContext: () => injectTraceContext,
246
262
  invoke: () => invoke,
247
263
  isTemplateFormat: () => isTemplateFormat,
248
264
  loadParameters: () => loadParameters,
@@ -874,6 +890,7 @@ function wrapAISDK(aiSDK, options = {}) {
874
890
  case "Agent":
875
891
  case "Experimental_Agent":
876
892
  case "ToolLoopAgent":
893
+ case "WorkflowAgent":
877
894
  return original ? wrapAgentClass(original, options) : original;
878
895
  }
879
896
  return original;
@@ -892,10 +909,10 @@ var wrapAgentClass = (AgentClass, options = {}) => {
892
909
  return new Proxy(instance, {
893
910
  get(instanceTarget, prop, instanceReceiver) {
894
911
  const original = Reflect.get(instanceTarget, prop, instanceTarget);
895
- if (prop === "generate") {
912
+ if (prop === "generate" && typeof original === "function" && instanceTarget.constructor.name !== "WorkflowAgent") {
896
913
  return wrapAgentGenerate(original, instanceTarget, options);
897
914
  }
898
- if (prop === "stream") {
915
+ if (prop === "stream" && typeof original === "function") {
899
916
  return wrapAgentStream(original, instanceTarget, options);
900
917
  }
901
918
  if (typeof original === "function") {
@@ -910,7 +927,7 @@ var wrapAgentClass = (AgentClass, options = {}) => {
910
927
  var wrapAgentGenerate = (generate, instance, options = {}) => {
911
928
  const defaultName = `${instance.constructor.name}.generate`;
912
929
  return async (params) => makeGenerateTextWrapper(
913
- aiSDKChannels.generateText,
930
+ generateChannelForAgent(instance.constructor.name),
914
931
  defaultName,
915
932
  generate.bind(instance),
916
933
  {
@@ -920,19 +937,42 @@ var wrapAgentGenerate = (generate, instance, options = {}) => {
920
937
  options
921
938
  )({ ...instance.settings, ...params });
922
939
  };
940
+ function generateChannelForAgent(agentName) {
941
+ if (agentName === "ToolLoopAgent") {
942
+ return aiSDKChannels.toolLoopAgentGenerate;
943
+ }
944
+ return aiSDKChannels.agentGenerate;
945
+ }
923
946
  var wrapAgentStream = (stream, instance, options = {}) => {
924
947
  const defaultName = `${instance.constructor.name}.stream`;
925
- return (params) => makeStreamWrapper(
926
- aiSDKChannels.agentStream,
927
- defaultName,
928
- stream.bind(instance),
929
- {
930
- self: instance,
931
- spanType: "function" /* FUNCTION */
932
- },
933
- options
934
- )({ ...instance.settings, ...params });
948
+ return (params) => {
949
+ const workflowAgent = instance.constructor.name === "WorkflowAgent";
950
+ if (workflowAgent && currentWorkflowAgentWrapperSpan()) {
951
+ const { span_info: _spanInfo, ...cleanParams } = params;
952
+ return stream.call(instance, { ...instance.settings, ...cleanParams });
953
+ }
954
+ const trace = () => makeStreamWrapper(
955
+ streamChannelForAgent(instance.constructor.name),
956
+ defaultName,
957
+ stream.bind(instance),
958
+ {
959
+ self: instance,
960
+ spanType: "function" /* FUNCTION */
961
+ },
962
+ options
963
+ )({ ...instance.settings, ...params });
964
+ return trace();
965
+ };
935
966
  };
967
+ function streamChannelForAgent(agentName) {
968
+ if (agentName === "ToolLoopAgent") {
969
+ return aiSDKChannels.toolLoopAgentStream;
970
+ }
971
+ if (agentName === "WorkflowAgent") {
972
+ return aiSDKChannels.workflowAgentStream;
973
+ }
974
+ return aiSDKChannels.agentStream;
975
+ }
936
976
  var makeGenerateTextWrapper = (channel, name, generateText, contextOptions = {}, options = {}) => {
937
977
  const wrapper = async function(allParams) {
938
978
  const { span_info, ...params } = allParams;
@@ -1194,7 +1234,7 @@ var BraintrustLanguageModelWrapper = class {
1194
1234
  let time_to_first_token = void 0;
1195
1235
  let usage = void 0;
1196
1236
  let fullText = void 0;
1197
- const toolCalls = {};
1237
+ const toolCalls2 = {};
1198
1238
  let finishReason = void 0;
1199
1239
  return {
1200
1240
  ...ret,
@@ -1213,7 +1253,7 @@ var BraintrustLanguageModelWrapper = class {
1213
1253
  fullText += chunk.textDelta;
1214
1254
  break;
1215
1255
  case "tool-call":
1216
- toolCalls[chunk.toolCallId] = {
1256
+ toolCalls2[chunk.toolCallId] = {
1217
1257
  toolCallType: chunk.toolCallType,
1218
1258
  toolCallId: chunk.toolCallId,
1219
1259
  toolName: chunk.toolName,
@@ -1221,15 +1261,15 @@ var BraintrustLanguageModelWrapper = class {
1221
1261
  };
1222
1262
  break;
1223
1263
  case "tool-call-delta":
1224
- if (toolCalls[chunk.toolCallId] === void 0) {
1225
- toolCalls[chunk.toolCallId] = {
1264
+ if (toolCalls2[chunk.toolCallId] === void 0) {
1265
+ toolCalls2[chunk.toolCallId] = {
1226
1266
  toolCallType: chunk.toolCallType,
1227
1267
  toolCallId: chunk.toolCallId,
1228
1268
  toolName: chunk.toolName,
1229
1269
  args: ""
1230
1270
  };
1231
1271
  }
1232
- toolCalls[chunk.toolCallId].args += chunk.argsTextDelta;
1272
+ toolCalls2[chunk.toolCallId].args += chunk.argsTextDelta;
1233
1273
  break;
1234
1274
  case "finish":
1235
1275
  usage = chunk.usage;
@@ -1242,7 +1282,7 @@ var BraintrustLanguageModelWrapper = class {
1242
1282
  span.log({
1243
1283
  output: postProcessOutput(
1244
1284
  fullText,
1245
- Object.keys(toolCalls).length > 0 ? Object.values(toolCalls) : void 0,
1285
+ Object.keys(toolCalls2).length > 0 ? Object.values(toolCalls2) : void 0,
1246
1286
  finishReason
1247
1287
  ),
1248
1288
  metrics: {
@@ -1343,15 +1383,15 @@ function postProcessPrompt(prompt) {
1343
1383
  }
1344
1384
  });
1345
1385
  }
1346
- function postProcessOutput(text, toolCalls, finishReason) {
1386
+ function postProcessOutput(text, toolCalls2, finishReason) {
1347
1387
  return [
1348
1388
  {
1349
1389
  index: 0,
1350
1390
  message: {
1351
1391
  role: "assistant",
1352
1392
  content: text ?? "",
1353
- ...toolCalls && toolCalls.length > 0 ? {
1354
- tool_calls: toolCalls.map((toolCall) => ({
1393
+ ...toolCalls2 && toolCalls2.length > 0 ? {
1394
+ tool_calls: toolCalls2.map((toolCall) => ({
1355
1395
  id: toolCall.toolCallId,
1356
1396
  function: {
1357
1397
  name: toolCall.toolName,
@@ -1450,29 +1490,29 @@ function normalizeFinishReason(reason) {
1450
1490
  if (typeof reason !== "string") return void 0;
1451
1491
  return reason.replace(/-/g, "_");
1452
1492
  }
1453
- function buildAssistantOutputWithToolCalls(result, toolCalls) {
1493
+ function buildAssistantOutputWithToolCalls(result, toolCalls2) {
1454
1494
  return [
1455
1495
  {
1456
1496
  index: 0,
1457
1497
  logprobs: null,
1458
- finish_reason: normalizeFinishReason(result?.finishReason) ?? (toolCalls.length ? "tool_calls" : void 0),
1498
+ finish_reason: normalizeFinishReason(result?.finishReason) ?? (toolCalls2.length ? "tool_calls" : void 0),
1459
1499
  message: {
1460
1500
  role: "assistant",
1461
- tool_calls: toolCalls.length > 0 ? toolCalls : void 0
1501
+ tool_calls: toolCalls2.length > 0 ? toolCalls2 : void 0
1462
1502
  }
1463
1503
  }
1464
1504
  ];
1465
1505
  }
1466
1506
  function extractToolCallsFromSteps(steps) {
1467
- const toolCalls = [];
1468
- if (!Array.isArray(steps)) return toolCalls;
1507
+ const toolCalls2 = [];
1508
+ if (!Array.isArray(steps)) return toolCalls2;
1469
1509
  let idx = 0;
1470
1510
  for (const step of steps) {
1471
1511
  const blocks = step?.content;
1472
1512
  if (!Array.isArray(blocks)) continue;
1473
1513
  for (const block of blocks) {
1474
1514
  if (block && typeof block === "object" && block.type === "tool-call") {
1475
- toolCalls.push({
1515
+ toolCalls2.push({
1476
1516
  id: block.toolCallId,
1477
1517
  type: "function",
1478
1518
  index: idx++,
@@ -1484,7 +1524,7 @@ function extractToolCallsFromSteps(steps) {
1484
1524
  }
1485
1525
  }
1486
1526
  }
1487
- return toolCalls;
1527
+ return toolCalls2;
1488
1528
  }
1489
1529
  function extractToolCallsFromBlocks(blocks) {
1490
1530
  if (!Array.isArray(blocks)) return [];
@@ -1550,12 +1590,12 @@ function BraintrustMiddleware(config = {}) {
1550
1590
  metadata.model = modelId;
1551
1591
  }
1552
1592
  }
1553
- let toolCalls = extractToolCallsFromSteps(result?.steps);
1554
- if (!toolCalls || toolCalls.length === 0) {
1555
- toolCalls = extractToolCallsFromBlocks(result?.content);
1593
+ let toolCalls2 = extractToolCallsFromSteps(result?.steps);
1594
+ if (!toolCalls2 || toolCalls2.length === 0) {
1595
+ toolCalls2 = extractToolCallsFromBlocks(result?.content);
1556
1596
  }
1557
1597
  span.log({
1558
- output: toolCalls.length > 0 ? buildAssistantOutputWithToolCalls(result, toolCalls) : result?.content,
1598
+ output: toolCalls2.length > 0 ? buildAssistantOutputWithToolCalls(result, toolCalls2) : result?.content,
1559
1599
  metadata,
1560
1600
  metrics: normalizeUsageMetrics(
1561
1601
  result.usage,
@@ -1644,13 +1684,13 @@ function BraintrustMiddleware(config = {}) {
1644
1684
  metadata.model = model;
1645
1685
  }
1646
1686
  if (toolBlocks.length > 0) {
1647
- const toolCalls = extractToolCallsFromSteps([
1687
+ const toolCalls2 = extractToolCallsFromSteps([
1648
1688
  { content: toolBlocks }
1649
1689
  ]);
1650
- if (toolCalls.length > 0) {
1690
+ if (toolCalls2.length > 0) {
1651
1691
  output = buildAssistantOutputWithToolCalls(
1652
1692
  resultForDetection,
1653
- toolCalls
1693
+ toolCalls2
1654
1694
  );
1655
1695
  }
1656
1696
  }
@@ -5599,15 +5639,21 @@ var waterfall$1 = awaitify(waterfall);
5599
5639
 
5600
5640
  // src/trace.ts
5601
5641
  var SpanFetcher = class _SpanFetcher extends ObjectFetcher {
5602
- constructor(objectType, _objectId, rootSpanId, _state, spanTypeFilter, includeScorers = false) {
5642
+ constructor(objectType, _objectId, rootSpanId, _state, spanTypeFilter, includeScorers = false, brainstoreRealtime = true) {
5603
5643
  const filterExpr = _SpanFetcher.buildFilter(
5604
5644
  rootSpanId,
5605
5645
  spanTypeFilter,
5606
5646
  includeScorers
5607
5647
  );
5608
- super(objectType, void 0, void 0, {
5609
- filter: filterExpr
5610
- });
5648
+ super(
5649
+ objectType,
5650
+ void 0,
5651
+ void 0,
5652
+ {
5653
+ filter: filterExpr
5654
+ },
5655
+ brainstoreRealtime
5656
+ );
5611
5657
  this._objectId = _objectId;
5612
5658
  this.rootSpanId = rootSpanId;
5613
5659
  this._state = _state;
@@ -5661,7 +5707,7 @@ var CachedSpanFetcher = class {
5661
5707
  spanCache = /* @__PURE__ */ new Map();
5662
5708
  allFetched = false;
5663
5709
  fetchFn;
5664
- constructor(objectTypeOrFetchFn, objectId, rootSpanId, getState) {
5710
+ constructor(objectTypeOrFetchFn, objectId, rootSpanId, getState, brainstoreRealtime = true) {
5665
5711
  if (typeof objectTypeOrFetchFn === "function") {
5666
5712
  this.fetchFn = (spanType) => objectTypeOrFetchFn(spanType);
5667
5713
  } else {
@@ -5674,7 +5720,8 @@ var CachedSpanFetcher = class {
5674
5720
  rootSpanId,
5675
5721
  state,
5676
5722
  spanType,
5677
- includeScorers
5723
+ includeScorers,
5724
+ brainstoreRealtime
5678
5725
  );
5679
5726
  const rows = await fetcher.fetchedData();
5680
5727
  return rows.map((row) => ({
@@ -6504,7 +6551,7 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
6504
6551
  }
6505
6552
  };
6506
6553
  const parentStr = state.currentParent.getStore();
6507
- const parentComponents = parentStr ? SpanComponentsV3.fromStr(parentStr) : null;
6554
+ const parentComponents = typeof parentStr === "string" ? SpanComponentsV4.fromStr(parentStr) : null;
6508
6555
  const trace = state ? new LocalTrace({
6509
6556
  objectType: parentComponents ? spanObjectTypeV3ToTypedString(
6510
6557
  parentComponents.data.object_type
@@ -7030,6 +7077,350 @@ var defaultReporter = {
7030
7077
  }
7031
7078
  };
7032
7079
 
7080
+ // src/agent-assertions.ts
7081
+ function agentAssertionScorer(callback, options = {}) {
7082
+ return async (args) => {
7083
+ const { trace: _trace, ...callbackArgs } = args;
7084
+ const callbackMetadata = callbackArgs.metadata ?? {};
7085
+ const assertions = await callback({
7086
+ ...callbackArgs,
7087
+ metadata: callbackMetadata,
7088
+ assert: agentAssertionHelpers
7089
+ });
7090
+ const resources = {};
7091
+ if (assertions.some((assertion) => assertion.requiresTrace)) {
7092
+ resources.spans = await args.trace?.getSpans({ spanType: ["tool"] });
7093
+ }
7094
+ const results = await Promise.all(
7095
+ assertions.map(async (assertion) => {
7096
+ const result = await assertion.evaluate(resources);
7097
+ return {
7098
+ name: assertion.name,
7099
+ passed: result.passed,
7100
+ failure: result.failure
7101
+ };
7102
+ })
7103
+ );
7104
+ const passed = results.filter((result) => result.passed).length;
7105
+ const total = results.length;
7106
+ const failed = results.filter((result) => !result.passed).map(
7107
+ (result) => `${result.name}: ${result.failure ?? "assertion did not pass"}`
7108
+ );
7109
+ return {
7110
+ name: options.name ?? "assertions",
7111
+ score: total === 0 ? 1 : passed / total,
7112
+ metadata: {
7113
+ assertions: results.map(({ name, passed: passed2 }) => ({ name, passed: passed2 })),
7114
+ failed
7115
+ }
7116
+ };
7117
+ };
7118
+ }
7119
+ var agentAssertionHelpers = {
7120
+ equals: (actual, expected, name = "equals") => ({
7121
+ name,
7122
+ evaluate: () => {
7123
+ const passed = deepEqual(actual, expected);
7124
+ return {
7125
+ passed,
7126
+ failure: passed ? void 0 : `expected ${formatValue(actual)} to equal ${formatValue(expected)}`
7127
+ };
7128
+ }
7129
+ }),
7130
+ notEquals: (actual, expected, name = "not equals") => ({
7131
+ name,
7132
+ evaluate: () => {
7133
+ const passed = !deepEqual(actual, expected);
7134
+ return {
7135
+ passed,
7136
+ failure: passed ? void 0 : `expected ${formatValue(actual)} not to equal ${formatValue(expected)}`
7137
+ };
7138
+ }
7139
+ }),
7140
+ contains: (value, expected, name = "contains") => ({
7141
+ name,
7142
+ evaluate: () => {
7143
+ const searchedValue = typeof value === "string" ? value : formatValue(value);
7144
+ const passed = expected instanceof RegExp ? testRegex(expected, value) : searchedValue.includes(expected);
7145
+ return {
7146
+ passed,
7147
+ failure: passed ? void 0 : `expected ${formatValue(value)} to contain ${formatValue(expected)}`
7148
+ };
7149
+ }
7150
+ }),
7151
+ matches: (value, schema, name = "matches schema") => ({
7152
+ name,
7153
+ evaluate: async () => {
7154
+ const result = await validateSchema(schema, value);
7155
+ return {
7156
+ passed: result.passed,
7157
+ failure: result.passed ? void 0 : `expected value to match schema: ${result.message}`
7158
+ };
7159
+ }
7160
+ }),
7161
+ calledTool: (toolName, options = {}, name = `called tool ${toolName}`) => ({
7162
+ name,
7163
+ requiresTrace: true,
7164
+ evaluate: ({ spans }) => {
7165
+ const calls = matchingToolCalls(spans ?? [], toolName, options);
7166
+ const passed = options.times === void 0 ? calls.length > 0 : calls.length === options.times;
7167
+ return {
7168
+ passed,
7169
+ failure: passed ? void 0 : options.times === void 0 ? `expected tool "${toolName}" to be called; found ${calls.length} matching call${calls.length === 1 ? "" : "s"}` : `expected tool "${toolName}" to be called ${options.times} time${options.times === 1 ? "" : "s"}; found ${calls.length} matching call${calls.length === 1 ? "" : "s"}`
7170
+ };
7171
+ }
7172
+ }),
7173
+ notCalledTool: (toolName, name = `did not call tool ${toolName}`) => ({
7174
+ name,
7175
+ requiresTrace: true,
7176
+ evaluate: ({ spans }) => {
7177
+ const calls = toolCalls(spans ?? []).filter(
7178
+ (span) => getToolName(span) === toolName
7179
+ );
7180
+ const passed = calls.length === 0;
7181
+ return {
7182
+ passed,
7183
+ failure: passed ? void 0 : `expected tool "${toolName}" not to be called; found ${calls.length} call${calls.length === 1 ? "" : "s"}`
7184
+ };
7185
+ }
7186
+ }),
7187
+ toolOrder: (toolNames, name = "tool order") => ({
7188
+ name,
7189
+ requiresTrace: true,
7190
+ evaluate: ({ spans }) => {
7191
+ const observed = toolCalls(spans ?? []).map(getToolName).filter((toolName) => toolName !== void 0);
7192
+ let fromIndex = 0;
7193
+ const passed = toolNames.every((toolName) => {
7194
+ const index = observed.indexOf(toolName, fromIndex);
7195
+ if (index === -1) return false;
7196
+ fromIndex = index + 1;
7197
+ return true;
7198
+ });
7199
+ return {
7200
+ passed,
7201
+ failure: passed ? void 0 : `expected tool order ${toolNames.join(" -> ")}; observed ${observed.join(" -> ") || "no tools"}`
7202
+ };
7203
+ }
7204
+ }),
7205
+ usedNoTools: (name = "used no tools") => ({
7206
+ name,
7207
+ requiresTrace: true,
7208
+ evaluate: ({ spans }) => {
7209
+ const calls = toolCalls(spans ?? []);
7210
+ const passed = calls.length === 0;
7211
+ return {
7212
+ passed,
7213
+ failure: passed ? void 0 : `expected no tool calls; found ${calls.length}`
7214
+ };
7215
+ }
7216
+ }),
7217
+ maxToolCalls: (max, name = `at most ${max} tool calls`) => ({
7218
+ name,
7219
+ requiresTrace: true,
7220
+ evaluate: ({ spans }) => {
7221
+ const calls = toolCalls(spans ?? []);
7222
+ const passed = calls.length <= max;
7223
+ return {
7224
+ passed,
7225
+ failure: passed ? void 0 : `expected at most ${max} tool call${max === 1 ? "" : "s"}; found ${calls.length}`
7226
+ };
7227
+ }
7228
+ })
7229
+ };
7230
+ async function validateSchema(schema, value) {
7231
+ try {
7232
+ if ("safeParse" in schema) {
7233
+ const result2 = schema.safeParse(value);
7234
+ return result2.success ? { passed: true, message: "" } : { passed: false, message: formatSchemaError(result2.error) };
7235
+ }
7236
+ if ("parse" in schema) {
7237
+ schema.parse(value);
7238
+ return { passed: true, message: "" };
7239
+ }
7240
+ const result = await schema["~standard"].validate(value);
7241
+ if (typeof result === "object" && result !== null && "issues" in result && Array.isArray(result.issues) && result.issues.length > 0) {
7242
+ return { passed: false, message: formatValue(result.issues) };
7243
+ }
7244
+ return { passed: true, message: "" };
7245
+ } catch (e) {
7246
+ return { passed: false, message: formatSchemaError(e) };
7247
+ }
7248
+ }
7249
+ function toolCalls(spans) {
7250
+ return spans.filter((span) => span.span_attributes?.type === "tool");
7251
+ }
7252
+ function matchingToolCalls(spans, toolName, options) {
7253
+ return toolCalls(spans).filter((span) => {
7254
+ if (getToolName(span) !== toolName) return false;
7255
+ if (Object.prototype.hasOwnProperty.call(options, "input") && !matchesValue(span.input, options.input)) {
7256
+ return false;
7257
+ }
7258
+ if (Object.prototype.hasOwnProperty.call(options, "output") && !matchesValue(span.output, options.output)) {
7259
+ return false;
7260
+ }
7261
+ if (options.isError !== void 0 && Boolean(span.error) !== options.isError) {
7262
+ return false;
7263
+ }
7264
+ return true;
7265
+ });
7266
+ }
7267
+ function getToolName(span) {
7268
+ const spanName = [span.span_attributes?.name, span.name].map(normalizeToolName).find((value) => value !== void 0);
7269
+ if (spanName?.includes("/")) {
7270
+ return spanName;
7271
+ }
7272
+ const metadataName = [
7273
+ span.metadata?.tool_name,
7274
+ span.metadata?.["gen_ai.tool.name"]
7275
+ ].map(normalizeToolName).find((value) => value !== void 0);
7276
+ const mcpServer = [
7277
+ span.metadata?.["mcp.server"],
7278
+ span.metadata?.["openai_codex.mcp.server"]
7279
+ ].find((value) => typeof value === "string" && value !== "");
7280
+ if (metadataName && mcpServer) {
7281
+ return `${mcpServer}/${metadataName}`;
7282
+ }
7283
+ return metadataName ?? spanName;
7284
+ }
7285
+ function normalizeToolName(value) {
7286
+ if (typeof value !== "string" || value === "") {
7287
+ return void 0;
7288
+ }
7289
+ return value.startsWith("tool:") ? value.slice("tool:".length).trim() : value;
7290
+ }
7291
+ function matchesValue(actual, matcher) {
7292
+ if (matcher instanceof RegExp) {
7293
+ return testRegex(matcher, actual);
7294
+ }
7295
+ if (typeof matcher === "function") {
7296
+ return matcher(actual);
7297
+ }
7298
+ if (Array.isArray(matcher)) {
7299
+ return Array.isArray(actual) && actual.length === matcher.length && matcher.every((value, index) => matchesValue(actual[index], value));
7300
+ }
7301
+ if (isPlainObject(matcher) && isPlainObject(actual)) {
7302
+ return Object.entries(matcher).every(
7303
+ ([key, value]) => Object.prototype.hasOwnProperty.call(actual, key) && matchesValue(actual[key], value)
7304
+ );
7305
+ }
7306
+ return deepEqual(actual, matcher);
7307
+ }
7308
+ function testRegex(matcher, value) {
7309
+ matcher.lastIndex = 0;
7310
+ return matcher.test(typeof value === "string" ? value : formatValue(value));
7311
+ }
7312
+ function deepEqual(left, right) {
7313
+ if (Object.is(left, right)) return true;
7314
+ if (Array.isArray(left) && Array.isArray(right)) {
7315
+ return left.length === right.length && left.every((item, index) => deepEqual(item, right[index]));
7316
+ }
7317
+ if (isPlainObject(left) && isPlainObject(right)) {
7318
+ const leftKeys = Object.keys(left);
7319
+ const rightKeys = Object.keys(right);
7320
+ return leftKeys.length === rightKeys.length && leftKeys.every(
7321
+ (key) => Object.prototype.hasOwnProperty.call(right, key) && deepEqual(left[key], right[key])
7322
+ );
7323
+ }
7324
+ return false;
7325
+ }
7326
+ function isPlainObject(value) {
7327
+ return typeof value === "object" && value !== null && !Array.isArray(value) && Object.getPrototypeOf(value) === Object.prototype;
7328
+ }
7329
+ function formatSchemaError(error) {
7330
+ if (error instanceof Error) {
7331
+ return error.message;
7332
+ }
7333
+ return formatValue(error);
7334
+ }
7335
+ function formatValue(value) {
7336
+ if (value instanceof RegExp) {
7337
+ return value.toString();
7338
+ }
7339
+ if (typeof value === "string") {
7340
+ return JSON.stringify(value);
7341
+ }
7342
+ try {
7343
+ const serialized = JSON.stringify(value);
7344
+ if (serialized !== void 0 && !hasUndefinedJsonValue(value)) {
7345
+ return serialized;
7346
+ }
7347
+ return formatValueWithUndefined(value, /* @__PURE__ */ new Set());
7348
+ } catch {
7349
+ return String(value);
7350
+ }
7351
+ }
7352
+ function hasUndefinedJsonValue(value, seen = /* @__PURE__ */ new Set()) {
7353
+ if (value === void 0) {
7354
+ return true;
7355
+ }
7356
+ if (typeof value !== "object" || value === null) {
7357
+ return false;
7358
+ }
7359
+ if (seen.has(value)) {
7360
+ return false;
7361
+ }
7362
+ seen.add(value);
7363
+ if (Array.isArray(value)) {
7364
+ return Array.from({ length: value.length }).some(
7365
+ (_, index) => hasUndefinedJsonValue(value[index], seen)
7366
+ );
7367
+ }
7368
+ return Object.keys(value).some(
7369
+ (key) => hasUndefinedJsonValue(value[key], seen)
7370
+ );
7371
+ }
7372
+ function formatValueWithUndefined(value, seen) {
7373
+ if (value === void 0) {
7374
+ return "undefined";
7375
+ }
7376
+ if (value === null) {
7377
+ return "null";
7378
+ }
7379
+ if (typeof value === "string") {
7380
+ return JSON.stringify(value);
7381
+ }
7382
+ if (typeof value === "number" || typeof value === "boolean") {
7383
+ return JSON.stringify(value);
7384
+ }
7385
+ if (typeof value === "bigint" || typeof value === "function") {
7386
+ return String(value);
7387
+ }
7388
+ if (typeof value === "symbol") {
7389
+ return String(value);
7390
+ }
7391
+ if (value instanceof RegExp) {
7392
+ return value.toString();
7393
+ }
7394
+ if (typeof value !== "object") {
7395
+ return String(value);
7396
+ }
7397
+ if (seen.has(value)) {
7398
+ return '"[Circular]"';
7399
+ }
7400
+ seen.add(value);
7401
+ if (typeof value.toJSON === "function") {
7402
+ const jsonValue = value.toJSON();
7403
+ seen.delete(value);
7404
+ return formatValueWithUndefined(jsonValue, seen);
7405
+ }
7406
+ if (Array.isArray(value)) {
7407
+ const formatted2 = Array.from(
7408
+ { length: value.length },
7409
+ (_, index) => formatValueWithUndefined(value[index], seen)
7410
+ );
7411
+ seen.delete(value);
7412
+ return `[${formatted2.join(",")}]`;
7413
+ }
7414
+ const formatted = Object.keys(value).map(
7415
+ (key) => `${JSON.stringify(key)}:${formatValueWithUndefined(
7416
+ value[key],
7417
+ seen
7418
+ )}`
7419
+ );
7420
+ seen.delete(value);
7421
+ return `{${formatted.join(",")}}`;
7422
+ }
7423
+
7033
7424
  // src/dataset-pipeline.ts
7034
7425
  function DatasetPipeline(definition) {
7035
7426
  if (!globalThis.__braintrust_dataset_pipelines) {
@@ -7606,8 +7997,10 @@ configureNode();
7606
7997
  export {
7607
7998
  Attachment,
7608
7999
  AttachmentReference,
8000
+ BAGGAGE_HEADER,
7609
8001
  BRAINTRUST_CURRENT_SPAN_STORE,
7610
8002
  BRAINTRUST_LANGCHAIN_CALLBACK_HANDLER_NAME,
8003
+ BRAINTRUST_PARENT_KEY,
7611
8004
  BaseAttachment,
7612
8005
  BaseExperiment,
7613
8006
  BraintrustLangChainCallbackHandler,
@@ -7640,6 +8033,7 @@ export {
7640
8033
  NOOP_SPAN,
7641
8034
  NOOP_SPAN_PERMALINK,
7642
8035
  NoopSpan,
8036
+ OTELIDGenerator,
7643
8037
  ObjectFetcher,
7644
8038
  Project2 as Project,
7645
8039
  ProjectNameIdMap,
@@ -7651,6 +8045,8 @@ export {
7651
8045
  ScorerBuilder,
7652
8046
  SpanFetcher,
7653
8047
  SpanImpl,
8048
+ TRACEPARENT_HEADER,
8049
+ TRACESTATE_HEADER,
7654
8050
  TestBackgroundLogger,
7655
8051
  ToolBuilder,
7656
8052
  UUIDGenerator,
@@ -7660,6 +8056,7 @@ export {
7660
8056
  isomorph_default as _internalIso,
7661
8057
  _internalSetInitialState,
7662
8058
  addAzureBlobHeaders,
8059
+ agentAssertionScorer,
7663
8060
  braintrustAISDKTelemetry,
7664
8061
  braintrustFlueInstrumentation,
7665
8062
  braintrustFlueObserver,
@@ -7678,6 +8075,7 @@ export {
7678
8075
  devNullWritableStream,
7679
8076
  evaluatorDefinitionSchema,
7680
8077
  evaluatorDefinitionsSchema,
8078
+ extractTraceContextFromHeaders,
7681
8079
  flush,
7682
8080
  getContextManager,
7683
8081
  getIdGenerator,
@@ -7691,6 +8089,7 @@ export {
7691
8089
  initFunction,
7692
8090
  initLogger,
7693
8091
  initNodeTestSuite,
8092
+ injectTraceContext,
7694
8093
  invoke,
7695
8094
  isTemplateFormat,
7696
8095
  loadParameters,