braintrust 2.2.1-rc.0 → 2.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dev/dist/index.d.mts +1 -1
- package/dev/dist/index.d.ts +1 -1
- package/dev/dist/index.js +7 -7
- package/dev/dist/index.mjs +13 -13
- package/dist/browser.d.mts +239 -239
- package/dist/browser.d.ts +239 -239
- package/dist/browser.js +220 -57
- package/dist/browser.mjs +192 -29
- package/dist/cli.js +1214 -718
- package/dist/index.d.mts +239 -239
- package/dist/index.d.ts +239 -239
- package/dist/index.js +220 -57
- package/dist/index.mjs +198 -35
- package/package.json +23 -23
- package/util/dist/index.d.mts +3 -2
- package/util/dist/index.d.ts +3 -2
- package/util/dist/index.js +3 -1
- package/util/dist/index.mjs +3 -1
- package/LICENSE +0 -201
- package/dist/eval-runner.js +0 -12680
package/dist/index.mjs
CHANGED
|
@@ -5,12 +5,12 @@ var __export = (target, all) => {
|
|
|
5
5
|
};
|
|
6
6
|
|
|
7
7
|
// src/node.ts
|
|
8
|
-
import { AsyncLocalStorage } from "async_hooks";
|
|
9
|
-
import * as path from "path";
|
|
10
|
-
import * as fs from "fs/promises";
|
|
11
|
-
import * as os from "os";
|
|
12
|
-
import * as fsSync from "fs";
|
|
13
|
-
import * as crypto from "crypto";
|
|
8
|
+
import { AsyncLocalStorage } from "node:async_hooks";
|
|
9
|
+
import * as path from "node:path";
|
|
10
|
+
import * as fs from "node:fs/promises";
|
|
11
|
+
import * as os from "node:os";
|
|
12
|
+
import * as fsSync from "node:fs";
|
|
13
|
+
import * as crypto from "node:crypto";
|
|
14
14
|
|
|
15
15
|
// src/isomorph.ts
|
|
16
16
|
var DefaultAsyncLocalStorage = class {
|
|
@@ -2150,7 +2150,8 @@ var SpanType = z6.union([
|
|
|
2150
2150
|
"automation",
|
|
2151
2151
|
"facet",
|
|
2152
2152
|
"preprocessor",
|
|
2153
|
-
"classifier"
|
|
2153
|
+
"classifier",
|
|
2154
|
+
"review"
|
|
2154
2155
|
]),
|
|
2155
2156
|
z6.null()
|
|
2156
2157
|
]);
|
|
@@ -12922,14 +12923,17 @@ function parseToolName(rawToolName) {
|
|
|
12922
12923
|
rawToolName
|
|
12923
12924
|
};
|
|
12924
12925
|
}
|
|
12925
|
-
function createToolTracingHooks(
|
|
12926
|
+
function createToolTracingHooks(resolveParentSpan, activeToolSpans, mcpServers, subAgentSpans, endedSubAgentSpans) {
|
|
12926
12927
|
const preToolUse = async (input, toolUseID) => {
|
|
12927
12928
|
if (input.hook_event_name !== "PreToolUse" || !toolUseID) {
|
|
12928
12929
|
return {};
|
|
12929
12930
|
}
|
|
12931
|
+
if (input.tool_name === "Task") {
|
|
12932
|
+
return {};
|
|
12933
|
+
}
|
|
12930
12934
|
const parsed = parseToolName(input.tool_name);
|
|
12931
12935
|
const mcpMetadata = getMcpServerMetadata(parsed.mcpServer, mcpServers);
|
|
12932
|
-
const parentExport = await
|
|
12936
|
+
const parentExport = await resolveParentSpan(toolUseID);
|
|
12933
12937
|
const toolSpan = startSpan({
|
|
12934
12938
|
name: parsed.displayName,
|
|
12935
12939
|
spanAttributes: { type: "tool" /* TOOL */ },
|
|
@@ -12957,6 +12961,30 @@ function createToolTracingHooks(parentSpanExportPromise, activeToolSpans, mcpSer
|
|
|
12957
12961
|
if (input.hook_event_name !== "PostToolUse" || !toolUseID) {
|
|
12958
12962
|
return {};
|
|
12959
12963
|
}
|
|
12964
|
+
const subAgentSpan = subAgentSpans.get(toolUseID);
|
|
12965
|
+
if (subAgentSpan) {
|
|
12966
|
+
try {
|
|
12967
|
+
const response = input.tool_response;
|
|
12968
|
+
const metadata = {};
|
|
12969
|
+
if (response?.status) {
|
|
12970
|
+
metadata["claude_agent_sdk.status"] = response.status;
|
|
12971
|
+
}
|
|
12972
|
+
if (response?.totalDurationMs) {
|
|
12973
|
+
metadata["claude_agent_sdk.duration_ms"] = response.totalDurationMs;
|
|
12974
|
+
}
|
|
12975
|
+
if (response?.totalToolUseCount !== void 0) {
|
|
12976
|
+
metadata["claude_agent_sdk.tool_use_count"] = response.totalToolUseCount;
|
|
12977
|
+
}
|
|
12978
|
+
subAgentSpan.log({
|
|
12979
|
+
output: response?.content,
|
|
12980
|
+
metadata
|
|
12981
|
+
});
|
|
12982
|
+
} finally {
|
|
12983
|
+
subAgentSpan.end();
|
|
12984
|
+
endedSubAgentSpans.add(toolUseID);
|
|
12985
|
+
}
|
|
12986
|
+
return {};
|
|
12987
|
+
}
|
|
12960
12988
|
const toolSpan = activeToolSpans.get(toolUseID);
|
|
12961
12989
|
if (!toolSpan) {
|
|
12962
12990
|
return {};
|
|
@@ -12973,6 +13001,16 @@ function createToolTracingHooks(parentSpanExportPromise, activeToolSpans, mcpSer
|
|
|
12973
13001
|
if (input.hook_event_name !== "PostToolUseFailure" || !toolUseID) {
|
|
12974
13002
|
return {};
|
|
12975
13003
|
}
|
|
13004
|
+
const subAgentSpan = subAgentSpans.get(toolUseID);
|
|
13005
|
+
if (subAgentSpan) {
|
|
13006
|
+
try {
|
|
13007
|
+
subAgentSpan.log({ error: input.error });
|
|
13008
|
+
} finally {
|
|
13009
|
+
subAgentSpan.end();
|
|
13010
|
+
endedSubAgentSpans.add(toolUseID);
|
|
13011
|
+
}
|
|
13012
|
+
return {};
|
|
13013
|
+
}
|
|
12976
13014
|
const toolSpan = activeToolSpans.get(toolUseID);
|
|
12977
13015
|
if (!toolSpan) {
|
|
12978
13016
|
return {};
|
|
@@ -12997,12 +13035,14 @@ function createToolTracingHooks(parentSpanExportPromise, activeToolSpans, mcpSer
|
|
|
12997
13035
|
};
|
|
12998
13036
|
return { preToolUse, postToolUse, postToolUseFailure };
|
|
12999
13037
|
}
|
|
13000
|
-
function injectTracingHooks(options,
|
|
13038
|
+
function injectTracingHooks(options, resolveParentSpan, activeToolSpans, subAgentSpans, endedSubAgentSpans) {
|
|
13001
13039
|
const mcpServers = options.mcpServers;
|
|
13002
13040
|
const { preToolUse, postToolUse, postToolUseFailure } = createToolTracingHooks(
|
|
13003
|
-
|
|
13041
|
+
resolveParentSpan,
|
|
13004
13042
|
activeToolSpans,
|
|
13005
|
-
mcpServers
|
|
13043
|
+
mcpServers,
|
|
13044
|
+
subAgentSpans,
|
|
13045
|
+
endedSubAgentSpans
|
|
13006
13046
|
);
|
|
13007
13047
|
const existingHooks = options.hooks ?? {};
|
|
13008
13048
|
return {
|
|
@@ -13048,18 +13088,45 @@ function filterSerializableOptions(options) {
|
|
|
13048
13088
|
}
|
|
13049
13089
|
return filtered;
|
|
13050
13090
|
}
|
|
13091
|
+
function isAsyncIterable(value) {
|
|
13092
|
+
return value !== null && value !== void 0 && typeof value[Symbol.asyncIterator] === "function";
|
|
13093
|
+
}
|
|
13051
13094
|
function wrapClaudeAgentQuery(queryFn, defaultThis) {
|
|
13052
13095
|
const proxy = new Proxy(queryFn, {
|
|
13053
13096
|
apply(target, thisArg, argArray) {
|
|
13054
13097
|
const params = argArray[0] ?? {};
|
|
13055
13098
|
const { prompt, options = {} } = params;
|
|
13099
|
+
const promptIsAsyncIterable = isAsyncIterable(prompt);
|
|
13100
|
+
let capturedPromptMessages;
|
|
13101
|
+
let promptForQuery = prompt;
|
|
13102
|
+
let promptStarted = false;
|
|
13103
|
+
let resolvePromptDone;
|
|
13104
|
+
const promptDone = new Promise((resolve) => {
|
|
13105
|
+
resolvePromptDone = resolve;
|
|
13106
|
+
});
|
|
13107
|
+
if (promptIsAsyncIterable) {
|
|
13108
|
+
capturedPromptMessages = [];
|
|
13109
|
+
const originalPrompt = prompt;
|
|
13110
|
+
const capturingPrompt = (async function* () {
|
|
13111
|
+
promptStarted = true;
|
|
13112
|
+
try {
|
|
13113
|
+
for await (const msg of originalPrompt) {
|
|
13114
|
+
capturedPromptMessages.push(msg);
|
|
13115
|
+
yield msg;
|
|
13116
|
+
}
|
|
13117
|
+
} finally {
|
|
13118
|
+
resolvePromptDone?.();
|
|
13119
|
+
}
|
|
13120
|
+
})();
|
|
13121
|
+
promptForQuery = capturingPrompt;
|
|
13122
|
+
}
|
|
13056
13123
|
const span = startSpan({
|
|
13057
13124
|
name: "Claude Agent",
|
|
13058
13125
|
spanAttributes: {
|
|
13059
13126
|
type: "task" /* TASK */
|
|
13060
13127
|
},
|
|
13061
13128
|
event: {
|
|
13062
|
-
input: typeof prompt === "string" ? prompt :
|
|
13129
|
+
input: typeof prompt === "string" ? prompt : promptIsAsyncIterable ? void 0 : prompt !== void 0 ? String(prompt) : void 0,
|
|
13063
13130
|
metadata: filterSerializableOptions(options)
|
|
13064
13131
|
}
|
|
13065
13132
|
});
|
|
@@ -13070,13 +13137,22 @@ function wrapClaudeAgentQuery(queryFn, defaultThis) {
|
|
|
13070
13137
|
let currentMessageStartTime = getCurrentUnixTimestamp();
|
|
13071
13138
|
const currentMessages = [];
|
|
13072
13139
|
const createLLMSpan = async () => {
|
|
13140
|
+
const parentToolUseId = currentMessages[0]?.parent_tool_use_id ?? null;
|
|
13141
|
+
let parentSpanExport;
|
|
13142
|
+
if (parentToolUseId) {
|
|
13143
|
+
const subAgentSpan = subAgentSpans.get(parentToolUseId);
|
|
13144
|
+
parentSpanExport = subAgentSpan ? await subAgentSpan.export() : await span.export();
|
|
13145
|
+
} else {
|
|
13146
|
+
parentSpanExport = await span.export();
|
|
13147
|
+
}
|
|
13073
13148
|
const finalMessageContent = await _createLLMSpanForMessages(
|
|
13074
13149
|
currentMessages,
|
|
13075
13150
|
prompt,
|
|
13076
13151
|
finalResults,
|
|
13077
13152
|
options,
|
|
13078
13153
|
currentMessageStartTime,
|
|
13079
|
-
|
|
13154
|
+
capturedPromptMessages,
|
|
13155
|
+
parentSpanExport
|
|
13080
13156
|
);
|
|
13081
13157
|
if (finalMessageContent) {
|
|
13082
13158
|
finalResults.push(finalMessageContent);
|
|
@@ -13090,12 +13166,34 @@ function wrapClaudeAgentQuery(queryFn, defaultThis) {
|
|
|
13090
13166
|
};
|
|
13091
13167
|
const invocationTarget = thisArg === proxy || thisArg === void 0 ? defaultThis ?? thisArg : thisArg;
|
|
13092
13168
|
const activeToolSpans = /* @__PURE__ */ new Map();
|
|
13169
|
+
const subAgentSpans = /* @__PURE__ */ new Map();
|
|
13170
|
+
const endedSubAgentSpans = /* @__PURE__ */ new Set();
|
|
13171
|
+
const toolUseToParent = /* @__PURE__ */ new Map();
|
|
13172
|
+
const pendingSubAgentNames = /* @__PURE__ */ new Map();
|
|
13173
|
+
const resolveParentSpan = async (toolUseID) => {
|
|
13174
|
+
const parentToolUseId = toolUseToParent.get(toolUseID);
|
|
13175
|
+
if (parentToolUseId) {
|
|
13176
|
+
const subAgentSpan = subAgentSpans.get(parentToolUseId);
|
|
13177
|
+
if (subAgentSpan) {
|
|
13178
|
+
return subAgentSpan.export();
|
|
13179
|
+
}
|
|
13180
|
+
}
|
|
13181
|
+
return span.export();
|
|
13182
|
+
};
|
|
13093
13183
|
const optionsWithHooks = injectTracingHooks(
|
|
13094
13184
|
options,
|
|
13095
|
-
|
|
13096
|
-
activeToolSpans
|
|
13185
|
+
resolveParentSpan,
|
|
13186
|
+
activeToolSpans,
|
|
13187
|
+
subAgentSpans,
|
|
13188
|
+
endedSubAgentSpans
|
|
13097
13189
|
);
|
|
13098
|
-
const modifiedArgArray = [
|
|
13190
|
+
const modifiedArgArray = [
|
|
13191
|
+
{
|
|
13192
|
+
...params,
|
|
13193
|
+
...promptForQuery !== void 0 ? { prompt: promptForQuery } : {},
|
|
13194
|
+
options: optionsWithHooks
|
|
13195
|
+
}
|
|
13196
|
+
];
|
|
13099
13197
|
const originalGenerator = withCurrent(
|
|
13100
13198
|
span,
|
|
13101
13199
|
() => Reflect.apply(target, invocationTarget, modifiedArgArray)
|
|
@@ -13104,6 +13202,41 @@ function wrapClaudeAgentQuery(queryFn, defaultThis) {
|
|
|
13104
13202
|
try {
|
|
13105
13203
|
for await (const message of originalGenerator) {
|
|
13106
13204
|
const currentTime = getCurrentUnixTimestamp();
|
|
13205
|
+
if (message.type === "assistant" && Array.isArray(message.message?.content)) {
|
|
13206
|
+
const parentToolUseId = message.parent_tool_use_id ?? null;
|
|
13207
|
+
for (const block of message.message.content) {
|
|
13208
|
+
if (block.type === "tool_use" && block.id) {
|
|
13209
|
+
toolUseToParent.set(block.id, parentToolUseId);
|
|
13210
|
+
if (block.name === "Task" && block.input?.subagent_type) {
|
|
13211
|
+
pendingSubAgentNames.set(
|
|
13212
|
+
block.id,
|
|
13213
|
+
block.input.subagent_type
|
|
13214
|
+
);
|
|
13215
|
+
}
|
|
13216
|
+
}
|
|
13217
|
+
}
|
|
13218
|
+
}
|
|
13219
|
+
if ("parent_tool_use_id" in message) {
|
|
13220
|
+
const parentToolUseId = message.parent_tool_use_id;
|
|
13221
|
+
if (parentToolUseId && !subAgentSpans.has(parentToolUseId)) {
|
|
13222
|
+
const agentName = pendingSubAgentNames.get(parentToolUseId);
|
|
13223
|
+
const spanName = agentName ? `Agent: ${agentName}` : "Agent: sub-agent";
|
|
13224
|
+
const parentExport = await span.export();
|
|
13225
|
+
const subAgentSpan = startSpan({
|
|
13226
|
+
name: spanName,
|
|
13227
|
+
spanAttributes: { type: "task" /* TASK */ },
|
|
13228
|
+
event: {
|
|
13229
|
+
metadata: {
|
|
13230
|
+
...agentName && {
|
|
13231
|
+
"claude_agent_sdk.agent_type": agentName
|
|
13232
|
+
}
|
|
13233
|
+
}
|
|
13234
|
+
},
|
|
13235
|
+
parent: parentExport
|
|
13236
|
+
});
|
|
13237
|
+
subAgentSpans.set(parentToolUseId, subAgentSpan);
|
|
13238
|
+
}
|
|
13239
|
+
}
|
|
13107
13240
|
const messageId = message.message?.id;
|
|
13108
13241
|
if (messageId && messageId !== currentMessageId) {
|
|
13109
13242
|
await createLLMSpan();
|
|
@@ -13149,6 +13282,22 @@ function wrapClaudeAgentQuery(queryFn, defaultThis) {
|
|
|
13149
13282
|
});
|
|
13150
13283
|
throw error;
|
|
13151
13284
|
} finally {
|
|
13285
|
+
for (const [id, subSpan] of subAgentSpans) {
|
|
13286
|
+
if (!endedSubAgentSpans.has(id)) {
|
|
13287
|
+
subSpan.end();
|
|
13288
|
+
}
|
|
13289
|
+
}
|
|
13290
|
+
subAgentSpans.clear();
|
|
13291
|
+
if (capturedPromptMessages) {
|
|
13292
|
+
if (promptStarted) {
|
|
13293
|
+
await promptDone;
|
|
13294
|
+
}
|
|
13295
|
+
if (capturedPromptMessages.length > 0) {
|
|
13296
|
+
span.log({
|
|
13297
|
+
input: _formatCapturedMessages(capturedPromptMessages)
|
|
13298
|
+
});
|
|
13299
|
+
}
|
|
13300
|
+
}
|
|
13152
13301
|
span.end();
|
|
13153
13302
|
}
|
|
13154
13303
|
})();
|
|
@@ -13176,14 +13325,25 @@ function wrapClaudeAgentQuery(queryFn, defaultThis) {
|
|
|
13176
13325
|
});
|
|
13177
13326
|
return proxy;
|
|
13178
13327
|
}
|
|
13179
|
-
function _buildLLMInput(prompt, conversationHistory) {
|
|
13180
|
-
const
|
|
13181
|
-
|
|
13182
|
-
|
|
13183
|
-
|
|
13184
|
-
|
|
13328
|
+
function _buildLLMInput(prompt, conversationHistory, capturedPromptMessages) {
|
|
13329
|
+
const promptMessages = [];
|
|
13330
|
+
if (typeof prompt === "string") {
|
|
13331
|
+
promptMessages.push({ content: prompt, role: "user" });
|
|
13332
|
+
} else if (capturedPromptMessages && capturedPromptMessages.length > 0) {
|
|
13333
|
+
for (const msg of capturedPromptMessages) {
|
|
13334
|
+
const role = msg.message?.role;
|
|
13335
|
+
const content = msg.message?.content;
|
|
13336
|
+
if (role && content !== void 0) {
|
|
13337
|
+
promptMessages.push({ content, role });
|
|
13338
|
+
}
|
|
13339
|
+
}
|
|
13340
|
+
}
|
|
13341
|
+
const inputParts = [...promptMessages, ...conversationHistory];
|
|
13185
13342
|
return inputParts.length > 0 ? inputParts : void 0;
|
|
13186
13343
|
}
|
|
13344
|
+
function _formatCapturedMessages(messages) {
|
|
13345
|
+
return messages.length > 0 ? messages : [];
|
|
13346
|
+
}
|
|
13187
13347
|
function _extractUsageFromMessage(message) {
|
|
13188
13348
|
const metrics = {};
|
|
13189
13349
|
let usage;
|
|
@@ -13217,7 +13377,7 @@ function _extractUsageFromMessage(message) {
|
|
|
13217
13377
|
}
|
|
13218
13378
|
return metrics;
|
|
13219
13379
|
}
|
|
13220
|
-
async function _createLLMSpanForMessages(messages, prompt, conversationHistory, options, startTime, parentSpan) {
|
|
13380
|
+
async function _createLLMSpanForMessages(messages, prompt, conversationHistory, options, startTime, capturedPromptMessages, parentSpan) {
|
|
13221
13381
|
if (messages.length === 0) return void 0;
|
|
13222
13382
|
const lastMessage = messages[messages.length - 1];
|
|
13223
13383
|
if (lastMessage.type !== "assistant" || !lastMessage.message?.usage) {
|
|
@@ -13225,7 +13385,11 @@ async function _createLLMSpanForMessages(messages, prompt, conversationHistory,
|
|
|
13225
13385
|
}
|
|
13226
13386
|
const model = lastMessage.message.model || options.model;
|
|
13227
13387
|
const usage = _extractUsageFromMessage(lastMessage);
|
|
13228
|
-
const input = _buildLLMInput(
|
|
13388
|
+
const input = _buildLLMInput(
|
|
13389
|
+
prompt,
|
|
13390
|
+
conversationHistory,
|
|
13391
|
+
capturedPromptMessages
|
|
13392
|
+
);
|
|
13229
13393
|
const outputs = messages.map(
|
|
13230
13394
|
(m) => m.message?.content && m.message?.role ? { content: m.message.content, role: m.message.role } : void 0
|
|
13231
13395
|
).filter((c) => c !== void 0);
|
|
@@ -13982,7 +14146,7 @@ function unescapePath(path2) {
|
|
|
13982
14146
|
}
|
|
13983
14147
|
var graph_framework_default = { createGraph };
|
|
13984
14148
|
|
|
13985
|
-
// ../node_modules
|
|
14149
|
+
// ../node_modules/async/dist/async.mjs
|
|
13986
14150
|
function initialParams(fn) {
|
|
13987
14151
|
return function(...args) {
|
|
13988
14152
|
var callback = args.pop();
|
|
@@ -14053,7 +14217,7 @@ function isAsync(fn) {
|
|
|
14053
14217
|
function isAsyncGenerator2(fn) {
|
|
14054
14218
|
return fn[Symbol.toStringTag] === "AsyncGenerator";
|
|
14055
14219
|
}
|
|
14056
|
-
function
|
|
14220
|
+
function isAsyncIterable2(obj) {
|
|
14057
14221
|
return typeof obj[Symbol.asyncIterator] === "function";
|
|
14058
14222
|
}
|
|
14059
14223
|
function wrapAsync(asyncFn) {
|
|
@@ -14107,7 +14271,6 @@ function isArrayLike(value) {
|
|
|
14107
14271
|
return value && typeof value.length === "number" && value.length >= 0 && value.length % 1 === 0;
|
|
14108
14272
|
}
|
|
14109
14273
|
var breakLoop = {};
|
|
14110
|
-
var breakLoop$1 = breakLoop;
|
|
14111
14274
|
function once(fn) {
|
|
14112
14275
|
function wrapper(...args) {
|
|
14113
14276
|
if (fn === null) return;
|
|
@@ -14199,7 +14362,7 @@ function asyncEachOfLimit(generator, limit, iteratee, callback) {
|
|
|
14199
14362
|
canceled = true;
|
|
14200
14363
|
return;
|
|
14201
14364
|
}
|
|
14202
|
-
if (result === breakLoop
|
|
14365
|
+
if (result === breakLoop || done && running <= 0) {
|
|
14203
14366
|
done = true;
|
|
14204
14367
|
return callback(null);
|
|
14205
14368
|
}
|
|
@@ -14225,7 +14388,7 @@ var eachOfLimit$2 = (limit) => {
|
|
|
14225
14388
|
if (isAsyncGenerator2(obj)) {
|
|
14226
14389
|
return asyncEachOfLimit(obj, limit, iteratee, callback);
|
|
14227
14390
|
}
|
|
14228
|
-
if (
|
|
14391
|
+
if (isAsyncIterable2(obj)) {
|
|
14229
14392
|
return asyncEachOfLimit(obj[Symbol.asyncIterator](), limit, iteratee, callback);
|
|
14230
14393
|
}
|
|
14231
14394
|
var nextElem = createIterator(obj);
|
|
@@ -14242,7 +14405,7 @@ var eachOfLimit$2 = (limit) => {
|
|
|
14242
14405
|
} else if (err === false) {
|
|
14243
14406
|
done = true;
|
|
14244
14407
|
canceled = true;
|
|
14245
|
-
} else if (value === breakLoop
|
|
14408
|
+
} else if (value === breakLoop || done && running <= 0) {
|
|
14246
14409
|
done = true;
|
|
14247
14410
|
return callback(null);
|
|
14248
14411
|
} else if (!looping) {
|
|
@@ -14285,7 +14448,7 @@ function eachOfArrayLike(coll, iteratee, callback) {
|
|
|
14285
14448
|
if (canceled === true) return;
|
|
14286
14449
|
if (err) {
|
|
14287
14450
|
callback(err);
|
|
14288
|
-
} else if (++completed === length || value === breakLoop
|
|
14451
|
+
} else if (++completed === length || value === breakLoop) {
|
|
14289
14452
|
callback(null);
|
|
14290
14453
|
}
|
|
14291
14454
|
}
|
|
@@ -14681,7 +14844,7 @@ function _createTester(check, getResult) {
|
|
|
14681
14844
|
if (check(result) && !testResult) {
|
|
14682
14845
|
testPassed = true;
|
|
14683
14846
|
testResult = getResult(true, value);
|
|
14684
|
-
return callback(null, breakLoop
|
|
14847
|
+
return callback(null, breakLoop);
|
|
14685
14848
|
}
|
|
14686
14849
|
callback();
|
|
14687
14850
|
});
|
|
@@ -15459,7 +15622,7 @@ function callEvaluatorData(data) {
|
|
|
15459
15622
|
baseExperiment
|
|
15460
15623
|
};
|
|
15461
15624
|
}
|
|
15462
|
-
function
|
|
15625
|
+
function isAsyncIterable3(value) {
|
|
15463
15626
|
return typeof value === "object" && value !== null && typeof value[Symbol.asyncIterator] === "function";
|
|
15464
15627
|
}
|
|
15465
15628
|
function isIterable(value) {
|
|
@@ -15680,7 +15843,7 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
|
|
|
15680
15843
|
}
|
|
15681
15844
|
const resolvedDataResult = dataResult instanceof Promise ? await dataResult : dataResult;
|
|
15682
15845
|
const dataIterable = (() => {
|
|
15683
|
-
if (
|
|
15846
|
+
if (isAsyncIterable3(resolvedDataResult)) {
|
|
15684
15847
|
return resolvedDataResult;
|
|
15685
15848
|
}
|
|
15686
15849
|
if (Array.isArray(resolvedDataResult) || isIterable(resolvedDataResult)) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "braintrust",
|
|
3
|
-
"version": "2.2.1
|
|
3
|
+
"version": "2.2.1",
|
|
4
4
|
"description": "SDK for integrating Braintrust",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -51,6 +51,27 @@
|
|
|
51
51
|
"dev/dist/**/*",
|
|
52
52
|
"util/dist/**/*"
|
|
53
53
|
],
|
|
54
|
+
"scripts": {
|
|
55
|
+
"build": "cross-env NODE_OPTIONS=\"--max-old-space-size=8192\" tsup",
|
|
56
|
+
"watch": "tsup --watch",
|
|
57
|
+
"clean": "rm -r dist/* && rm -r dev/dist/*",
|
|
58
|
+
"docs": "npx typedoc --options typedoc.json src/index.ts",
|
|
59
|
+
"test": "vitest run --exclude \"src/wrappers/**/*.test.ts\" --exclude \"src/otel/**/*.test.ts\" --exclude \"smoke/**/*.test.ts\" --exclude \"src/zod/**/*.test.ts\" --exclude \"tests/api-compatibility/**\"",
|
|
60
|
+
"test:api-compat": "vitest run tests/api-compatibility/api-compatibility.test.ts",
|
|
61
|
+
"test:anthropic": "vitest run src/wrappers/anthropic.test.ts",
|
|
62
|
+
"test:openai": "vitest run src/wrappers/oai.test.ts",
|
|
63
|
+
"test:otel": "vitest run --dir src/otel",
|
|
64
|
+
"test:otel-no-deps": "vitest run src/otel/otel-no-deps.test.ts --reporter=verbose",
|
|
65
|
+
"test:google-genai": "vitest run src/wrappers/google-genai.test.ts",
|
|
66
|
+
"test:ai-sdk-v1": "vitest run src/wrappers/ai-sdk-v1.test.ts",
|
|
67
|
+
"test:ai-sdk-v2": "vitest run src/wrappers/ai-sdk-v2.test.ts src/wrappers/ai-sdk-v1.test.ts",
|
|
68
|
+
"test:ai-sdk-v3": "vitest run src/wrappers/ai-sdk-v3.test.ts",
|
|
69
|
+
"test:zod-v3": "vitest run src/zod/zod-v3-serialization.test.ts",
|
|
70
|
+
"test:zod-v4": "vitest run src/zod/zod-v4-serialization.test.ts",
|
|
71
|
+
"test:mastra": "vitest run src/wrappers/mastra.test.ts",
|
|
72
|
+
"test:claude-agent-sdk": "vitest run src/wrappers/claude-agent-sdk.test.ts",
|
|
73
|
+
"test:output": "tsx scripts/test-output.ts --with-comparison --with-metrics --with-progress"
|
|
74
|
+
},
|
|
54
75
|
"author": "",
|
|
55
76
|
"license": "MIT",
|
|
56
77
|
"devDependencies": {
|
|
@@ -123,26 +144,5 @@
|
|
|
123
144
|
},
|
|
124
145
|
"peerDependencies": {
|
|
125
146
|
"zod": "^3.25.34 || ^4.0"
|
|
126
|
-
},
|
|
127
|
-
"scripts": {
|
|
128
|
-
"build": "cross-env NODE_OPTIONS=\"--max-old-space-size=8192\" tsup",
|
|
129
|
-
"watch": "tsup --watch",
|
|
130
|
-
"clean": "rm -r dist/* && rm -r dev/dist/*",
|
|
131
|
-
"docs": "npx typedoc --options typedoc.json src/index.ts",
|
|
132
|
-
"test": "vitest run --exclude \"src/wrappers/**/*.test.ts\" --exclude \"src/otel/**/*.test.ts\" --exclude \"smoke/**/*.test.ts\" --exclude \"src/zod/**/*.test.ts\" --exclude \"tests/api-compatibility/**\"",
|
|
133
|
-
"test:api-compat": "vitest run tests/api-compatibility/api-compatibility.test.ts",
|
|
134
|
-
"test:anthropic": "vitest run src/wrappers/anthropic.test.ts",
|
|
135
|
-
"test:openai": "vitest run src/wrappers/oai.test.ts",
|
|
136
|
-
"test:otel": "vitest run --dir src/otel",
|
|
137
|
-
"test:otel-no-deps": "vitest run src/otel/otel-no-deps.test.ts --reporter=verbose",
|
|
138
|
-
"test:google-genai": "vitest run src/wrappers/google-genai.test.ts",
|
|
139
|
-
"test:ai-sdk-v1": "vitest run src/wrappers/ai-sdk-v1.test.ts",
|
|
140
|
-
"test:ai-sdk-v2": "vitest run src/wrappers/ai-sdk-v2.test.ts src/wrappers/ai-sdk-v1.test.ts",
|
|
141
|
-
"test:ai-sdk-v3": "vitest run src/wrappers/ai-sdk-v3.test.ts",
|
|
142
|
-
"test:zod-v3": "vitest run src/zod/zod-v3-serialization.test.ts",
|
|
143
|
-
"test:zod-v4": "vitest run src/zod/zod-v4-serialization.test.ts",
|
|
144
|
-
"test:mastra": "vitest run src/wrappers/mastra.test.ts",
|
|
145
|
-
"test:claude-agent-sdk": "vitest run src/wrappers/claude-agent-sdk.test.ts",
|
|
146
|
-
"test:output": "tsx scripts/test-output.ts --with-comparison --with-metrics --with-progress"
|
|
147
147
|
}
|
|
148
|
-
}
|
|
148
|
+
}
|
package/util/dist/index.d.mts
CHANGED
|
@@ -5290,7 +5290,7 @@ declare function parseParent(parent: InvokeFunctionType["parent"]): string | und
|
|
|
5290
5290
|
*/
|
|
5291
5291
|
declare function makeScorerPropagatedEvent(parent: string | undefined): Record<string, unknown>;
|
|
5292
5292
|
|
|
5293
|
-
declare const spanTypeAttributeValues: readonly ["llm", "score", "function", "eval", "task", "tool", "automation", "facet", "preprocessor", "classifier"];
|
|
5293
|
+
declare const spanTypeAttributeValues: readonly ["llm", "score", "function", "eval", "task", "tool", "automation", "facet", "preprocessor", "classifier", "review"];
|
|
5294
5294
|
declare enum SpanTypeAttribute {
|
|
5295
5295
|
LLM = "llm",
|
|
5296
5296
|
SCORE = "score",
|
|
@@ -5301,7 +5301,8 @@ declare enum SpanTypeAttribute {
|
|
|
5301
5301
|
AUTOMATION = "automation",
|
|
5302
5302
|
FACET = "facet",
|
|
5303
5303
|
PREPROCESSOR = "preprocessor",
|
|
5304
|
-
CLASSIFIER = "classifier"
|
|
5304
|
+
CLASSIFIER = "classifier",
|
|
5305
|
+
REVIEW = "review"
|
|
5305
5306
|
}
|
|
5306
5307
|
type SpanType = (typeof spanTypeAttributeValues)[number];
|
|
5307
5308
|
declare const spanPurposeAttributeValues: readonly ["scorer"];
|
package/util/dist/index.d.ts
CHANGED
|
@@ -5290,7 +5290,7 @@ declare function parseParent(parent: InvokeFunctionType["parent"]): string | und
|
|
|
5290
5290
|
*/
|
|
5291
5291
|
declare function makeScorerPropagatedEvent(parent: string | undefined): Record<string, unknown>;
|
|
5292
5292
|
|
|
5293
|
-
declare const spanTypeAttributeValues: readonly ["llm", "score", "function", "eval", "task", "tool", "automation", "facet", "preprocessor", "classifier"];
|
|
5293
|
+
declare const spanTypeAttributeValues: readonly ["llm", "score", "function", "eval", "task", "tool", "automation", "facet", "preprocessor", "classifier", "review"];
|
|
5294
5294
|
declare enum SpanTypeAttribute {
|
|
5295
5295
|
LLM = "llm",
|
|
5296
5296
|
SCORE = "score",
|
|
@@ -5301,7 +5301,8 @@ declare enum SpanTypeAttribute {
|
|
|
5301
5301
|
AUTOMATION = "automation",
|
|
5302
5302
|
FACET = "facet",
|
|
5303
5303
|
PREPROCESSOR = "preprocessor",
|
|
5304
|
-
CLASSIFIER = "classifier"
|
|
5304
|
+
CLASSIFIER = "classifier",
|
|
5305
|
+
REVIEW = "review"
|
|
5305
5306
|
}
|
|
5306
5307
|
type SpanType = (typeof spanTypeAttributeValues)[number];
|
|
5307
5308
|
declare const spanPurposeAttributeValues: readonly ["scorer"];
|
package/util/dist/index.js
CHANGED
|
@@ -1226,7 +1226,8 @@ var spanTypeAttributeValues = [
|
|
|
1226
1226
|
"automation",
|
|
1227
1227
|
"facet",
|
|
1228
1228
|
"preprocessor",
|
|
1229
|
-
"classifier"
|
|
1229
|
+
"classifier",
|
|
1230
|
+
"review"
|
|
1230
1231
|
];
|
|
1231
1232
|
var SpanTypeAttribute = /* @__PURE__ */ ((SpanTypeAttribute2) => {
|
|
1232
1233
|
SpanTypeAttribute2["LLM"] = "llm";
|
|
@@ -1239,6 +1240,7 @@ var SpanTypeAttribute = /* @__PURE__ */ ((SpanTypeAttribute2) => {
|
|
|
1239
1240
|
SpanTypeAttribute2["FACET"] = "facet";
|
|
1240
1241
|
SpanTypeAttribute2["PREPROCESSOR"] = "preprocessor";
|
|
1241
1242
|
SpanTypeAttribute2["CLASSIFIER"] = "classifier";
|
|
1243
|
+
SpanTypeAttribute2["REVIEW"] = "review";
|
|
1242
1244
|
return SpanTypeAttribute2;
|
|
1243
1245
|
})(SpanTypeAttribute || {});
|
|
1244
1246
|
var spanPurposeAttributeValues = ["scorer"];
|
package/util/dist/index.mjs
CHANGED
|
@@ -1226,7 +1226,8 @@ var spanTypeAttributeValues = [
|
|
|
1226
1226
|
"automation",
|
|
1227
1227
|
"facet",
|
|
1228
1228
|
"preprocessor",
|
|
1229
|
-
"classifier"
|
|
1229
|
+
"classifier",
|
|
1230
|
+
"review"
|
|
1230
1231
|
];
|
|
1231
1232
|
var SpanTypeAttribute = /* @__PURE__ */ ((SpanTypeAttribute2) => {
|
|
1232
1233
|
SpanTypeAttribute2["LLM"] = "llm";
|
|
@@ -1239,6 +1240,7 @@ var SpanTypeAttribute = /* @__PURE__ */ ((SpanTypeAttribute2) => {
|
|
|
1239
1240
|
SpanTypeAttribute2["FACET"] = "facet";
|
|
1240
1241
|
SpanTypeAttribute2["PREPROCESSOR"] = "preprocessor";
|
|
1241
1242
|
SpanTypeAttribute2["CLASSIFIER"] = "classifier";
|
|
1243
|
+
SpanTypeAttribute2["REVIEW"] = "review";
|
|
1242
1244
|
return SpanTypeAttribute2;
|
|
1243
1245
|
})(SpanTypeAttribute || {});
|
|
1244
1246
|
var spanPurposeAttributeValues = ["scorer"];
|