braintrust 1.0.2 → 1.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dev/dist/index.d.mts +30 -25
- package/dev/dist/index.d.ts +30 -25
- package/dev/dist/index.js +336 -325
- package/dev/dist/index.mjs +259 -248
- package/dist/browser.d.mts +2313 -322
- package/dist/browser.d.ts +2313 -322
- package/dist/browser.js +2819 -198
- package/dist/browser.mjs +2681 -60
- package/dist/cli.js +358 -160
- package/dist/index.d.mts +254 -284
- package/dist/index.d.ts +254 -284
- package/dist/index.js +484 -470
- package/dist/index.mjs +335 -321
- package/package.json +6 -2
package/dist/index.mjs
CHANGED
|
@@ -32,7 +32,9 @@ var iso = {
|
|
|
32
32
|
getCallerLocation: () => void 0,
|
|
33
33
|
newAsyncLocalStorage: () => new DefaultAsyncLocalStorage(),
|
|
34
34
|
processOn: (_0, _1) => {
|
|
35
|
-
}
|
|
35
|
+
},
|
|
36
|
+
basename: (filepath) => filepath.split(/[\\/]/).pop() || filepath,
|
|
37
|
+
writeln: (text) => console.log(text)
|
|
36
38
|
};
|
|
37
39
|
var isomorph_default = iso;
|
|
38
40
|
|
|
@@ -125,7 +127,7 @@ async function getPastNAncestors(n = 1e3, remote = void 0) {
|
|
|
125
127
|
return [];
|
|
126
128
|
}
|
|
127
129
|
const commits = await git.log({ from: ancestor, to: "HEAD", maxCount: n });
|
|
128
|
-
return commits.all.map((c) => c.hash);
|
|
130
|
+
return commits.all.slice(0, n).map((c) => c.hash);
|
|
129
131
|
}
|
|
130
132
|
async function attempt(fn) {
|
|
131
133
|
try {
|
|
@@ -964,11 +966,11 @@ function mergeDictsWithPaths({
|
|
|
964
966
|
function mergeDictsWithPathsHelper({
|
|
965
967
|
mergeInto,
|
|
966
968
|
mergeFrom,
|
|
967
|
-
path:
|
|
969
|
+
path: path2,
|
|
968
970
|
mergePaths
|
|
969
971
|
}) {
|
|
970
972
|
Object.entries(mergeFrom).forEach(([k, mergeFromV]) => {
|
|
971
|
-
const fullPath =
|
|
973
|
+
const fullPath = path2.concat([k]);
|
|
972
974
|
const fullPathSerialized = JSON.stringify(fullPath);
|
|
973
975
|
const mergeIntoV = recordFind(mergeInto, k);
|
|
974
976
|
if (isObject(mergeIntoV) && isObject(mergeFromV) && !mergePaths.has(fullPathSerialized)) {
|
|
@@ -999,9 +1001,9 @@ function mapAt(m, k) {
|
|
|
999
1001
|
function recordFind(m, k) {
|
|
1000
1002
|
return m[k];
|
|
1001
1003
|
}
|
|
1002
|
-
function getObjValueByPath(row,
|
|
1004
|
+
function getObjValueByPath(row, path2) {
|
|
1003
1005
|
let curr = row;
|
|
1004
|
-
for (const p of
|
|
1006
|
+
for (const p of path2) {
|
|
1005
1007
|
if (!isObjectOrArray(curr)) {
|
|
1006
1008
|
return null;
|
|
1007
1009
|
}
|
|
@@ -1276,6 +1278,93 @@ function _urljoin(...parts) {
|
|
|
1276
1278
|
(x, i) => x.replace(/^\//, "").replace(i < parts.length - 1 ? /\/$/ : "", "")
|
|
1277
1279
|
).filter((x) => x.trim() !== "").join("/");
|
|
1278
1280
|
}
|
|
1281
|
+
function slugify(text, options) {
|
|
1282
|
+
if (typeof text !== "string") {
|
|
1283
|
+
throw new Error("slugify: string argument expected");
|
|
1284
|
+
}
|
|
1285
|
+
const charMap = {
|
|
1286
|
+
// Currency and symbols
|
|
1287
|
+
$: "dollar",
|
|
1288
|
+
"%": "percent",
|
|
1289
|
+
"&": "and",
|
|
1290
|
+
// Latin characters
|
|
1291
|
+
\u00C0: "A",
|
|
1292
|
+
\u00C1: "A",
|
|
1293
|
+
\u00C2: "A",
|
|
1294
|
+
\u00C3: "A",
|
|
1295
|
+
\u00C4: "A",
|
|
1296
|
+
\u00C5: "A",
|
|
1297
|
+
\u00C6: "AE",
|
|
1298
|
+
\u00C7: "C",
|
|
1299
|
+
\u00C8: "E",
|
|
1300
|
+
\u00C9: "E",
|
|
1301
|
+
\u00CA: "E",
|
|
1302
|
+
\u00CB: "E",
|
|
1303
|
+
\u00CC: "I",
|
|
1304
|
+
\u00CD: "I",
|
|
1305
|
+
\u00CE: "I",
|
|
1306
|
+
\u00CF: "I",
|
|
1307
|
+
\u00D1: "N",
|
|
1308
|
+
\u00D2: "O",
|
|
1309
|
+
\u00D3: "O",
|
|
1310
|
+
\u00D4: "O",
|
|
1311
|
+
\u00D5: "O",
|
|
1312
|
+
\u00D6: "O",
|
|
1313
|
+
\u00D8: "O",
|
|
1314
|
+
\u00D9: "U",
|
|
1315
|
+
\u00DA: "U",
|
|
1316
|
+
\u00DB: "U",
|
|
1317
|
+
\u00DC: "U",
|
|
1318
|
+
\u00DD: "Y",
|
|
1319
|
+
\u00E0: "a",
|
|
1320
|
+
\u00E1: "a",
|
|
1321
|
+
\u00E2: "a",
|
|
1322
|
+
\u00E3: "a",
|
|
1323
|
+
\u00E4: "a",
|
|
1324
|
+
\u00E5: "a",
|
|
1325
|
+
\u00E6: "ae",
|
|
1326
|
+
\u00E7: "c",
|
|
1327
|
+
\u00E8: "e",
|
|
1328
|
+
\u00E9: "e",
|
|
1329
|
+
\u00EA: "e",
|
|
1330
|
+
\u00EB: "e",
|
|
1331
|
+
\u00EC: "i",
|
|
1332
|
+
\u00ED: "i",
|
|
1333
|
+
\u00EE: "i",
|
|
1334
|
+
\u00EF: "i",
|
|
1335
|
+
\u00F1: "n",
|
|
1336
|
+
\u00F2: "o",
|
|
1337
|
+
\u00F3: "o",
|
|
1338
|
+
\u00F4: "o",
|
|
1339
|
+
\u00F5: "o",
|
|
1340
|
+
\u00F6: "o",
|
|
1341
|
+
\u00F8: "o",
|
|
1342
|
+
\u00F9: "u",
|
|
1343
|
+
\u00FA: "u",
|
|
1344
|
+
\u00FB: "u",
|
|
1345
|
+
\u00FC: "u",
|
|
1346
|
+
\u00FD: "y",
|
|
1347
|
+
\u00FF: "y"
|
|
1348
|
+
};
|
|
1349
|
+
const replacement = "-";
|
|
1350
|
+
const trim = options?.trim !== false;
|
|
1351
|
+
let slug = text.normalize().split("").reduce((result, ch) => {
|
|
1352
|
+
const mapped = charMap[ch] || ch;
|
|
1353
|
+
const appendChar = mapped === replacement ? " " : mapped;
|
|
1354
|
+
return result + appendChar.replace(/[^\w\s$*_+~.()'"!\-:@]+/g, "");
|
|
1355
|
+
}, "");
|
|
1356
|
+
if (options?.strict) {
|
|
1357
|
+
slug = slug.replace(/[^A-Za-z0-9\s]/g, "");
|
|
1358
|
+
}
|
|
1359
|
+
if (trim) {
|
|
1360
|
+
slug = slug.trim();
|
|
1361
|
+
}
|
|
1362
|
+
slug = slug.replace(/\s+/g, replacement);
|
|
1363
|
+
if (options?.lower) {
|
|
1364
|
+
slug = slug.toLowerCase();
|
|
1365
|
+
}
|
|
1366
|
+
return slug;
|
|
1367
|
+
}
|
|
1279
1368
|
|
|
1280
1369
|
// util/span_identifier_v4.ts
|
|
1281
1370
|
import { z as z4 } from "zod/v3";
|
|
@@ -2916,9 +3005,9 @@ var BraintrustStream = class _BraintrustStream {
|
|
|
2916
3005
|
reader.releaseLock();
|
|
2917
3006
|
return { done: true, value: void 0 };
|
|
2918
3007
|
},
|
|
2919
|
-
async throw(
|
|
3008
|
+
async throw(error) {
|
|
2920
3009
|
reader.releaseLock();
|
|
2921
|
-
throw
|
|
3010
|
+
throw error;
|
|
2922
3011
|
}
|
|
2923
3012
|
};
|
|
2924
3013
|
}
|
|
@@ -3232,10 +3321,10 @@ var DiskCache = class {
|
|
|
3232
3321
|
return;
|
|
3233
3322
|
}
|
|
3234
3323
|
const stats = await Promise.all(
|
|
3235
|
-
paths.map(async (
|
|
3236
|
-
const stat2 = await isomorph_default.stat(
|
|
3324
|
+
paths.map(async (path2) => {
|
|
3325
|
+
const stat2 = await isomorph_default.stat(path2);
|
|
3237
3326
|
return {
|
|
3238
|
-
path:
|
|
3327
|
+
path: path2,
|
|
3239
3328
|
mtime: stat2.mtime.getTime()
|
|
3240
3329
|
};
|
|
3241
3330
|
})
|
|
@@ -3376,7 +3465,7 @@ function runCatchFinally(f, catchF, finallyF) {
|
|
|
3376
3465
|
function getCurrentUnixTimestamp() {
|
|
3377
3466
|
return (/* @__PURE__ */ new Date()).getTime() / 1e3;
|
|
3378
3467
|
}
|
|
3379
|
-
function
|
|
3468
|
+
function isEmpty2(a) {
|
|
3380
3469
|
return a === void 0 || a === null;
|
|
3381
3470
|
}
|
|
3382
3471
|
var LazyValue = class {
|
|
@@ -3513,8 +3602,8 @@ var MaskingError = class {
|
|
|
3513
3602
|
function applyMaskingToField(maskingFunction, data, fieldName) {
|
|
3514
3603
|
try {
|
|
3515
3604
|
return maskingFunction(data);
|
|
3516
|
-
} catch (
|
|
3517
|
-
const errorType =
|
|
3605
|
+
} catch (error) {
|
|
3606
|
+
const errorType = error instanceof Error ? error.constructor.name : "Error";
|
|
3518
3607
|
if (fieldName === "scores" || fieldName === "metrics") {
|
|
3519
3608
|
return new MaskingError(fieldName, errorType);
|
|
3520
3609
|
}
|
|
@@ -3801,7 +3890,7 @@ var BraintrustState = class _BraintrustState {
|
|
|
3801
3890
|
const newState = await loginToState({
|
|
3802
3891
|
...this.loginParams,
|
|
3803
3892
|
...Object.fromEntries(
|
|
3804
|
-
Object.entries(loginParams).filter(([k, v]) => !
|
|
3893
|
+
Object.entries(loginParams).filter(([k, v]) => !isEmpty2(v))
|
|
3805
3894
|
)
|
|
3806
3895
|
});
|
|
3807
3896
|
this.copyLoginInfo(newState);
|
|
@@ -3990,9 +4079,9 @@ var HTTPConnection = class _HTTPConnection {
|
|
|
3990
4079
|
this.headers["Authorization"] = `Bearer ${this.token}`;
|
|
3991
4080
|
}
|
|
3992
4081
|
}
|
|
3993
|
-
async get(
|
|
4082
|
+
async get(path2, params = void 0, config) {
|
|
3994
4083
|
const { headers, ...rest } = config || {};
|
|
3995
|
-
const url = new URL(_urljoin(this.base_url,
|
|
4084
|
+
const url = new URL(_urljoin(this.base_url, path2));
|
|
3996
4085
|
url.search = new URLSearchParams(
|
|
3997
4086
|
params ? Object.entries(params).filter(([_, v]) => v !== void 0).flatMap(
|
|
3998
4087
|
([k, v]) => v !== void 0 ? typeof v === "string" ? [[k, v]] : v.map((x) => [k, x]) : []
|
|
@@ -4013,13 +4102,13 @@ var HTTPConnection = class _HTTPConnection {
|
|
|
4013
4102
|
})
|
|
4014
4103
|
);
|
|
4015
4104
|
}
|
|
4016
|
-
async post(
|
|
4105
|
+
async post(path2, params, config) {
|
|
4017
4106
|
const { headers, ...rest } = config || {};
|
|
4018
4107
|
const this_fetch = this.fetch;
|
|
4019
4108
|
const this_base_url = this.base_url;
|
|
4020
4109
|
const this_headers = this.headers;
|
|
4021
4110
|
return await checkResponse(
|
|
4022
|
-
await this_fetch(_urljoin(this_base_url,
|
|
4111
|
+
await this_fetch(_urljoin(this_base_url, path2), {
|
|
4023
4112
|
method: "POST",
|
|
4024
4113
|
headers: {
|
|
4025
4114
|
Accept: "application/json",
|
|
@@ -4171,12 +4260,12 @@ var Attachment = class extends BaseAttachment {
|
|
|
4171
4260
|
signedUrl: z8.string().url(),
|
|
4172
4261
|
headers: z8.record(z8.string())
|
|
4173
4262
|
}).parse(await metadataResponse.json()));
|
|
4174
|
-
} catch (
|
|
4175
|
-
if (
|
|
4176
|
-
const errorStr = JSON.stringify(
|
|
4263
|
+
} catch (error) {
|
|
4264
|
+
if (error instanceof ZodError) {
|
|
4265
|
+
const errorStr = JSON.stringify(error.flatten());
|
|
4177
4266
|
throw new Error(`Invalid response from API server: ${errorStr}`);
|
|
4178
4267
|
}
|
|
4179
|
-
throw
|
|
4268
|
+
throw error;
|
|
4180
4269
|
}
|
|
4181
4270
|
addAzureBlobHeaders(headers, signedUrl);
|
|
4182
4271
|
let objectStoreResponse;
|
|
@@ -4188,13 +4277,13 @@ var Attachment = class extends BaseAttachment {
|
|
|
4188
4277
|
body: data
|
|
4189
4278
|
})
|
|
4190
4279
|
);
|
|
4191
|
-
} catch (
|
|
4192
|
-
if (
|
|
4280
|
+
} catch (error) {
|
|
4281
|
+
if (error instanceof FailedHTTPResponse) {
|
|
4193
4282
|
throw new Error(
|
|
4194
|
-
`Failed to upload attachment to object store: ${
|
|
4283
|
+
`Failed to upload attachment to object store: ${error.status} ${error.text} ${error.data}`
|
|
4195
4284
|
);
|
|
4196
4285
|
}
|
|
4197
|
-
throw
|
|
4286
|
+
throw error;
|
|
4198
4287
|
}
|
|
4199
4288
|
return { signedUrl, metadataResponse, objectStoreResponse };
|
|
4200
4289
|
};
|
|
@@ -4206,9 +4295,9 @@ var Attachment = class extends BaseAttachment {
|
|
|
4206
4295
|
const orgId = state.orgId ?? "";
|
|
4207
4296
|
try {
|
|
4208
4297
|
await doUpload(conn, orgId);
|
|
4209
|
-
} catch (
|
|
4298
|
+
} catch (error) {
|
|
4210
4299
|
status.upload_status = "error";
|
|
4211
|
-
status.error_message =
|
|
4300
|
+
status.error_message = error instanceof Error ? error.message : JSON.stringify(error);
|
|
4212
4301
|
}
|
|
4213
4302
|
const requestParams = {
|
|
4214
4303
|
key: this.reference.key,
|
|
@@ -4407,8 +4496,8 @@ var ReadonlyAttachment = class {
|
|
|
4407
4496
|
}
|
|
4408
4497
|
const objResponse = await fetch(downloadUrl);
|
|
4409
4498
|
if (objResponse.status !== 200) {
|
|
4410
|
-
const
|
|
4411
|
-
throw new Error(`Couldn't download attachment: ${
|
|
4499
|
+
const error = await objResponse.text();
|
|
4500
|
+
throw new Error(`Couldn't download attachment: ${error}`);
|
|
4412
4501
|
}
|
|
4413
4502
|
return await objResponse.blob();
|
|
4414
4503
|
};
|
|
@@ -4466,7 +4555,7 @@ function logFeedbackImpl(state, parentObjectType, parentObjectId, {
|
|
|
4466
4555
|
if (!VALID_SOURCES.includes(source)) {
|
|
4467
4556
|
throw new Error(`source must be one of ${VALID_SOURCES}`);
|
|
4468
4557
|
}
|
|
4469
|
-
if (
|
|
4558
|
+
if (isEmpty2(scores) && isEmpty2(expected) && isEmpty2(tags) && isEmpty2(comment)) {
|
|
4470
4559
|
throw new Error(
|
|
4471
4560
|
"At least one of scores, expected, tags, or comment must be specified"
|
|
4472
4561
|
);
|
|
@@ -4479,7 +4568,7 @@ function logFeedbackImpl(state, parentObjectType, parentObjectId, {
|
|
|
4479
4568
|
});
|
|
4480
4569
|
let { metadata, ...updateEvent } = deepCopyEvent(validatedEvent);
|
|
4481
4570
|
updateEvent = Object.fromEntries(
|
|
4482
|
-
Object.entries(updateEvent).filter(([_, v]) => !
|
|
4571
|
+
Object.entries(updateEvent).filter(([_, v]) => !isEmpty2(v))
|
|
4483
4572
|
);
|
|
4484
4573
|
const parentIds = async () => new SpanComponentsV3({
|
|
4485
4574
|
object_type: parentObjectType,
|
|
@@ -4498,7 +4587,7 @@ function logFeedbackImpl(state, parentObjectType, parentObjectId, {
|
|
|
4498
4587
|
});
|
|
4499
4588
|
state.bgLogger().log([record]);
|
|
4500
4589
|
}
|
|
4501
|
-
if (!
|
|
4590
|
+
if (!isEmpty2(comment)) {
|
|
4502
4591
|
const record = new LazyValue(async () => {
|
|
4503
4592
|
return {
|
|
4504
4593
|
id: uuidv42(),
|
|
@@ -5135,8 +5224,8 @@ var HTTPBackgroundLogger = class _HTTPBackgroundLogger {
|
|
|
5135
5224
|
if (result.upload_status === "error") {
|
|
5136
5225
|
throw new Error(result.error_message);
|
|
5137
5226
|
}
|
|
5138
|
-
} catch (
|
|
5139
|
-
attachmentErrors.push(
|
|
5227
|
+
} catch (error) {
|
|
5228
|
+
attachmentErrors.push(error);
|
|
5140
5229
|
}
|
|
5141
5230
|
}
|
|
5142
5231
|
if (attachmentErrors.length === 1) {
|
|
@@ -5219,22 +5308,22 @@ var HTTPBackgroundLogger = class _HTTPBackgroundLogger {
|
|
|
5219
5308
|
}
|
|
5220
5309
|
for (let i = 0; i < this.numTries; i++) {
|
|
5221
5310
|
const startTime = now();
|
|
5222
|
-
let
|
|
5311
|
+
let error = void 0;
|
|
5223
5312
|
try {
|
|
5224
5313
|
await conn.post_json("logs3", dataStr);
|
|
5225
5314
|
} catch (e) {
|
|
5226
|
-
|
|
5315
|
+
error = e;
|
|
5227
5316
|
}
|
|
5228
|
-
if (
|
|
5317
|
+
if (error === void 0) {
|
|
5229
5318
|
return;
|
|
5230
5319
|
}
|
|
5231
5320
|
const isRetrying = i + 1 < this.numTries;
|
|
5232
5321
|
const retryingText = isRetrying ? "" : " Retrying";
|
|
5233
5322
|
const errorText = (() => {
|
|
5234
|
-
if (
|
|
5235
|
-
return `${
|
|
5323
|
+
if (error instanceof FailedHTTPResponse) {
|
|
5324
|
+
return `${error.status} (${error.text}): ${error.data}`;
|
|
5236
5325
|
} else {
|
|
5237
|
-
return `${
|
|
5326
|
+
return `${error}`;
|
|
5238
5327
|
}
|
|
5239
5328
|
})();
|
|
5240
5329
|
const errMsg = `log request failed. Elapsed time: ${(now() - startTime) / 1e3} seconds. Payload size: ${dataStr.length}.${retryingText}
|
|
@@ -5410,7 +5499,7 @@ function init(projectOrOptions, optionalOptions) {
|
|
|
5410
5499
|
const state = stateArg ?? _globalState;
|
|
5411
5500
|
state.enforceQueueSizeLimit(false);
|
|
5412
5501
|
if (open) {
|
|
5413
|
-
if (
|
|
5502
|
+
if (isEmpty2(experiment)) {
|
|
5414
5503
|
throw new Error(`Cannot open an experiment without specifying its name`);
|
|
5415
5504
|
}
|
|
5416
5505
|
const lazyMetadata2 = new LazyValue(
|
|
@@ -5649,7 +5738,7 @@ async function computeLoggerMetadata(state, {
|
|
|
5649
5738
|
}) {
|
|
5650
5739
|
await state.login({});
|
|
5651
5740
|
const org_id = state.orgId;
|
|
5652
|
-
if (
|
|
5741
|
+
if (isEmpty2(project_id)) {
|
|
5653
5742
|
const response = await state.appConn().post_json("api/project/register", {
|
|
5654
5743
|
project_name: project_name || GLOBAL_PROJECT,
|
|
5655
5744
|
org_id
|
|
@@ -5662,7 +5751,7 @@ async function computeLoggerMetadata(state, {
|
|
|
5662
5751
|
fullInfo: response.project
|
|
5663
5752
|
}
|
|
5664
5753
|
};
|
|
5665
|
-
} else if (
|
|
5754
|
+
} else if (isEmpty2(project_name)) {
|
|
5666
5755
|
const response = await state.appConn().get_json("api/project", {
|
|
5667
5756
|
id: project_id
|
|
5668
5757
|
});
|
|
@@ -5743,9 +5832,9 @@ async function loadPrompt({
|
|
|
5743
5832
|
);
|
|
5744
5833
|
}
|
|
5745
5834
|
if (id) {
|
|
5746
|
-
} else if (
|
|
5835
|
+
} else if (isEmpty2(projectName) && isEmpty2(projectId)) {
|
|
5747
5836
|
throw new Error("Must specify either projectName or projectId");
|
|
5748
|
-
} else if (
|
|
5837
|
+
} else if (isEmpty2(slug)) {
|
|
5749
5838
|
throw new Error("Must specify slug");
|
|
5750
5839
|
}
|
|
5751
5840
|
const state = stateArg ?? _globalState;
|
|
@@ -5847,7 +5936,7 @@ async function login(options = {}) {
|
|
|
5847
5936
|
const { forceLogin = false } = options || {};
|
|
5848
5937
|
if (_globalState.loggedIn && !forceLogin) {
|
|
5849
5938
|
let checkUpdatedParam2 = function(varname, arg, orig) {
|
|
5850
|
-
if (!
|
|
5939
|
+
if (!isEmpty2(arg) && !isEmpty2(orig) && arg !== orig) {
|
|
5851
5940
|
throw new Error(
|
|
5852
5941
|
`Re-logging in with different ${varname} (${arg}) than original (${orig}). To force re-login, pass \`forceLogin: true\``
|
|
5853
5942
|
);
|
|
@@ -5985,14 +6074,14 @@ function getSpanParentObject(options) {
|
|
|
5985
6074
|
}
|
|
5986
6075
|
return NOOP_SPAN;
|
|
5987
6076
|
}
|
|
5988
|
-
function logError(span,
|
|
6077
|
+
function logError(span, error) {
|
|
5989
6078
|
let errorMessage = "<error>";
|
|
5990
6079
|
let stackTrace = "";
|
|
5991
|
-
if (
|
|
5992
|
-
errorMessage =
|
|
5993
|
-
stackTrace =
|
|
6080
|
+
if (error instanceof Error) {
|
|
6081
|
+
errorMessage = error.message;
|
|
6082
|
+
stackTrace = error.stack || "";
|
|
5994
6083
|
} else {
|
|
5995
|
-
errorMessage = String(
|
|
6084
|
+
errorMessage = String(error);
|
|
5996
6085
|
}
|
|
5997
6086
|
span.log({ error: `${errorMessage}
|
|
5998
6087
|
|
|
@@ -6061,12 +6150,12 @@ function wrapTracedSyncGenerator(fn, spanArgs, noTraceIO) {
|
|
|
6061
6150
|
if (!truncated) {
|
|
6062
6151
|
span.log({ output: collected });
|
|
6063
6152
|
}
|
|
6064
|
-
} catch (
|
|
6065
|
-
logError(span,
|
|
6153
|
+
} catch (error) {
|
|
6154
|
+
logError(span, error);
|
|
6066
6155
|
if (!truncated && collected.length > 0) {
|
|
6067
6156
|
span.log({ output: collected });
|
|
6068
6157
|
}
|
|
6069
|
-
throw
|
|
6158
|
+
throw error;
|
|
6070
6159
|
}
|
|
6071
6160
|
} else {
|
|
6072
6161
|
const gen = generatorWithCurrent(span, fn.apply(this, fnArgs));
|
|
@@ -6110,12 +6199,12 @@ function wrapTracedAsyncGenerator(fn, spanArgs, noTraceIO) {
|
|
|
6110
6199
|
if (!truncated) {
|
|
6111
6200
|
span.log({ output: collected });
|
|
6112
6201
|
}
|
|
6113
|
-
} catch (
|
|
6114
|
-
logError(span,
|
|
6202
|
+
} catch (error) {
|
|
6203
|
+
logError(span, error);
|
|
6115
6204
|
if (!truncated && collected.length > 0) {
|
|
6116
6205
|
span.log({ output: collected });
|
|
6117
6206
|
}
|
|
6118
|
-
throw
|
|
6207
|
+
throw error;
|
|
6119
6208
|
}
|
|
6120
6209
|
} else {
|
|
6121
6210
|
const gen = asyncGeneratorWithCurrent(span, fn.apply(this, fnArgs));
|
|
@@ -6456,15 +6545,15 @@ async function resolveAttachmentsToBase64(event, state) {
|
|
|
6456
6545
|
return event;
|
|
6457
6546
|
}
|
|
6458
6547
|
function validateAndSanitizeExperimentLogFullArgs(event, hasDataset) {
|
|
6459
|
-
if ("input" in event && !
|
|
6548
|
+
if ("input" in event && !isEmpty2(event.input) && "inputs" in event && !isEmpty2(event.inputs) || !("input" in event) && !("inputs" in event)) {
|
|
6460
6549
|
throw new Error(
|
|
6461
6550
|
"Exactly one of input or inputs (deprecated) must be specified. Prefer input."
|
|
6462
6551
|
);
|
|
6463
6552
|
}
|
|
6464
|
-
if (
|
|
6553
|
+
if (isEmpty2(event.output)) {
|
|
6465
6554
|
throw new Error("output must be specified");
|
|
6466
6555
|
}
|
|
6467
|
-
if (
|
|
6556
|
+
if (isEmpty2(event.scores)) {
|
|
6468
6557
|
throw new Error("scores must be specified");
|
|
6469
6558
|
}
|
|
6470
6559
|
if (hasDataset && event.datasetRecordId === void 0) {
|
|
@@ -7524,7 +7613,7 @@ function renderMessage(render, message) {
|
|
|
7524
7613
|
return {
|
|
7525
7614
|
...message,
|
|
7526
7615
|
..."content" in message ? {
|
|
7527
|
-
content:
|
|
7616
|
+
content: isEmpty2(message.content) ? void 0 : typeof message.content === "string" ? render(message.content) : message.content.map((c) => {
|
|
7528
7617
|
switch (c.type) {
|
|
7529
7618
|
case "text":
|
|
7530
7619
|
return { ...c, text: render(c.text) };
|
|
@@ -7561,7 +7650,7 @@ function renderMessage(render, message) {
|
|
|
7561
7650
|
})
|
|
7562
7651
|
} : {},
|
|
7563
7652
|
..."tool_calls" in message ? {
|
|
7564
|
-
tool_calls:
|
|
7653
|
+
tool_calls: isEmpty2(message.tool_calls) ? void 0 : message.tool_calls.map((t) => {
|
|
7565
7654
|
return {
|
|
7566
7655
|
type: t.type,
|
|
7567
7656
|
id: render(t.id),
|
|
@@ -7710,11 +7799,11 @@ var Prompt2 = class _Prompt {
|
|
|
7710
7799
|
([k, _v]) => !BRAINTRUST_PARAMS.includes(k)
|
|
7711
7800
|
)
|
|
7712
7801
|
),
|
|
7713
|
-
...!
|
|
7802
|
+
...!isEmpty2(this.options.model) ? {
|
|
7714
7803
|
model: this.options.model
|
|
7715
7804
|
} : {}
|
|
7716
7805
|
};
|
|
7717
|
-
if (!("model" in params) ||
|
|
7806
|
+
if (!("model" in params) || isEmpty2(params.model)) {
|
|
7718
7807
|
throw new Error(
|
|
7719
7808
|
"No model specified. Either specify it in the prompt or as a default"
|
|
7720
7809
|
);
|
|
@@ -7964,6 +8053,8 @@ function configureNode() {
|
|
|
7964
8053
|
isomorph_default.processOn = (event, handler) => {
|
|
7965
8054
|
process.on(event, handler);
|
|
7966
8055
|
};
|
|
8056
|
+
isomorph_default.basename = path.basename;
|
|
8057
|
+
isomorph_default.writeln = (text) => process.stdout.write(text + "\n");
|
|
7967
8058
|
isomorph_default.pathJoin = path.join;
|
|
7968
8059
|
isomorph_default.pathDirname = path.dirname;
|
|
7969
8060
|
isomorph_default.mkdir = fs.mkdir;
|
|
@@ -7981,10 +8072,11 @@ function configureNode() {
|
|
|
7981
8072
|
_internalSetInitialState();
|
|
7982
8073
|
}
|
|
7983
8074
|
|
|
7984
|
-
// src/exports
|
|
7985
|
-
var
|
|
7986
|
-
__export(
|
|
8075
|
+
// src/exports.ts
|
|
8076
|
+
var exports_exports = {};
|
|
8077
|
+
__export(exports_exports, {
|
|
7987
8078
|
Attachment: () => Attachment,
|
|
8079
|
+
AttachmentReference: () => AttachmentReference,
|
|
7988
8080
|
BaseAttachment: () => BaseAttachment,
|
|
7989
8081
|
BaseExperiment: () => BaseExperiment,
|
|
7990
8082
|
BraintrustMiddleware: () => BraintrustMiddleware,
|
|
@@ -8036,6 +8128,8 @@ __export(exports_node_exports, {
|
|
|
8036
8128
|
defaultErrorScoreHandler: () => defaultErrorScoreHandler,
|
|
8037
8129
|
deserializePlainStringAsJSON: () => deserializePlainStringAsJSON,
|
|
8038
8130
|
devNullWritableStream: () => devNullWritableStream,
|
|
8131
|
+
evaluatorDefinitionSchema: () => evaluatorDefinitionSchema,
|
|
8132
|
+
evaluatorDefinitionsSchema: () => evaluatorDefinitionsSchema,
|
|
8039
8133
|
flush: () => flush,
|
|
8040
8134
|
getContextManager: () => getContextManager,
|
|
8041
8135
|
getIdGenerator: () => getIdGenerator,
|
|
@@ -8662,13 +8756,13 @@ function wrapBetaChatCompletionStream(completion) {
|
|
|
8662
8756
|
var LEGACY_CACHED_HEADER = "x-cached";
|
|
8663
8757
|
var X_CACHED_HEADER = "x-bt-cached";
|
|
8664
8758
|
function parseCachedHeader(value) {
|
|
8665
|
-
return
|
|
8759
|
+
return isEmpty2(value) ? void 0 : ["true", "hit"].includes(value.toLowerCase()) ? 1 : 0;
|
|
8666
8760
|
}
|
|
8667
8761
|
function logHeaders(response, span) {
|
|
8668
8762
|
const cachedHeader = response.headers.get(X_CACHED_HEADER);
|
|
8669
|
-
if (
|
|
8763
|
+
if (isEmpty2(cachedHeader)) {
|
|
8670
8764
|
const legacyCacheHeader = response.headers.get(LEGACY_CACHED_HEADER);
|
|
8671
|
-
if (!
|
|
8765
|
+
if (!isEmpty2(legacyCacheHeader)) {
|
|
8672
8766
|
span.log({
|
|
8673
8767
|
metrics: {
|
|
8674
8768
|
cached: parseCachedHeader(legacyCacheHeader)
|
|
@@ -8984,7 +9078,7 @@ function convertDataToBlob(data, mediaType) {
|
|
|
8984
9078
|
} else if (typeof Buffer !== "undefined" && data instanceof Buffer) {
|
|
8985
9079
|
return new Blob([data], { type: mediaType });
|
|
8986
9080
|
}
|
|
8987
|
-
} catch (
|
|
9081
|
+
} catch (error) {
|
|
8988
9082
|
return null;
|
|
8989
9083
|
}
|
|
8990
9084
|
return null;
|
|
@@ -9448,12 +9542,12 @@ var makeStreamTextWrapper = (name, options, streamText, aiSDK) => {
|
|
|
9448
9542
|
});
|
|
9449
9543
|
}
|
|
9450
9544
|
return result;
|
|
9451
|
-
} catch (
|
|
9545
|
+
} catch (error) {
|
|
9452
9546
|
span.log({
|
|
9453
|
-
error: serializeError(
|
|
9547
|
+
error: serializeError(error)
|
|
9454
9548
|
});
|
|
9455
9549
|
span.end();
|
|
9456
|
-
throw
|
|
9550
|
+
throw error;
|
|
9457
9551
|
}
|
|
9458
9552
|
};
|
|
9459
9553
|
Object.defineProperty(wrapper, "name", { value: name, writable: false });
|
|
@@ -9546,12 +9640,12 @@ var wrapStreamObject = (streamObject, options = {}, aiSDK) => {
|
|
|
9546
9640
|
});
|
|
9547
9641
|
}
|
|
9548
9642
|
return result;
|
|
9549
|
-
} catch (
|
|
9643
|
+
} catch (error) {
|
|
9550
9644
|
span.log({
|
|
9551
|
-
error: serializeError(
|
|
9645
|
+
error: serializeError(error)
|
|
9552
9646
|
});
|
|
9553
9647
|
span.end();
|
|
9554
|
-
throw
|
|
9648
|
+
throw error;
|
|
9555
9649
|
}
|
|
9556
9650
|
};
|
|
9557
9651
|
};
|
|
@@ -9633,17 +9727,17 @@ var wrapToolExecute = (tool, name) => {
|
|
|
9633
9727
|
}
|
|
9634
9728
|
return tool;
|
|
9635
9729
|
};
|
|
9636
|
-
var serializeError = (
|
|
9637
|
-
if (
|
|
9638
|
-
return
|
|
9730
|
+
var serializeError = (error) => {
|
|
9731
|
+
if (error instanceof Error) {
|
|
9732
|
+
return error;
|
|
9639
9733
|
}
|
|
9640
|
-
if (typeof
|
|
9734
|
+
if (typeof error === "object" && error !== null) {
|
|
9641
9735
|
try {
|
|
9642
|
-
return JSON.stringify(
|
|
9736
|
+
return JSON.stringify(error);
|
|
9643
9737
|
} catch {
|
|
9644
9738
|
}
|
|
9645
9739
|
}
|
|
9646
|
-
return String(
|
|
9740
|
+
return String(error);
|
|
9647
9741
|
};
|
|
9648
9742
|
var serializeModel = (model) => {
|
|
9649
9743
|
return typeof model === "string" ? model : model?.modelId;
|
|
@@ -9784,8 +9878,8 @@ var processContentPart = (part) => {
|
|
|
9784
9878
|
}
|
|
9785
9879
|
}
|
|
9786
9880
|
}
|
|
9787
|
-
} catch (
|
|
9788
|
-
console.warn("Error processing content part:",
|
|
9881
|
+
} catch (error) {
|
|
9882
|
+
console.warn("Error processing content part:", error);
|
|
9789
9883
|
}
|
|
9790
9884
|
return part;
|
|
9791
9885
|
};
|
|
@@ -9831,8 +9925,8 @@ var convertImageToAttachment = (image, explicitMimeType) => {
|
|
|
9831
9925
|
if (image instanceof Attachment) {
|
|
9832
9926
|
return image;
|
|
9833
9927
|
}
|
|
9834
|
-
} catch (
|
|
9835
|
-
console.warn("Error converting image to attachment:",
|
|
9928
|
+
} catch (error) {
|
|
9929
|
+
console.warn("Error converting image to attachment:", error);
|
|
9836
9930
|
}
|
|
9837
9931
|
return null;
|
|
9838
9932
|
};
|
|
@@ -9861,8 +9955,8 @@ var convertDataToAttachment = (data, mimeType, filename) => {
|
|
|
9861
9955
|
contentType: mimeType
|
|
9862
9956
|
});
|
|
9863
9957
|
}
|
|
9864
|
-
} catch (
|
|
9865
|
-
console.warn("Error converting data to attachment:",
|
|
9958
|
+
} catch (error) {
|
|
9959
|
+
console.warn("Error converting data to attachment:", error);
|
|
9866
9960
|
}
|
|
9867
9961
|
return null;
|
|
9868
9962
|
};
|
|
@@ -9899,8 +9993,8 @@ var processOutput = async (output, denyOutputPaths) => {
|
|
|
9899
9993
|
var processOutputAttachments = async (output) => {
|
|
9900
9994
|
try {
|
|
9901
9995
|
return await doProcessOutputAttachments(output);
|
|
9902
|
-
} catch (
|
|
9903
|
-
console.error("Error processing output attachments:",
|
|
9996
|
+
} catch (error) {
|
|
9997
|
+
console.error("Error processing output attachments:", error);
|
|
9904
9998
|
return output;
|
|
9905
9999
|
}
|
|
9906
10000
|
};
|
|
@@ -9945,8 +10039,8 @@ var convertFileToAttachment = (file, index) => {
|
|
|
9945
10039
|
filename,
|
|
9946
10040
|
contentType: mediaType
|
|
9947
10041
|
});
|
|
9948
|
-
} catch (
|
|
9949
|
-
console.warn(`Error processing file at index ${index}:`,
|
|
10042
|
+
} catch (error) {
|
|
10043
|
+
console.warn(`Error processing file at index ${index}:`, error);
|
|
9950
10044
|
return file;
|
|
9951
10045
|
}
|
|
9952
10046
|
};
|
|
@@ -10002,11 +10096,11 @@ function extractTokenMetrics(result) {
|
|
|
10002
10096
|
var deepCopy = (obj) => {
|
|
10003
10097
|
return JSON.parse(JSON.stringify(obj));
|
|
10004
10098
|
};
|
|
10005
|
-
var parsePath = (
|
|
10099
|
+
var parsePath = (path2) => {
|
|
10006
10100
|
const keys = [];
|
|
10007
10101
|
let current = "";
|
|
10008
|
-
for (let i = 0; i <
|
|
10009
|
-
const char =
|
|
10102
|
+
for (let i = 0; i < path2.length; i++) {
|
|
10103
|
+
const char = path2[i];
|
|
10010
10104
|
if (char === ".") {
|
|
10011
10105
|
if (current) {
|
|
10012
10106
|
keys.push(current);
|
|
@@ -10019,8 +10113,8 @@ var parsePath = (path3) => {
|
|
|
10019
10113
|
}
|
|
10020
10114
|
let bracketContent = "";
|
|
10021
10115
|
i++;
|
|
10022
|
-
while (i <
|
|
10023
|
-
bracketContent +=
|
|
10116
|
+
while (i < path2.length && path2[i] !== "]") {
|
|
10117
|
+
bracketContent += path2[i];
|
|
10024
10118
|
i++;
|
|
10025
10119
|
}
|
|
10026
10120
|
if (bracketContent === "") {
|
|
@@ -10062,8 +10156,8 @@ var omitAtPath = (obj, keys) => {
|
|
|
10062
10156
|
};
|
|
10063
10157
|
var omit = (obj, paths) => {
|
|
10064
10158
|
const result = deepCopy(obj);
|
|
10065
|
-
for (const
|
|
10066
|
-
const keys = parsePath(
|
|
10159
|
+
for (const path2 of paths) {
|
|
10160
|
+
const keys = parsePath(path2);
|
|
10067
10161
|
omitAtPath(result, keys);
|
|
10068
10162
|
}
|
|
10069
10163
|
return result;
|
|
@@ -10128,7 +10222,7 @@ var BraintrustLanguageModelWrapper = class {
|
|
|
10128
10222
|
output: postProcessOutput(ret.text, ret.toolCalls, ret.finishReason),
|
|
10129
10223
|
metrics: {
|
|
10130
10224
|
time_to_first_token: getCurrentUnixTimestamp() - startTime,
|
|
10131
|
-
tokens: !
|
|
10225
|
+
tokens: !isEmpty2(ret.usage) ? ret.usage.promptTokens + ret.usage.completionTokens : void 0,
|
|
10132
10226
|
prompt_tokens: ret.usage?.promptTokens,
|
|
10133
10227
|
completion_tokens: ret.usage?.completionTokens,
|
|
10134
10228
|
cached: parseCachedHeader(
|
|
@@ -10223,7 +10317,7 @@ var BraintrustLanguageModelWrapper = class {
|
|
|
10223
10317
|
),
|
|
10224
10318
|
metrics: {
|
|
10225
10319
|
time_to_first_token,
|
|
10226
|
-
tokens: !
|
|
10320
|
+
tokens: !isEmpty2(usage) ? usage.promptTokens + usage.completionTokens : void 0,
|
|
10227
10321
|
prompt_tokens: usage?.promptTokens,
|
|
10228
10322
|
completion_tokens: usage?.completionTokens,
|
|
10229
10323
|
cached: parseCachedHeader(
|
|
@@ -10560,11 +10654,11 @@ function BraintrustMiddleware(config = {}) {
|
|
|
10560
10654
|
)
|
|
10561
10655
|
});
|
|
10562
10656
|
return result;
|
|
10563
|
-
} catch (
|
|
10657
|
+
} catch (error) {
|
|
10564
10658
|
span.log({
|
|
10565
|
-
error:
|
|
10659
|
+
error: error instanceof Error ? error.message : String(error)
|
|
10566
10660
|
});
|
|
10567
|
-
throw
|
|
10661
|
+
throw error;
|
|
10568
10662
|
} finally {
|
|
10569
10663
|
span.end();
|
|
10570
10664
|
}
|
|
@@ -10609,12 +10703,12 @@ function BraintrustMiddleware(config = {}) {
|
|
|
10609
10703
|
providerMetadata = chunk.providerMetadata || {};
|
|
10610
10704
|
}
|
|
10611
10705
|
controller.enqueue(chunk);
|
|
10612
|
-
} catch (
|
|
10706
|
+
} catch (error) {
|
|
10613
10707
|
span.log({
|
|
10614
|
-
error:
|
|
10708
|
+
error: error instanceof Error ? error.message : String(error)
|
|
10615
10709
|
});
|
|
10616
10710
|
span.end();
|
|
10617
|
-
controller.error(
|
|
10711
|
+
controller.error(error);
|
|
10618
10712
|
}
|
|
10619
10713
|
},
|
|
10620
10714
|
flush() {
|
|
@@ -10660,12 +10754,12 @@ function BraintrustMiddleware(config = {}) {
|
|
|
10660
10754
|
)
|
|
10661
10755
|
});
|
|
10662
10756
|
span.end();
|
|
10663
|
-
} catch (
|
|
10757
|
+
} catch (error) {
|
|
10664
10758
|
span.log({
|
|
10665
|
-
error:
|
|
10759
|
+
error: error instanceof Error ? error.message : String(error)
|
|
10666
10760
|
});
|
|
10667
10761
|
span.end();
|
|
10668
|
-
throw
|
|
10762
|
+
throw error;
|
|
10669
10763
|
}
|
|
10670
10764
|
}
|
|
10671
10765
|
});
|
|
@@ -10673,12 +10767,12 @@ function BraintrustMiddleware(config = {}) {
|
|
|
10673
10767
|
stream: stream.pipeThrough(transformStream),
|
|
10674
10768
|
...rest
|
|
10675
10769
|
};
|
|
10676
|
-
} catch (
|
|
10770
|
+
} catch (error) {
|
|
10677
10771
|
span.log({
|
|
10678
|
-
error:
|
|
10772
|
+
error: error instanceof Error ? error.message : String(error)
|
|
10679
10773
|
});
|
|
10680
10774
|
span.end();
|
|
10681
|
-
throw
|
|
10775
|
+
throw error;
|
|
10682
10776
|
}
|
|
10683
10777
|
}
|
|
10684
10778
|
};
|
|
@@ -10786,8 +10880,8 @@ function apiPromiseProxy2(apiPromise, span, onThen) {
|
|
|
10786
10880
|
try {
|
|
10787
10881
|
const processed = onThen(result);
|
|
10788
10882
|
return onFulfilled ? onFulfilled(processed) : processed;
|
|
10789
|
-
} catch (
|
|
10790
|
-
return onRejected ? onRejected(
|
|
10883
|
+
} catch (error) {
|
|
10884
|
+
return onRejected ? onRejected(error) : Promise.reject(error);
|
|
10791
10885
|
}
|
|
10792
10886
|
},
|
|
10793
10887
|
onRejected
|
|
@@ -11096,11 +11190,11 @@ function wrapClaudeAgentQuery(queryFn, defaultThis) {
|
|
|
11096
11190
|
span.log({
|
|
11097
11191
|
output: finalResults.length > 0 ? finalResults[finalResults.length - 1] : void 0
|
|
11098
11192
|
});
|
|
11099
|
-
} catch (
|
|
11193
|
+
} catch (error) {
|
|
11100
11194
|
span.log({
|
|
11101
|
-
error:
|
|
11195
|
+
error: error instanceof Error ? error.message : String(error)
|
|
11102
11196
|
});
|
|
11103
|
-
throw
|
|
11197
|
+
throw error;
|
|
11104
11198
|
} finally {
|
|
11105
11199
|
span.end();
|
|
11106
11200
|
}
|
|
@@ -11335,11 +11429,11 @@ function wrapGenerateContent(original) {
|
|
|
11335
11429
|
metrics: cleanMetrics(metrics)
|
|
11336
11430
|
});
|
|
11337
11431
|
return result;
|
|
11338
|
-
} catch (
|
|
11432
|
+
} catch (error) {
|
|
11339
11433
|
span.log({
|
|
11340
|
-
error:
|
|
11434
|
+
error: error instanceof Error ? error.message : String(error)
|
|
11341
11435
|
});
|
|
11342
|
-
throw
|
|
11436
|
+
throw error;
|
|
11343
11437
|
}
|
|
11344
11438
|
},
|
|
11345
11439
|
{
|
|
@@ -11416,14 +11510,14 @@ function asyncGeneratorProxy(generator, input, metadata) {
|
|
|
11416
11510
|
span.end();
|
|
11417
11511
|
}
|
|
11418
11512
|
return result;
|
|
11419
|
-
} catch (
|
|
11513
|
+
} catch (error) {
|
|
11420
11514
|
if (span) {
|
|
11421
11515
|
span.log({
|
|
11422
|
-
error:
|
|
11516
|
+
error: error instanceof Error ? error.message : String(error)
|
|
11423
11517
|
});
|
|
11424
11518
|
span.end();
|
|
11425
11519
|
}
|
|
11426
|
-
throw
|
|
11520
|
+
throw error;
|
|
11427
11521
|
}
|
|
11428
11522
|
};
|
|
11429
11523
|
}
|
|
@@ -11949,9 +12043,9 @@ function escapePath(parts) {
|
|
|
11949
12043
|
return part;
|
|
11950
12044
|
}).join(".");
|
|
11951
12045
|
}
|
|
11952
|
-
function unescapePath(
|
|
12046
|
+
function unescapePath(path2) {
|
|
11953
12047
|
const regex = /"((?:\\["\\]|[^"\\])*)"|([^\.]+)/g;
|
|
11954
|
-
const matches =
|
|
12048
|
+
const matches = path2.match(regex);
|
|
11955
12049
|
return matches ? matches.map((match) => {
|
|
11956
12050
|
if (match.startsWith('"')) {
|
|
11957
12051
|
return match.slice(1, -1).replace(/\\(["\\])/g, "$1");
|
|
@@ -12017,9 +12111,9 @@ function handlePromise(promise, callback) {
|
|
|
12017
12111
|
invokeCallback(callback, err && (err instanceof Error || err.message) ? err : new Error(err));
|
|
12018
12112
|
});
|
|
12019
12113
|
}
|
|
12020
|
-
function invokeCallback(callback,
|
|
12114
|
+
function invokeCallback(callback, error, value) {
|
|
12021
12115
|
try {
|
|
12022
|
-
callback(
|
|
12116
|
+
callback(error, value);
|
|
12023
12117
|
} catch (err) {
|
|
12024
12118
|
setImmediate$1((e) => {
|
|
12025
12119
|
throw e;
|
|
@@ -12941,7 +13035,7 @@ function sortBy(coll, iteratee, callback) {
|
|
|
12941
13035
|
}
|
|
12942
13036
|
var sortBy$1 = awaitify(sortBy, 3);
|
|
12943
13037
|
function tryEach(tasks, callback) {
|
|
12944
|
-
var
|
|
13038
|
+
var error = null;
|
|
12945
13039
|
var result;
|
|
12946
13040
|
return eachSeries$1(tasks, (task, taskCb) => {
|
|
12947
13041
|
wrapAsync(task)((err, ...args) => {
|
|
@@ -12951,10 +13045,10 @@ function tryEach(tasks, callback) {
|
|
|
12951
13045
|
} else {
|
|
12952
13046
|
result = args;
|
|
12953
13047
|
}
|
|
12954
|
-
|
|
13048
|
+
error = err;
|
|
12955
13049
|
taskCb(err ? null : {});
|
|
12956
13050
|
});
|
|
12957
|
-
}, () => callback(
|
|
13051
|
+
}, () => callback(error, result));
|
|
12958
13052
|
}
|
|
12959
13053
|
var tryEach$1 = awaitify(tryEach);
|
|
12960
13054
|
function whilst(test, iteratee, callback) {
|
|
@@ -12997,53 +13091,16 @@ function waterfall(tasks, callback) {
|
|
|
12997
13091
|
}
|
|
12998
13092
|
var waterfall$1 = awaitify(waterfall);
|
|
12999
13093
|
|
|
13000
|
-
// src/
|
|
13001
|
-
|
|
13002
|
-
|
|
13003
|
-
|
|
13004
|
-
import pluralize from "pluralize";
|
|
13005
|
-
import Table from "cli-table3";
|
|
13006
|
-
|
|
13007
|
-
// src/progress.ts
|
|
13008
|
-
import chalk from "chalk";
|
|
13009
|
-
import * as cliProgress from "cli-progress";
|
|
13010
|
-
var MAX_NAME_LENGTH = 40;
|
|
13011
|
-
function fitNameToSpaces(name, length) {
|
|
13012
|
-
const padded = name.padEnd(length);
|
|
13013
|
-
if (padded.length <= length) {
|
|
13014
|
-
return padded;
|
|
13015
|
-
}
|
|
13016
|
-
return padded.substring(0, length - 3) + "...";
|
|
13017
|
-
}
|
|
13018
|
-
var BarProgressReporter = class {
|
|
13019
|
-
multiBar;
|
|
13020
|
-
bars = {};
|
|
13021
|
-
constructor() {
|
|
13022
|
-
this.multiBar = new cliProgress.MultiBar(
|
|
13023
|
-
{
|
|
13024
|
-
// clearOnComplete: true,
|
|
13025
|
-
format: `${chalk.blueBright("{bar}")} ${chalk.blue("{evaluator}")} {percentage}% ${chalk.gray("{value}/{total} {eta_formatted}")}`,
|
|
13026
|
-
// autopadding: true,
|
|
13027
|
-
hideCursor: true,
|
|
13028
|
-
barsize: 10
|
|
13029
|
-
},
|
|
13030
|
-
cliProgress.Presets.shades_grey
|
|
13031
|
-
);
|
|
13032
|
-
}
|
|
13033
|
-
start(name, total) {
|
|
13034
|
-
const bar = this.multiBar.create(total, 0);
|
|
13035
|
-
this.bars[name] = bar;
|
|
13094
|
+
// src/reporters/progress.ts
|
|
13095
|
+
var SimpleProgressReporter = class {
|
|
13096
|
+
start(name, _total) {
|
|
13097
|
+
console.log(`Running evaluator ${name}`);
|
|
13036
13098
|
}
|
|
13037
13099
|
stop() {
|
|
13038
|
-
this.multiBar.stop();
|
|
13039
13100
|
}
|
|
13040
|
-
increment(
|
|
13041
|
-
this.bars[name].increment({
|
|
13042
|
-
evaluator: fitNameToSpaces(name, MAX_NAME_LENGTH)
|
|
13043
|
-
});
|
|
13101
|
+
increment(_name) {
|
|
13044
13102
|
}
|
|
13045
|
-
setTotal(
|
|
13046
|
-
this.bars[name].setTotal(total);
|
|
13103
|
+
setTotal(_name, _total) {
|
|
13047
13104
|
}
|
|
13048
13105
|
};
|
|
13049
13106
|
|
|
@@ -13051,9 +13108,8 @@ var BarProgressReporter = class {
|
|
|
13051
13108
|
import { z as z10 } from "zod/v3";
|
|
13052
13109
|
|
|
13053
13110
|
// src/framework2.ts
|
|
13054
|
-
import path2 from "path";
|
|
13055
|
-
import slugifyLib from "slugify";
|
|
13056
13111
|
import { z as z9 } from "zod/v3";
|
|
13112
|
+
var currentFilename = typeof __filename !== "undefined" ? __filename : "unknown";
|
|
13057
13113
|
var ProjectBuilder = class {
|
|
13058
13114
|
create(opts) {
|
|
13059
13115
|
return new Project2(opts);
|
|
@@ -13125,12 +13181,12 @@ var ToolBuilder = class {
|
|
|
13125
13181
|
const { handler, name, slug, parameters, returns, ...rest } = opts;
|
|
13126
13182
|
let resolvedName = name ?? handler.name;
|
|
13127
13183
|
if (resolvedName.trim().length === 0) {
|
|
13128
|
-
resolvedName = `Tool ${
|
|
13184
|
+
resolvedName = `Tool ${isomorph_default.basename(currentFilename)} ${this.taskCounter}`;
|
|
13129
13185
|
}
|
|
13130
13186
|
const tool = new CodeFunction(this.project, {
|
|
13131
13187
|
handler,
|
|
13132
13188
|
name: resolvedName,
|
|
13133
|
-
slug: slug ??
|
|
13189
|
+
slug: slug ?? slugify(resolvedName, { lower: true, strict: true }),
|
|
13134
13190
|
type: "tool",
|
|
13135
13191
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/consistent-type-assertions
|
|
13136
13192
|
parameters,
|
|
@@ -13154,9 +13210,9 @@ var ScorerBuilder = class {
|
|
|
13154
13210
|
resolvedName = opts.handler.name;
|
|
13155
13211
|
}
|
|
13156
13212
|
if (!resolvedName || resolvedName.trim().length === 0) {
|
|
13157
|
-
resolvedName = `Scorer ${
|
|
13213
|
+
resolvedName = `Scorer ${isomorph_default.basename(currentFilename)} ${this.taskCounter}`;
|
|
13158
13214
|
}
|
|
13159
|
-
const slug = opts.slug ??
|
|
13215
|
+
const slug = opts.slug ?? slugify(resolvedName, { lower: true, strict: true });
|
|
13160
13216
|
if ("handler" in opts) {
|
|
13161
13217
|
const scorer = new CodeFunction(this.project, {
|
|
13162
13218
|
...opts,
|
|
@@ -13327,7 +13383,7 @@ var PromptBuilder = class {
|
|
|
13327
13383
|
rawTools.push(tool);
|
|
13328
13384
|
}
|
|
13329
13385
|
}
|
|
13330
|
-
const slug = opts.slug ??
|
|
13386
|
+
const slug = opts.slug ?? slugify(opts.name, { lower: true, strict: true });
|
|
13331
13387
|
const promptData = promptDefinitionToPromptData(opts, rawTools);
|
|
13332
13388
|
const promptRow = {
|
|
13333
13389
|
id: opts.id,
|
|
@@ -13455,8 +13511,11 @@ var EvalResultWithSummary = class {
|
|
|
13455
13511
|
this.summary = summary;
|
|
13456
13512
|
this.results = results;
|
|
13457
13513
|
}
|
|
13514
|
+
/**
|
|
13515
|
+
* @deprecated Use `summary` instead.
|
|
13516
|
+
*/
|
|
13458
13517
|
toString() {
|
|
13459
|
-
return
|
|
13518
|
+
return JSON.stringify(this.summary);
|
|
13460
13519
|
}
|
|
13461
13520
|
[Symbol.for("nodejs.util.inspect.custom")]() {
|
|
13462
13521
|
return `EvalResultWithSummary(summary="...", results=[...])`;
|
|
@@ -13509,7 +13568,7 @@ function _initializeSpanContext() {
|
|
|
13509
13568
|
globalThis._spanContext = { currentSpan, withCurrent, startSpan, NOOP_SPAN };
|
|
13510
13569
|
}
|
|
13511
13570
|
async function Eval(name, evaluator, reporterOrOpts) {
|
|
13512
|
-
const options =
|
|
13571
|
+
const options = isEmpty2(reporterOrOpts) ? {} : typeof reporterOrOpts === "string" ? { reporter: reporterOrOpts } : "name" in reporterOrOpts ? { reporter: reporterOrOpts } : reporterOrOpts;
|
|
13513
13572
|
let evalName = makeEvalName(name, evaluator.experimentName);
|
|
13514
13573
|
if (globalThis._evals.evaluators[evalName]) {
|
|
13515
13574
|
evalName = `${evalName}_${Object.keys(_evals).length}`;
|
|
@@ -13535,7 +13594,7 @@ async function Eval(name, evaluator, reporterOrOpts) {
|
|
|
13535
13594
|
[]
|
|
13536
13595
|
);
|
|
13537
13596
|
}
|
|
13538
|
-
const progressReporter = options.progress ?? new
|
|
13597
|
+
const progressReporter = options.progress ?? new SimpleProgressReporter();
|
|
13539
13598
|
const shouldCollectResults = options.returnResults ?? true;
|
|
13540
13599
|
if (typeof options.reporter === "string") {
|
|
13541
13600
|
throw new Error(
|
|
@@ -13635,8 +13694,8 @@ function serializeJSONWithPlainString(v) {
|
|
|
13635
13694
|
}
|
|
13636
13695
|
}
|
|
13637
13696
|
function evaluateFilter(object, filter2) {
|
|
13638
|
-
const { path:
|
|
13639
|
-
const key =
|
|
13697
|
+
const { path: path2, pattern } = filter2;
|
|
13698
|
+
const key = path2.reduce(
|
|
13640
13699
|
(acc, p) => typeof acc === "object" && acc !== null ? (
|
|
13641
13700
|
// eslint-disable-next-line @typescript-eslint/consistent-type-assertions
|
|
13642
13701
|
acc[p]
|
|
@@ -13687,7 +13746,7 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
|
|
|
13687
13746
|
);
|
|
13688
13747
|
}
|
|
13689
13748
|
let name = dataResult.name;
|
|
13690
|
-
if (
|
|
13749
|
+
if (isEmpty2(name)) {
|
|
13691
13750
|
const baseExperiment = await experiment.fetchBaseExperiment();
|
|
13692
13751
|
if (!baseExperiment) {
|
|
13693
13752
|
throw new Error("BaseExperiment() failed to fetch base experiment");
|
|
@@ -13756,7 +13815,7 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
|
|
|
13756
13815
|
};
|
|
13757
13816
|
const expected = "expected" in datum ? datum.expected : void 0;
|
|
13758
13817
|
let output = void 0;
|
|
13759
|
-
let
|
|
13818
|
+
let error = void 0;
|
|
13760
13819
|
let tags = [...datum.tags ?? []];
|
|
13761
13820
|
const scores = {};
|
|
13762
13821
|
const scorerNames = evaluator.scores.map(scorerName);
|
|
@@ -13820,7 +13879,7 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
|
|
|
13820
13879
|
}
|
|
13821
13880
|
if (Array.isArray(scoreValue)) {
|
|
13822
13881
|
for (const s of scoreValue) {
|
|
13823
|
-
if (!(typeof s === "object" && !
|
|
13882
|
+
if (!(typeof s === "object" && !isEmpty2(s))) {
|
|
13824
13883
|
throw new Error(
|
|
13825
13884
|
`When returning an array of scores, each score must be a non-empty object. Got: ${JSON.stringify(
|
|
13826
13885
|
s
|
|
@@ -13829,7 +13888,7 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
|
|
|
13829
13888
|
}
|
|
13830
13889
|
}
|
|
13831
13890
|
}
|
|
13832
|
-
const results2 = Array.isArray(scoreValue) ? scoreValue : typeof scoreValue === "object" && !
|
|
13891
|
+
const results2 = Array.isArray(scoreValue) ? scoreValue : typeof scoreValue === "object" && !isEmpty2(scoreValue) ? [scoreValue] : [
|
|
13833
13892
|
{
|
|
13834
13893
|
name: scorerNames[score_idx],
|
|
13835
13894
|
score: scoreValue
|
|
@@ -13887,9 +13946,9 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
|
|
|
13887
13946
|
unhandledScores = null;
|
|
13888
13947
|
if (failingScorersAndResults.length) {
|
|
13889
13948
|
const scorerErrors = Object.fromEntries(
|
|
13890
|
-
failingScorersAndResults.map(({ name, error:
|
|
13949
|
+
failingScorersAndResults.map(({ name, error: error2 }) => [
|
|
13891
13950
|
name,
|
|
13892
|
-
|
|
13951
|
+
error2 instanceof Error ? error2.stack : `${error2}`
|
|
13893
13952
|
])
|
|
13894
13953
|
);
|
|
13895
13954
|
metadata["scorer_errors"] = scorerErrors;
|
|
@@ -13906,7 +13965,7 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
|
|
|
13906
13965
|
}
|
|
13907
13966
|
} catch (e) {
|
|
13908
13967
|
logError(rootSpan, e);
|
|
13909
|
-
|
|
13968
|
+
error = e;
|
|
13910
13969
|
} finally {
|
|
13911
13970
|
progressReporter.increment(evaluator.evalName);
|
|
13912
13971
|
}
|
|
@@ -13929,7 +13988,7 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
|
|
|
13929
13988
|
tags: tags.length ? tags : void 0,
|
|
13930
13989
|
metadata,
|
|
13931
13990
|
scores: mergedScores,
|
|
13932
|
-
error
|
|
13991
|
+
error,
|
|
13933
13992
|
origin: baseEvent.event?.origin
|
|
13934
13993
|
});
|
|
13935
13994
|
}
|
|
@@ -13976,7 +14035,7 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
|
|
|
13976
14035
|
}
|
|
13977
14036
|
let timeoutId;
|
|
13978
14037
|
let abortHandler;
|
|
13979
|
-
const rejectOnce = (
|
|
14038
|
+
const rejectOnce = (error) => {
|
|
13980
14039
|
if (cancelled) {
|
|
13981
14040
|
return;
|
|
13982
14041
|
}
|
|
@@ -13988,7 +14047,7 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
|
|
|
13988
14047
|
if (abortHandler && evaluator.signal) {
|
|
13989
14048
|
evaluator.signal.removeEventListener("abort", abortHandler);
|
|
13990
14049
|
}
|
|
13991
|
-
reject2(
|
|
14050
|
+
reject2(error);
|
|
13992
14051
|
};
|
|
13993
14052
|
if (evaluator.timeout) {
|
|
13994
14053
|
timeoutId = setTimeout(() => {
|
|
@@ -14015,7 +14074,7 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
|
|
|
14015
14074
|
} catch (e) {
|
|
14016
14075
|
q.kill();
|
|
14017
14076
|
if (e instanceof InternalAbortError) {
|
|
14018
|
-
if (
|
|
14077
|
+
if (isomorph_default.getEnv("BRAINTRUST_VERBOSE")) {
|
|
14019
14078
|
console.warn("Evaluator cancelled:", e.message);
|
|
14020
14079
|
}
|
|
14021
14080
|
}
|
|
@@ -14035,8 +14094,7 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
|
|
|
14035
14094
|
collectResults ? collectedResults : []
|
|
14036
14095
|
);
|
|
14037
14096
|
}
|
|
14038
|
-
var
|
|
14039
|
-
var warning = chalk2.yellow;
|
|
14097
|
+
var warning = (text) => `Warning: ${text}`;
|
|
14040
14098
|
function logError2(e, verbose) {
|
|
14041
14099
|
if (!verbose) {
|
|
14042
14100
|
console.error(`${e}`);
|
|
@@ -14085,11 +14143,7 @@ function reportFailures(evaluator, failingResults, { verbose, jsonl }) {
|
|
|
14085
14143
|
if (failingResults.length > 0) {
|
|
14086
14144
|
console.error(
|
|
14087
14145
|
warning(
|
|
14088
|
-
`Evaluator ${evaluator.evalName} failed with ${
|
|
14089
|
-
"error",
|
|
14090
|
-
failingResults.length,
|
|
14091
|
-
true
|
|
14092
|
-
)}. This evaluation ("${evaluator.evalName}") will not be fully logged.`
|
|
14146
|
+
`Evaluator ${evaluator.evalName} failed with ${failingResults.length} error${failingResults.length === 1 ? "" : "s"}. This evaluation ("${evaluator.evalName}") will not be fully logged.`
|
|
14093
14147
|
)
|
|
14094
14148
|
);
|
|
14095
14149
|
if (jsonl) {
|
|
@@ -14121,124 +14175,84 @@ var defaultReporter = {
|
|
|
14121
14175
|
if (failingResults.length > 0) {
|
|
14122
14176
|
reportFailures(evaluator, failingResults, { verbose, jsonl });
|
|
14123
14177
|
}
|
|
14124
|
-
|
|
14125
|
-
|
|
14126
|
-
|
|
14127
|
-
|
|
14178
|
+
if (jsonl) {
|
|
14179
|
+
isomorph_default.writeln(JSON.stringify(summary));
|
|
14180
|
+
} else {
|
|
14181
|
+
isomorph_default.writeln("Experiment summary");
|
|
14182
|
+
isomorph_default.writeln("==================");
|
|
14183
|
+
if (summary.comparisonExperimentName) {
|
|
14184
|
+
isomorph_default.writeln(
|
|
14185
|
+
`${summary.comparisonExperimentName} (baseline) <- ${summary.experimentName} (comparison)`
|
|
14186
|
+
);
|
|
14187
|
+
isomorph_default.writeln("");
|
|
14188
|
+
}
|
|
14189
|
+
const hasScores = Object.keys(summary.scores).length > 0;
|
|
14190
|
+
const hasMetrics = Object.keys(summary.metrics ?? {}).length > 0;
|
|
14191
|
+
const hasComparison = !!summary.comparisonExperimentName;
|
|
14192
|
+
if (hasScores || hasMetrics) {
|
|
14193
|
+
if (hasComparison) {
|
|
14194
|
+
isomorph_default.writeln(
|
|
14195
|
+
"Name Value Change Improvements Regressions"
|
|
14196
|
+
);
|
|
14197
|
+
isomorph_default.writeln(
|
|
14198
|
+
"----------------------------------------------------------------"
|
|
14199
|
+
);
|
|
14200
|
+
}
|
|
14201
|
+
for (const score of Object.values(summary.scores)) {
|
|
14202
|
+
const scorePercent = (score.score * 100).toFixed(2);
|
|
14203
|
+
const scoreValue = `${scorePercent}%`;
|
|
14204
|
+
if (hasComparison) {
|
|
14205
|
+
let diffString = "-";
|
|
14206
|
+
if (!isEmpty2(score.diff)) {
|
|
14207
|
+
const diffPercent = (score.diff * 100).toFixed(2);
|
|
14208
|
+
const diffSign = score.diff > 0 ? "+" : "";
|
|
14209
|
+
diffString = `${diffSign}${diffPercent}%`;
|
|
14210
|
+
}
|
|
14211
|
+
const improvements = score.improvements > 0 ? score.improvements.toString() : "-";
|
|
14212
|
+
const regressions = score.regressions > 0 ? score.regressions.toString() : "-";
|
|
14213
|
+
isomorph_default.writeln(
|
|
14214
|
+
`${score.name.padEnd(18)} ${scoreValue.padStart(10)} ${diffString.padStart(10)} ${improvements.padStart(12)} ${regressions.padStart(11)}`
|
|
14215
|
+
);
|
|
14216
|
+
} else {
|
|
14217
|
+
isomorph_default.writeln(`${score.name.padEnd(20)} ${scoreValue.padStart(15)}`);
|
|
14218
|
+
}
|
|
14219
|
+
}
|
|
14220
|
+
for (const metric of Object.values(summary.metrics ?? {})) {
|
|
14221
|
+
const fractionDigits = Number.isInteger(metric.metric) ? 0 : 2;
|
|
14222
|
+
const formattedValue = metric.metric.toFixed(fractionDigits);
|
|
14223
|
+
const metricValue = metric.unit === "$" ? `${metric.unit}${formattedValue}` : `${formattedValue}${metric.unit}`;
|
|
14224
|
+
if (hasComparison) {
|
|
14225
|
+
let diffString = "-";
|
|
14226
|
+
if (!isEmpty2(metric.diff)) {
|
|
14227
|
+
const diffPercent = (metric.diff * 100).toFixed(2);
|
|
14228
|
+
const diffSign = metric.diff > 0 ? "+" : "";
|
|
14229
|
+
diffString = `${diffSign}${diffPercent}%`;
|
|
14230
|
+
}
|
|
14231
|
+
const improvements = metric.improvements > 0 ? metric.improvements.toString() : "-";
|
|
14232
|
+
const regressions = metric.regressions > 0 ? metric.regressions.toString() : "-";
|
|
14233
|
+
isomorph_default.writeln(
|
|
14234
|
+
`${metric.name.padEnd(18)} ${metricValue.padStart(10)} ${diffString.padStart(10)} ${improvements.padStart(12)} ${regressions.padStart(11)}`
|
|
14235
|
+
);
|
|
14236
|
+
} else {
|
|
14237
|
+
isomorph_default.writeln(
|
|
14238
|
+
`${metric.name.padEnd(20)} ${metricValue.padStart(15)}`
|
|
14239
|
+
);
|
|
14240
|
+
}
|
|
14241
|
+
}
|
|
14242
|
+
}
|
|
14243
|
+
if (summary.experimentUrl) {
|
|
14244
|
+
isomorph_default.writeln("");
|
|
14245
|
+
isomorph_default.writeln(`View results for ${summary.experimentName}`);
|
|
14246
|
+
isomorph_default.writeln(`See results at ${summary.experimentUrl}`);
|
|
14247
|
+
}
|
|
14248
|
+
}
|
|
14249
|
+
isomorph_default.writeln("");
|
|
14128
14250
|
return failingResults.length === 0;
|
|
14129
14251
|
},
|
|
14130
14252
|
async reportRun(evalReports) {
|
|
14131
14253
|
return evalReports.every((r) => r);
|
|
14132
14254
|
}
|
|
14133
14255
|
};
|
|
14134
|
-
function formatExperimentSummary(summary) {
|
|
14135
|
-
let comparisonLine = "";
|
|
14136
|
-
if (summary.comparisonExperimentName) {
|
|
14137
|
-
comparisonLine = `${summary.comparisonExperimentName} ${chalk2.gray("(baseline)")} \u2190 ${summary.experimentName} ${chalk2.gray("(comparison)")}
|
|
14138
|
-
|
|
14139
|
-
`;
|
|
14140
|
-
}
|
|
14141
|
-
const tableParts = [];
|
|
14142
|
-
const hasScores = Object.keys(summary.scores).length > 0;
|
|
14143
|
-
const hasMetrics = Object.keys(summary.metrics ?? {}).length > 0;
|
|
14144
|
-
const hasComparison = !!summary.comparisonExperimentName;
|
|
14145
|
-
if (hasScores || hasMetrics) {
|
|
14146
|
-
const headers = [chalk2.gray("Name"), chalk2.gray("Value")];
|
|
14147
|
-
if (hasComparison) {
|
|
14148
|
-
headers.push(
|
|
14149
|
-
chalk2.gray("Change"),
|
|
14150
|
-
chalk2.gray("Improvements"),
|
|
14151
|
-
chalk2.gray("Regressions")
|
|
14152
|
-
);
|
|
14153
|
-
}
|
|
14154
|
-
const combinedTable = new Table({
|
|
14155
|
-
head: hasComparison ? headers : [],
|
|
14156
|
-
style: { head: [], "padding-left": 0, "padding-right": 0, border: [] },
|
|
14157
|
-
chars: {
|
|
14158
|
-
top: "",
|
|
14159
|
-
"top-mid": "",
|
|
14160
|
-
"top-left": "",
|
|
14161
|
-
"top-right": "",
|
|
14162
|
-
bottom: "",
|
|
14163
|
-
"bottom-mid": "",
|
|
14164
|
-
"bottom-left": "",
|
|
14165
|
-
"bottom-right": "",
|
|
14166
|
-
left: "",
|
|
14167
|
-
"left-mid": "",
|
|
14168
|
-
mid: "",
|
|
14169
|
-
"mid-mid": "",
|
|
14170
|
-
right: "",
|
|
14171
|
-
"right-mid": "",
|
|
14172
|
-
middle: " "
|
|
14173
|
-
},
|
|
14174
|
-
colWidths: hasComparison ? [18, 10, 10, 13, 12] : [20, 15],
|
|
14175
|
-
colAligns: hasComparison ? ["left", "right", "right", "right", "right"] : ["left", "right"],
|
|
14176
|
-
wordWrap: false
|
|
14177
|
-
});
|
|
14178
|
-
const scoreValues = Object.values(summary.scores);
|
|
14179
|
-
for (let i = 0; i < scoreValues.length; i++) {
|
|
14180
|
-
const score = scoreValues[i];
|
|
14181
|
-
const scorePercent = (score.score * 100).toFixed(2);
|
|
14182
|
-
const scoreValue = chalk2.white(`${scorePercent}%`);
|
|
14183
|
-
let diffString = "";
|
|
14184
|
-
if (!isEmpty(score.diff)) {
|
|
14185
|
-
const diffPercent = (score.diff * 100).toFixed(2);
|
|
14186
|
-
const diffSign = score.diff > 0 ? "+" : "";
|
|
14187
|
-
const diffColor = score.diff > 0 ? chalk2.green : chalk2.red;
|
|
14188
|
-
diffString = diffColor(`${diffSign}${diffPercent}%`);
|
|
14189
|
-
} else {
|
|
14190
|
-
diffString = chalk2.gray("-");
|
|
14191
|
-
}
|
|
14192
|
-
const improvements = score.improvements > 0 ? chalk2.dim.green(score.improvements) : chalk2.gray("-");
|
|
14193
|
-
const regressions = score.regressions > 0 ? chalk2.dim.red(score.regressions) : chalk2.gray("-");
|
|
14194
|
-
const row = [`${chalk2.blue("\u25EF")} ${score.name}`, scoreValue];
|
|
14195
|
-
if (hasComparison) {
|
|
14196
|
-
row.push(diffString, improvements, regressions);
|
|
14197
|
-
}
|
|
14198
|
-
combinedTable.push(row);
|
|
14199
|
-
}
|
|
14200
|
-
const metricValues = Object.values(summary.metrics ?? {});
|
|
14201
|
-
for (let i = 0; i < metricValues.length; i++) {
|
|
14202
|
-
const metric = metricValues[i];
|
|
14203
|
-
const fractionDigits = Number.isInteger(metric.metric) ? 0 : 2;
|
|
14204
|
-
const formattedValue = metric.metric.toFixed(fractionDigits);
|
|
14205
|
-
const metricValue = chalk2.white(
|
|
14206
|
-
metric.unit === "$" ? `${metric.unit}${formattedValue}` : `${formattedValue}${metric.unit}`
|
|
14207
|
-
);
|
|
14208
|
-
let diffString = "";
|
|
14209
|
-
if (!isEmpty(metric.diff)) {
|
|
14210
|
-
const diffPercent = (metric.diff * 100).toFixed(2);
|
|
14211
|
-
const diffSign = metric.diff > 0 ? "+" : "";
|
|
14212
|
-
const diffColor = metric.diff > 0 ? chalk2.green : chalk2.red;
|
|
14213
|
-
diffString = diffColor(`${diffSign}${diffPercent}%`);
|
|
14214
|
-
} else {
|
|
14215
|
-
diffString = chalk2.gray("-");
|
|
14216
|
-
}
|
|
14217
|
-
const improvements = metric.improvements > 0 ? chalk2.dim.green(metric.improvements) : chalk2.gray("-");
|
|
14218
|
-
const regressions = metric.regressions > 0 ? chalk2.dim.red(metric.regressions) : chalk2.gray("-");
|
|
14219
|
-
const row = [`${chalk2.magenta("\u25EF")} ${metric.name}`, metricValue];
|
|
14220
|
-
if (hasComparison) {
|
|
14221
|
-
row.push(diffString, improvements, regressions);
|
|
14222
|
-
}
|
|
14223
|
-
combinedTable.push(row);
|
|
14224
|
-
}
|
|
14225
|
-
tableParts.push(combinedTable.toString());
|
|
14226
|
-
}
|
|
14227
|
-
const content = [comparisonLine, ...tableParts].filter(Boolean).join("\n");
|
|
14228
|
-
const footer = summary.experimentUrl ? terminalLink(
|
|
14229
|
-
`View results for ${summary.experimentName}`,
|
|
14230
|
-
summary.experimentUrl,
|
|
14231
|
-
{ fallback: () => `See results at ${summary.experimentUrl}` }
|
|
14232
|
-
) : "";
|
|
14233
|
-
const boxContent = [content, footer].filter(Boolean).join("\n\n");
|
|
14234
|
-
return "\n" + boxen(boxContent, {
|
|
14235
|
-
title: chalk2.gray("Experiment summary"),
|
|
14236
|
-
titleAlignment: "left",
|
|
14237
|
-
padding: 0.5,
|
|
14238
|
-
borderColor: "gray",
|
|
14239
|
-
borderStyle: "round"
|
|
14240
|
-
});
|
|
14241
|
-
}
|
|
14242
14256
|
|
|
14243
14257
|
// dev/types.ts
|
|
14244
14258
|
import { z as z11 } from "zod/v3";
|
|
@@ -14284,9 +14298,9 @@ var evaluatorDefinitionsSchema = z11.record(
|
|
|
14284
14298
|
|
|
14285
14299
|
// src/index.ts
|
|
14286
14300
|
configureNode();
|
|
14287
|
-
var index_default = exports_node_exports;
|
|
14288
14301
|
export {
|
|
14289
14302
|
Attachment,
|
|
14303
|
+
AttachmentReference,
|
|
14290
14304
|
BaseAttachment,
|
|
14291
14305
|
BaseExperiment,
|
|
14292
14306
|
BraintrustMiddleware,
|
|
@@ -14335,7 +14349,7 @@ export {
|
|
|
14335
14349
|
currentLogger,
|
|
14336
14350
|
currentSpan,
|
|
14337
14351
|
deepCopyEvent,
|
|
14338
|
-
|
|
14352
|
+
exports_exports as default,
|
|
14339
14353
|
defaultErrorScoreHandler,
|
|
14340
14354
|
deserializePlainStringAsJSON,
|
|
14341
14355
|
devNullWritableStream,
|