braintrust 1.0.2 → 1.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.mjs CHANGED
@@ -32,7 +32,9 @@ var iso = {
32
32
  getCallerLocation: () => void 0,
33
33
  newAsyncLocalStorage: () => new DefaultAsyncLocalStorage(),
34
34
  processOn: (_0, _1) => {
35
- }
35
+ },
36
+ basename: (filepath) => filepath.split(/[\\/]/).pop() || filepath,
37
+ writeln: (text) => console.log(text)
36
38
  };
37
39
  var isomorph_default = iso;
38
40
 
@@ -125,7 +127,7 @@ async function getPastNAncestors(n = 1e3, remote = void 0) {
125
127
  return [];
126
128
  }
127
129
  const commits = await git.log({ from: ancestor, to: "HEAD", maxCount: n });
128
- return commits.all.map((c) => c.hash);
130
+ return commits.all.slice(0, n).map((c) => c.hash);
129
131
  }
130
132
  async function attempt(fn) {
131
133
  try {
@@ -964,11 +966,11 @@ function mergeDictsWithPaths({
964
966
  function mergeDictsWithPathsHelper({
965
967
  mergeInto,
966
968
  mergeFrom,
967
- path: path3,
969
+ path: path2,
968
970
  mergePaths
969
971
  }) {
970
972
  Object.entries(mergeFrom).forEach(([k, mergeFromV]) => {
971
- const fullPath = path3.concat([k]);
973
+ const fullPath = path2.concat([k]);
972
974
  const fullPathSerialized = JSON.stringify(fullPath);
973
975
  const mergeIntoV = recordFind(mergeInto, k);
974
976
  if (isObject(mergeIntoV) && isObject(mergeFromV) && !mergePaths.has(fullPathSerialized)) {
@@ -999,9 +1001,9 @@ function mapAt(m, k) {
999
1001
  function recordFind(m, k) {
1000
1002
  return m[k];
1001
1003
  }
1002
- function getObjValueByPath(row, path3) {
1004
+ function getObjValueByPath(row, path2) {
1003
1005
  let curr = row;
1004
- for (const p of path3) {
1006
+ for (const p of path2) {
1005
1007
  if (!isObjectOrArray(curr)) {
1006
1008
  return null;
1007
1009
  }
@@ -1276,6 +1278,93 @@ function _urljoin(...parts) {
1276
1278
  (x, i) => x.replace(/^\//, "").replace(i < parts.length - 1 ? /\/$/ : "", "")
1277
1279
  ).filter((x) => x.trim() !== "").join("/");
1278
1280
  }
1281
+ function slugify(text, options) {
1282
+ if (typeof text !== "string") {
1283
+ throw new Error("slugify: string argument expected");
1284
+ }
1285
+ const charMap = {
1286
+ // Currency and symbols
1287
+ $: "dollar",
1288
+ "%": "percent",
1289
+ "&": "and",
1290
+ // Latin characters
1291
+ \u00C0: "A",
1292
+ \u00C1: "A",
1293
+ \u00C2: "A",
1294
+ \u00C3: "A",
1295
+ \u00C4: "A",
1296
+ \u00C5: "A",
1297
+ \u00C6: "AE",
1298
+ \u00C7: "C",
1299
+ \u00C8: "E",
1300
+ \u00C9: "E",
1301
+ \u00CA: "E",
1302
+ \u00CB: "E",
1303
+ \u00CC: "I",
1304
+ \u00CD: "I",
1305
+ \u00CE: "I",
1306
+ \u00CF: "I",
1307
+ \u00D1: "N",
1308
+ \u00D2: "O",
1309
+ \u00D3: "O",
1310
+ \u00D4: "O",
1311
+ \u00D5: "O",
1312
+ \u00D6: "O",
1313
+ \u00D8: "O",
1314
+ \u00D9: "U",
1315
+ \u00DA: "U",
1316
+ \u00DB: "U",
1317
+ \u00DC: "U",
1318
+ \u00DD: "Y",
1319
+ \u00E0: "a",
1320
+ \u00E1: "a",
1321
+ \u00E2: "a",
1322
+ \u00E3: "a",
1323
+ \u00E4: "a",
1324
+ \u00E5: "a",
1325
+ \u00E6: "ae",
1326
+ \u00E7: "c",
1327
+ \u00E8: "e",
1328
+ \u00E9: "e",
1329
+ \u00EA: "e",
1330
+ \u00EB: "e",
1331
+ \u00EC: "i",
1332
+ \u00ED: "i",
1333
+ \u00EE: "i",
1334
+ \u00EF: "i",
1335
+ \u00F1: "n",
1336
+ \u00F2: "o",
1337
+ \u00F3: "o",
1338
+ \u00F4: "o",
1339
+ \u00F5: "o",
1340
+ \u00F6: "o",
1341
+ \u00F8: "o",
1342
+ \u00F9: "u",
1343
+ \u00FA: "u",
1344
+ \u00FB: "u",
1345
+ \u00FC: "u",
1346
+ \u00FD: "y",
1347
+ \u00FF: "y"
1348
+ };
1349
+ const replacement = "-";
1350
+ const trim = options?.trim !== false;
1351
+ let slug = text.normalize().split("").reduce((result, ch) => {
1352
+ const mapped = charMap[ch] || ch;
1353
+ const appendChar = mapped === replacement ? " " : mapped;
1354
+ return result + appendChar.replace(/[^\w\s$*_+~.()'"!\-:@]+/g, "");
1355
+ }, "");
1356
+ if (options?.strict) {
1357
+ slug = slug.replace(/[^A-Za-z0-9\s]/g, "");
1358
+ }
1359
+ if (trim) {
1360
+ slug = slug.trim();
1361
+ }
1362
+ slug = slug.replace(/\s+/g, replacement);
1363
+ if (options?.lower) {
1364
+ slug = slug.toLowerCase();
1365
+ }
1366
+ return slug;
1367
+ }
1279
1368
 
1280
1369
  // util/span_identifier_v4.ts
1281
1370
  import { z as z4 } from "zod/v3";
@@ -2916,9 +3005,9 @@ var BraintrustStream = class _BraintrustStream {
2916
3005
  reader.releaseLock();
2917
3006
  return { done: true, value: void 0 };
2918
3007
  },
2919
- async throw(error2) {
3008
+ async throw(error) {
2920
3009
  reader.releaseLock();
2921
- throw error2;
3010
+ throw error;
2922
3011
  }
2923
3012
  };
2924
3013
  }
@@ -3232,10 +3321,10 @@ var DiskCache = class {
3232
3321
  return;
3233
3322
  }
3234
3323
  const stats = await Promise.all(
3235
- paths.map(async (path3) => {
3236
- const stat2 = await isomorph_default.stat(path3);
3324
+ paths.map(async (path2) => {
3325
+ const stat2 = await isomorph_default.stat(path2);
3237
3326
  return {
3238
- path: path3,
3327
+ path: path2,
3239
3328
  mtime: stat2.mtime.getTime()
3240
3329
  };
3241
3330
  })
@@ -3376,7 +3465,7 @@ function runCatchFinally(f, catchF, finallyF) {
3376
3465
  function getCurrentUnixTimestamp() {
3377
3466
  return (/* @__PURE__ */ new Date()).getTime() / 1e3;
3378
3467
  }
3379
- function isEmpty(a) {
3468
+ function isEmpty2(a) {
3380
3469
  return a === void 0 || a === null;
3381
3470
  }
3382
3471
  var LazyValue = class {
@@ -3513,8 +3602,8 @@ var MaskingError = class {
3513
3602
  function applyMaskingToField(maskingFunction, data, fieldName) {
3514
3603
  try {
3515
3604
  return maskingFunction(data);
3516
- } catch (error2) {
3517
- const errorType = error2 instanceof Error ? error2.constructor.name : "Error";
3605
+ } catch (error) {
3606
+ const errorType = error instanceof Error ? error.constructor.name : "Error";
3518
3607
  if (fieldName === "scores" || fieldName === "metrics") {
3519
3608
  return new MaskingError(fieldName, errorType);
3520
3609
  }
@@ -3801,7 +3890,7 @@ var BraintrustState = class _BraintrustState {
3801
3890
  const newState = await loginToState({
3802
3891
  ...this.loginParams,
3803
3892
  ...Object.fromEntries(
3804
- Object.entries(loginParams).filter(([k, v]) => !isEmpty(v))
3893
+ Object.entries(loginParams).filter(([k, v]) => !isEmpty2(v))
3805
3894
  )
3806
3895
  });
3807
3896
  this.copyLoginInfo(newState);
@@ -3990,9 +4079,9 @@ var HTTPConnection = class _HTTPConnection {
3990
4079
  this.headers["Authorization"] = `Bearer ${this.token}`;
3991
4080
  }
3992
4081
  }
3993
- async get(path3, params = void 0, config) {
4082
+ async get(path2, params = void 0, config) {
3994
4083
  const { headers, ...rest } = config || {};
3995
- const url = new URL(_urljoin(this.base_url, path3));
4084
+ const url = new URL(_urljoin(this.base_url, path2));
3996
4085
  url.search = new URLSearchParams(
3997
4086
  params ? Object.entries(params).filter(([_, v]) => v !== void 0).flatMap(
3998
4087
  ([k, v]) => v !== void 0 ? typeof v === "string" ? [[k, v]] : v.map((x) => [k, x]) : []
@@ -4013,13 +4102,13 @@ var HTTPConnection = class _HTTPConnection {
4013
4102
  })
4014
4103
  );
4015
4104
  }
4016
- async post(path3, params, config) {
4105
+ async post(path2, params, config) {
4017
4106
  const { headers, ...rest } = config || {};
4018
4107
  const this_fetch = this.fetch;
4019
4108
  const this_base_url = this.base_url;
4020
4109
  const this_headers = this.headers;
4021
4110
  return await checkResponse(
4022
- await this_fetch(_urljoin(this_base_url, path3), {
4111
+ await this_fetch(_urljoin(this_base_url, path2), {
4023
4112
  method: "POST",
4024
4113
  headers: {
4025
4114
  Accept: "application/json",
@@ -4171,12 +4260,12 @@ var Attachment = class extends BaseAttachment {
4171
4260
  signedUrl: z8.string().url(),
4172
4261
  headers: z8.record(z8.string())
4173
4262
  }).parse(await metadataResponse.json()));
4174
- } catch (error2) {
4175
- if (error2 instanceof ZodError) {
4176
- const errorStr = JSON.stringify(error2.flatten());
4263
+ } catch (error) {
4264
+ if (error instanceof ZodError) {
4265
+ const errorStr = JSON.stringify(error.flatten());
4177
4266
  throw new Error(`Invalid response from API server: ${errorStr}`);
4178
4267
  }
4179
- throw error2;
4268
+ throw error;
4180
4269
  }
4181
4270
  addAzureBlobHeaders(headers, signedUrl);
4182
4271
  let objectStoreResponse;
@@ -4188,13 +4277,13 @@ var Attachment = class extends BaseAttachment {
4188
4277
  body: data
4189
4278
  })
4190
4279
  );
4191
- } catch (error2) {
4192
- if (error2 instanceof FailedHTTPResponse) {
4280
+ } catch (error) {
4281
+ if (error instanceof FailedHTTPResponse) {
4193
4282
  throw new Error(
4194
- `Failed to upload attachment to object store: ${error2.status} ${error2.text} ${error2.data}`
4283
+ `Failed to upload attachment to object store: ${error.status} ${error.text} ${error.data}`
4195
4284
  );
4196
4285
  }
4197
- throw error2;
4286
+ throw error;
4198
4287
  }
4199
4288
  return { signedUrl, metadataResponse, objectStoreResponse };
4200
4289
  };
@@ -4206,9 +4295,9 @@ var Attachment = class extends BaseAttachment {
4206
4295
  const orgId = state.orgId ?? "";
4207
4296
  try {
4208
4297
  await doUpload(conn, orgId);
4209
- } catch (error2) {
4298
+ } catch (error) {
4210
4299
  status.upload_status = "error";
4211
- status.error_message = error2 instanceof Error ? error2.message : JSON.stringify(error2);
4300
+ status.error_message = error instanceof Error ? error.message : JSON.stringify(error);
4212
4301
  }
4213
4302
  const requestParams = {
4214
4303
  key: this.reference.key,
@@ -4407,8 +4496,8 @@ var ReadonlyAttachment = class {
4407
4496
  }
4408
4497
  const objResponse = await fetch(downloadUrl);
4409
4498
  if (objResponse.status !== 200) {
4410
- const error2 = await objResponse.text();
4411
- throw new Error(`Couldn't download attachment: ${error2}`);
4499
+ const error = await objResponse.text();
4500
+ throw new Error(`Couldn't download attachment: ${error}`);
4412
4501
  }
4413
4502
  return await objResponse.blob();
4414
4503
  };
@@ -4466,7 +4555,7 @@ function logFeedbackImpl(state, parentObjectType, parentObjectId, {
4466
4555
  if (!VALID_SOURCES.includes(source)) {
4467
4556
  throw new Error(`source must be one of ${VALID_SOURCES}`);
4468
4557
  }
4469
- if (isEmpty(scores) && isEmpty(expected) && isEmpty(tags) && isEmpty(comment)) {
4558
+ if (isEmpty2(scores) && isEmpty2(expected) && isEmpty2(tags) && isEmpty2(comment)) {
4470
4559
  throw new Error(
4471
4560
  "At least one of scores, expected, tags, or comment must be specified"
4472
4561
  );
@@ -4479,7 +4568,7 @@ function logFeedbackImpl(state, parentObjectType, parentObjectId, {
4479
4568
  });
4480
4569
  let { metadata, ...updateEvent } = deepCopyEvent(validatedEvent);
4481
4570
  updateEvent = Object.fromEntries(
4482
- Object.entries(updateEvent).filter(([_, v]) => !isEmpty(v))
4571
+ Object.entries(updateEvent).filter(([_, v]) => !isEmpty2(v))
4483
4572
  );
4484
4573
  const parentIds = async () => new SpanComponentsV3({
4485
4574
  object_type: parentObjectType,
@@ -4498,7 +4587,7 @@ function logFeedbackImpl(state, parentObjectType, parentObjectId, {
4498
4587
  });
4499
4588
  state.bgLogger().log([record]);
4500
4589
  }
4501
- if (!isEmpty(comment)) {
4590
+ if (!isEmpty2(comment)) {
4502
4591
  const record = new LazyValue(async () => {
4503
4592
  return {
4504
4593
  id: uuidv42(),
@@ -5135,8 +5224,8 @@ var HTTPBackgroundLogger = class _HTTPBackgroundLogger {
5135
5224
  if (result.upload_status === "error") {
5136
5225
  throw new Error(result.error_message);
5137
5226
  }
5138
- } catch (error2) {
5139
- attachmentErrors.push(error2);
5227
+ } catch (error) {
5228
+ attachmentErrors.push(error);
5140
5229
  }
5141
5230
  }
5142
5231
  if (attachmentErrors.length === 1) {
@@ -5219,22 +5308,22 @@ var HTTPBackgroundLogger = class _HTTPBackgroundLogger {
5219
5308
  }
5220
5309
  for (let i = 0; i < this.numTries; i++) {
5221
5310
  const startTime = now();
5222
- let error2 = void 0;
5311
+ let error = void 0;
5223
5312
  try {
5224
5313
  await conn.post_json("logs3", dataStr);
5225
5314
  } catch (e) {
5226
- error2 = e;
5315
+ error = e;
5227
5316
  }
5228
- if (error2 === void 0) {
5317
+ if (error === void 0) {
5229
5318
  return;
5230
5319
  }
5231
5320
  const isRetrying = i + 1 < this.numTries;
5232
5321
  const retryingText = isRetrying ? "" : " Retrying";
5233
5322
  const errorText = (() => {
5234
- if (error2 instanceof FailedHTTPResponse) {
5235
- return `${error2.status} (${error2.text}): ${error2.data}`;
5323
+ if (error instanceof FailedHTTPResponse) {
5324
+ return `${error.status} (${error.text}): ${error.data}`;
5236
5325
  } else {
5237
- return `${error2}`;
5326
+ return `${error}`;
5238
5327
  }
5239
5328
  })();
5240
5329
  const errMsg = `log request failed. Elapsed time: ${(now() - startTime) / 1e3} seconds. Payload size: ${dataStr.length}.${retryingText}
@@ -5410,7 +5499,7 @@ function init(projectOrOptions, optionalOptions) {
5410
5499
  const state = stateArg ?? _globalState;
5411
5500
  state.enforceQueueSizeLimit(false);
5412
5501
  if (open) {
5413
- if (isEmpty(experiment)) {
5502
+ if (isEmpty2(experiment)) {
5414
5503
  throw new Error(`Cannot open an experiment without specifying its name`);
5415
5504
  }
5416
5505
  const lazyMetadata2 = new LazyValue(
@@ -5649,7 +5738,7 @@ async function computeLoggerMetadata(state, {
5649
5738
  }) {
5650
5739
  await state.login({});
5651
5740
  const org_id = state.orgId;
5652
- if (isEmpty(project_id)) {
5741
+ if (isEmpty2(project_id)) {
5653
5742
  const response = await state.appConn().post_json("api/project/register", {
5654
5743
  project_name: project_name || GLOBAL_PROJECT,
5655
5744
  org_id
@@ -5662,7 +5751,7 @@ async function computeLoggerMetadata(state, {
5662
5751
  fullInfo: response.project
5663
5752
  }
5664
5753
  };
5665
- } else if (isEmpty(project_name)) {
5754
+ } else if (isEmpty2(project_name)) {
5666
5755
  const response = await state.appConn().get_json("api/project", {
5667
5756
  id: project_id
5668
5757
  });
@@ -5743,9 +5832,9 @@ async function loadPrompt({
5743
5832
  );
5744
5833
  }
5745
5834
  if (id) {
5746
- } else if (isEmpty(projectName) && isEmpty(projectId)) {
5835
+ } else if (isEmpty2(projectName) && isEmpty2(projectId)) {
5747
5836
  throw new Error("Must specify either projectName or projectId");
5748
- } else if (isEmpty(slug)) {
5837
+ } else if (isEmpty2(slug)) {
5749
5838
  throw new Error("Must specify slug");
5750
5839
  }
5751
5840
  const state = stateArg ?? _globalState;
@@ -5847,7 +5936,7 @@ async function login(options = {}) {
5847
5936
  const { forceLogin = false } = options || {};
5848
5937
  if (_globalState.loggedIn && !forceLogin) {
5849
5938
  let checkUpdatedParam2 = function(varname, arg, orig) {
5850
- if (!isEmpty(arg) && !isEmpty(orig) && arg !== orig) {
5939
+ if (!isEmpty2(arg) && !isEmpty2(orig) && arg !== orig) {
5851
5940
  throw new Error(
5852
5941
  `Re-logging in with different ${varname} (${arg}) than original (${orig}). To force re-login, pass \`forceLogin: true\``
5853
5942
  );
@@ -5985,14 +6074,14 @@ function getSpanParentObject(options) {
5985
6074
  }
5986
6075
  return NOOP_SPAN;
5987
6076
  }
5988
- function logError(span, error2) {
6077
+ function logError(span, error) {
5989
6078
  let errorMessage = "<error>";
5990
6079
  let stackTrace = "";
5991
- if (error2 instanceof Error) {
5992
- errorMessage = error2.message;
5993
- stackTrace = error2.stack || "";
6080
+ if (error instanceof Error) {
6081
+ errorMessage = error.message;
6082
+ stackTrace = error.stack || "";
5994
6083
  } else {
5995
- errorMessage = String(error2);
6084
+ errorMessage = String(error);
5996
6085
  }
5997
6086
  span.log({ error: `${errorMessage}
5998
6087
 
@@ -6061,12 +6150,12 @@ function wrapTracedSyncGenerator(fn, spanArgs, noTraceIO) {
6061
6150
  if (!truncated) {
6062
6151
  span.log({ output: collected });
6063
6152
  }
6064
- } catch (error2) {
6065
- logError(span, error2);
6153
+ } catch (error) {
6154
+ logError(span, error);
6066
6155
  if (!truncated && collected.length > 0) {
6067
6156
  span.log({ output: collected });
6068
6157
  }
6069
- throw error2;
6158
+ throw error;
6070
6159
  }
6071
6160
  } else {
6072
6161
  const gen = generatorWithCurrent(span, fn.apply(this, fnArgs));
@@ -6110,12 +6199,12 @@ function wrapTracedAsyncGenerator(fn, spanArgs, noTraceIO) {
6110
6199
  if (!truncated) {
6111
6200
  span.log({ output: collected });
6112
6201
  }
6113
- } catch (error2) {
6114
- logError(span, error2);
6202
+ } catch (error) {
6203
+ logError(span, error);
6115
6204
  if (!truncated && collected.length > 0) {
6116
6205
  span.log({ output: collected });
6117
6206
  }
6118
- throw error2;
6207
+ throw error;
6119
6208
  }
6120
6209
  } else {
6121
6210
  const gen = asyncGeneratorWithCurrent(span, fn.apply(this, fnArgs));
@@ -6456,15 +6545,15 @@ async function resolveAttachmentsToBase64(event, state) {
6456
6545
  return event;
6457
6546
  }
6458
6547
  function validateAndSanitizeExperimentLogFullArgs(event, hasDataset) {
6459
- if ("input" in event && !isEmpty(event.input) && "inputs" in event && !isEmpty(event.inputs) || !("input" in event) && !("inputs" in event)) {
6548
+ if ("input" in event && !isEmpty2(event.input) && "inputs" in event && !isEmpty2(event.inputs) || !("input" in event) && !("inputs" in event)) {
6460
6549
  throw new Error(
6461
6550
  "Exactly one of input or inputs (deprecated) must be specified. Prefer input."
6462
6551
  );
6463
6552
  }
6464
- if (isEmpty(event.output)) {
6553
+ if (isEmpty2(event.output)) {
6465
6554
  throw new Error("output must be specified");
6466
6555
  }
6467
- if (isEmpty(event.scores)) {
6556
+ if (isEmpty2(event.scores)) {
6468
6557
  throw new Error("scores must be specified");
6469
6558
  }
6470
6559
  if (hasDataset && event.datasetRecordId === void 0) {
@@ -7524,7 +7613,7 @@ function renderMessage(render, message) {
7524
7613
  return {
7525
7614
  ...message,
7526
7615
  ..."content" in message ? {
7527
- content: isEmpty(message.content) ? void 0 : typeof message.content === "string" ? render(message.content) : message.content.map((c) => {
7616
+ content: isEmpty2(message.content) ? void 0 : typeof message.content === "string" ? render(message.content) : message.content.map((c) => {
7528
7617
  switch (c.type) {
7529
7618
  case "text":
7530
7619
  return { ...c, text: render(c.text) };
@@ -7561,7 +7650,7 @@ function renderMessage(render, message) {
7561
7650
  })
7562
7651
  } : {},
7563
7652
  ..."tool_calls" in message ? {
7564
- tool_calls: isEmpty(message.tool_calls) ? void 0 : message.tool_calls.map((t) => {
7653
+ tool_calls: isEmpty2(message.tool_calls) ? void 0 : message.tool_calls.map((t) => {
7565
7654
  return {
7566
7655
  type: t.type,
7567
7656
  id: render(t.id),
@@ -7710,11 +7799,11 @@ var Prompt2 = class _Prompt {
7710
7799
  ([k, _v]) => !BRAINTRUST_PARAMS.includes(k)
7711
7800
  )
7712
7801
  ),
7713
- ...!isEmpty(this.options.model) ? {
7802
+ ...!isEmpty2(this.options.model) ? {
7714
7803
  model: this.options.model
7715
7804
  } : {}
7716
7805
  };
7717
- if (!("model" in params) || isEmpty(params.model)) {
7806
+ if (!("model" in params) || isEmpty2(params.model)) {
7718
7807
  throw new Error(
7719
7808
  "No model specified. Either specify it in the prompt or as a default"
7720
7809
  );
@@ -7964,6 +8053,8 @@ function configureNode() {
7964
8053
  isomorph_default.processOn = (event, handler) => {
7965
8054
  process.on(event, handler);
7966
8055
  };
8056
+ isomorph_default.basename = path.basename;
8057
+ isomorph_default.writeln = (text) => process.stdout.write(text + "\n");
7967
8058
  isomorph_default.pathJoin = path.join;
7968
8059
  isomorph_default.pathDirname = path.dirname;
7969
8060
  isomorph_default.mkdir = fs.mkdir;
@@ -7981,10 +8072,11 @@ function configureNode() {
7981
8072
  _internalSetInitialState();
7982
8073
  }
7983
8074
 
7984
- // src/exports-node.ts
7985
- var exports_node_exports = {};
7986
- __export(exports_node_exports, {
8075
+ // src/exports.ts
8076
+ var exports_exports = {};
8077
+ __export(exports_exports, {
7987
8078
  Attachment: () => Attachment,
8079
+ AttachmentReference: () => AttachmentReference,
7988
8080
  BaseAttachment: () => BaseAttachment,
7989
8081
  BaseExperiment: () => BaseExperiment,
7990
8082
  BraintrustMiddleware: () => BraintrustMiddleware,
@@ -8036,6 +8128,8 @@ __export(exports_node_exports, {
8036
8128
  defaultErrorScoreHandler: () => defaultErrorScoreHandler,
8037
8129
  deserializePlainStringAsJSON: () => deserializePlainStringAsJSON,
8038
8130
  devNullWritableStream: () => devNullWritableStream,
8131
+ evaluatorDefinitionSchema: () => evaluatorDefinitionSchema,
8132
+ evaluatorDefinitionsSchema: () => evaluatorDefinitionsSchema,
8039
8133
  flush: () => flush,
8040
8134
  getContextManager: () => getContextManager,
8041
8135
  getIdGenerator: () => getIdGenerator,
@@ -8662,13 +8756,13 @@ function wrapBetaChatCompletionStream(completion) {
8662
8756
  var LEGACY_CACHED_HEADER = "x-cached";
8663
8757
  var X_CACHED_HEADER = "x-bt-cached";
8664
8758
  function parseCachedHeader(value) {
8665
- return isEmpty(value) ? void 0 : ["true", "hit"].includes(value.toLowerCase()) ? 1 : 0;
8759
+ return isEmpty2(value) ? void 0 : ["true", "hit"].includes(value.toLowerCase()) ? 1 : 0;
8666
8760
  }
8667
8761
  function logHeaders(response, span) {
8668
8762
  const cachedHeader = response.headers.get(X_CACHED_HEADER);
8669
- if (isEmpty(cachedHeader)) {
8763
+ if (isEmpty2(cachedHeader)) {
8670
8764
  const legacyCacheHeader = response.headers.get(LEGACY_CACHED_HEADER);
8671
- if (!isEmpty(legacyCacheHeader)) {
8765
+ if (!isEmpty2(legacyCacheHeader)) {
8672
8766
  span.log({
8673
8767
  metrics: {
8674
8768
  cached: parseCachedHeader(legacyCacheHeader)
@@ -8984,7 +9078,7 @@ function convertDataToBlob(data, mediaType) {
8984
9078
  } else if (typeof Buffer !== "undefined" && data instanceof Buffer) {
8985
9079
  return new Blob([data], { type: mediaType });
8986
9080
  }
8987
- } catch (error2) {
9081
+ } catch (error) {
8988
9082
  return null;
8989
9083
  }
8990
9084
  return null;
@@ -9448,12 +9542,12 @@ var makeStreamTextWrapper = (name, options, streamText, aiSDK) => {
9448
9542
  });
9449
9543
  }
9450
9544
  return result;
9451
- } catch (error2) {
9545
+ } catch (error) {
9452
9546
  span.log({
9453
- error: serializeError(error2)
9547
+ error: serializeError(error)
9454
9548
  });
9455
9549
  span.end();
9456
- throw error2;
9550
+ throw error;
9457
9551
  }
9458
9552
  };
9459
9553
  Object.defineProperty(wrapper, "name", { value: name, writable: false });
@@ -9546,12 +9640,12 @@ var wrapStreamObject = (streamObject, options = {}, aiSDK) => {
9546
9640
  });
9547
9641
  }
9548
9642
  return result;
9549
- } catch (error2) {
9643
+ } catch (error) {
9550
9644
  span.log({
9551
- error: serializeError(error2)
9645
+ error: serializeError(error)
9552
9646
  });
9553
9647
  span.end();
9554
- throw error2;
9648
+ throw error;
9555
9649
  }
9556
9650
  };
9557
9651
  };
@@ -9633,17 +9727,17 @@ var wrapToolExecute = (tool, name) => {
9633
9727
  }
9634
9728
  return tool;
9635
9729
  };
9636
- var serializeError = (error2) => {
9637
- if (error2 instanceof Error) {
9638
- return error2;
9730
+ var serializeError = (error) => {
9731
+ if (error instanceof Error) {
9732
+ return error;
9639
9733
  }
9640
- if (typeof error2 === "object" && error2 !== null) {
9734
+ if (typeof error === "object" && error !== null) {
9641
9735
  try {
9642
- return JSON.stringify(error2);
9736
+ return JSON.stringify(error);
9643
9737
  } catch {
9644
9738
  }
9645
9739
  }
9646
- return String(error2);
9740
+ return String(error);
9647
9741
  };
9648
9742
  var serializeModel = (model) => {
9649
9743
  return typeof model === "string" ? model : model?.modelId;
@@ -9784,8 +9878,8 @@ var processContentPart = (part) => {
9784
9878
  }
9785
9879
  }
9786
9880
  }
9787
- } catch (error2) {
9788
- console.warn("Error processing content part:", error2);
9881
+ } catch (error) {
9882
+ console.warn("Error processing content part:", error);
9789
9883
  }
9790
9884
  return part;
9791
9885
  };
@@ -9831,8 +9925,8 @@ var convertImageToAttachment = (image, explicitMimeType) => {
9831
9925
  if (image instanceof Attachment) {
9832
9926
  return image;
9833
9927
  }
9834
- } catch (error2) {
9835
- console.warn("Error converting image to attachment:", error2);
9928
+ } catch (error) {
9929
+ console.warn("Error converting image to attachment:", error);
9836
9930
  }
9837
9931
  return null;
9838
9932
  };
@@ -9861,8 +9955,8 @@ var convertDataToAttachment = (data, mimeType, filename) => {
9861
9955
  contentType: mimeType
9862
9956
  });
9863
9957
  }
9864
- } catch (error2) {
9865
- console.warn("Error converting data to attachment:", error2);
9958
+ } catch (error) {
9959
+ console.warn("Error converting data to attachment:", error);
9866
9960
  }
9867
9961
  return null;
9868
9962
  };
@@ -9899,8 +9993,8 @@ var processOutput = async (output, denyOutputPaths) => {
9899
9993
  var processOutputAttachments = async (output) => {
9900
9994
  try {
9901
9995
  return await doProcessOutputAttachments(output);
9902
- } catch (error2) {
9903
- console.error("Error processing output attachments:", error2);
9996
+ } catch (error) {
9997
+ console.error("Error processing output attachments:", error);
9904
9998
  return output;
9905
9999
  }
9906
10000
  };
@@ -9945,8 +10039,8 @@ var convertFileToAttachment = (file, index) => {
9945
10039
  filename,
9946
10040
  contentType: mediaType
9947
10041
  });
9948
- } catch (error2) {
9949
- console.warn(`Error processing file at index ${index}:`, error2);
10042
+ } catch (error) {
10043
+ console.warn(`Error processing file at index ${index}:`, error);
9950
10044
  return file;
9951
10045
  }
9952
10046
  };
@@ -10002,11 +10096,11 @@ function extractTokenMetrics(result) {
10002
10096
  var deepCopy = (obj) => {
10003
10097
  return JSON.parse(JSON.stringify(obj));
10004
10098
  };
10005
- var parsePath = (path3) => {
10099
+ var parsePath = (path2) => {
10006
10100
  const keys = [];
10007
10101
  let current = "";
10008
- for (let i = 0; i < path3.length; i++) {
10009
- const char = path3[i];
10102
+ for (let i = 0; i < path2.length; i++) {
10103
+ const char = path2[i];
10010
10104
  if (char === ".") {
10011
10105
  if (current) {
10012
10106
  keys.push(current);
@@ -10019,8 +10113,8 @@ var parsePath = (path3) => {
10019
10113
  }
10020
10114
  let bracketContent = "";
10021
10115
  i++;
10022
- while (i < path3.length && path3[i] !== "]") {
10023
- bracketContent += path3[i];
10116
+ while (i < path2.length && path2[i] !== "]") {
10117
+ bracketContent += path2[i];
10024
10118
  i++;
10025
10119
  }
10026
10120
  if (bracketContent === "") {
@@ -10062,8 +10156,8 @@ var omitAtPath = (obj, keys) => {
10062
10156
  };
10063
10157
  var omit = (obj, paths) => {
10064
10158
  const result = deepCopy(obj);
10065
- for (const path3 of paths) {
10066
- const keys = parsePath(path3);
10159
+ for (const path2 of paths) {
10160
+ const keys = parsePath(path2);
10067
10161
  omitAtPath(result, keys);
10068
10162
  }
10069
10163
  return result;
@@ -10128,7 +10222,7 @@ var BraintrustLanguageModelWrapper = class {
10128
10222
  output: postProcessOutput(ret.text, ret.toolCalls, ret.finishReason),
10129
10223
  metrics: {
10130
10224
  time_to_first_token: getCurrentUnixTimestamp() - startTime,
10131
- tokens: !isEmpty(ret.usage) ? ret.usage.promptTokens + ret.usage.completionTokens : void 0,
10225
+ tokens: !isEmpty2(ret.usage) ? ret.usage.promptTokens + ret.usage.completionTokens : void 0,
10132
10226
  prompt_tokens: ret.usage?.promptTokens,
10133
10227
  completion_tokens: ret.usage?.completionTokens,
10134
10228
  cached: parseCachedHeader(
@@ -10223,7 +10317,7 @@ var BraintrustLanguageModelWrapper = class {
10223
10317
  ),
10224
10318
  metrics: {
10225
10319
  time_to_first_token,
10226
- tokens: !isEmpty(usage) ? usage.promptTokens + usage.completionTokens : void 0,
10320
+ tokens: !isEmpty2(usage) ? usage.promptTokens + usage.completionTokens : void 0,
10227
10321
  prompt_tokens: usage?.promptTokens,
10228
10322
  completion_tokens: usage?.completionTokens,
10229
10323
  cached: parseCachedHeader(
@@ -10560,11 +10654,11 @@ function BraintrustMiddleware(config = {}) {
10560
10654
  )
10561
10655
  });
10562
10656
  return result;
10563
- } catch (error2) {
10657
+ } catch (error) {
10564
10658
  span.log({
10565
- error: error2 instanceof Error ? error2.message : String(error2)
10659
+ error: error instanceof Error ? error.message : String(error)
10566
10660
  });
10567
- throw error2;
10661
+ throw error;
10568
10662
  } finally {
10569
10663
  span.end();
10570
10664
  }
@@ -10609,12 +10703,12 @@ function BraintrustMiddleware(config = {}) {
10609
10703
  providerMetadata = chunk.providerMetadata || {};
10610
10704
  }
10611
10705
  controller.enqueue(chunk);
10612
- } catch (error2) {
10706
+ } catch (error) {
10613
10707
  span.log({
10614
- error: error2 instanceof Error ? error2.message : String(error2)
10708
+ error: error instanceof Error ? error.message : String(error)
10615
10709
  });
10616
10710
  span.end();
10617
- controller.error(error2);
10711
+ controller.error(error);
10618
10712
  }
10619
10713
  },
10620
10714
  flush() {
@@ -10660,12 +10754,12 @@ function BraintrustMiddleware(config = {}) {
10660
10754
  )
10661
10755
  });
10662
10756
  span.end();
10663
- } catch (error2) {
10757
+ } catch (error) {
10664
10758
  span.log({
10665
- error: error2 instanceof Error ? error2.message : String(error2)
10759
+ error: error instanceof Error ? error.message : String(error)
10666
10760
  });
10667
10761
  span.end();
10668
- throw error2;
10762
+ throw error;
10669
10763
  }
10670
10764
  }
10671
10765
  });
@@ -10673,12 +10767,12 @@ function BraintrustMiddleware(config = {}) {
10673
10767
  stream: stream.pipeThrough(transformStream),
10674
10768
  ...rest
10675
10769
  };
10676
- } catch (error2) {
10770
+ } catch (error) {
10677
10771
  span.log({
10678
- error: error2 instanceof Error ? error2.message : String(error2)
10772
+ error: error instanceof Error ? error.message : String(error)
10679
10773
  });
10680
10774
  span.end();
10681
- throw error2;
10775
+ throw error;
10682
10776
  }
10683
10777
  }
10684
10778
  };
@@ -10786,8 +10880,8 @@ function apiPromiseProxy2(apiPromise, span, onThen) {
10786
10880
  try {
10787
10881
  const processed = onThen(result);
10788
10882
  return onFulfilled ? onFulfilled(processed) : processed;
10789
- } catch (error2) {
10790
- return onRejected ? onRejected(error2) : Promise.reject(error2);
10883
+ } catch (error) {
10884
+ return onRejected ? onRejected(error) : Promise.reject(error);
10791
10885
  }
10792
10886
  },
10793
10887
  onRejected
@@ -11096,11 +11190,11 @@ function wrapClaudeAgentQuery(queryFn, defaultThis) {
11096
11190
  span.log({
11097
11191
  output: finalResults.length > 0 ? finalResults[finalResults.length - 1] : void 0
11098
11192
  });
11099
- } catch (error2) {
11193
+ } catch (error) {
11100
11194
  span.log({
11101
- error: error2 instanceof Error ? error2.message : String(error2)
11195
+ error: error instanceof Error ? error.message : String(error)
11102
11196
  });
11103
- throw error2;
11197
+ throw error;
11104
11198
  } finally {
11105
11199
  span.end();
11106
11200
  }
@@ -11335,11 +11429,11 @@ function wrapGenerateContent(original) {
11335
11429
  metrics: cleanMetrics(metrics)
11336
11430
  });
11337
11431
  return result;
11338
- } catch (error2) {
11432
+ } catch (error) {
11339
11433
  span.log({
11340
- error: error2 instanceof Error ? error2.message : String(error2)
11434
+ error: error instanceof Error ? error.message : String(error)
11341
11435
  });
11342
- throw error2;
11436
+ throw error;
11343
11437
  }
11344
11438
  },
11345
11439
  {
@@ -11416,14 +11510,14 @@ function asyncGeneratorProxy(generator, input, metadata) {
11416
11510
  span.end();
11417
11511
  }
11418
11512
  return result;
11419
- } catch (error2) {
11513
+ } catch (error) {
11420
11514
  if (span) {
11421
11515
  span.log({
11422
- error: error2 instanceof Error ? error2.message : String(error2)
11516
+ error: error instanceof Error ? error.message : String(error)
11423
11517
  });
11424
11518
  span.end();
11425
11519
  }
11426
- throw error2;
11520
+ throw error;
11427
11521
  }
11428
11522
  };
11429
11523
  }
@@ -11949,9 +12043,9 @@ function escapePath(parts) {
11949
12043
  return part;
11950
12044
  }).join(".");
11951
12045
  }
11952
- function unescapePath(path3) {
12046
+ function unescapePath(path2) {
11953
12047
  const regex = /"((?:\\["\\]|[^"\\])*)"|([^\.]+)/g;
11954
- const matches = path3.match(regex);
12048
+ const matches = path2.match(regex);
11955
12049
  return matches ? matches.map((match) => {
11956
12050
  if (match.startsWith('"')) {
11957
12051
  return match.slice(1, -1).replace(/\\(["\\])/g, "$1");
@@ -12017,9 +12111,9 @@ function handlePromise(promise, callback) {
12017
12111
  invokeCallback(callback, err && (err instanceof Error || err.message) ? err : new Error(err));
12018
12112
  });
12019
12113
  }
12020
- function invokeCallback(callback, error2, value) {
12114
+ function invokeCallback(callback, error, value) {
12021
12115
  try {
12022
- callback(error2, value);
12116
+ callback(error, value);
12023
12117
  } catch (err) {
12024
12118
  setImmediate$1((e) => {
12025
12119
  throw e;
@@ -12941,7 +13035,7 @@ function sortBy(coll, iteratee, callback) {
12941
13035
  }
12942
13036
  var sortBy$1 = awaitify(sortBy, 3);
12943
13037
  function tryEach(tasks, callback) {
12944
- var error2 = null;
13038
+ var error = null;
12945
13039
  var result;
12946
13040
  return eachSeries$1(tasks, (task, taskCb) => {
12947
13041
  wrapAsync(task)((err, ...args) => {
@@ -12951,10 +13045,10 @@ function tryEach(tasks, callback) {
12951
13045
  } else {
12952
13046
  result = args;
12953
13047
  }
12954
- error2 = err;
13048
+ error = err;
12955
13049
  taskCb(err ? null : {});
12956
13050
  });
12957
- }, () => callback(error2, result));
13051
+ }, () => callback(error, result));
12958
13052
  }
12959
13053
  var tryEach$1 = awaitify(tryEach);
12960
13054
  function whilst(test, iteratee, callback) {
@@ -12997,53 +13091,16 @@ function waterfall(tasks, callback) {
12997
13091
  }
12998
13092
  var waterfall$1 = awaitify(waterfall);
12999
13093
 
13000
- // src/framework.ts
13001
- import chalk2 from "chalk";
13002
- import { terminalLink } from "termi-link";
13003
- import boxen from "boxen";
13004
- import pluralize from "pluralize";
13005
- import Table from "cli-table3";
13006
-
13007
- // src/progress.ts
13008
- import chalk from "chalk";
13009
- import * as cliProgress from "cli-progress";
13010
- var MAX_NAME_LENGTH = 40;
13011
- function fitNameToSpaces(name, length) {
13012
- const padded = name.padEnd(length);
13013
- if (padded.length <= length) {
13014
- return padded;
13015
- }
13016
- return padded.substring(0, length - 3) + "...";
13017
- }
13018
- var BarProgressReporter = class {
13019
- multiBar;
13020
- bars = {};
13021
- constructor() {
13022
- this.multiBar = new cliProgress.MultiBar(
13023
- {
13024
- // clearOnComplete: true,
13025
- format: `${chalk.blueBright("{bar}")} ${chalk.blue("{evaluator}")} {percentage}% ${chalk.gray("{value}/{total} {eta_formatted}")}`,
13026
- // autopadding: true,
13027
- hideCursor: true,
13028
- barsize: 10
13029
- },
13030
- cliProgress.Presets.shades_grey
13031
- );
13032
- }
13033
- start(name, total) {
13034
- const bar = this.multiBar.create(total, 0);
13035
- this.bars[name] = bar;
13094
+ // src/reporters/progress.ts
13095
+ var SimpleProgressReporter = class {
13096
+ start(name, _total) {
13097
+ console.log(`Running evaluator ${name}`);
13036
13098
  }
13037
13099
  stop() {
13038
- this.multiBar.stop();
13039
13100
  }
13040
- increment(name) {
13041
- this.bars[name].increment({
13042
- evaluator: fitNameToSpaces(name, MAX_NAME_LENGTH)
13043
- });
13101
+ increment(_name) {
13044
13102
  }
13045
- setTotal(name, total) {
13046
- this.bars[name].setTotal(total);
13103
+ setTotal(_name, _total) {
13047
13104
  }
13048
13105
  };
13049
13106
 
@@ -13051,9 +13108,8 @@ var BarProgressReporter = class {
13051
13108
  import { z as z10 } from "zod/v3";
13052
13109
 
13053
13110
  // src/framework2.ts
13054
- import path2 from "path";
13055
- import slugifyLib from "slugify";
13056
13111
  import { z as z9 } from "zod/v3";
13112
+ var currentFilename = typeof __filename !== "undefined" ? __filename : "unknown";
13057
13113
  var ProjectBuilder = class {
13058
13114
  create(opts) {
13059
13115
  return new Project2(opts);
@@ -13125,12 +13181,12 @@ var ToolBuilder = class {
13125
13181
  const { handler, name, slug, parameters, returns, ...rest } = opts;
13126
13182
  let resolvedName = name ?? handler.name;
13127
13183
  if (resolvedName.trim().length === 0) {
13128
- resolvedName = `Tool ${path2.basename(__filename)} ${this.taskCounter}`;
13184
+ resolvedName = `Tool ${isomorph_default.basename(currentFilename)} ${this.taskCounter}`;
13129
13185
  }
13130
13186
  const tool = new CodeFunction(this.project, {
13131
13187
  handler,
13132
13188
  name: resolvedName,
13133
- slug: slug ?? slugifyLib(resolvedName, { lower: true, strict: true }),
13189
+ slug: slug ?? slugify(resolvedName, { lower: true, strict: true }),
13134
13190
  type: "tool",
13135
13191
  // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/consistent-type-assertions
13136
13192
  parameters,
@@ -13154,9 +13210,9 @@ var ScorerBuilder = class {
13154
13210
  resolvedName = opts.handler.name;
13155
13211
  }
13156
13212
  if (!resolvedName || resolvedName.trim().length === 0) {
13157
- resolvedName = `Scorer ${path2.basename(__filename)} ${this.taskCounter}`;
13213
+ resolvedName = `Scorer ${isomorph_default.basename(currentFilename)} ${this.taskCounter}`;
13158
13214
  }
13159
- const slug = opts.slug ?? slugifyLib(resolvedName, { lower: true, strict: true });
13215
+ const slug = opts.slug ?? slugify(resolvedName, { lower: true, strict: true });
13160
13216
  if ("handler" in opts) {
13161
13217
  const scorer = new CodeFunction(this.project, {
13162
13218
  ...opts,
@@ -13327,7 +13383,7 @@ var PromptBuilder = class {
13327
13383
  rawTools.push(tool);
13328
13384
  }
13329
13385
  }
13330
- const slug = opts.slug ?? slugifyLib(opts.name, { lower: true, strict: true });
13386
+ const slug = opts.slug ?? slugify(opts.name, { lower: true, strict: true });
13331
13387
  const promptData = promptDefinitionToPromptData(opts, rawTools);
13332
13388
  const promptRow = {
13333
13389
  id: opts.id,
@@ -13455,8 +13511,11 @@ var EvalResultWithSummary = class {
13455
13511
  this.summary = summary;
13456
13512
  this.results = results;
13457
13513
  }
13514
+ /**
13515
+ * @deprecated Use `summary` instead.
13516
+ */
13458
13517
  toString() {
13459
- return formatExperimentSummary(this.summary);
13518
+ return JSON.stringify(this.summary);
13460
13519
  }
13461
13520
  [Symbol.for("nodejs.util.inspect.custom")]() {
13462
13521
  return `EvalResultWithSummary(summary="...", results=[...])`;
@@ -13509,7 +13568,7 @@ function _initializeSpanContext() {
13509
13568
  globalThis._spanContext = { currentSpan, withCurrent, startSpan, NOOP_SPAN };
13510
13569
  }
13511
13570
  async function Eval(name, evaluator, reporterOrOpts) {
13512
- const options = isEmpty(reporterOrOpts) ? {} : typeof reporterOrOpts === "string" ? { reporter: reporterOrOpts } : "name" in reporterOrOpts ? { reporter: reporterOrOpts } : reporterOrOpts;
13571
+ const options = isEmpty2(reporterOrOpts) ? {} : typeof reporterOrOpts === "string" ? { reporter: reporterOrOpts } : "name" in reporterOrOpts ? { reporter: reporterOrOpts } : reporterOrOpts;
13513
13572
  let evalName = makeEvalName(name, evaluator.experimentName);
13514
13573
  if (globalThis._evals.evaluators[evalName]) {
13515
13574
  evalName = `${evalName}_${Object.keys(_evals).length}`;
@@ -13535,7 +13594,7 @@ async function Eval(name, evaluator, reporterOrOpts) {
13535
13594
  []
13536
13595
  );
13537
13596
  }
13538
- const progressReporter = options.progress ?? new BarProgressReporter();
13597
+ const progressReporter = options.progress ?? new SimpleProgressReporter();
13539
13598
  const shouldCollectResults = options.returnResults ?? true;
13540
13599
  if (typeof options.reporter === "string") {
13541
13600
  throw new Error(
@@ -13635,8 +13694,8 @@ function serializeJSONWithPlainString(v) {
13635
13694
  }
13636
13695
  }
13637
13696
  function evaluateFilter(object, filter2) {
13638
- const { path: path3, pattern } = filter2;
13639
- const key = path3.reduce(
13697
+ const { path: path2, pattern } = filter2;
13698
+ const key = path2.reduce(
13640
13699
  (acc, p) => typeof acc === "object" && acc !== null ? (
13641
13700
  // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
13642
13701
  acc[p]
@@ -13687,7 +13746,7 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
13687
13746
  );
13688
13747
  }
13689
13748
  let name = dataResult.name;
13690
- if (isEmpty(name)) {
13749
+ if (isEmpty2(name)) {
13691
13750
  const baseExperiment = await experiment.fetchBaseExperiment();
13692
13751
  if (!baseExperiment) {
13693
13752
  throw new Error("BaseExperiment() failed to fetch base experiment");
@@ -13756,7 +13815,7 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
13756
13815
  };
13757
13816
  const expected = "expected" in datum ? datum.expected : void 0;
13758
13817
  let output = void 0;
13759
- let error2 = void 0;
13818
+ let error = void 0;
13760
13819
  let tags = [...datum.tags ?? []];
13761
13820
  const scores = {};
13762
13821
  const scorerNames = evaluator.scores.map(scorerName);
@@ -13820,7 +13879,7 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
13820
13879
  }
13821
13880
  if (Array.isArray(scoreValue)) {
13822
13881
  for (const s of scoreValue) {
13823
- if (!(typeof s === "object" && !isEmpty(s))) {
13882
+ if (!(typeof s === "object" && !isEmpty2(s))) {
13824
13883
  throw new Error(
13825
13884
  `When returning an array of scores, each score must be a non-empty object. Got: ${JSON.stringify(
13826
13885
  s
@@ -13829,7 +13888,7 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
13829
13888
  }
13830
13889
  }
13831
13890
  }
13832
- const results2 = Array.isArray(scoreValue) ? scoreValue : typeof scoreValue === "object" && !isEmpty(scoreValue) ? [scoreValue] : [
13891
+ const results2 = Array.isArray(scoreValue) ? scoreValue : typeof scoreValue === "object" && !isEmpty2(scoreValue) ? [scoreValue] : [
13833
13892
  {
13834
13893
  name: scorerNames[score_idx],
13835
13894
  score: scoreValue
@@ -13887,9 +13946,9 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
13887
13946
  unhandledScores = null;
13888
13947
  if (failingScorersAndResults.length) {
13889
13948
  const scorerErrors = Object.fromEntries(
13890
- failingScorersAndResults.map(({ name, error: error3 }) => [
13949
+ failingScorersAndResults.map(({ name, error: error2 }) => [
13891
13950
  name,
13892
- error3 instanceof Error ? error3.stack : `${error3}`
13951
+ error2 instanceof Error ? error2.stack : `${error2}`
13893
13952
  ])
13894
13953
  );
13895
13954
  metadata["scorer_errors"] = scorerErrors;
@@ -13906,7 +13965,7 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
13906
13965
  }
13907
13966
  } catch (e) {
13908
13967
  logError(rootSpan, e);
13909
- error2 = e;
13968
+ error = e;
13910
13969
  } finally {
13911
13970
  progressReporter.increment(evaluator.evalName);
13912
13971
  }
@@ -13929,7 +13988,7 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
13929
13988
  tags: tags.length ? tags : void 0,
13930
13989
  metadata,
13931
13990
  scores: mergedScores,
13932
- error: error2,
13991
+ error,
13933
13992
  origin: baseEvent.event?.origin
13934
13993
  });
13935
13994
  }
@@ -13976,7 +14035,7 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
13976
14035
  }
13977
14036
  let timeoutId;
13978
14037
  let abortHandler;
13979
- const rejectOnce = (error2) => {
14038
+ const rejectOnce = (error) => {
13980
14039
  if (cancelled) {
13981
14040
  return;
13982
14041
  }
@@ -13988,7 +14047,7 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
13988
14047
  if (abortHandler && evaluator.signal) {
13989
14048
  evaluator.signal.removeEventListener("abort", abortHandler);
13990
14049
  }
13991
- reject2(error2);
14050
+ reject2(error);
13992
14051
  };
13993
14052
  if (evaluator.timeout) {
13994
14053
  timeoutId = setTimeout(() => {
@@ -14015,7 +14074,7 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
14015
14074
  } catch (e) {
14016
14075
  q.kill();
14017
14076
  if (e instanceof InternalAbortError) {
14018
- if (process.env.BRAINTRUST_VERBOSE) {
14077
+ if (isomorph_default.getEnv("BRAINTRUST_VERBOSE")) {
14019
14078
  console.warn("Evaluator cancelled:", e.message);
14020
14079
  }
14021
14080
  }
@@ -14035,8 +14094,7 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
14035
14094
  collectResults ? collectedResults : []
14036
14095
  );
14037
14096
  }
14038
- var error = chalk2.red;
14039
- var warning = chalk2.yellow;
14097
+ var warning = (text) => `Warning: ${text}`;
14040
14098
  function logError2(e, verbose) {
14041
14099
  if (!verbose) {
14042
14100
  console.error(`${e}`);
@@ -14085,11 +14143,7 @@ function reportFailures(evaluator, failingResults, { verbose, jsonl }) {
14085
14143
  if (failingResults.length > 0) {
14086
14144
  console.error(
14087
14145
  warning(
14088
- `Evaluator ${evaluator.evalName} failed with ${pluralize(
14089
- "error",
14090
- failingResults.length,
14091
- true
14092
- )}. This evaluation ("${evaluator.evalName}") will not be fully logged.`
14146
+ `Evaluator ${evaluator.evalName} failed with ${failingResults.length} error${failingResults.length === 1 ? "" : "s"}. This evaluation ("${evaluator.evalName}") will not be fully logged.`
14093
14147
  )
14094
14148
  );
14095
14149
  if (jsonl) {
@@ -14121,124 +14175,84 @@ var defaultReporter = {
14121
14175
  if (failingResults.length > 0) {
14122
14176
  reportFailures(evaluator, failingResults, { verbose, jsonl });
14123
14177
  }
14124
- process.stdout.write(
14125
- jsonl ? JSON.stringify(summary) : formatExperimentSummary(summary)
14126
- );
14127
- process.stdout.write("\n");
14178
+ if (jsonl) {
14179
+ isomorph_default.writeln(JSON.stringify(summary));
14180
+ } else {
14181
+ isomorph_default.writeln("Experiment summary");
14182
+ isomorph_default.writeln("==================");
14183
+ if (summary.comparisonExperimentName) {
14184
+ isomorph_default.writeln(
14185
+ `${summary.comparisonExperimentName} (baseline) <- ${summary.experimentName} (comparison)`
14186
+ );
14187
+ isomorph_default.writeln("");
14188
+ }
14189
+ const hasScores = Object.keys(summary.scores).length > 0;
14190
+ const hasMetrics = Object.keys(summary.metrics ?? {}).length > 0;
14191
+ const hasComparison = !!summary.comparisonExperimentName;
14192
+ if (hasScores || hasMetrics) {
14193
+ if (hasComparison) {
14194
+ isomorph_default.writeln(
14195
+ "Name Value Change Improvements Regressions"
14196
+ );
14197
+ isomorph_default.writeln(
14198
+ "----------------------------------------------------------------"
14199
+ );
14200
+ }
14201
+ for (const score of Object.values(summary.scores)) {
14202
+ const scorePercent = (score.score * 100).toFixed(2);
14203
+ const scoreValue = `${scorePercent}%`;
14204
+ if (hasComparison) {
14205
+ let diffString = "-";
14206
+ if (!isEmpty2(score.diff)) {
14207
+ const diffPercent = (score.diff * 100).toFixed(2);
14208
+ const diffSign = score.diff > 0 ? "+" : "";
14209
+ diffString = `${diffSign}${diffPercent}%`;
14210
+ }
14211
+ const improvements = score.improvements > 0 ? score.improvements.toString() : "-";
14212
+ const regressions = score.regressions > 0 ? score.regressions.toString() : "-";
14213
+ isomorph_default.writeln(
14214
+ `${score.name.padEnd(18)} ${scoreValue.padStart(10)} ${diffString.padStart(10)} ${improvements.padStart(12)} ${regressions.padStart(11)}`
14215
+ );
14216
+ } else {
14217
+ isomorph_default.writeln(`${score.name.padEnd(20)} ${scoreValue.padStart(15)}`);
14218
+ }
14219
+ }
14220
+ for (const metric of Object.values(summary.metrics ?? {})) {
14221
+ const fractionDigits = Number.isInteger(metric.metric) ? 0 : 2;
14222
+ const formattedValue = metric.metric.toFixed(fractionDigits);
14223
+ const metricValue = metric.unit === "$" ? `${metric.unit}${formattedValue}` : `${formattedValue}${metric.unit}`;
14224
+ if (hasComparison) {
14225
+ let diffString = "-";
14226
+ if (!isEmpty2(metric.diff)) {
14227
+ const diffPercent = (metric.diff * 100).toFixed(2);
14228
+ const diffSign = metric.diff > 0 ? "+" : "";
14229
+ diffString = `${diffSign}${diffPercent}%`;
14230
+ }
14231
+ const improvements = metric.improvements > 0 ? metric.improvements.toString() : "-";
14232
+ const regressions = metric.regressions > 0 ? metric.regressions.toString() : "-";
14233
+ isomorph_default.writeln(
14234
+ `${metric.name.padEnd(18)} ${metricValue.padStart(10)} ${diffString.padStart(10)} ${improvements.padStart(12)} ${regressions.padStart(11)}`
14235
+ );
14236
+ } else {
14237
+ isomorph_default.writeln(
14238
+ `${metric.name.padEnd(20)} ${metricValue.padStart(15)}`
14239
+ );
14240
+ }
14241
+ }
14242
+ }
14243
+ if (summary.experimentUrl) {
14244
+ isomorph_default.writeln("");
14245
+ isomorph_default.writeln(`View results for ${summary.experimentName}`);
14246
+ isomorph_default.writeln(`See results at ${summary.experimentUrl}`);
14247
+ }
14248
+ }
14249
+ isomorph_default.writeln("");
14128
14250
  return failingResults.length === 0;
14129
14251
  },
14130
14252
  async reportRun(evalReports) {
14131
14253
  return evalReports.every((r) => r);
14132
14254
  }
14133
14255
  };
14134
- function formatExperimentSummary(summary) {
14135
- let comparisonLine = "";
14136
- if (summary.comparisonExperimentName) {
14137
- comparisonLine = `${summary.comparisonExperimentName} ${chalk2.gray("(baseline)")} \u2190 ${summary.experimentName} ${chalk2.gray("(comparison)")}
14138
-
14139
- `;
14140
- }
14141
- const tableParts = [];
14142
- const hasScores = Object.keys(summary.scores).length > 0;
14143
- const hasMetrics = Object.keys(summary.metrics ?? {}).length > 0;
14144
- const hasComparison = !!summary.comparisonExperimentName;
14145
- if (hasScores || hasMetrics) {
14146
- const headers = [chalk2.gray("Name"), chalk2.gray("Value")];
14147
- if (hasComparison) {
14148
- headers.push(
14149
- chalk2.gray("Change"),
14150
- chalk2.gray("Improvements"),
14151
- chalk2.gray("Regressions")
14152
- );
14153
- }
14154
- const combinedTable = new Table({
14155
- head: hasComparison ? headers : [],
14156
- style: { head: [], "padding-left": 0, "padding-right": 0, border: [] },
14157
- chars: {
14158
- top: "",
14159
- "top-mid": "",
14160
- "top-left": "",
14161
- "top-right": "",
14162
- bottom: "",
14163
- "bottom-mid": "",
14164
- "bottom-left": "",
14165
- "bottom-right": "",
14166
- left: "",
14167
- "left-mid": "",
14168
- mid: "",
14169
- "mid-mid": "",
14170
- right: "",
14171
- "right-mid": "",
14172
- middle: " "
14173
- },
14174
- colWidths: hasComparison ? [18, 10, 10, 13, 12] : [20, 15],
14175
- colAligns: hasComparison ? ["left", "right", "right", "right", "right"] : ["left", "right"],
14176
- wordWrap: false
14177
- });
14178
- const scoreValues = Object.values(summary.scores);
14179
- for (let i = 0; i < scoreValues.length; i++) {
14180
- const score = scoreValues[i];
14181
- const scorePercent = (score.score * 100).toFixed(2);
14182
- const scoreValue = chalk2.white(`${scorePercent}%`);
14183
- let diffString = "";
14184
- if (!isEmpty(score.diff)) {
14185
- const diffPercent = (score.diff * 100).toFixed(2);
14186
- const diffSign = score.diff > 0 ? "+" : "";
14187
- const diffColor = score.diff > 0 ? chalk2.green : chalk2.red;
14188
- diffString = diffColor(`${diffSign}${diffPercent}%`);
14189
- } else {
14190
- diffString = chalk2.gray("-");
14191
- }
14192
- const improvements = score.improvements > 0 ? chalk2.dim.green(score.improvements) : chalk2.gray("-");
14193
- const regressions = score.regressions > 0 ? chalk2.dim.red(score.regressions) : chalk2.gray("-");
14194
- const row = [`${chalk2.blue("\u25EF")} ${score.name}`, scoreValue];
14195
- if (hasComparison) {
14196
- row.push(diffString, improvements, regressions);
14197
- }
14198
- combinedTable.push(row);
14199
- }
14200
- const metricValues = Object.values(summary.metrics ?? {});
14201
- for (let i = 0; i < metricValues.length; i++) {
14202
- const metric = metricValues[i];
14203
- const fractionDigits = Number.isInteger(metric.metric) ? 0 : 2;
14204
- const formattedValue = metric.metric.toFixed(fractionDigits);
14205
- const metricValue = chalk2.white(
14206
- metric.unit === "$" ? `${metric.unit}${formattedValue}` : `${formattedValue}${metric.unit}`
14207
- );
14208
- let diffString = "";
14209
- if (!isEmpty(metric.diff)) {
14210
- const diffPercent = (metric.diff * 100).toFixed(2);
14211
- const diffSign = metric.diff > 0 ? "+" : "";
14212
- const diffColor = metric.diff > 0 ? chalk2.green : chalk2.red;
14213
- diffString = diffColor(`${diffSign}${diffPercent}%`);
14214
- } else {
14215
- diffString = chalk2.gray("-");
14216
- }
14217
- const improvements = metric.improvements > 0 ? chalk2.dim.green(metric.improvements) : chalk2.gray("-");
14218
- const regressions = metric.regressions > 0 ? chalk2.dim.red(metric.regressions) : chalk2.gray("-");
14219
- const row = [`${chalk2.magenta("\u25EF")} ${metric.name}`, metricValue];
14220
- if (hasComparison) {
14221
- row.push(diffString, improvements, regressions);
14222
- }
14223
- combinedTable.push(row);
14224
- }
14225
- tableParts.push(combinedTable.toString());
14226
- }
14227
- const content = [comparisonLine, ...tableParts].filter(Boolean).join("\n");
14228
- const footer = summary.experimentUrl ? terminalLink(
14229
- `View results for ${summary.experimentName}`,
14230
- summary.experimentUrl,
14231
- { fallback: () => `See results at ${summary.experimentUrl}` }
14232
- ) : "";
14233
- const boxContent = [content, footer].filter(Boolean).join("\n\n");
14234
- return "\n" + boxen(boxContent, {
14235
- title: chalk2.gray("Experiment summary"),
14236
- titleAlignment: "left",
14237
- padding: 0.5,
14238
- borderColor: "gray",
14239
- borderStyle: "round"
14240
- });
14241
- }
14242
14256
 
14243
14257
  // dev/types.ts
14244
14258
  import { z as z11 } from "zod/v3";
@@ -14284,9 +14298,9 @@ var evaluatorDefinitionsSchema = z11.record(
14284
14298
 
14285
14299
  // src/index.ts
14286
14300
  configureNode();
14287
- var index_default = exports_node_exports;
14288
14301
  export {
14289
14302
  Attachment,
14303
+ AttachmentReference,
14290
14304
  BaseAttachment,
14291
14305
  BaseExperiment,
14292
14306
  BraintrustMiddleware,
@@ -14335,7 +14349,7 @@ export {
14335
14349
  currentLogger,
14336
14350
  currentSpan,
14337
14351
  deepCopyEvent,
14338
- index_default as default,
14352
+ exports_exports as default,
14339
14353
  defaultErrorScoreHandler,
14340
14354
  deserializePlainStringAsJSON,
14341
14355
  devNullWritableStream,