braintrust 2.2.1-rc.0 → 2.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (20) hide show
  1. package/dev/dist/index.d.mts +1 -1
  2. package/dev/dist/index.d.ts +1 -1
  3. package/dev/dist/index.js +34 -12
  4. package/dev/dist/index.mjs +40 -18
  5. package/dist/browser.d.mts +240 -240
  6. package/dist/browser.d.ts +240 -240
  7. package/dist/browser.js +249 -62
  8. package/dist/browser.mjs +221 -34
  9. package/dist/cli.js +1241 -723
  10. package/dist/index.d.mts +240 -240
  11. package/dist/index.d.ts +240 -240
  12. package/dist/index.js +249 -62
  13. package/dist/index.mjs +227 -40
  14. package/package.json +23 -23
  15. package/util/dist/index.d.mts +3 -2
  16. package/util/dist/index.d.ts +3 -2
  17. package/util/dist/index.js +3 -1
  18. package/util/dist/index.mjs +3 -1
  19. package/LICENSE +0 -201
  20. package/dist/eval-runner.js +0 -12680
package/dist/cli.js CHANGED
@@ -31,14 +31,14 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
31
31
  ));
32
32
  var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
33
33
 
34
- // ../node_modules/.pnpm/@nodelib+fs.stat@2.0.5/node_modules/@nodelib/fs.stat/out/providers/async.js
34
+ // ../node_modules/@nodelib/fs.stat/out/providers/async.js
35
35
  var require_async = __commonJS({
36
- "../node_modules/.pnpm/@nodelib+fs.stat@2.0.5/node_modules/@nodelib/fs.stat/out/providers/async.js"(exports2) {
36
+ "../node_modules/@nodelib/fs.stat/out/providers/async.js"(exports2) {
37
37
  "use strict";
38
38
  Object.defineProperty(exports2, "__esModule", { value: true });
39
39
  exports2.read = void 0;
40
- function read(path9, settings, callback) {
41
- settings.fs.lstat(path9, (lstatError, lstat) => {
40
+ function read(path8, settings, callback) {
41
+ settings.fs.lstat(path8, (lstatError, lstat) => {
42
42
  if (lstatError !== null) {
43
43
  callFailureCallback(callback, lstatError);
44
44
  return;
@@ -47,7 +47,7 @@ var require_async = __commonJS({
47
47
  callSuccessCallback(callback, lstat);
48
48
  return;
49
49
  }
50
- settings.fs.stat(path9, (statError, stat2) => {
50
+ settings.fs.stat(path8, (statError, stat2) => {
51
51
  if (statError !== null) {
52
52
  if (settings.throwErrorOnBrokenSymbolicLink) {
53
53
  callFailureCallback(callback, statError);
@@ -73,19 +73,19 @@ var require_async = __commonJS({
73
73
  }
74
74
  });
75
75
 
76
- // ../node_modules/.pnpm/@nodelib+fs.stat@2.0.5/node_modules/@nodelib/fs.stat/out/providers/sync.js
76
+ // ../node_modules/@nodelib/fs.stat/out/providers/sync.js
77
77
  var require_sync = __commonJS({
78
- "../node_modules/.pnpm/@nodelib+fs.stat@2.0.5/node_modules/@nodelib/fs.stat/out/providers/sync.js"(exports2) {
78
+ "../node_modules/@nodelib/fs.stat/out/providers/sync.js"(exports2) {
79
79
  "use strict";
80
80
  Object.defineProperty(exports2, "__esModule", { value: true });
81
81
  exports2.read = void 0;
82
- function read(path9, settings) {
83
- const lstat = settings.fs.lstatSync(path9);
82
+ function read(path8, settings) {
83
+ const lstat = settings.fs.lstatSync(path8);
84
84
  if (!lstat.isSymbolicLink() || !settings.followSymbolicLink) {
85
85
  return lstat;
86
86
  }
87
87
  try {
88
- const stat2 = settings.fs.statSync(path9);
88
+ const stat2 = settings.fs.statSync(path8);
89
89
  if (settings.markSymbolicLink) {
90
90
  stat2.isSymbolicLink = () => true;
91
91
  }
@@ -101,9 +101,9 @@ var require_sync = __commonJS({
101
101
  }
102
102
  });
103
103
 
104
- // ../node_modules/.pnpm/@nodelib+fs.stat@2.0.5/node_modules/@nodelib/fs.stat/out/adapters/fs.js
104
+ // ../node_modules/@nodelib/fs.stat/out/adapters/fs.js
105
105
  var require_fs = __commonJS({
106
- "../node_modules/.pnpm/@nodelib+fs.stat@2.0.5/node_modules/@nodelib/fs.stat/out/adapters/fs.js"(exports2) {
106
+ "../node_modules/@nodelib/fs.stat/out/adapters/fs.js"(exports2) {
107
107
  "use strict";
108
108
  Object.defineProperty(exports2, "__esModule", { value: true });
109
109
  exports2.createFileSystemAdapter = exports2.FILE_SYSTEM_ADAPTER = void 0;
@@ -124,9 +124,9 @@ var require_fs = __commonJS({
124
124
  }
125
125
  });
126
126
 
127
- // ../node_modules/.pnpm/@nodelib+fs.stat@2.0.5/node_modules/@nodelib/fs.stat/out/settings.js
127
+ // ../node_modules/@nodelib/fs.stat/out/settings.js
128
128
  var require_settings = __commonJS({
129
- "../node_modules/.pnpm/@nodelib+fs.stat@2.0.5/node_modules/@nodelib/fs.stat/out/settings.js"(exports2) {
129
+ "../node_modules/@nodelib/fs.stat/out/settings.js"(exports2) {
130
130
  "use strict";
131
131
  Object.defineProperty(exports2, "__esModule", { value: true });
132
132
  var fs6 = require_fs();
@@ -146,9 +146,9 @@ var require_settings = __commonJS({
146
146
  }
147
147
  });
148
148
 
149
- // ../node_modules/.pnpm/@nodelib+fs.stat@2.0.5/node_modules/@nodelib/fs.stat/out/index.js
149
+ // ../node_modules/@nodelib/fs.stat/out/index.js
150
150
  var require_out = __commonJS({
151
- "../node_modules/.pnpm/@nodelib+fs.stat@2.0.5/node_modules/@nodelib/fs.stat/out/index.js"(exports2) {
151
+ "../node_modules/@nodelib/fs.stat/out/index.js"(exports2) {
152
152
  "use strict";
153
153
  Object.defineProperty(exports2, "__esModule", { value: true });
154
154
  exports2.statSync = exports2.stat = exports2.Settings = void 0;
@@ -156,17 +156,17 @@ var require_out = __commonJS({
156
156
  var sync = require_sync();
157
157
  var settings_1 = require_settings();
158
158
  exports2.Settings = settings_1.default;
159
- function stat2(path9, optionsOrSettingsOrCallback, callback) {
159
+ function stat2(path8, optionsOrSettingsOrCallback, callback) {
160
160
  if (typeof optionsOrSettingsOrCallback === "function") {
161
- async.read(path9, getSettings(), optionsOrSettingsOrCallback);
161
+ async.read(path8, getSettings(), optionsOrSettingsOrCallback);
162
162
  return;
163
163
  }
164
- async.read(path9, getSettings(optionsOrSettingsOrCallback), callback);
164
+ async.read(path8, getSettings(optionsOrSettingsOrCallback), callback);
165
165
  }
166
166
  exports2.stat = stat2;
167
- function statSync2(path9, optionsOrSettings) {
167
+ function statSync2(path8, optionsOrSettings) {
168
168
  const settings = getSettings(optionsOrSettings);
169
- return sync.read(path9, settings);
169
+ return sync.read(path8, settings);
170
170
  }
171
171
  exports2.statSync = statSync2;
172
172
  function getSettings(settingsOrOptions = {}) {
@@ -178,9 +178,9 @@ var require_out = __commonJS({
178
178
  }
179
179
  });
180
180
 
181
- // ../node_modules/.pnpm/queue-microtask@1.2.3/node_modules/queue-microtask/index.js
181
+ // ../node_modules/queue-microtask/index.js
182
182
  var require_queue_microtask = __commonJS({
183
- "../node_modules/.pnpm/queue-microtask@1.2.3/node_modules/queue-microtask/index.js"(exports2, module2) {
183
+ "../node_modules/queue-microtask/index.js"(exports2, module2) {
184
184
  "use strict";
185
185
  var promise;
186
186
  module2.exports = typeof queueMicrotask === "function" ? queueMicrotask.bind(typeof window !== "undefined" ? window : global) : (cb) => (promise || (promise = Promise.resolve())).then(cb).catch((err) => setTimeout(() => {
@@ -189,9 +189,9 @@ var require_queue_microtask = __commonJS({
189
189
  }
190
190
  });
191
191
 
192
- // ../node_modules/.pnpm/run-parallel@1.2.0/node_modules/run-parallel/index.js
192
+ // ../node_modules/run-parallel/index.js
193
193
  var require_run_parallel = __commonJS({
194
- "../node_modules/.pnpm/run-parallel@1.2.0/node_modules/run-parallel/index.js"(exports2, module2) {
194
+ "../node_modules/run-parallel/index.js"(exports2, module2) {
195
195
  "use strict";
196
196
  module2.exports = runParallel;
197
197
  var queueMicrotask2 = require_queue_microtask();
@@ -240,9 +240,9 @@ var require_run_parallel = __commonJS({
240
240
  }
241
241
  });
242
242
 
243
- // ../node_modules/.pnpm/@nodelib+fs.scandir@2.1.5/node_modules/@nodelib/fs.scandir/out/constants.js
243
+ // ../node_modules/@nodelib/fs.scandir/out/constants.js
244
244
  var require_constants = __commonJS({
245
- "../node_modules/.pnpm/@nodelib+fs.scandir@2.1.5/node_modules/@nodelib/fs.scandir/out/constants.js"(exports2) {
245
+ "../node_modules/@nodelib/fs.scandir/out/constants.js"(exports2) {
246
246
  "use strict";
247
247
  Object.defineProperty(exports2, "__esModule", { value: true });
248
248
  exports2.IS_SUPPORT_READDIR_WITH_FILE_TYPES = void 0;
@@ -260,9 +260,9 @@ var require_constants = __commonJS({
260
260
  }
261
261
  });
262
262
 
263
- // ../node_modules/.pnpm/@nodelib+fs.scandir@2.1.5/node_modules/@nodelib/fs.scandir/out/utils/fs.js
263
+ // ../node_modules/@nodelib/fs.scandir/out/utils/fs.js
264
264
  var require_fs2 = __commonJS({
265
- "../node_modules/.pnpm/@nodelib+fs.scandir@2.1.5/node_modules/@nodelib/fs.scandir/out/utils/fs.js"(exports2) {
265
+ "../node_modules/@nodelib/fs.scandir/out/utils/fs.js"(exports2) {
266
266
  "use strict";
267
267
  Object.defineProperty(exports2, "__esModule", { value: true });
268
268
  exports2.createDirentFromStats = void 0;
@@ -285,9 +285,9 @@ var require_fs2 = __commonJS({
285
285
  }
286
286
  });
287
287
 
288
- // ../node_modules/.pnpm/@nodelib+fs.scandir@2.1.5/node_modules/@nodelib/fs.scandir/out/utils/index.js
288
+ // ../node_modules/@nodelib/fs.scandir/out/utils/index.js
289
289
  var require_utils = __commonJS({
290
- "../node_modules/.pnpm/@nodelib+fs.scandir@2.1.5/node_modules/@nodelib/fs.scandir/out/utils/index.js"(exports2) {
290
+ "../node_modules/@nodelib/fs.scandir/out/utils/index.js"(exports2) {
291
291
  "use strict";
292
292
  Object.defineProperty(exports2, "__esModule", { value: true });
293
293
  exports2.fs = void 0;
@@ -296,9 +296,9 @@ var require_utils = __commonJS({
296
296
  }
297
297
  });
298
298
 
299
- // ../node_modules/.pnpm/@nodelib+fs.scandir@2.1.5/node_modules/@nodelib/fs.scandir/out/providers/common.js
299
+ // ../node_modules/@nodelib/fs.scandir/out/providers/common.js
300
300
  var require_common = __commonJS({
301
- "../node_modules/.pnpm/@nodelib+fs.scandir@2.1.5/node_modules/@nodelib/fs.scandir/out/providers/common.js"(exports2) {
301
+ "../node_modules/@nodelib/fs.scandir/out/providers/common.js"(exports2) {
302
302
  "use strict";
303
303
  Object.defineProperty(exports2, "__esModule", { value: true });
304
304
  exports2.joinPathSegments = void 0;
@@ -312,9 +312,9 @@ var require_common = __commonJS({
312
312
  }
313
313
  });
314
314
 
315
- // ../node_modules/.pnpm/@nodelib+fs.scandir@2.1.5/node_modules/@nodelib/fs.scandir/out/providers/async.js
315
+ // ../node_modules/@nodelib/fs.scandir/out/providers/async.js
316
316
  var require_async2 = __commonJS({
317
- "../node_modules/.pnpm/@nodelib+fs.scandir@2.1.5/node_modules/@nodelib/fs.scandir/out/providers/async.js"(exports2) {
317
+ "../node_modules/@nodelib/fs.scandir/out/providers/async.js"(exports2) {
318
318
  "use strict";
319
319
  Object.defineProperty(exports2, "__esModule", { value: true });
320
320
  exports2.readdir = exports2.readdirWithFileTypes = exports2.read = void 0;
@@ -384,16 +384,16 @@ var require_async2 = __commonJS({
384
384
  return;
385
385
  }
386
386
  const tasks = names.map((name) => {
387
- const path9 = common.joinPathSegments(directory, name, settings.pathSegmentSeparator);
387
+ const path8 = common.joinPathSegments(directory, name, settings.pathSegmentSeparator);
388
388
  return (done) => {
389
- fsStat.stat(path9, settings.fsStatSettings, (error2, stats) => {
389
+ fsStat.stat(path8, settings.fsStatSettings, (error2, stats) => {
390
390
  if (error2 !== null) {
391
391
  done(error2);
392
392
  return;
393
393
  }
394
394
  const entry = {
395
395
  name,
396
- path: path9,
396
+ path: path8,
397
397
  dirent: utils.fs.createDirentFromStats(name, stats)
398
398
  };
399
399
  if (settings.stats) {
@@ -422,9 +422,9 @@ var require_async2 = __commonJS({
422
422
  }
423
423
  });
424
424
 
425
- // ../node_modules/.pnpm/@nodelib+fs.scandir@2.1.5/node_modules/@nodelib/fs.scandir/out/providers/sync.js
425
+ // ../node_modules/@nodelib/fs.scandir/out/providers/sync.js
426
426
  var require_sync2 = __commonJS({
427
- "../node_modules/.pnpm/@nodelib+fs.scandir@2.1.5/node_modules/@nodelib/fs.scandir/out/providers/sync.js"(exports2) {
427
+ "../node_modules/@nodelib/fs.scandir/out/providers/sync.js"(exports2) {
428
428
  "use strict";
429
429
  Object.defineProperty(exports2, "__esModule", { value: true });
430
430
  exports2.readdir = exports2.readdirWithFileTypes = exports2.read = void 0;
@@ -481,9 +481,9 @@ var require_sync2 = __commonJS({
481
481
  }
482
482
  });
483
483
 
484
- // ../node_modules/.pnpm/@nodelib+fs.scandir@2.1.5/node_modules/@nodelib/fs.scandir/out/adapters/fs.js
484
+ // ../node_modules/@nodelib/fs.scandir/out/adapters/fs.js
485
485
  var require_fs3 = __commonJS({
486
- "../node_modules/.pnpm/@nodelib+fs.scandir@2.1.5/node_modules/@nodelib/fs.scandir/out/adapters/fs.js"(exports2) {
486
+ "../node_modules/@nodelib/fs.scandir/out/adapters/fs.js"(exports2) {
487
487
  "use strict";
488
488
  Object.defineProperty(exports2, "__esModule", { value: true });
489
489
  exports2.createFileSystemAdapter = exports2.FILE_SYSTEM_ADAPTER = void 0;
@@ -506,12 +506,12 @@ var require_fs3 = __commonJS({
506
506
  }
507
507
  });
508
508
 
509
- // ../node_modules/.pnpm/@nodelib+fs.scandir@2.1.5/node_modules/@nodelib/fs.scandir/out/settings.js
509
+ // ../node_modules/@nodelib/fs.scandir/out/settings.js
510
510
  var require_settings2 = __commonJS({
511
- "../node_modules/.pnpm/@nodelib+fs.scandir@2.1.5/node_modules/@nodelib/fs.scandir/out/settings.js"(exports2) {
511
+ "../node_modules/@nodelib/fs.scandir/out/settings.js"(exports2) {
512
512
  "use strict";
513
513
  Object.defineProperty(exports2, "__esModule", { value: true });
514
- var path9 = require("path");
514
+ var path8 = require("path");
515
515
  var fsStat = require_out();
516
516
  var fs6 = require_fs3();
517
517
  var Settings = class {
@@ -519,7 +519,7 @@ var require_settings2 = __commonJS({
519
519
  this._options = _options;
520
520
  this.followSymbolicLinks = this._getValue(this._options.followSymbolicLinks, false);
521
521
  this.fs = fs6.createFileSystemAdapter(this._options.fs);
522
- this.pathSegmentSeparator = this._getValue(this._options.pathSegmentSeparator, path9.sep);
522
+ this.pathSegmentSeparator = this._getValue(this._options.pathSegmentSeparator, path8.sep);
523
523
  this.stats = this._getValue(this._options.stats, false);
524
524
  this.throwErrorOnBrokenSymbolicLink = this._getValue(this._options.throwErrorOnBrokenSymbolicLink, true);
525
525
  this.fsStatSettings = new fsStat.Settings({
@@ -536,9 +536,9 @@ var require_settings2 = __commonJS({
536
536
  }
537
537
  });
538
538
 
539
- // ../node_modules/.pnpm/@nodelib+fs.scandir@2.1.5/node_modules/@nodelib/fs.scandir/out/index.js
539
+ // ../node_modules/@nodelib/fs.scandir/out/index.js
540
540
  var require_out2 = __commonJS({
541
- "../node_modules/.pnpm/@nodelib+fs.scandir@2.1.5/node_modules/@nodelib/fs.scandir/out/index.js"(exports2) {
541
+ "../node_modules/@nodelib/fs.scandir/out/index.js"(exports2) {
542
542
  "use strict";
543
543
  Object.defineProperty(exports2, "__esModule", { value: true });
544
544
  exports2.Settings = exports2.scandirSync = exports2.scandir = void 0;
@@ -546,17 +546,17 @@ var require_out2 = __commonJS({
546
546
  var sync = require_sync2();
547
547
  var settings_1 = require_settings2();
548
548
  exports2.Settings = settings_1.default;
549
- function scandir(path9, optionsOrSettingsOrCallback, callback) {
549
+ function scandir(path8, optionsOrSettingsOrCallback, callback) {
550
550
  if (typeof optionsOrSettingsOrCallback === "function") {
551
- async.read(path9, getSettings(), optionsOrSettingsOrCallback);
551
+ async.read(path8, getSettings(), optionsOrSettingsOrCallback);
552
552
  return;
553
553
  }
554
- async.read(path9, getSettings(optionsOrSettingsOrCallback), callback);
554
+ async.read(path8, getSettings(optionsOrSettingsOrCallback), callback);
555
555
  }
556
556
  exports2.scandir = scandir;
557
- function scandirSync(path9, optionsOrSettings) {
557
+ function scandirSync(path8, optionsOrSettings) {
558
558
  const settings = getSettings(optionsOrSettings);
559
- return sync.read(path9, settings);
559
+ return sync.read(path8, settings);
560
560
  }
561
561
  exports2.scandirSync = scandirSync;
562
562
  function getSettings(settingsOrOptions = {}) {
@@ -568,9 +568,9 @@ var require_out2 = __commonJS({
568
568
  }
569
569
  });
570
570
 
571
- // ../node_modules/.pnpm/reusify@1.0.4/node_modules/reusify/reusify.js
571
+ // ../node_modules/reusify/reusify.js
572
572
  var require_reusify = __commonJS({
573
- "../node_modules/.pnpm/reusify@1.0.4/node_modules/reusify/reusify.js"(exports2, module2) {
573
+ "../node_modules/reusify/reusify.js"(exports2, module2) {
574
574
  "use strict";
575
575
  function reusify(Constructor) {
576
576
  var head = new Constructor();
@@ -599,32 +599,45 @@ var require_reusify = __commonJS({
599
599
  }
600
600
  });
601
601
 
602
- // ../node_modules/.pnpm/fastq@1.16.0/node_modules/fastq/queue.js
602
+ // ../node_modules/fastq/queue.js
603
603
  var require_queue = __commonJS({
604
- "../node_modules/.pnpm/fastq@1.16.0/node_modules/fastq/queue.js"(exports2, module2) {
604
+ "../node_modules/fastq/queue.js"(exports2, module2) {
605
605
  "use strict";
606
606
  var reusify = require_reusify();
607
- function fastqueue(context2, worker, concurrency) {
607
+ function fastqueue(context2, worker, _concurrency) {
608
608
  if (typeof context2 === "function") {
609
- concurrency = worker;
609
+ _concurrency = worker;
610
610
  worker = context2;
611
611
  context2 = null;
612
612
  }
613
- if (concurrency < 1) {
614
- throw new Error("fastqueue concurrency must be greater than 1");
613
+ if (!(_concurrency >= 1)) {
614
+ throw new Error("fastqueue concurrency must be equal to or greater than 1");
615
615
  }
616
616
  var cache = reusify(Task);
617
617
  var queueHead = null;
618
618
  var queueTail = null;
619
619
  var _running = 0;
620
- var errorHandler = null;
620
+ var errorHandler2 = null;
621
621
  var self = {
622
622
  push,
623
623
  drain: noop,
624
624
  saturated: noop,
625
625
  pause,
626
626
  paused: false,
627
- concurrency,
627
+ get concurrency() {
628
+ return _concurrency;
629
+ },
630
+ set concurrency(value) {
631
+ if (!(value >= 1)) {
632
+ throw new Error("fastqueue concurrency must be equal to or greater than 1");
633
+ }
634
+ _concurrency = value;
635
+ if (self.paused) return;
636
+ for (; queueHead && _running < _concurrency; ) {
637
+ _running++;
638
+ release();
639
+ }
640
+ },
628
641
  running,
629
642
  resume,
630
643
  idle,
@@ -634,7 +647,8 @@ var require_queue = __commonJS({
634
647
  empty: noop,
635
648
  kill,
636
649
  killAndDrain,
637
- error: error2
650
+ error: error2,
651
+ abort
638
652
  };
639
653
  return self;
640
654
  function running() {
@@ -664,7 +678,12 @@ var require_queue = __commonJS({
664
678
  function resume() {
665
679
  if (!self.paused) return;
666
680
  self.paused = false;
667
- for (var i = 0; i < self.concurrency; i++) {
681
+ if (queueHead === null) {
682
+ _running++;
683
+ release();
684
+ return;
685
+ }
686
+ for (; queueHead && _running < _concurrency; ) {
668
687
  _running++;
669
688
  release();
670
689
  }
@@ -678,8 +697,8 @@ var require_queue = __commonJS({
678
697
  current.release = release;
679
698
  current.value = value;
680
699
  current.callback = done || noop;
681
- current.errorHandler = errorHandler;
682
- if (_running === self.concurrency || self.paused) {
700
+ current.errorHandler = errorHandler2;
701
+ if (_running >= _concurrency || self.paused) {
683
702
  if (queueTail) {
684
703
  queueTail.next = current;
685
704
  queueTail = current;
@@ -699,8 +718,8 @@ var require_queue = __commonJS({
699
718
  current.release = release;
700
719
  current.value = value;
701
720
  current.callback = done || noop;
702
- current.errorHandler = errorHandler;
703
- if (_running === self.concurrency || self.paused) {
721
+ current.errorHandler = errorHandler2;
722
+ if (_running >= _concurrency || self.paused) {
704
723
  if (queueHead) {
705
724
  current.next = queueHead;
706
725
  queueHead = current;
@@ -719,7 +738,7 @@ var require_queue = __commonJS({
719
738
  cache.release(holder);
720
739
  }
721
740
  var next = queueHead;
722
- if (next) {
741
+ if (next && _running <= _concurrency) {
723
742
  if (!self.paused) {
724
743
  if (queueTail === queueHead) {
725
744
  queueTail = null;
@@ -748,8 +767,30 @@ var require_queue = __commonJS({
748
767
  self.drain();
749
768
  self.drain = noop;
750
769
  }
770
+ function abort() {
771
+ var current = queueHead;
772
+ queueHead = null;
773
+ queueTail = null;
774
+ while (current) {
775
+ var next = current.next;
776
+ var callback = current.callback;
777
+ var errorHandler3 = current.errorHandler;
778
+ var val = current.value;
779
+ var context3 = current.context;
780
+ current.value = null;
781
+ current.callback = noop;
782
+ current.errorHandler = null;
783
+ if (errorHandler3) {
784
+ errorHandler3(new Error("abort"), val);
785
+ }
786
+ callback.call(context3, new Error("abort"));
787
+ current.release(current);
788
+ current = next;
789
+ }
790
+ self.drain = noop;
791
+ }
751
792
  function error2(handler) {
752
- errorHandler = handler;
793
+ errorHandler2 = handler;
753
794
  }
754
795
  }
755
796
  function noop() {
@@ -764,20 +805,20 @@ var require_queue = __commonJS({
764
805
  var self = this;
765
806
  this.worked = function worked(err, result) {
766
807
  var callback = self.callback;
767
- var errorHandler = self.errorHandler;
808
+ var errorHandler2 = self.errorHandler;
768
809
  var val = self.value;
769
810
  self.value = null;
770
811
  self.callback = noop;
771
812
  if (self.errorHandler) {
772
- errorHandler(err, val);
813
+ errorHandler2(err, val);
773
814
  }
774
815
  callback.call(self.context, err, result);
775
816
  self.release(self);
776
817
  };
777
818
  }
778
- function queueAsPromised(context2, worker, concurrency) {
819
+ function queueAsPromised(context2, worker, _concurrency) {
779
820
  if (typeof context2 === "function") {
780
- concurrency = worker;
821
+ _concurrency = worker;
781
822
  worker = context2;
782
823
  context2 = null;
783
824
  }
@@ -786,7 +827,7 @@ var require_queue = __commonJS({
786
827
  cb(null, res);
787
828
  }, cb);
788
829
  }
789
- var queue2 = fastqueue(context2, asyncWrapper, concurrency);
830
+ var queue2 = fastqueue(context2, asyncWrapper, _concurrency);
790
831
  var pushCb = queue2.push;
791
832
  var unshiftCb = queue2.unshift;
792
833
  queue2.push = push;
@@ -820,17 +861,19 @@ var require_queue = __commonJS({
820
861
  return p;
821
862
  }
822
863
  function drained() {
823
- if (queue2.idle()) {
824
- return new Promise(function(resolve2) {
825
- resolve2();
826
- });
827
- }
828
- var previousDrain = queue2.drain;
829
864
  var p = new Promise(function(resolve2) {
830
- queue2.drain = function() {
831
- previousDrain();
832
- resolve2();
833
- };
865
+ process.nextTick(function() {
866
+ if (queue2.idle()) {
867
+ resolve2();
868
+ } else {
869
+ var previousDrain = queue2.drain;
870
+ queue2.drain = function() {
871
+ if (typeof previousDrain === "function") previousDrain();
872
+ resolve2();
873
+ queue2.drain = previousDrain;
874
+ };
875
+ }
876
+ });
834
877
  });
835
878
  return p;
836
879
  }
@@ -840,9 +883,9 @@ var require_queue = __commonJS({
840
883
  }
841
884
  });
842
885
 
843
- // ../node_modules/.pnpm/@nodelib+fs.walk@1.2.8/node_modules/@nodelib/fs.walk/out/readers/common.js
886
+ // ../node_modules/@nodelib/fs.walk/out/readers/common.js
844
887
  var require_common2 = __commonJS({
845
- "../node_modules/.pnpm/@nodelib+fs.walk@1.2.8/node_modules/@nodelib/fs.walk/out/readers/common.js"(exports2) {
888
+ "../node_modules/@nodelib/fs.walk/out/readers/common.js"(exports2) {
846
889
  "use strict";
847
890
  Object.defineProperty(exports2, "__esModule", { value: true });
848
891
  exports2.joinPathSegments = exports2.replacePathSegmentSeparator = exports2.isAppliedFilter = exports2.isFatalError = void 0;
@@ -874,9 +917,9 @@ var require_common2 = __commonJS({
874
917
  }
875
918
  });
876
919
 
877
- // ../node_modules/.pnpm/@nodelib+fs.walk@1.2.8/node_modules/@nodelib/fs.walk/out/readers/reader.js
920
+ // ../node_modules/@nodelib/fs.walk/out/readers/reader.js
878
921
  var require_reader = __commonJS({
879
- "../node_modules/.pnpm/@nodelib+fs.walk@1.2.8/node_modules/@nodelib/fs.walk/out/readers/reader.js"(exports2) {
922
+ "../node_modules/@nodelib/fs.walk/out/readers/reader.js"(exports2) {
880
923
  "use strict";
881
924
  Object.defineProperty(exports2, "__esModule", { value: true });
882
925
  var common = require_common2();
@@ -891,9 +934,9 @@ var require_reader = __commonJS({
891
934
  }
892
935
  });
893
936
 
894
- // ../node_modules/.pnpm/@nodelib+fs.walk@1.2.8/node_modules/@nodelib/fs.walk/out/readers/async.js
937
+ // ../node_modules/@nodelib/fs.walk/out/readers/async.js
895
938
  var require_async3 = __commonJS({
896
- "../node_modules/.pnpm/@nodelib+fs.walk@1.2.8/node_modules/@nodelib/fs.walk/out/readers/async.js"(exports2) {
939
+ "../node_modules/@nodelib/fs.walk/out/readers/async.js"(exports2) {
897
940
  "use strict";
898
941
  Object.defineProperty(exports2, "__esModule", { value: true });
899
942
  var events_1 = require("events");
@@ -994,9 +1037,9 @@ var require_async3 = __commonJS({
994
1037
  }
995
1038
  });
996
1039
 
997
- // ../node_modules/.pnpm/@nodelib+fs.walk@1.2.8/node_modules/@nodelib/fs.walk/out/providers/async.js
1040
+ // ../node_modules/@nodelib/fs.walk/out/providers/async.js
998
1041
  var require_async4 = __commonJS({
999
- "../node_modules/.pnpm/@nodelib+fs.walk@1.2.8/node_modules/@nodelib/fs.walk/out/providers/async.js"(exports2) {
1042
+ "../node_modules/@nodelib/fs.walk/out/providers/async.js"(exports2) {
1000
1043
  "use strict";
1001
1044
  Object.defineProperty(exports2, "__esModule", { value: true });
1002
1045
  var async_1 = require_async3();
@@ -1030,9 +1073,9 @@ var require_async4 = __commonJS({
1030
1073
  }
1031
1074
  });
1032
1075
 
1033
- // ../node_modules/.pnpm/@nodelib+fs.walk@1.2.8/node_modules/@nodelib/fs.walk/out/providers/stream.js
1076
+ // ../node_modules/@nodelib/fs.walk/out/providers/stream.js
1034
1077
  var require_stream = __commonJS({
1035
- "../node_modules/.pnpm/@nodelib+fs.walk@1.2.8/node_modules/@nodelib/fs.walk/out/providers/stream.js"(exports2) {
1078
+ "../node_modules/@nodelib/fs.walk/out/providers/stream.js"(exports2) {
1036
1079
  "use strict";
1037
1080
  Object.defineProperty(exports2, "__esModule", { value: true });
1038
1081
  var stream_1 = require("stream");
@@ -1071,9 +1114,9 @@ var require_stream = __commonJS({
1071
1114
  }
1072
1115
  });
1073
1116
 
1074
- // ../node_modules/.pnpm/@nodelib+fs.walk@1.2.8/node_modules/@nodelib/fs.walk/out/readers/sync.js
1117
+ // ../node_modules/@nodelib/fs.walk/out/readers/sync.js
1075
1118
  var require_sync3 = __commonJS({
1076
- "../node_modules/.pnpm/@nodelib+fs.walk@1.2.8/node_modules/@nodelib/fs.walk/out/readers/sync.js"(exports2) {
1119
+ "../node_modules/@nodelib/fs.walk/out/readers/sync.js"(exports2) {
1077
1120
  "use strict";
1078
1121
  Object.defineProperty(exports2, "__esModule", { value: true });
1079
1122
  var fsScandir = require_out2();
@@ -1135,9 +1178,9 @@ var require_sync3 = __commonJS({
1135
1178
  }
1136
1179
  });
1137
1180
 
1138
- // ../node_modules/.pnpm/@nodelib+fs.walk@1.2.8/node_modules/@nodelib/fs.walk/out/providers/sync.js
1181
+ // ../node_modules/@nodelib/fs.walk/out/providers/sync.js
1139
1182
  var require_sync4 = __commonJS({
1140
- "../node_modules/.pnpm/@nodelib+fs.walk@1.2.8/node_modules/@nodelib/fs.walk/out/providers/sync.js"(exports2) {
1183
+ "../node_modules/@nodelib/fs.walk/out/providers/sync.js"(exports2) {
1141
1184
  "use strict";
1142
1185
  Object.defineProperty(exports2, "__esModule", { value: true });
1143
1186
  var sync_1 = require_sync3();
@@ -1155,12 +1198,12 @@ var require_sync4 = __commonJS({
1155
1198
  }
1156
1199
  });
1157
1200
 
1158
- // ../node_modules/.pnpm/@nodelib+fs.walk@1.2.8/node_modules/@nodelib/fs.walk/out/settings.js
1201
+ // ../node_modules/@nodelib/fs.walk/out/settings.js
1159
1202
  var require_settings3 = __commonJS({
1160
- "../node_modules/.pnpm/@nodelib+fs.walk@1.2.8/node_modules/@nodelib/fs.walk/out/settings.js"(exports2) {
1203
+ "../node_modules/@nodelib/fs.walk/out/settings.js"(exports2) {
1161
1204
  "use strict";
1162
1205
  Object.defineProperty(exports2, "__esModule", { value: true });
1163
- var path9 = require("path");
1206
+ var path8 = require("path");
1164
1207
  var fsScandir = require_out2();
1165
1208
  var Settings = class {
1166
1209
  constructor(_options = {}) {
@@ -1170,7 +1213,7 @@ var require_settings3 = __commonJS({
1170
1213
  this.deepFilter = this._getValue(this._options.deepFilter, null);
1171
1214
  this.entryFilter = this._getValue(this._options.entryFilter, null);
1172
1215
  this.errorFilter = this._getValue(this._options.errorFilter, null);
1173
- this.pathSegmentSeparator = this._getValue(this._options.pathSegmentSeparator, path9.sep);
1216
+ this.pathSegmentSeparator = this._getValue(this._options.pathSegmentSeparator, path8.sep);
1174
1217
  this.fsScandirSettings = new fsScandir.Settings({
1175
1218
  followSymbolicLinks: this._options.followSymbolicLinks,
1176
1219
  fs: this._options.fs,
@@ -1187,9 +1230,9 @@ var require_settings3 = __commonJS({
1187
1230
  }
1188
1231
  });
1189
1232
 
1190
- // ../node_modules/.pnpm/@nodelib+fs.walk@1.2.8/node_modules/@nodelib/fs.walk/out/index.js
1233
+ // ../node_modules/@nodelib/fs.walk/out/index.js
1191
1234
  var require_out3 = __commonJS({
1192
- "../node_modules/.pnpm/@nodelib+fs.walk@1.2.8/node_modules/@nodelib/fs.walk/out/index.js"(exports2) {
1235
+ "../node_modules/@nodelib/fs.walk/out/index.js"(exports2) {
1193
1236
  "use strict";
1194
1237
  Object.defineProperty(exports2, "__esModule", { value: true });
1195
1238
  exports2.Settings = exports2.walkStream = exports2.walkSync = exports2.walk = void 0;
@@ -1206,12 +1249,12 @@ var require_out3 = __commonJS({
1206
1249
  new async_1.default(directory, getSettings(optionsOrSettingsOrCallback)).read(callback);
1207
1250
  }
1208
1251
  exports2.walk = walk2;
1209
- function walkSync2(directory, optionsOrSettings) {
1252
+ function walkSync(directory, optionsOrSettings) {
1210
1253
  const settings = getSettings(optionsOrSettings);
1211
1254
  const provider = new sync_1.default(directory, settings);
1212
1255
  return provider.read();
1213
1256
  }
1214
- exports2.walkSync = walkSync2;
1257
+ exports2.walkSync = walkSync;
1215
1258
  function walkStream(directory, optionsOrSettings) {
1216
1259
  const settings = getSettings(optionsOrSettings);
1217
1260
  const provider = new stream_1.default(directory, settings);
@@ -1232,7 +1275,7 @@ var require_package = __commonJS({
1232
1275
  "package.json"(exports2, module2) {
1233
1276
  module2.exports = {
1234
1277
  name: "braintrust",
1235
- version: "2.2.1-rc.0",
1278
+ version: "2.2.2",
1236
1279
  description: "SDK for integrating Braintrust",
1237
1280
  repository: {
1238
1281
  type: "git",
@@ -1392,15 +1435,14 @@ module.exports = __toCommonJS(cli_exports);
1392
1435
  var esbuild = __toESM(require("esbuild"));
1393
1436
  var dotenv2 = __toESM(require("dotenv"));
1394
1437
  var import_fs2 = __toESM(require("fs"));
1395
- var import_os2 = __toESM(require("os"));
1396
- var import_path6 = __toESM(require("path"));
1397
- var import_util15 = __toESM(require("util"));
1438
+ var import_os = __toESM(require("os"));
1439
+ var import_path5 = __toESM(require("path"));
1440
+ var import_util16 = __toESM(require("util"));
1398
1441
  var fsWalk = __toESM(require_out3());
1399
1442
  var import_minimatch = require("minimatch");
1400
1443
  var import_argparse = require("argparse");
1401
1444
  var import_uuid3 = require("uuid");
1402
1445
  var import_pluralize4 = __toESM(require("pluralize"));
1403
- var import_child_process = require("child_process");
1404
1446
 
1405
1447
  // src/logger.ts
1406
1448
  var import_uuid2 = require("uuid");
@@ -2097,6 +2139,10 @@ var SpanComponentsV3 = class _SpanComponentsV3 {
2097
2139
  }
2098
2140
  };
2099
2141
 
2142
+ // util/http_headers.ts
2143
+ var BT_FOUND_EXISTING_HEADER = "x-bt-found-existing";
2144
+ var BT_CURSOR_HEADER = "x-bt-cursor";
2145
+
2100
2146
  // util/type_util.ts
2101
2147
  function isObject(value) {
2102
2148
  return value instanceof Object && !(value instanceof Array);
@@ -2131,14 +2177,14 @@ function mergeDictsWithPaths({
2131
2177
  function mergeDictsWithPathsHelper({
2132
2178
  mergeInto,
2133
2179
  mergeFrom,
2134
- path: path9,
2180
+ path: path8,
2135
2181
  mergePaths
2136
2182
  }) {
2137
2183
  Object.entries(mergeFrom).forEach(([k, mergeFromV]) => {
2138
- const fullPath = path9.concat([k]);
2184
+ const fullPath = path8.concat([k]);
2139
2185
  const fullPathSerialized = JSON.stringify(fullPath);
2140
2186
  const mergeIntoV = recordFind(mergeInto, k);
2141
- const isSetUnionField = path9.length === 0 && SET_UNION_FIELDS.has(k) && !mergePaths.has(fullPathSerialized);
2187
+ const isSetUnionField = path8.length === 0 && SET_UNION_FIELDS.has(k) && !mergePaths.has(fullPathSerialized);
2142
2188
  if (isSetUnionField && isArray(mergeIntoV) && isArray(mergeFromV)) {
2143
2189
  const seen = /* @__PURE__ */ new Set();
2144
2190
  const combined = [];
@@ -2171,9 +2217,9 @@ function mergeDicts(mergeInto, mergeFrom) {
2171
2217
  function recordFind(m, k) {
2172
2218
  return m[k];
2173
2219
  }
2174
- function getObjValueByPath(row, path9) {
2220
+ function getObjValueByPath(row, path8) {
2175
2221
  let curr = row;
2176
- for (const p of path9) {
2222
+ for (const p of path8) {
2177
2223
  if (!isObjectOrArray(curr)) {
2178
2224
  return null;
2179
2225
  }
@@ -2623,6 +2669,22 @@ var SpanComponentsV4 = class _SpanComponentsV4 {
2623
2669
  return new _SpanComponentsV4(spanComponentsV4Schema.parse(jsonObj));
2624
2670
  }
2625
2671
  };
2672
+ function parseParent(parent) {
2673
+ return typeof parent === "string" ? parent : parent ? new SpanComponentsV4({
2674
+ object_type: parent.object_type === "experiment" ? 1 /* EXPERIMENT */ : parent.object_type === "playground_logs" ? 3 /* PLAYGROUND_LOGS */ : 2 /* PROJECT_LOGS */,
2675
+ object_id: parent.object_id,
2676
+ ...parent.row_ids ? {
2677
+ row_id: parent.row_ids.id,
2678
+ span_id: parent.row_ids.span_id,
2679
+ root_span_id: parent.row_ids.root_span_id
2680
+ } : {
2681
+ row_id: void 0,
2682
+ span_id: void 0,
2683
+ root_span_id: void 0
2684
+ },
2685
+ propagated_event: parent.propagated_event
2686
+ }).toStr() : void 0;
2687
+ }
2626
2688
  function makeScorerPropagatedEvent(parent) {
2627
2689
  const parentPropagatedEvent = parent ? SpanComponentsV4.fromStr(parent).data.propagated_event ?? {} : {};
2628
2690
  return mergeDicts(
@@ -3307,7 +3369,8 @@ var SpanType = import_v36.z.union([
3307
3369
  "automation",
3308
3370
  "facet",
3309
3371
  "preprocessor",
3310
- "classifier"
3372
+ "classifier",
3373
+ "review"
3311
3374
  ]),
3312
3375
  import_v36.z.null()
3313
3376
  ]);
@@ -5030,10 +5093,10 @@ var DiskCache = class {
5030
5093
  return;
5031
5094
  }
5032
5095
  const stats = await Promise.all(
5033
- paths.map(async (path9) => {
5034
- const stat2 = await isomorph_default.stat(path9);
5096
+ paths.map(async (path8) => {
5097
+ const stat2 = await isomorph_default.stat(path8);
5035
5098
  return {
5036
- path: path9,
5099
+ path: path8,
5037
5100
  mtime: stat2.mtime.getTime()
5038
5101
  };
5039
5102
  })
@@ -6016,9 +6079,9 @@ var HTTPConnection = class _HTTPConnection {
6016
6079
  this.headers["Authorization"] = `Bearer ${this.token}`;
6017
6080
  }
6018
6081
  }
6019
- async get(path9, params = void 0, config3) {
6082
+ async get(path8, params = void 0, config3) {
6020
6083
  const { headers, ...rest } = config3 || {};
6021
- const url = new URL(_urljoin(this.base_url, path9));
6084
+ const url = new URL(_urljoin(this.base_url, path8));
6022
6085
  url.search = new URLSearchParams(
6023
6086
  params ? Object.entries(params).filter(([_, v]) => v !== void 0).flatMap(
6024
6087
  ([k, v]) => v !== void 0 ? typeof v === "string" ? [[k, v]] : v.map((x) => [k, x]) : []
@@ -6039,13 +6102,13 @@ var HTTPConnection = class _HTTPConnection {
6039
6102
  })
6040
6103
  );
6041
6104
  }
6042
- async post(path9, params, config3) {
6105
+ async post(path8, params, config3) {
6043
6106
  const { headers, ...rest } = config3 || {};
6044
6107
  const this_fetch = this.fetch;
6045
6108
  const this_base_url = this.base_url;
6046
6109
  const this_headers = this.headers;
6047
6110
  return await checkResponse(
6048
- await this_fetch(_urljoin(this_base_url, path9), {
6111
+ await this_fetch(_urljoin(this_base_url, path8), {
6049
6112
  method: "POST",
6050
6113
  headers: {
6051
6114
  Accept: "application/json",
@@ -6445,12 +6508,19 @@ function updateSpanImpl({
6445
6508
  parentObjectType,
6446
6509
  parentObjectId,
6447
6510
  id,
6511
+ root_span_id,
6512
+ span_id,
6448
6513
  event
6449
6514
  }) {
6515
+ if (isEmpty2(root_span_id) !== isEmpty2(span_id)) {
6516
+ throw new Error("both root_span_id and span_id must be set, or neither");
6517
+ }
6518
+ const hasExplicitSpanIds = root_span_id !== void 0 && span_id !== void 0;
6450
6519
  const updateEvent = deepCopyEvent(
6451
6520
  validateAndSanitizeExperimentLogPartialArgs({
6521
+ ...event,
6452
6522
  id,
6453
- ...event
6523
+ ...hasExplicitSpanIds ? { root_span_id, span_id } : {}
6454
6524
  })
6455
6525
  );
6456
6526
  const parentIds = async () => new SpanComponentsV3({
@@ -6773,7 +6843,7 @@ var Logger = class {
6773
6843
  * @param event The event data to update the span with. Must include `id`. See {@link Experiment.log} for a full list of valid fields.
6774
6844
  */
6775
6845
  updateSpan(event) {
6776
- const { id, ...eventRest } = event;
6846
+ const { id, root_span_id, span_id, ...eventRest } = event;
6777
6847
  if (!id) {
6778
6848
  throw new Error("Span id is required to update a span");
6779
6849
  }
@@ -6782,6 +6852,8 @@ var Logger = class {
6782
6852
  parentObjectType: this.parentObjectType(),
6783
6853
  parentObjectId: this.lazyId,
6784
6854
  id,
6855
+ root_span_id,
6856
+ span_id,
6785
6857
  event: eventRest
6786
6858
  });
6787
6859
  }
@@ -7588,6 +7660,76 @@ function init(projectOrOptions, optionalOptions) {
7588
7660
  }
7589
7661
  return ret;
7590
7662
  }
7663
+ function initDataset(projectOrOptions, optionalOptions) {
7664
+ const options = (() => {
7665
+ if (typeof projectOrOptions === "string") {
7666
+ return { ...optionalOptions, project: projectOrOptions };
7667
+ } else {
7668
+ if (optionalOptions !== void 0) {
7669
+ throw new Error(
7670
+ "Cannot specify options struct as both parameters. Must call either initDataset(project, options) or initDataset(options)."
7671
+ );
7672
+ }
7673
+ return projectOrOptions;
7674
+ }
7675
+ })();
7676
+ const {
7677
+ project,
7678
+ dataset,
7679
+ description,
7680
+ version: version2,
7681
+ appUrl,
7682
+ apiKey,
7683
+ orgName,
7684
+ fetch: fetch2,
7685
+ forceLogin,
7686
+ projectId,
7687
+ metadata,
7688
+ useOutput: legacy,
7689
+ state: stateArg,
7690
+ _internal_btql
7691
+ } = options;
7692
+ const state = stateArg ?? _globalState;
7693
+ const lazyMetadata = new LazyValue(
7694
+ async () => {
7695
+ await state.login({
7696
+ orgName,
7697
+ apiKey,
7698
+ appUrl,
7699
+ fetch: fetch2,
7700
+ forceLogin
7701
+ });
7702
+ const args = {
7703
+ org_id: state.orgId,
7704
+ project_name: project,
7705
+ project_id: projectId,
7706
+ dataset_name: dataset,
7707
+ description,
7708
+ metadata
7709
+ };
7710
+ const response = await state.appConn().post_json("api/dataset/register", args);
7711
+ return {
7712
+ project: {
7713
+ id: response.project.id,
7714
+ name: response.project.name,
7715
+ fullInfo: response.project
7716
+ },
7717
+ dataset: {
7718
+ id: response.dataset.id,
7719
+ name: response.dataset.name,
7720
+ fullInfo: response.dataset
7721
+ }
7722
+ };
7723
+ }
7724
+ );
7725
+ return new Dataset2(
7726
+ stateArg ?? _globalState,
7727
+ lazyMetadata,
7728
+ version2,
7729
+ legacy,
7730
+ _internal_btql
7731
+ );
7732
+ }
7591
7733
  async function computeLoggerMetadata(state, {
7592
7734
  project_name,
7593
7735
  project_id
@@ -7838,6 +7980,9 @@ function withCurrent(span, callback, state = void 0) {
7838
7980
  const currentState = state ?? _globalState;
7839
7981
  return currentState.contextManager.runInContext(span, () => callback(span));
7840
7982
  }
7983
+ function withParent(parent, callback, state = void 0) {
7984
+ return (state ?? _globalState).currentParent.run(parent, () => callback());
7985
+ }
7841
7986
  function _saveOrgInfo(state, org_info, org_name) {
7842
7987
  if (org_info.length === 0) {
7843
7988
  throw new LoginInvalidOrgError(
@@ -8392,7 +8537,7 @@ View complete results in Braintrust or run experiment.summarize() again.`
8392
8537
  * @param event The event data to update the span with. Must include `id`. See {@link Experiment.log} for a full list of valid fields.
8393
8538
  */
8394
8539
  updateSpan(event) {
8395
- const { id, ...eventRest } = event;
8540
+ const { id, root_span_id, span_id, ...eventRest } = event;
8396
8541
  if (!id) {
8397
8542
  throw new Error("Span id is required to update a span");
8398
8543
  }
@@ -8401,6 +8546,8 @@ View complete results in Braintrust or run experiment.summarize() again.`
8401
8546
  parentObjectType: this.parentObjectType(),
8402
8547
  parentObjectId: this.lazyId,
8403
8548
  id,
8549
+ root_span_id,
8550
+ span_id,
8404
8551
  event: eventRest
8405
8552
  });
8406
8553
  }
@@ -9579,7 +9726,7 @@ var BarProgressReporter = class {
9579
9726
  var import_chalk3 = __toESM(require("chalk"));
9580
9727
  var import_termi_link2 = require("termi-link");
9581
9728
 
9582
- // ../node_modules/.pnpm/async@3.2.5/node_modules/async/dist/async.mjs
9729
+ // ../node_modules/async/dist/async.mjs
9583
9730
  function initialParams(fn) {
9584
9731
  return function(...args) {
9585
9732
  var callback = args.pop();
@@ -9704,7 +9851,6 @@ function isArrayLike(value) {
9704
9851
  return value && typeof value.length === "number" && value.length >= 0 && value.length % 1 === 0;
9705
9852
  }
9706
9853
  var breakLoop = {};
9707
- var breakLoop$1 = breakLoop;
9708
9854
  function once(fn) {
9709
9855
  function wrapper(...args) {
9710
9856
  if (fn === null) return;
@@ -9796,7 +9942,7 @@ function asyncEachOfLimit(generator, limit, iteratee, callback) {
9796
9942
  canceled = true;
9797
9943
  return;
9798
9944
  }
9799
- if (result === breakLoop$1 || done && running <= 0) {
9945
+ if (result === breakLoop || done && running <= 0) {
9800
9946
  done = true;
9801
9947
  return callback(null);
9802
9948
  }
@@ -9839,7 +9985,7 @@ var eachOfLimit$2 = (limit) => {
9839
9985
  } else if (err === false) {
9840
9986
  done = true;
9841
9987
  canceled = true;
9842
- } else if (value === breakLoop$1 || done && running <= 0) {
9988
+ } else if (value === breakLoop || done && running <= 0) {
9843
9989
  done = true;
9844
9990
  return callback(null);
9845
9991
  } else if (!looping) {
@@ -9882,7 +10028,7 @@ function eachOfArrayLike(coll, iteratee, callback) {
9882
10028
  if (canceled === true) return;
9883
10029
  if (err) {
9884
10030
  callback(err);
9885
- } else if (++completed === length || value === breakLoop$1) {
10031
+ } else if (++completed === length || value === breakLoop) {
9886
10032
  callback(null);
9887
10033
  }
9888
10034
  }
@@ -10278,7 +10424,7 @@ function _createTester(check, getResult) {
10278
10424
  if (check(result) && !testResult) {
10279
10425
  testPassed = true;
10280
10426
  testResult = getResult(true, value);
10281
- return callback(null, breakLoop$1);
10427
+ return callback(null, breakLoop);
10282
10428
  }
10283
10429
  callback();
10284
10430
  });
@@ -11064,8 +11210,8 @@ function validateParametersWithJsonSchema(parameters, schema) {
11064
11210
  const validate = ajv.compile(schema);
11065
11211
  if (!validate(parameters)) {
11066
11212
  const errorMessages = validate.errors?.map((err) => {
11067
- const path9 = err.instancePath || "root";
11068
- return `${path9}: ${err.message}`;
11213
+ const path8 = err.instancePath || "root";
11214
+ return `${path8}: ${err.message}`;
11069
11215
  }).join(", ");
11070
11216
  throw Error(`Invalid parameters: ${errorMessages}`);
11071
11217
  }
@@ -11094,6 +11240,13 @@ var EvalResultWithSummary = class {
11094
11240
  };
11095
11241
  }
11096
11242
  };
11243
+ function makeEvalName(projectName, experimentName) {
11244
+ let out = projectName;
11245
+ if (experimentName) {
11246
+ out += ` [experimentName=${experimentName}]`;
11247
+ }
11248
+ return out;
11249
+ }
11097
11250
  function initExperiment(state, options = {}) {
11098
11251
  return init({
11099
11252
  state,
@@ -11128,6 +11281,118 @@ globalThis._evals = {
11128
11281
  function _initializeSpanContext() {
11129
11282
  globalThis._spanContext = { currentSpan, withCurrent, startSpan, NOOP_SPAN };
11130
11283
  }
11284
+ async function Eval(name, evaluator, reporterOrOpts) {
11285
+ const options = isEmpty2(reporterOrOpts) ? {} : typeof reporterOrOpts === "string" ? { reporter: reporterOrOpts } : "name" in reporterOrOpts ? { reporter: reporterOrOpts } : reporterOrOpts;
11286
+ let evalName = makeEvalName(name, evaluator.experimentName);
11287
+ if (globalThis._evals.evaluators[evalName]) {
11288
+ evalName = `${evalName}_${Object.keys(_evals).length}`;
11289
+ }
11290
+ if (globalThis._lazy_load) {
11291
+ globalThis._evals.evaluators[evalName] = {
11292
+ // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
11293
+ evaluator: {
11294
+ evalName,
11295
+ projectName: name,
11296
+ ...evaluator
11297
+ },
11298
+ reporter: options.reporter
11299
+ };
11300
+ _initializeSpanContext();
11301
+ return new EvalResultWithSummary(
11302
+ {
11303
+ scores: {},
11304
+ metrics: {},
11305
+ projectName: "",
11306
+ experimentName: ""
11307
+ },
11308
+ []
11309
+ );
11310
+ }
11311
+ const progressReporter = options.progress ?? new SimpleProgressReporter();
11312
+ const shouldCollectResults = options.returnResults ?? true;
11313
+ if (typeof options.reporter === "string") {
11314
+ throw new Error(
11315
+ "Must specify a reporter object, not a name. Can only specify reporter names when running 'braintrust eval'"
11316
+ );
11317
+ }
11318
+ const resolvedReporter = options.reporter || defaultReporter;
11319
+ try {
11320
+ const { data, baseExperiment: defaultBaseExperiment } = callEvaluatorData(
11321
+ evaluator.data
11322
+ );
11323
+ const experiment = options.parent || options.noSendLogs ? null : initExperiment(evaluator.state, {
11324
+ ...evaluator.projectId ? { projectId: evaluator.projectId } : { project: name },
11325
+ experiment: evaluator.experimentName,
11326
+ description: evaluator.description,
11327
+ metadata: evaluator.metadata,
11328
+ isPublic: evaluator.isPublic,
11329
+ update: evaluator.update,
11330
+ baseExperiment: evaluator.baseExperimentName ?? defaultBaseExperiment,
11331
+ baseExperimentId: evaluator.baseExperimentId,
11332
+ gitMetadataSettings: evaluator.gitMetadataSettings,
11333
+ repoInfo: evaluator.repoInfo,
11334
+ dataset: Dataset2.isDataset(data) ? data : void 0
11335
+ });
11336
+ if (experiment && typeof process !== "undefined" && globalThis.BRAINTRUST_CONTEXT_MANAGER !== void 0) {
11337
+ await experiment._waitForId();
11338
+ }
11339
+ if (experiment && options.onStart) {
11340
+ const summary = await experiment.summarize({ summarizeScores: false });
11341
+ options.onStart(summary);
11342
+ }
11343
+ try {
11344
+ const evalDef = {
11345
+ evalName,
11346
+ projectName: name,
11347
+ ...evaluator,
11348
+ data
11349
+ };
11350
+ const enableCache = options.enableCache ?? true;
11351
+ let ret;
11352
+ if (options.parent) {
11353
+ ret = await withParent(
11354
+ options.parent,
11355
+ () => runEvaluator(
11356
+ null,
11357
+ evalDef,
11358
+ progressReporter,
11359
+ [],
11360
+ options.stream,
11361
+ options.parameters,
11362
+ shouldCollectResults,
11363
+ enableCache
11364
+ ),
11365
+ evaluator.state
11366
+ );
11367
+ } else {
11368
+ ret = await runEvaluator(
11369
+ experiment,
11370
+ evalDef,
11371
+ progressReporter,
11372
+ [],
11373
+ options.stream,
11374
+ options.parameters,
11375
+ shouldCollectResults,
11376
+ enableCache
11377
+ );
11378
+ }
11379
+ progressReporter.stop();
11380
+ resolvedReporter.reportEval(evalDef, ret, {
11381
+ verbose: true,
11382
+ jsonl: false
11383
+ });
11384
+ return ret;
11385
+ } finally {
11386
+ if (experiment) {
11387
+ await experiment.flush().catch(console.error);
11388
+ } else if (options.parent) {
11389
+ await flush().catch(console.error);
11390
+ }
11391
+ }
11392
+ } finally {
11393
+ progressReporter.stop();
11394
+ }
11395
+ }
11131
11396
  function serializeJSONWithPlainString(v) {
11132
11397
  if (typeof v === "string") {
11133
11398
  return v;
@@ -11149,21 +11414,21 @@ function parseFilters(filters) {
11149
11414
  if (equalsIdx === -1) {
11150
11415
  throw new Error(`Invalid filter ${f}`);
11151
11416
  }
11152
- const [path9, value] = [f.slice(0, equalsIdx), f.slice(equalsIdx + 1)];
11417
+ const [path8, value] = [f.slice(0, equalsIdx), f.slice(equalsIdx + 1)];
11153
11418
  let deserializedValue = deserializePlainStringAsJSON2(value).value;
11154
11419
  if (typeof deserializedValue !== "string") {
11155
11420
  deserializedValue = value;
11156
11421
  }
11157
11422
  result.push({
11158
- path: path9.split("."),
11423
+ path: path8.split("."),
11159
11424
  pattern: new RegExp(deserializedValue)
11160
11425
  });
11161
11426
  }
11162
11427
  return result;
11163
11428
  }
11164
11429
  function evaluateFilter(object, filter2) {
11165
- const { path: path9, pattern } = filter2;
11166
- const key = path9.reduce(
11430
+ const { path: path8, pattern } = filter2;
11431
+ const key = path8.reduce(
11167
11432
  (acc, p) => typeof acc === "object" && acc !== null ? (
11168
11433
  // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
11169
11434
  acc[p]
@@ -11656,6 +11921,120 @@ function buildLocalSummary(evaluator, results, precomputedScores) {
11656
11921
  )
11657
11922
  };
11658
11923
  }
11924
+ function reportFailures(evaluator, failingResults, { verbose, jsonl }) {
11925
+ if (failingResults.length > 0) {
11926
+ console.error(
11927
+ warning(
11928
+ `Evaluator ${evaluator.evalName} failed with ${failingResults.length} error${failingResults.length === 1 ? "" : "s"}. This evaluation ("${evaluator.evalName}") will not be fully logged.`
11929
+ )
11930
+ );
11931
+ if (jsonl) {
11932
+ console.log(
11933
+ JSON.stringify({
11934
+ evaluatorName: evaluator.evalName,
11935
+ errors: failingResults.map(
11936
+ (r) => `${r.error instanceof Error ? r.error.stack : r.error}`
11937
+ )
11938
+ })
11939
+ );
11940
+ } else {
11941
+ for (const result of failingResults) {
11942
+ logError2(result.error, verbose);
11943
+ }
11944
+ }
11945
+ if (!verbose && !jsonl) {
11946
+ console.error(warning("Add --verbose to see full stack traces."));
11947
+ }
11948
+ }
11949
+ }
11950
+ var defaultReporter = {
11951
+ name: "Braintrust default reporter",
11952
+ async reportEval(evaluator, result, { verbose, jsonl }) {
11953
+ const { results, summary } = result;
11954
+ const failingResults = results.filter(
11955
+ (r) => r.error !== void 0
11956
+ );
11957
+ if (failingResults.length > 0) {
11958
+ reportFailures(evaluator, failingResults, { verbose, jsonl });
11959
+ }
11960
+ if (jsonl) {
11961
+ isomorph_default.writeln(JSON.stringify(summary));
11962
+ } else {
11963
+ isomorph_default.writeln("Experiment summary");
11964
+ isomorph_default.writeln("==================");
11965
+ if (summary.comparisonExperimentName) {
11966
+ isomorph_default.writeln(
11967
+ `${summary.comparisonExperimentName} (baseline) <- ${summary.experimentName} (comparison)`
11968
+ );
11969
+ isomorph_default.writeln("");
11970
+ }
11971
+ const hasScores = Object.keys(summary.scores).length > 0;
11972
+ const hasMetrics = Object.keys(summary.metrics ?? {}).length > 0;
11973
+ const hasComparison = !!summary.comparisonExperimentName;
11974
+ if (hasScores || hasMetrics) {
11975
+ if (hasComparison) {
11976
+ isomorph_default.writeln(
11977
+ "Name Value Change Improvements Regressions"
11978
+ );
11979
+ isomorph_default.writeln(
11980
+ "----------------------------------------------------------------"
11981
+ );
11982
+ }
11983
+ for (const score of Object.values(summary.scores)) {
11984
+ const scorePercent = (score.score * 100).toFixed(2);
11985
+ const scoreValue = `${scorePercent}%`;
11986
+ if (hasComparison) {
11987
+ let diffString = "-";
11988
+ if (!isEmpty2(score.diff)) {
11989
+ const diffPercent = (score.diff * 100).toFixed(2);
11990
+ const diffSign = score.diff > 0 ? "+" : "";
11991
+ diffString = `${diffSign}${diffPercent}%`;
11992
+ }
11993
+ const improvements = score.improvements > 0 ? score.improvements.toString() : "-";
11994
+ const regressions = score.regressions > 0 ? score.regressions.toString() : "-";
11995
+ isomorph_default.writeln(
11996
+ `${score.name.padEnd(18)} ${scoreValue.padStart(10)} ${diffString.padStart(10)} ${improvements.padStart(12)} ${regressions.padStart(11)}`
11997
+ );
11998
+ } else {
11999
+ isomorph_default.writeln(`${score.name.padEnd(20)} ${scoreValue.padStart(15)}`);
12000
+ }
12001
+ }
12002
+ for (const metric of Object.values(summary.metrics ?? {})) {
12003
+ const fractionDigits = Number.isInteger(metric.metric) ? 0 : 2;
12004
+ const formattedValue = metric.metric.toFixed(fractionDigits);
12005
+ const metricValue = metric.unit === "$" ? `${metric.unit}${formattedValue}` : `${formattedValue}${metric.unit}`;
12006
+ if (hasComparison) {
12007
+ let diffString = "-";
12008
+ if (!isEmpty2(metric.diff)) {
12009
+ const diffPercent = (metric.diff * 100).toFixed(2);
12010
+ const diffSign = metric.diff > 0 ? "+" : "";
12011
+ diffString = `${diffSign}${diffPercent}%`;
12012
+ }
12013
+ const improvements = metric.improvements > 0 ? metric.improvements.toString() : "-";
12014
+ const regressions = metric.regressions > 0 ? metric.regressions.toString() : "-";
12015
+ isomorph_default.writeln(
12016
+ `${metric.name.padEnd(18)} ${metricValue.padStart(10)} ${diffString.padStart(10)} ${improvements.padStart(12)} ${regressions.padStart(11)}`
12017
+ );
12018
+ } else {
12019
+ isomorph_default.writeln(
12020
+ `${metric.name.padEnd(20)} ${metricValue.padStart(15)}`
12021
+ );
12022
+ }
12023
+ }
12024
+ }
12025
+ if (summary.experimentUrl) {
12026
+ isomorph_default.writeln("");
12027
+ isomorph_default.writeln(`View results for ${summary.experimentName}`);
12028
+ isomorph_default.writeln(`See results at ${summary.experimentUrl}`);
12029
+ }
12030
+ }
12031
+ isomorph_default.writeln("");
12032
+ return failingResults.length === 0;
12033
+ },
12034
+ async reportRun(evalReports) {
12035
+ return evalReports.every((r) => r);
12036
+ }
12037
+ };
11659
12038
 
11660
12039
  // src/cli/reporters/eval.ts
11661
12040
  var import_chalk2 = __toESM(require("chalk"));
@@ -12086,15 +12465,15 @@ var path2 = __toESM(require("path"));
12086
12465
 
12087
12466
  // src/cli/jest/tryRealpath.ts
12088
12467
  var import_graceful_fs = require("graceful-fs");
12089
- function tryRealpath(path9) {
12468
+ function tryRealpath(path8) {
12090
12469
  try {
12091
- path9 = import_graceful_fs.realpathSync.native(path9);
12470
+ path8 = import_graceful_fs.realpathSync.native(path8);
12092
12471
  } catch (error2) {
12093
12472
  if (error2.code !== "ENOENT" && error2.code !== "EISDIR") {
12094
12473
  throw error2;
12095
12474
  }
12096
12475
  }
12097
- return path9;
12476
+ return path8;
12098
12477
  }
12099
12478
 
12100
12479
  // src/cli/jest/nodeModulesPaths.ts
@@ -12174,12 +12553,12 @@ function loadModule({
12174
12553
  globalThis._lazy_load = true;
12175
12554
  globalThis.__inherited_braintrust_state = _internalGetGlobalState();
12176
12555
  const __filename2 = inFile;
12177
- const __dirname2 = (0, import_path.dirname)(__filename2);
12556
+ const __dirname = (0, import_path.dirname)(__filename2);
12178
12557
  new Function("require", "module", "__filename", "__dirname", moduleText)(
12179
12558
  require,
12180
12559
  module,
12181
12560
  __filename2,
12182
- __dirname2
12561
+ __dirname
12183
12562
  );
12184
12563
  return { ...globalThis._evals };
12185
12564
  });
@@ -12660,6 +13039,7 @@ var CodeParameters = class {
12660
13039
  this.metadata = opts.metadata;
12661
13040
  }
12662
13041
  async toFunctionDefinition(projectNameToId) {
13042
+ const schema = serializeEvalParameterstoParametersSchema(this.schema);
12663
13043
  return {
12664
13044
  project_id: await projectNameToId.resolve(this.project),
12665
13045
  name: this.name,
@@ -12668,8 +13048,8 @@ var CodeParameters = class {
12668
13048
  function_type: "parameters",
12669
13049
  function_data: {
12670
13050
  type: "parameters",
12671
- data: {},
12672
- __schema: serializeEvalParameterstoParametersSchema(this.schema)
13051
+ data: getDefaultDataFromParametersSchema(schema),
13052
+ __schema: schema
12673
13053
  },
12674
13054
  if_exists: this.ifExists,
12675
13055
  metadata: this.metadata
@@ -12694,6 +13074,33 @@ var ParametersBuilder = class {
12694
13074
  return opts.schema;
12695
13075
  }
12696
13076
  };
13077
+ function serializeEvalParametersToStaticParametersSchema(parameters) {
13078
+ return Object.fromEntries(
13079
+ Object.entries(parameters).map(([name, value]) => {
13080
+ if ("type" in value && value.type === "prompt") {
13081
+ return [
13082
+ name,
13083
+ {
13084
+ type: "prompt",
13085
+ default: value.default ? promptDefinitionToPromptData(value.default) : void 0,
13086
+ description: value.description
13087
+ }
13088
+ ];
13089
+ } else {
13090
+ const schemaObj = zodToJsonSchema(value);
13091
+ return [
13092
+ name,
13093
+ {
13094
+ type: "data",
13095
+ schema: schemaObj,
13096
+ default: schemaObj.default,
13097
+ description: schemaObj.description
13098
+ }
13099
+ ];
13100
+ }
13101
+ })
13102
+ );
13103
+ }
12697
13104
  function serializeEvalParameterstoParametersSchema(parameters) {
12698
13105
  const properties = {};
12699
13106
  const required = [];
@@ -12724,6 +13131,37 @@ function serializeEvalParameterstoParametersSchema(parameters) {
12724
13131
  additionalProperties: true
12725
13132
  };
12726
13133
  }
13134
+ function getDefaultDataFromParametersSchema(schema) {
13135
+ return Object.fromEntries(
13136
+ Object.entries(schema.properties).flatMap(([name, value]) => {
13137
+ if (!("default" in value)) {
13138
+ return [];
13139
+ }
13140
+ return [[name, value.default]];
13141
+ })
13142
+ );
13143
+ }
13144
+ function serializeRemoteEvalParametersContainer(parameters) {
13145
+ if (RemoteEvalParameters.isParameters(parameters)) {
13146
+ return {
13147
+ type: "braintrust.parameters",
13148
+ // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
13149
+ schema: parameters.schema,
13150
+ source: {
13151
+ parametersId: parameters.id,
13152
+ slug: parameters.slug,
13153
+ name: parameters.name,
13154
+ projectId: parameters.projectId,
13155
+ version: parameters.version
13156
+ }
13157
+ };
13158
+ }
13159
+ return {
13160
+ type: "braintrust.staticParameters",
13161
+ schema: serializeEvalParametersToStaticParametersSchema(parameters),
13162
+ source: null
13163
+ };
13164
+ }
12727
13165
  var ProjectNameIdMap = class {
12728
13166
  nameToId = {};
12729
13167
  idToName = {};
@@ -13396,303 +13834,543 @@ function safeStringify(obj) {
13396
13834
  }
13397
13835
  }
13398
13836
 
13399
- // src/cli/util/sse.ts
13400
- var import_net = __toESM(require("net"));
13401
- var import_os = __toESM(require("os"));
13402
- var import_path5 = __toESM(require("path"));
13403
- var import_readline = require("readline");
13404
- function buildSseSocketPath() {
13405
- const pid = process.pid;
13406
- const now2 = Date.now();
13407
- return import_path5.default.join(import_os.default.tmpdir(), `bt-eval-${pid}-${now2}.sock`);
13408
- }
13409
- function createSseServer(socketPath, onEvent) {
13410
- const server = import_net.default.createServer((socket) => {
13411
- let currentEvent = null;
13412
- let dataLines = [];
13413
- const rl = (0, import_readline.createInterface)({ input: socket });
13414
- rl.on("line", (line) => {
13415
- if (line === "") {
13416
- if (currentEvent !== null || dataLines.length > 0) {
13417
- const data = dataLines.join("\n");
13418
- dispatchSseEvent(currentEvent, data, onEvent);
13419
- currentEvent = null;
13420
- dataLines = [];
13421
- }
13422
- return;
13423
- }
13424
- if (line.startsWith("event:")) {
13425
- currentEvent = line.slice("event:".length).trim();
13426
- } else if (line.startsWith("data:")) {
13427
- dataLines.push(line.slice("data:".length).trimStart());
13837
+ // dev/server.ts
13838
+ var import_express = __toESM(require("express"));
13839
+ var import_cors = __toESM(require("cors"));
13840
+
13841
+ // dev/errorHandler.ts
13842
+ var import_v314 = require("zod/v3");
13843
+ var errorHandler = (err, req, res, next) => {
13844
+ if ("status" in err) {
13845
+ res.status(err.status).json({
13846
+ error: {
13847
+ message: err.message,
13848
+ status: err.status
13428
13849
  }
13429
13850
  });
13430
- rl.on("close", () => {
13431
- if (currentEvent !== null || dataLines.length > 0) {
13432
- const data = dataLines.join("\n");
13433
- dispatchSseEvent(currentEvent, data, onEvent);
13851
+ return;
13852
+ }
13853
+ if (err instanceof import_v314.z.ZodError) {
13854
+ res.status(400).json({
13855
+ error: {
13856
+ message: "Invalid request",
13857
+ errors: err.errors
13434
13858
  }
13435
13859
  });
13860
+ return;
13861
+ }
13862
+ console.error("Internal server error", err);
13863
+ res.status(500).json({
13864
+ error: {
13865
+ message: "Internal server error",
13866
+ status: 500
13867
+ }
13436
13868
  });
13437
- server.listen(socketPath);
13438
- return {
13439
- server,
13440
- close: () => {
13441
- server.close();
13442
- try {
13443
- require("fs").unlinkSync(socketPath);
13444
- } catch {
13869
+ };
13870
+
13871
+ // dev/authorize.ts
13872
+ var import_http_errors = __toESM(require("http-errors"));
13873
+ function authorizeRequest(req, res, next) {
13874
+ try {
13875
+ const ctx = {
13876
+ appOrigin: extractAllowedOrigin(req.headers[ORIGIN_HEADER]),
13877
+ token: void 0,
13878
+ state: void 0,
13879
+ projectId: parseHeader(req.headers, PROJECT_ID_HEADER)
13880
+ };
13881
+ if (req.headers.authorization || req.headers[BRAINTRUST_AUTH_TOKEN_HEADER]) {
13882
+ const tokenText = parseBraintrustAuthHeader(req.headers);
13883
+ if (!tokenText) {
13884
+ return next((0, import_http_errors.default)(400, "Invalid authorization token format"));
13445
13885
  }
13886
+ ctx.token = tokenText.toLowerCase() === "null" ? void 0 : tokenText;
13446
13887
  }
13447
- };
13888
+ req.ctx = ctx;
13889
+ next();
13890
+ } catch (e) {
13891
+ next(e);
13892
+ }
13448
13893
  }
13449
- function dispatchSseEvent(eventName, data, onEvent) {
13450
- const name = eventName ?? "";
13451
- switch (name) {
13452
- case "start": {
13453
- try {
13454
- const parsed = JSON.parse(data);
13455
- onEvent({ type: "start", data: parsed });
13456
- } catch {
13457
- }
13458
- break;
13459
- }
13460
- case "summary": {
13461
- try {
13462
- const parsed = JSON.parse(data);
13463
- onEvent({ type: "summary", data: parsed });
13464
- } catch {
13465
- }
13466
- break;
13467
- }
13468
- case "progress": {
13469
- try {
13470
- const parsed = JSON.parse(data);
13471
- onEvent({ type: "progress", data: parsed });
13472
- } catch {
13473
- }
13474
- break;
13475
- }
13476
- case "console": {
13477
- try {
13478
- const parsed = JSON.parse(data);
13479
- onEvent({ type: "console", data: parsed });
13480
- } catch {
13481
- }
13482
- break;
13483
- }
13484
- case "error": {
13485
- try {
13486
- const parsed = JSON.parse(data);
13487
- onEvent({ type: "error", data: parsed });
13488
- } catch {
13489
- onEvent({ type: "error", data: { message: data } });
13490
- }
13491
- break;
13894
+ var loginCache = new LRUCache({
13895
+ max: 32
13896
+ // TODO: Make this configurable
13897
+ });
13898
+ async function cachedLogin(options) {
13899
+ const key = JSON.stringify(options);
13900
+ const cached = loginCache.get(key);
13901
+ if (cached) {
13902
+ return cached;
13903
+ }
13904
+ const state = await loginToState(options);
13905
+ loginCache.set(key, state);
13906
+ return state;
13907
+ }
13908
+ function makeCheckAuthorized(allowedOrgName) {
13909
+ return async (req, _res, next) => {
13910
+ if (!req.ctx?.token) {
13911
+ return next((0, import_http_errors.default)(401, "Unauthorized"));
13492
13912
  }
13493
- case "list": {
13494
- try {
13495
- const parsed = JSON.parse(data);
13496
- onEvent({ type: "list", data: parsed });
13497
- } catch {
13913
+ try {
13914
+ const orgName = parseHeader(req.headers, "x-bt-org-name");
13915
+ if (!orgName) {
13916
+ return next((0, import_http_errors.default)(400, "Missing x-bt-org-name header"));
13498
13917
  }
13499
- break;
13500
- }
13501
- case "dev-ready": {
13502
- try {
13503
- const parsed = JSON.parse(data);
13504
- onEvent({ type: "dev-ready", data: parsed });
13505
- } catch {
13918
+ if (allowedOrgName && allowedOrgName !== orgName) {
13919
+ const errorMessage = `Org '${orgName}' is not allowed. Only org '${allowedOrgName}' is allowed.`;
13920
+ return next((0, import_http_errors.default)(403, errorMessage));
13506
13921
  }
13507
- break;
13508
- }
13509
- case "done": {
13510
- onEvent({ type: "done", data });
13511
- break;
13922
+ const state = await cachedLogin({
13923
+ apiKey: req.ctx?.token,
13924
+ orgName
13925
+ });
13926
+ req.ctx.state = state;
13927
+ next();
13928
+ } catch (e) {
13929
+ console.error("Authorization error:", e);
13930
+ return next((0, import_http_errors.default)(401, "Unauthorized"));
13512
13931
  }
13513
- default:
13514
- break;
13932
+ };
13933
+ }
13934
+ function parseBraintrustAuthHeader(headers) {
13935
+ const tokenString = parseHeader(headers, BRAINTRUST_AUTH_TOKEN_HEADER);
13936
+ return tokenString ?? parseAuthHeader(headers) ?? void 0;
13937
+ }
13938
+ function parseHeader(headers, headerName) {
13939
+ const token = headers[headerName];
13940
+ let tokenString;
13941
+ if (typeof token === "string") {
13942
+ tokenString = token;
13943
+ } else if (Array.isArray(token) && token.length > 0) {
13944
+ tokenString = token[0];
13515
13945
  }
13946
+ return tokenString;
13516
13947
  }
13517
- function findProjectRoot(startDir) {
13518
- const fs6 = require("fs");
13519
- const homeDir = require("os").homedir();
13520
- let current = import_path5.default.resolve(startDir);
13521
- let depth = 0;
13522
- const maxDepth = 8;
13523
- while (depth < maxDepth) {
13524
- if (current === "/" || current === homeDir) {
13525
- return null;
13948
+ function checkOrigin(requestOrigin, callback) {
13949
+ if (!requestOrigin) {
13950
+ return callback(null, true);
13951
+ }
13952
+ for (const origin of WHITELISTED_ORIGINS || []) {
13953
+ if (origin instanceof RegExp && origin.test(requestOrigin) || origin === requestOrigin) {
13954
+ return callback(null, requestOrigin);
13526
13955
  }
13527
- try {
13528
- const entries = fs6.readdirSync(current);
13529
- const hasPackageJson = entries.includes("package.json");
13530
- const hasLockfile = entries.includes("package-lock.json") || entries.includes("pnpm-lock.yaml") || entries.includes("yarn.lock") || entries.includes("bun.lock");
13531
- const hasGit = entries.includes(".git");
13532
- const hasDenoJson = entries.includes("deno.json") || entries.includes("deno.jsonc");
13533
- if (hasPackageJson || hasLockfile || hasGit || hasDenoJson) {
13534
- return current;
13535
- }
13536
- } catch {
13537
- }
13538
- const parent = import_path5.default.dirname(current);
13539
- if (parent === current) {
13540
- return null;
13541
- }
13542
- current = parent;
13543
- depth++;
13544
- }
13545
- return null;
13546
- }
13547
- function classifyEnvironment(projectRoot) {
13548
- const fs6 = require("fs");
13549
- const ctx = {
13550
- root: projectRoot,
13551
- hasVite: false,
13552
- hasTsNode: false,
13553
- hasTypeModule: false,
13554
- hasTsx: false,
13555
- hasBun: false,
13556
- hasDeno: false,
13557
- nodeModulesBin: import_path5.default.join(projectRoot, "node_modules", ".bin")
13558
- };
13559
- const denoJsonPath = import_path5.default.join(projectRoot, "deno.json");
13560
- const denoJsoncPath = import_path5.default.join(projectRoot, "deno.jsonc");
13561
- try {
13562
- if (fs6.existsSync(denoJsonPath) || fs6.existsSync(denoJsoncPath)) {
13563
- ctx.hasDeno = true;
13564
- return ctx;
13565
- }
13566
- } catch {
13567
13956
  }
13568
- const bunLockPath = import_path5.default.join(projectRoot, "bun.lock");
13569
- try {
13570
- if (fs6.existsSync(bunLockPath)) {
13571
- ctx.hasBun = true;
13572
- return ctx;
13957
+ return callback(null, false);
13958
+ }
13959
+ var BRAINTRUST_AUTH_TOKEN_HEADER = "x-bt-auth-token";
13960
+ var ORIGIN_HEADER = "origin";
13961
+ var PROJECT_ID_HEADER = "x-bt-project-id";
13962
+ function extractAllowedOrigin(originHeader) {
13963
+ let allowedOrigin = MAIN_ORIGIN;
13964
+ checkOrigin(originHeader, (err, origin) => {
13965
+ if (!err && originHeader && origin) {
13966
+ allowedOrigin = originHeader;
13573
13967
  }
13574
- } catch {
13968
+ });
13969
+ return allowedOrigin;
13970
+ }
13971
+ var MAIN_ORIGIN = "https://www.braintrust.dev";
13972
+ var WHITELISTED_ORIGINS = [
13973
+ MAIN_ORIGIN,
13974
+ "https://www.braintrustdata.com",
13975
+ new RegExp("https://.*.preview.braintrust.dev")
13976
+ ].concat(
13977
+ process.env.WHITELISTED_ORIGIN ? [process.env.WHITELISTED_ORIGIN] : []
13978
+ ).concat(
13979
+ process.env.BRAINTRUST_APP_URL ? [process.env.BRAINTRUST_APP_URL] : []
13980
+ );
13981
+ function parseAuthHeader(headers) {
13982
+ const authHeader = headers["authorization"];
13983
+ let authValue = null;
13984
+ if (Array.isArray(authHeader)) {
13985
+ authValue = authHeader[authHeader.length - 1];
13986
+ } else {
13987
+ authValue = authHeader;
13575
13988
  }
13576
- const packageJsonPath = import_path5.default.join(projectRoot, "package.json");
13577
- try {
13578
- if (fs6.existsSync(packageJsonPath)) {
13579
- const packageJson = JSON.parse(fs6.readFileSync(packageJsonPath, "utf-8"));
13580
- const allDeps = {
13581
- ...packageJson.dependencies,
13582
- ...packageJson.devDependencies
13583
- };
13584
- if (allDeps.vite || allDeps.vitest || fs6.existsSync(import_path5.default.join(projectRoot, "vite.config.ts")) || fs6.existsSync(import_path5.default.join(projectRoot, "vite.config.js")) || fs6.existsSync(import_path5.default.join(projectRoot, "vitest.config.ts")) || fs6.existsSync(import_path5.default.join(projectRoot, "vitest.config.js"))) {
13585
- ctx.hasVite = true;
13989
+ if (!authValue) {
13990
+ return null;
13991
+ }
13992
+ const parts = authValue.split(" ");
13993
+ if (parts.length !== 2) {
13994
+ return null;
13995
+ }
13996
+ return parts[1];
13997
+ }
13998
+ var baseAllowedHeaders = [
13999
+ "Content-Type",
14000
+ "X-Amz-Date",
14001
+ "Authorization",
14002
+ "X-Api-Key",
14003
+ "X-Amz-Security-Token",
14004
+ "x-bt-auth-token",
14005
+ "x-bt-parent",
14006
+ // These are eval-specific
14007
+ "x-bt-org-name",
14008
+ "x-bt-project-id",
14009
+ "x-bt-stream-fmt",
14010
+ "x-bt-use-cache",
14011
+ "x-stainless-os",
14012
+ "x-stainless-lang",
14013
+ "x-stainless-package-version",
14014
+ "x-stainless-runtime",
14015
+ "x-stainless-runtime-version",
14016
+ "x-stainless-arch"
14017
+ ];
14018
+
14019
+ // dev/stream.ts
14020
+ function serializeSSEEvent(event) {
14021
+ return Object.entries(event).filter(([_key, value]) => value !== void 0).map(([key, value]) => `${key}: ${value}`).join("\n") + "\n\n";
14022
+ }
14023
+
14024
+ // dev/types.ts
14025
+ var import_v315 = require("zod/v3");
14026
+ var evalBodySchema = import_v315.z.object({
14027
+ name: import_v315.z.string(),
14028
+ parameters: import_v315.z.record(import_v315.z.string(), import_v315.z.unknown()).nullish(),
14029
+ data: RunEval.shape.data,
14030
+ scores: import_v315.z.array(
14031
+ import_v315.z.object({
14032
+ function_id: FunctionId,
14033
+ name: import_v315.z.string()
14034
+ })
14035
+ ).nullish(),
14036
+ experiment_name: import_v315.z.string().nullish(),
14037
+ project_id: import_v315.z.string().nullish(),
14038
+ parent: InvokeParent.optional(),
14039
+ stream: import_v315.z.boolean().optional()
14040
+ });
14041
+ var staticParametersSchema = import_v315.z.record(
14042
+ import_v315.z.string(),
14043
+ import_v315.z.union([
14044
+ import_v315.z.object({
14045
+ type: import_v315.z.literal("prompt"),
14046
+ default: PromptData.optional(),
14047
+ description: import_v315.z.string().optional()
14048
+ }),
14049
+ import_v315.z.object({
14050
+ type: import_v315.z.literal("data"),
14051
+ schema: import_v315.z.record(import_v315.z.unknown()),
14052
+ default: import_v315.z.unknown().optional(),
14053
+ description: import_v315.z.string().optional()
14054
+ })
14055
+ ])
14056
+ );
14057
+ var parametersSchema = import_v315.z.object({
14058
+ type: import_v315.z.literal("object"),
14059
+ properties: import_v315.z.record(import_v315.z.string(), import_v315.z.record(import_v315.z.unknown())),
14060
+ required: import_v315.z.array(import_v315.z.string()).optional(),
14061
+ additionalProperties: import_v315.z.boolean().optional()
14062
+ });
14063
+ var parametersSourceSchema = import_v315.z.object({
14064
+ parametersId: import_v315.z.string().optional(),
14065
+ slug: import_v315.z.string(),
14066
+ name: import_v315.z.string(),
14067
+ projectId: import_v315.z.string().optional(),
14068
+ version: import_v315.z.string().optional()
14069
+ });
14070
+ var parametersContainerSchema = import_v315.z.object({
14071
+ type: import_v315.z.literal("braintrust.parameters"),
14072
+ schema: parametersSchema,
14073
+ source: parametersSourceSchema
14074
+ });
14075
+ var staticParametersContainerSchema = import_v315.z.object({
14076
+ type: import_v315.z.literal("braintrust.staticParameters"),
14077
+ schema: staticParametersSchema,
14078
+ source: import_v315.z.null()
14079
+ });
14080
+ var serializedParametersContainerSchema = import_v315.z.union([
14081
+ parametersContainerSchema,
14082
+ staticParametersContainerSchema,
14083
+ // keeping this type here since old versions of the SDK will still pass the unwrapped schema and we need to handle this in the app
14084
+ staticParametersSchema
14085
+ ]);
14086
+ var evaluatorDefinitionSchema = import_v315.z.object({
14087
+ parameters: serializedParametersContainerSchema.optional(),
14088
+ scores: import_v315.z.array(import_v315.z.object({ name: import_v315.z.string() })).optional()
14089
+ });
14090
+ var evaluatorDefinitionsSchema = import_v315.z.record(
14091
+ import_v315.z.string(),
14092
+ evaluatorDefinitionSchema
14093
+ );
14094
+
14095
+ // dev/server.ts
14096
+ var import_v316 = require("zod/v3");
14097
+ var import_ajv2 = require("ajv");
14098
+ function runDevServer(evaluators, opts) {
14099
+ const allEvaluators = Object.fromEntries(
14100
+ evaluators.map((evaluator) => [evaluator.evalName, evaluator])
14101
+ );
14102
+ globalThis._lazy_load = false;
14103
+ const app = (0, import_express.default)();
14104
+ app.use(import_express.default.json({ limit: "1gb" }));
14105
+ console.log("Starting server");
14106
+ app.use((req, res, next) => {
14107
+ if (req.headers["access-control-request-private-network"]) {
14108
+ res.setHeader("Access-Control-Allow-Private-Network", "true");
14109
+ }
14110
+ next();
14111
+ });
14112
+ const checkAuthorized = makeCheckAuthorized(opts.orgName);
14113
+ app.use(
14114
+ (0, import_cors.default)({
14115
+ origin: checkOrigin,
14116
+ methods: ["GET", "PATCH", "POST", "PUT", "DELETE", "OPTIONS"],
14117
+ allowedHeaders: baseAllowedHeaders,
14118
+ credentials: true,
14119
+ exposedHeaders: [
14120
+ BT_CURSOR_HEADER,
14121
+ BT_FOUND_EXISTING_HEADER,
14122
+ "x-bt-span-id",
14123
+ "x-bt-span-export"
14124
+ ],
14125
+ maxAge: 86400
14126
+ })
14127
+ );
14128
+ app.use(authorizeRequest);
14129
+ app.get("/", (req, res) => {
14130
+ res.send("Hello, world!");
14131
+ });
14132
+ app.get(
14133
+ "/list",
14134
+ checkAuthorized,
14135
+ asyncHandler(async (req, res) => {
14136
+ const evalDefs = {};
14137
+ for (const [name, evaluator] of Object.entries(allEvaluators)) {
14138
+ let parameters;
14139
+ if (evaluator.parameters) {
14140
+ const resolvedParams = await Promise.resolve(evaluator.parameters);
14141
+ parameters = serializeRemoteEvalParametersContainer(resolvedParams);
14142
+ }
14143
+ evalDefs[name] = {
14144
+ parameters,
14145
+ scores: evaluator.scores.map((score, idx) => ({
14146
+ name: scorerName(score, idx)
14147
+ }))
14148
+ };
13586
14149
  }
13587
- if (allDeps["ts-node"]) {
13588
- ctx.hasTsNode = true;
14150
+ res.json(evalDefs);
14151
+ })
14152
+ );
14153
+ app.post(
14154
+ "/eval",
14155
+ checkAuthorized,
14156
+ asyncHandler(async (req, res) => {
14157
+ const {
14158
+ name,
14159
+ parameters,
14160
+ parent,
14161
+ experiment_name,
14162
+ project_id,
14163
+ data,
14164
+ scores,
14165
+ stream
14166
+ } = evalBodySchema.parse(req.body);
14167
+ if (!req.ctx?.state) {
14168
+ res.status(500).json({ error: "Braintrust state not initialized in request" });
14169
+ return;
13589
14170
  }
13590
- if (allDeps.tsx) {
13591
- ctx.hasTsx = true;
14171
+ const state = req.ctx.state;
14172
+ const evaluator = allEvaluators[name];
14173
+ if (!evaluator) {
14174
+ res.status(404).json({ error: `Evaluator '${name}' not found` });
14175
+ return;
13592
14176
  }
13593
- if (packageJson.type === "module") {
13594
- ctx.hasTypeModule = true;
14177
+ if (evaluator.parameters) {
14178
+ try {
14179
+ await validateParameters(parameters ?? {}, evaluator.parameters);
14180
+ } catch (e) {
14181
+ console.error("Error validating parameters", e);
14182
+ if (e instanceof import_v316.z.ZodError || e instanceof import_ajv2.ValidationError || e instanceof Error) {
14183
+ res.status(400).json({
14184
+ error: e.message
14185
+ });
14186
+ return;
14187
+ }
14188
+ throw e;
14189
+ }
13595
14190
  }
13596
- }
13597
- } catch {
13598
- }
13599
- const tsconfigPath = import_path5.default.join(projectRoot, "tsconfig.json");
13600
- try {
13601
- if (fs6.existsSync(tsconfigPath)) {
13602
- const tsconfig = JSON.parse(fs6.readFileSync(tsconfigPath, "utf-8"));
13603
- if (tsconfig["ts-node"]) {
13604
- ctx.hasTsNode = true;
14191
+ const resolvedData = await getDataset(state, data);
14192
+ const evalData = callEvaluatorData(resolvedData);
14193
+ console.log("Starting eval", evaluator.evalName);
14194
+ if (stream) {
14195
+ res.setHeader("Content-Type", "text/event-stream");
14196
+ res.setHeader("Cache-Control", "no-cache");
14197
+ res.setHeader("Connection", "keep-alive");
14198
+ } else {
14199
+ res.setHeader("Content-Type", "application/json");
13605
14200
  }
13606
- }
13607
- } catch {
13608
- }
13609
- return ctx;
13610
- }
13611
- function selectRunner(ctx) {
13612
- const fs6 = require("fs");
13613
- if (ctx.hasDeno) {
13614
- const denoPath = findOnPath("deno");
13615
- if (denoPath) {
13616
- return { command: denoPath, prefixArgs: ["run", "-A"] };
13617
- }
13618
- }
13619
- if (ctx.hasBun) {
13620
- const bunPath = findOnPath("bun");
13621
- if (bunPath) {
13622
- return { command: bunPath, prefixArgs: [] };
13623
- }
13624
- const localBun = import_path5.default.join(ctx.nodeModulesBin, "bun");
13625
- try {
13626
- fs6.accessSync(localBun, fs6.constants.X_OK);
13627
- return { command: localBun, prefixArgs: [] };
13628
- } catch {
13629
- }
13630
- }
13631
- if (ctx.hasVite) {
13632
- const localViteNode = import_path5.default.join(ctx.nodeModulesBin, "vite-node");
13633
- try {
13634
- fs6.accessSync(localViteNode, fs6.constants.X_OK);
13635
- return { command: localViteNode, prefixArgs: [] };
13636
- } catch {
13637
- }
13638
- return { command: "npx", prefixArgs: ["--yes", "vite-node"] };
13639
- }
13640
- if (ctx.hasTsNode) {
13641
- const localTsNode = import_path5.default.join(ctx.nodeModulesBin, "ts-node");
13642
- try {
13643
- fs6.accessSync(localTsNode, fs6.constants.X_OK);
13644
- return { command: localTsNode, prefixArgs: [] };
13645
- } catch {
13646
- }
13647
- return { command: "npx", prefixArgs: ["--yes", "ts-node"] };
13648
- }
13649
- const localTsx = import_path5.default.join(ctx.nodeModulesBin, "tsx");
13650
- try {
13651
- fs6.accessSync(localTsx, fs6.constants.X_OK);
13652
- return { command: localTsx, prefixArgs: [] };
13653
- } catch {
13654
- }
13655
- const tsxOnPath = findOnPath("tsx");
13656
- if (tsxOnPath) {
13657
- return { command: tsxOnPath, prefixArgs: [] };
13658
- }
13659
- return { command: "npx", prefixArgs: ["--yes", "tsx"] };
14201
+ const task = async (input, hooks) => {
14202
+ const result = await evaluator.task(input, hooks);
14203
+ hooks.reportProgress({
14204
+ format: "code",
14205
+ output_type: "completion",
14206
+ event: "json_delta",
14207
+ data: JSON.stringify(result)
14208
+ });
14209
+ return result;
14210
+ };
14211
+ try {
14212
+ const summary = await Eval(
14213
+ "worker-thread",
14214
+ {
14215
+ ...evaluator,
14216
+ data: evalData.data,
14217
+ scores: evaluator.scores.concat(
14218
+ scores?.map(
14219
+ (score) => makeScorer(
14220
+ state,
14221
+ score.name,
14222
+ score.function_id,
14223
+ req.ctx?.projectId
14224
+ )
14225
+ ) ?? []
14226
+ ),
14227
+ task,
14228
+ state,
14229
+ experimentName: experiment_name ?? void 0,
14230
+ projectId: project_id ?? void 0
14231
+ },
14232
+ {
14233
+ // Avoid printing the bar to the console.
14234
+ progress: {
14235
+ start: () => {
14236
+ },
14237
+ stop: () => {
14238
+ console.log("Finished running experiment");
14239
+ },
14240
+ increment: () => {
14241
+ }
14242
+ },
14243
+ stream: (data2) => {
14244
+ if (stream) {
14245
+ res.write(
14246
+ serializeSSEEvent({
14247
+ event: "progress",
14248
+ data: JSON.stringify(data2)
14249
+ })
14250
+ );
14251
+ }
14252
+ },
14253
+ onStart: (metadata) => {
14254
+ if (stream) {
14255
+ res.write(
14256
+ serializeSSEEvent({
14257
+ event: "start",
14258
+ data: JSON.stringify(metadata)
14259
+ })
14260
+ );
14261
+ }
14262
+ },
14263
+ parent: parseParent(parent),
14264
+ parameters: parameters ?? {}
14265
+ }
14266
+ );
14267
+ if (stream) {
14268
+ res.write(
14269
+ serializeSSEEvent({
14270
+ event: "summary",
14271
+ data: JSON.stringify(summary.summary)
14272
+ })
14273
+ );
14274
+ res.write(
14275
+ serializeSSEEvent({
14276
+ event: "done",
14277
+ data: ""
14278
+ })
14279
+ );
14280
+ } else {
14281
+ res.json(summary.summary);
14282
+ }
14283
+ } catch (e) {
14284
+ console.error("Error running eval", e);
14285
+ if (stream) {
14286
+ res.write(
14287
+ serializeSSEEvent({
14288
+ event: "error",
14289
+ data: JSON.stringify(e)
14290
+ })
14291
+ );
14292
+ } else {
14293
+ res.status(500).json({ error: e });
14294
+ }
14295
+ } finally {
14296
+ res.end();
14297
+ }
14298
+ })
14299
+ );
14300
+ app.use(errorHandler);
14301
+ app.listen(opts.port, opts.host, () => {
14302
+ console.log(`Dev server running at http://${opts.host}:${opts.port}`);
14303
+ });
13660
14304
  }
13661
- function resolveRuntime(runnerOverride) {
13662
- if (runnerOverride) {
13663
- return { command: runnerOverride, prefixArgs: [] };
13664
- }
13665
- const envRunner = process.env.BT_EVAL_JS_RUNNER;
13666
- if (envRunner) {
13667
- return { command: envRunner, prefixArgs: [] };
13668
- }
13669
- const projectRoot = findProjectRoot(process.cwd());
13670
- if (!projectRoot) {
13671
- const tsxOnPath = findOnPath("tsx");
13672
- if (tsxOnPath) {
13673
- return { command: tsxOnPath, prefixArgs: [] };
13674
- }
13675
- return { command: "npx", prefixArgs: ["--yes", "tsx"] };
14305
+ var asyncHandler = (fn) => (req, res, next) => {
14306
+ Promise.resolve(fn(req, res, next)).catch(next);
14307
+ };
14308
+ async function getDataset(state, data) {
14309
+ if ("project_name" in data) {
14310
+ return initDataset({
14311
+ state,
14312
+ project: data.project_name,
14313
+ dataset: data.dataset_name,
14314
+ _internal_btql: data._internal_btql ?? void 0
14315
+ });
14316
+ } else if ("dataset_id" in data) {
14317
+ const datasetInfo = await getDatasetById({
14318
+ state,
14319
+ datasetId: data.dataset_id
14320
+ });
14321
+ return initDataset({
14322
+ state,
14323
+ projectId: datasetInfo.projectId,
14324
+ dataset: datasetInfo.dataset,
14325
+ _internal_btql: data._internal_btql ?? void 0
14326
+ });
14327
+ } else {
14328
+ return data.data;
13676
14329
  }
13677
- const ctx = classifyEnvironment(projectRoot);
13678
- return selectRunner(ctx);
13679
14330
  }
13680
- function findOnPath(binary) {
13681
- const pathEnv = process.env.PATH;
13682
- if (!pathEnv) {
13683
- return null;
13684
- }
13685
- const dirs = pathEnv.split(import_path5.default.delimiter);
13686
- for (const dir2 of dirs) {
13687
- const candidate = import_path5.default.join(dir2, binary);
13688
- try {
13689
- require("fs").accessSync(candidate);
13690
- return candidate;
13691
- } catch {
13692
- continue;
14331
+ var datasetFetchSchema = import_v316.z.object({
14332
+ project_id: import_v316.z.string(),
14333
+ name: import_v316.z.string()
14334
+ });
14335
+ async function getDatasetById({
14336
+ state,
14337
+ datasetId
14338
+ }) {
14339
+ const dataset = await state.appConn().post_json("api/dataset/get", {
14340
+ id: datasetId
14341
+ });
14342
+ const parsed = import_v316.z.array(datasetFetchSchema).parse(dataset);
14343
+ if (parsed.length === 0) {
14344
+ throw new Error(`Dataset '${datasetId}' not found`);
14345
+ }
14346
+ return { projectId: parsed[0].project_id, dataset: parsed[0].name };
14347
+ }
14348
+ function makeScorer(state, name, score, projectId) {
14349
+ const ret = async (input) => {
14350
+ const request = {
14351
+ ...score,
14352
+ input,
14353
+ parent: await getSpanParentObject().export(),
14354
+ stream: false,
14355
+ mode: "auto",
14356
+ strict: true
14357
+ };
14358
+ const headers = {
14359
+ Accept: "application/json"
14360
+ };
14361
+ if (projectId) {
14362
+ headers["x-bt-project-id"] = projectId;
13693
14363
  }
13694
- }
13695
- return null;
14364
+ const result = await state.proxyConn().post(`function/invoke`, request, {
14365
+ headers
14366
+ });
14367
+ const data = await result.json();
14368
+ return data;
14369
+ };
14370
+ Object.defineProperties(ret, {
14371
+ name: { value: `Remote eval scorer (${name})` }
14372
+ });
14373
+ return ret;
13696
14374
  }
13697
14375
 
13698
14376
  // src/cli/util/external-packages-plugin.ts
@@ -13802,6 +14480,89 @@ function addReport(evalReports, reporter, report) {
13802
14480
  }
13803
14481
  evalReports[reporter.name].results.push(report);
13804
14482
  }
14483
+ function buildWatchPluginForEvaluator(inFile, opts) {
14484
+ const evaluators = {
14485
+ evaluators: [],
14486
+ reporters: {}
14487
+ };
14488
+ const plugin = {
14489
+ name: "run-evalutator-on-end",
14490
+ setup(build2) {
14491
+ build2.onEnd(async (result) => {
14492
+ console.error(`Done building ${inFile}`);
14493
+ if (!result.outputFiles) {
14494
+ if (opts.verbose) {
14495
+ console.warn(`Failed to compile ${inFile}`);
14496
+ console.warn(result.errors);
14497
+ } else {
14498
+ console.warn(`Failed to compile ${inFile}: ${result.errors}`);
14499
+ }
14500
+ return;
14501
+ }
14502
+ const evalResult = evaluateBuildResults(inFile, result);
14503
+ if (!evalResult) {
14504
+ return;
14505
+ }
14506
+ evaluators.evaluators = evaluators.evaluators.filter(
14507
+ (e) => e.sourceFile !== inFile
14508
+ );
14509
+ for (const evaluator of Object.values(evalResult.evaluators)) {
14510
+ evaluators.evaluators.push({
14511
+ sourceFile: inFile,
14512
+ // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
14513
+ evaluator: evaluator.evaluator,
14514
+ reporter: evaluator.reporter
14515
+ });
14516
+ }
14517
+ for (const [reporterName, reporter] of Object.entries(
14518
+ evalResult.reporters
14519
+ )) {
14520
+ evaluators.reporters[reporterName] = reporter;
14521
+ }
14522
+ const evalReports = {};
14523
+ for (const evaluatorDef of Object.values(evalResult.evaluators)) {
14524
+ const { evaluator, reporter } = evaluatorDef;
14525
+ const evalData = callEvaluatorData(evaluator.data);
14526
+ const logger = opts.noSendLogs ? null : await initExperiment2(evaluator, evalData);
14527
+ const evaluatorResult = await runEvaluator(
14528
+ logger,
14529
+ {
14530
+ ...evaluator,
14531
+ data: evalData.data
14532
+ },
14533
+ opts.progressReporter,
14534
+ opts.filters,
14535
+ void 0,
14536
+ void 0
14537
+ );
14538
+ const resolvedReporter = resolveReporter(
14539
+ reporter,
14540
+ evaluators.reporters
14541
+ // Let these accumulate across all files.
14542
+ );
14543
+ const report = resolvedReporter.reportEval(
14544
+ evaluator,
14545
+ evaluatorResult,
14546
+ {
14547
+ verbose: opts.verbose,
14548
+ jsonl: opts.jsonl
14549
+ }
14550
+ );
14551
+ addReport(evalReports, resolvedReporter, report);
14552
+ }
14553
+ for (const [reporterName, { reporter, results }] of Object.entries(
14554
+ evalReports
14555
+ )) {
14556
+ const success = await reporter.reportRun(await Promise.all(results));
14557
+ if (!success) {
14558
+ console.error(error(`Reporter ${reporterName} failed.`));
14559
+ }
14560
+ }
14561
+ });
14562
+ }
14563
+ };
14564
+ return plugin;
14565
+ }
13805
14566
  async function initFile({
13806
14567
  inFile,
13807
14568
  outFile,
@@ -13918,6 +14679,26 @@ function updateEvaluators(evaluators, buildResults, opts) {
13918
14679
  }
13919
14680
  }
13920
14681
  }
14682
+ async function runAndWatch({
14683
+ handles,
14684
+ onExit
14685
+ }) {
14686
+ const count = Object.keys(handles).length;
14687
+ console.error(`Watching ${(0, import_pluralize4.default)("file", count, true)}...`);
14688
+ Object.values(handles).map((handle) => handle.watch());
14689
+ ["SIGINT", "SIGTERM"].forEach((signal) => {
14690
+ process.on(signal, function() {
14691
+ console.error("Stopped watching.");
14692
+ for (const handle of Object.values(handles)) {
14693
+ handle.destroy();
14694
+ }
14695
+ onExit?.();
14696
+ process.exit(0);
14697
+ });
14698
+ });
14699
+ await new Promise(() => {
14700
+ });
14701
+ }
13921
14702
  async function buildEvaluators(handles, opts) {
13922
14703
  const buildPromises = Object.values(handles).map(
13923
14704
  (handle) => handle.rebuild()
@@ -14020,7 +14801,7 @@ async function runOnce(handles, opts) {
14020
14801
  return allSuccess;
14021
14802
  }
14022
14803
  function checkMatch(pathInput, include_patterns, exclude_patterns) {
14023
- const p = import_path6.default.resolve(pathInput);
14804
+ const p = import_path5.default.resolve(pathInput);
14024
14805
  if (include_patterns !== null) {
14025
14806
  let include = false;
14026
14807
  for (const pattern of include_patterns) {
@@ -14069,7 +14850,7 @@ async function collectFiles(inputPath, mode) {
14069
14850
  }
14070
14851
  files.push(inputPath);
14071
14852
  } else {
14072
- const walked = await import_util15.default.promisify(fsWalk.walk)(inputPath, {
14853
+ const walked = await import_util16.default.promisify(fsWalk.walk)(inputPath, {
14073
14854
  deepFilter: (entry) => {
14074
14855
  return checkMatch(entry.path, null, EXCLUDE);
14075
14856
  },
@@ -14104,8 +14885,8 @@ var nativeNodeModulesPlugin = {
14104
14885
  };
14105
14886
  build2.onResolve({ filter: /\.node$/ }, (args) => {
14106
14887
  try {
14107
- const path9 = require.resolve(args.path, { paths: [args.resolveDir] });
14108
- const match = path9.match(
14888
+ const path8 = require.resolve(args.path, { paths: [args.resolveDir] });
14889
+ const match = path8.match(
14109
14890
  /node_modules[/\\]((?:@[^/\\]+[/\\])?[^/\\]+)/
14110
14891
  );
14111
14892
  if (match) {
@@ -14181,7 +14962,7 @@ async function initializeHandles({
14181
14962
  );
14182
14963
  }
14183
14964
  for (const file of newFiles) {
14184
- files[import_path6.default.resolve(file)] = true;
14965
+ files[import_path5.default.resolve(file)] = true;
14185
14966
  }
14186
14967
  }
14187
14968
  if (Object.keys(files).length == 0) {
@@ -14190,15 +14971,15 @@ async function initializeHandles({
14190
14971
  );
14191
14972
  process.exit(0);
14192
14973
  }
14193
- const tmpDir = import_path6.default.join(import_os2.default.tmpdir(), `btevals-${(0, import_uuid3.v4)().slice(0, 8)}`);
14974
+ const tmpDir = import_path5.default.join(import_os.default.tmpdir(), `btevals-${(0, import_uuid3.v4)().slice(0, 8)}`);
14194
14975
  const initPromises = [];
14195
14976
  for (const file of Object.keys(files)) {
14196
- const baseName = `${import_path6.default.basename(
14977
+ const baseName = `${import_path5.default.basename(
14197
14978
  file,
14198
- import_path6.default.extname(file)
14979
+ import_path5.default.extname(file)
14199
14980
  )}-${(0, import_uuid3.v4)().slice(0, 8)}`;
14200
- const outFile = import_path6.default.join(tmpDir, `${baseName}.${OUT_EXT}`);
14201
- const bundleFile = import_path6.default.join(tmpDir, `${baseName}.bundle.js`);
14981
+ const outFile = import_path5.default.join(tmpDir, `${baseName}.${OUT_EXT}`);
14982
+ const bundleFile = import_path5.default.join(tmpDir, `${baseName}.bundle.js`);
14202
14983
  initPromises.push(
14203
14984
  initFile({
14204
14985
  inFile: file,
@@ -14217,276 +14998,6 @@ async function initializeHandles({
14217
14998
  }
14218
14999
  return handles;
14219
15000
  }
14220
- function getEvalRunnerPath() {
14221
- return import_path6.default.resolve(__dirname, "eval-runner.js");
14222
- }
14223
- function buildRunnerEnv(args) {
14224
- const env = {};
14225
- if (args.api_key) {
14226
- env.BRAINTRUST_API_KEY = args.api_key;
14227
- }
14228
- if (args.app_url) {
14229
- env.BRAINTRUST_API_URL = args.app_url;
14230
- }
14231
- if (args.org_name) {
14232
- env.BRAINTRUST_ORG_NAME = args.org_name;
14233
- }
14234
- if (args.no_send_logs) {
14235
- env.BT_EVAL_NO_SEND_LOGS = "1";
14236
- env.BT_EVAL_LOCAL = "1";
14237
- }
14238
- if (args.filter && args.filter.length > 0) {
14239
- env.BT_EVAL_FILTERS = JSON.stringify(args.filter);
14240
- }
14241
- if (args.list) {
14242
- env.BT_EVAL_LIST = "1";
14243
- }
14244
- if (args.dev) {
14245
- env.BT_EVAL_DEV = "1";
14246
- env.BT_EVAL_DEV_HOST = args.dev_host || "localhost";
14247
- env.BT_EVAL_DEV_PORT = String(args.dev_port || 8300);
14248
- if (args.dev_org_name) {
14249
- env.BT_EVAL_DEV_ORG_NAME = args.dev_org_name;
14250
- }
14251
- }
14252
- return env;
14253
- }
14254
- function resolveEvalFiles(inputFiles) {
14255
- const inputPaths = inputFiles.length > 0 ? inputFiles : ["."];
14256
- const resolved = [];
14257
- for (const inputPath of inputPaths) {
14258
- const absPath = import_path6.default.resolve(inputPath);
14259
- let stat2;
14260
- try {
14261
- stat2 = import_fs2.default.lstatSync(absPath);
14262
- } catch {
14263
- console.warn(warning2(`Cannot read ${inputPath}, skipping...`));
14264
- continue;
14265
- }
14266
- if (!stat2.isDirectory()) {
14267
- resolved.push(absPath);
14268
- } else {
14269
- const walked = fsWalk.walkSync(absPath, {
14270
- deepFilter: (entry) => checkMatch(entry.path, null, EXCLUDE),
14271
- entryFilter: (entry) => entry.dirent.isFile() && checkMatch(entry.path, INCLUDE_EVAL, EXCLUDE)
14272
- });
14273
- for (const entry of walked) {
14274
- resolved.push(entry.path);
14275
- }
14276
- }
14277
- }
14278
- return resolved;
14279
- }
14280
- function spawnEvalRunner(args, socketPath, evalFiles) {
14281
- const evalRunnerPath = getEvalRunnerPath();
14282
- const { command, prefixArgs } = resolveRuntime(args.runner);
14283
- const childArgs = [...prefixArgs, evalRunnerPath, ...evalFiles];
14284
- const childEnv = {
14285
- ...process.env,
14286
- ...buildRunnerEnv(args),
14287
- BT_EVAL_SSE_SOCK: socketPath
14288
- };
14289
- const child = (0, import_child_process.spawn)(command, childArgs, {
14290
- env: childEnv,
14291
- stdio: ["ignore", "pipe", "pipe"],
14292
- cwd: process.cwd()
14293
- });
14294
- return child;
14295
- }
14296
- async function runWithRunner(args) {
14297
- const evalFiles = resolveEvalFiles(args.files);
14298
- if (evalFiles.length === 0) {
14299
- console.warn(
14300
- warning2("No eval files were found in any of the provided paths.")
14301
- );
14302
- return true;
14303
- }
14304
- const socketPath = buildSseSocketPath();
14305
- try {
14306
- import_fs2.default.unlinkSync(socketPath);
14307
- } catch {
14308
- }
14309
- const progressReporter = args.no_progress_bars ? new SimpleProgressReporter() : new BarProgressReporter();
14310
- let success = true;
14311
- let receivedDone = false;
14312
- return new Promise((resolve2) => {
14313
- const sseServer = createSseServer(socketPath, (event) => {
14314
- switch (event.type) {
14315
- case "start": {
14316
- const summary = event.data;
14317
- const linkText = summary.experimentUrl ? (0, import_termi_link2.terminalLink)(summary.experimentUrl, summary.experimentUrl, {
14318
- fallback: (_text, url) => url
14319
- }) : "locally";
14320
- console.error(
14321
- import_chalk3.default.cyan("\u25B6") + ` Experiment ${import_chalk3.default.bold(summary.experimentName)} is running at ${linkText}`
14322
- );
14323
- break;
14324
- }
14325
- case "progress": {
14326
- handleProgress(event.data, progressReporter);
14327
- break;
14328
- }
14329
- case "summary": {
14330
- const summary = event.data;
14331
- if (args.jsonl) {
14332
- process.stdout.write(JSON.stringify(summary));
14333
- process.stdout.write("\n");
14334
- } else {
14335
- const formatted = formatExperimentSummaryFancy(
14336
- summary
14337
- );
14338
- process.stdout.write(formatted);
14339
- process.stdout.write("\n");
14340
- }
14341
- break;
14342
- }
14343
- case "console": {
14344
- if (event.data.stream === "stderr") {
14345
- console.error(event.data.message);
14346
- } else {
14347
- console.log(event.data.message);
14348
- }
14349
- break;
14350
- }
14351
- case "error": {
14352
- success = false;
14353
- console.error(import_chalk3.default.red(event.data.message));
14354
- if (args.verbose && event.data.stack) {
14355
- for (const line of event.data.stack.split("\n")) {
14356
- console.error(import_chalk3.default.gray(line));
14357
- }
14358
- }
14359
- if (event.data.message.includes("Please specify an api key")) {
14360
- console.error(
14361
- import_chalk3.default.gray(
14362
- "Hint: pass --api-key or set BRAINTRUST_API_KEY, or use --no-send-logs for local evals."
14363
- )
14364
- );
14365
- }
14366
- break;
14367
- }
14368
- case "list": {
14369
- console.log(event.data.name);
14370
- break;
14371
- }
14372
- case "dev-ready": {
14373
- console.error(
14374
- import_chalk3.default.cyan("\u25B6") + ` Dev server is running at http://${event.data.host}:${event.data.port}`
14375
- );
14376
- break;
14377
- }
14378
- case "done": {
14379
- receivedDone = true;
14380
- progressReporter.stop();
14381
- break;
14382
- }
14383
- }
14384
- });
14385
- const child = spawnEvalRunner(args, socketPath, evalFiles);
14386
- child.stdout?.on("data", (data) => {
14387
- process.stderr.write(data);
14388
- });
14389
- child.stderr?.on("data", (data) => {
14390
- process.stderr.write(data);
14391
- });
14392
- child.on("close", (code) => {
14393
- setTimeout(() => {
14394
- progressReporter.stop();
14395
- sseServer.close();
14396
- if (code !== 0 && code !== null) {
14397
- success = false;
14398
- }
14399
- resolve2(success);
14400
- }, 100);
14401
- });
14402
- child.on("error", (err) => {
14403
- console.error(import_chalk3.default.red(`Failed to start eval runner: ${err.message}`));
14404
- progressReporter.stop();
14405
- sseServer.close();
14406
- resolve2(false);
14407
- });
14408
- });
14409
- }
14410
- function handleProgress(data, progressReporter) {
14411
- let payload;
14412
- try {
14413
- payload = JSON.parse(data.data);
14414
- } catch {
14415
- return;
14416
- }
14417
- if (payload.type !== "eval_progress") {
14418
- return;
14419
- }
14420
- switch (payload.kind) {
14421
- case "start":
14422
- progressReporter.start(data.name, payload.total ?? 0);
14423
- break;
14424
- case "increment":
14425
- progressReporter.increment(data.name);
14426
- break;
14427
- case "set_total":
14428
- if (progressReporter.setTotal && payload.total !== void 0) {
14429
- progressReporter.setTotal(data.name, payload.total);
14430
- }
14431
- break;
14432
- case "stop":
14433
- break;
14434
- }
14435
- }
14436
- async function runWithWatch(args) {
14437
- const evalFiles = resolveEvalFiles(args.files);
14438
- if (evalFiles.length === 0) {
14439
- console.warn(
14440
- warning2("No eval files were found in any of the provided paths.")
14441
- );
14442
- process.exit(0);
14443
- }
14444
- const watchDirs = /* @__PURE__ */ new Set();
14445
- for (const file of evalFiles) {
14446
- watchDirs.add(import_path6.default.dirname(file));
14447
- }
14448
- console.error(`Watching ${(0, import_pluralize4.default)("file", evalFiles.length, true)}...`);
14449
- await runWithRunner(args);
14450
- let debounceTimer = null;
14451
- const watchers = [];
14452
- for (const dir2 of watchDirs) {
14453
- try {
14454
- const watcher = import_fs2.default.watch(
14455
- dir2,
14456
- { recursive: false },
14457
- (_eventType, filename) => {
14458
- if (!filename) return;
14459
- const fullPath = import_path6.default.join(dir2, filename);
14460
- if (!evalFiles.includes(fullPath)) return;
14461
- if (debounceTimer) {
14462
- clearTimeout(debounceTimer);
14463
- }
14464
- debounceTimer = setTimeout(async () => {
14465
- console.error(
14466
- import_chalk3.default.dim(`
14467
- File changed: ${filename}. Re-running...`)
14468
- );
14469
- await runWithRunner(args);
14470
- }, 300);
14471
- }
14472
- );
14473
- watchers.push(watcher);
14474
- } catch {
14475
- }
14476
- }
14477
- const cleanup = () => {
14478
- console.error("Stopped watching.");
14479
- for (const w of watchers) {
14480
- w.close();
14481
- }
14482
- process.exit(0);
14483
- };
14484
- process.on("SIGINT", cleanup);
14485
- process.on("SIGTERM", cleanup);
14486
- await new Promise(() => {
14487
- });
14488
- process.exit(0);
14489
- }
14490
15001
  async function run(args) {
14491
15002
  (0, import_env2.loadEnvConfig)(process.cwd(), true);
14492
15003
  if (args.env_file) {
@@ -14496,23 +15007,6 @@ async function run(args) {
14496
15007
  process.exit(1);
14497
15008
  }
14498
15009
  }
14499
- if (args.bundle || args.push) {
14500
- return runWithEsbuild(args);
14501
- }
14502
- if (args.list && args.watch) {
14503
- console.error(error("Cannot specify both --list and --watch."));
14504
- process.exit(1);
14505
- }
14506
- if (args.watch) {
14507
- await runWithWatch(args);
14508
- return;
14509
- }
14510
- const success = await runWithRunner(args);
14511
- if (!success) {
14512
- process.exit(1);
14513
- }
14514
- }
14515
- async function runWithEsbuild(args) {
14516
15010
  const evaluatorOpts = {
14517
15011
  verbose: args.verbose,
14518
15012
  apiKey: args.api_key,
@@ -14522,19 +15016,38 @@ async function runWithEsbuild(args) {
14522
15016
  bundle: !!args.bundle || !!args.push,
14523
15017
  setCurrent: !!args.push,
14524
15018
  terminateOnFailure: !!args.terminate_on_failure,
14525
- watch: false,
15019
+ watch: !!args.watch,
14526
15020
  jsonl: args.jsonl,
14527
15021
  progressReporter: args.no_progress_bars ? new SimpleProgressReporter() : new BarProgressReporter(),
14528
15022
  filters: args.filter ? parseFilters(args.filter) : [],
14529
- list: false
15023
+ list: !!args.list
14530
15024
  };
15025
+ if (args.list && args.watch) {
15026
+ console.error(error("Cannot specify both --list and --watch."));
15027
+ process.exit(1);
15028
+ }
15029
+ const plugins = evaluatorOpts.watch ? [
15030
+ (fileName) => buildWatchPluginForEvaluator(fileName, evaluatorOpts)
15031
+ ] : [];
14531
15032
  const handles = await initializeHandles({
14532
15033
  files: args.files,
14533
15034
  mode: "eval",
14534
15035
  tsconfig: args.tsconfig,
14535
- plugins: [],
15036
+ plugins,
14536
15037
  externalPackages: args.external_packages
14537
15038
  });
15039
+ if (args.dev) {
15040
+ const { evaluators } = await buildEvaluators(handles, evaluatorOpts);
15041
+ const allEvaluators = Object.values(evaluators.evaluators).map(
15042
+ (e) => e.evaluator
15043
+ );
15044
+ runDevServer(allEvaluators, {
15045
+ host: args.dev_host,
15046
+ port: args.dev_port,
15047
+ orgName: args.dev_org_name
15048
+ });
15049
+ return;
15050
+ }
14538
15051
  let success = true;
14539
15052
  try {
14540
15053
  if (!evaluatorOpts.noSendLogs) {
@@ -14544,7 +15057,16 @@ async function runWithEsbuild(args) {
14544
15057
  appUrl: args.app_url
14545
15058
  });
14546
15059
  }
14547
- success = await runOnce(handles, evaluatorOpts);
15060
+ if (args.watch) {
15061
+ await runAndWatch({
15062
+ handles,
15063
+ onExit: () => {
15064
+ evaluatorOpts.progressReporter.stop();
15065
+ }
15066
+ });
15067
+ } else {
15068
+ success = await runOnce(handles, evaluatorOpts);
15069
+ }
14548
15070
  } finally {
14549
15071
  for (const handle of Object.values(handles)) {
14550
15072
  await handle.destroy();
@@ -14654,10 +15176,6 @@ async function main() {
14654
15176
  help: "Only allow users that belong this to this org name to run remote evals.",
14655
15177
  type: String
14656
15178
  });
14657
- parser_run.add_argument("--runner", {
14658
- help: "JS eval runner binary (e.g. tsx, bun, ts-node). Defaults to tsx if available.",
14659
- type: String
14660
- });
14661
15179
  parser_run.set_defaults({ func: run });
14662
15180
  const parser_push = subparser.add_parser("push", {
14663
15181
  help: "Bundle prompts, tools, scorers, and other resources into Braintrust"