braintrust 0.0.56 → 0.0.58
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser.js +5 -4
- package/dist/cli.js +859 -382
- package/dist/framework.d.ts +2 -1
- package/dist/index.js +733 -243
- package/dist/isomorph.d.ts +12 -0
- package/dist/logger.d.ts +331 -90
- package/dist/merge_row_batch.d.ts +1 -0
- package/dist/stackutil.d.ts +8 -0
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/dist/util.d.ts +4 -0
- package/package.json +2 -2
package/dist/cli.js
CHANGED
|
@@ -896,15 +896,15 @@ var require_route = __commonJS({
|
|
|
896
896
|
};
|
|
897
897
|
}
|
|
898
898
|
function wrapConversion(toModel, graph) {
|
|
899
|
-
const
|
|
899
|
+
const path5 = [graph[toModel].parent, toModel];
|
|
900
900
|
let fn = conversions[graph[toModel].parent][toModel];
|
|
901
901
|
let cur = graph[toModel].parent;
|
|
902
902
|
while (graph[cur].parent) {
|
|
903
|
-
|
|
903
|
+
path5.unshift(graph[cur].parent);
|
|
904
904
|
fn = link(conversions[graph[cur].parent][cur], fn);
|
|
905
905
|
cur = graph[cur].parent;
|
|
906
906
|
}
|
|
907
|
-
fn.conversion =
|
|
907
|
+
fn.conversion = path5;
|
|
908
908
|
return fn;
|
|
909
909
|
}
|
|
910
910
|
module2.exports = function(fromModel) {
|
|
@@ -1579,8 +1579,8 @@ var require_async = __commonJS({
|
|
|
1579
1579
|
"use strict";
|
|
1580
1580
|
Object.defineProperty(exports2, "__esModule", { value: true });
|
|
1581
1581
|
exports2.read = void 0;
|
|
1582
|
-
function read(
|
|
1583
|
-
settings.fs.lstat(
|
|
1582
|
+
function read(path5, settings, callback) {
|
|
1583
|
+
settings.fs.lstat(path5, (lstatError, lstat) => {
|
|
1584
1584
|
if (lstatError !== null) {
|
|
1585
1585
|
callFailureCallback(callback, lstatError);
|
|
1586
1586
|
return;
|
|
@@ -1589,7 +1589,7 @@ var require_async = __commonJS({
|
|
|
1589
1589
|
callSuccessCallback(callback, lstat);
|
|
1590
1590
|
return;
|
|
1591
1591
|
}
|
|
1592
|
-
settings.fs.stat(
|
|
1592
|
+
settings.fs.stat(path5, (statError, stat) => {
|
|
1593
1593
|
if (statError !== null) {
|
|
1594
1594
|
if (settings.throwErrorOnBrokenSymbolicLink) {
|
|
1595
1595
|
callFailureCallback(callback, statError);
|
|
@@ -1621,13 +1621,13 @@ var require_sync = __commonJS({
|
|
|
1621
1621
|
"use strict";
|
|
1622
1622
|
Object.defineProperty(exports2, "__esModule", { value: true });
|
|
1623
1623
|
exports2.read = void 0;
|
|
1624
|
-
function read(
|
|
1625
|
-
const lstat = settings.fs.lstatSync(
|
|
1624
|
+
function read(path5, settings) {
|
|
1625
|
+
const lstat = settings.fs.lstatSync(path5);
|
|
1626
1626
|
if (!lstat.isSymbolicLink() || !settings.followSymbolicLink) {
|
|
1627
1627
|
return lstat;
|
|
1628
1628
|
}
|
|
1629
1629
|
try {
|
|
1630
|
-
const stat = settings.fs.statSync(
|
|
1630
|
+
const stat = settings.fs.statSync(path5);
|
|
1631
1631
|
if (settings.markSymbolicLink) {
|
|
1632
1632
|
stat.isSymbolicLink = () => true;
|
|
1633
1633
|
}
|
|
@@ -1698,17 +1698,17 @@ var require_out = __commonJS({
|
|
|
1698
1698
|
var sync = require_sync();
|
|
1699
1699
|
var settings_1 = require_settings();
|
|
1700
1700
|
exports2.Settings = settings_1.default;
|
|
1701
|
-
function stat(
|
|
1701
|
+
function stat(path5, optionsOrSettingsOrCallback, callback) {
|
|
1702
1702
|
if (typeof optionsOrSettingsOrCallback === "function") {
|
|
1703
|
-
async.read(
|
|
1703
|
+
async.read(path5, getSettings(), optionsOrSettingsOrCallback);
|
|
1704
1704
|
return;
|
|
1705
1705
|
}
|
|
1706
|
-
async.read(
|
|
1706
|
+
async.read(path5, getSettings(optionsOrSettingsOrCallback), callback);
|
|
1707
1707
|
}
|
|
1708
1708
|
exports2.stat = stat;
|
|
1709
|
-
function statSync(
|
|
1709
|
+
function statSync(path5, optionsOrSettings) {
|
|
1710
1710
|
const settings = getSettings(optionsOrSettings);
|
|
1711
|
-
return sync.read(
|
|
1711
|
+
return sync.read(path5, settings);
|
|
1712
1712
|
}
|
|
1713
1713
|
exports2.statSync = statSync;
|
|
1714
1714
|
function getSettings(settingsOrOptions = {}) {
|
|
@@ -1927,16 +1927,16 @@ var require_async2 = __commonJS({
|
|
|
1927
1927
|
return;
|
|
1928
1928
|
}
|
|
1929
1929
|
const tasks = names.map((name) => {
|
|
1930
|
-
const
|
|
1930
|
+
const path5 = common.joinPathSegments(directory, name, settings.pathSegmentSeparator);
|
|
1931
1931
|
return (done) => {
|
|
1932
|
-
fsStat.stat(
|
|
1932
|
+
fsStat.stat(path5, settings.fsStatSettings, (error2, stats) => {
|
|
1933
1933
|
if (error2 !== null) {
|
|
1934
1934
|
done(error2);
|
|
1935
1935
|
return;
|
|
1936
1936
|
}
|
|
1937
1937
|
const entry = {
|
|
1938
1938
|
name,
|
|
1939
|
-
path:
|
|
1939
|
+
path: path5,
|
|
1940
1940
|
dirent: utils.fs.createDirentFromStats(name, stats)
|
|
1941
1941
|
};
|
|
1942
1942
|
if (settings.stats) {
|
|
@@ -2054,7 +2054,7 @@ var require_settings2 = __commonJS({
|
|
|
2054
2054
|
"node_modules/@nodelib/fs.scandir/out/settings.js"(exports2) {
|
|
2055
2055
|
"use strict";
|
|
2056
2056
|
Object.defineProperty(exports2, "__esModule", { value: true });
|
|
2057
|
-
var
|
|
2057
|
+
var path5 = require("path");
|
|
2058
2058
|
var fsStat = require_out();
|
|
2059
2059
|
var fs2 = require_fs3();
|
|
2060
2060
|
var Settings = class {
|
|
@@ -2062,7 +2062,7 @@ var require_settings2 = __commonJS({
|
|
|
2062
2062
|
this._options = _options;
|
|
2063
2063
|
this.followSymbolicLinks = this._getValue(this._options.followSymbolicLinks, false);
|
|
2064
2064
|
this.fs = fs2.createFileSystemAdapter(this._options.fs);
|
|
2065
|
-
this.pathSegmentSeparator = this._getValue(this._options.pathSegmentSeparator,
|
|
2065
|
+
this.pathSegmentSeparator = this._getValue(this._options.pathSegmentSeparator, path5.sep);
|
|
2066
2066
|
this.stats = this._getValue(this._options.stats, false);
|
|
2067
2067
|
this.throwErrorOnBrokenSymbolicLink = this._getValue(this._options.throwErrorOnBrokenSymbolicLink, true);
|
|
2068
2068
|
this.fsStatSettings = new fsStat.Settings({
|
|
@@ -2089,17 +2089,17 @@ var require_out2 = __commonJS({
|
|
|
2089
2089
|
var sync = require_sync2();
|
|
2090
2090
|
var settings_1 = require_settings2();
|
|
2091
2091
|
exports2.Settings = settings_1.default;
|
|
2092
|
-
function scandir(
|
|
2092
|
+
function scandir(path5, optionsOrSettingsOrCallback, callback) {
|
|
2093
2093
|
if (typeof optionsOrSettingsOrCallback === "function") {
|
|
2094
|
-
async.read(
|
|
2094
|
+
async.read(path5, getSettings(), optionsOrSettingsOrCallback);
|
|
2095
2095
|
return;
|
|
2096
2096
|
}
|
|
2097
|
-
async.read(
|
|
2097
|
+
async.read(path5, getSettings(optionsOrSettingsOrCallback), callback);
|
|
2098
2098
|
}
|
|
2099
2099
|
exports2.scandir = scandir;
|
|
2100
|
-
function scandirSync(
|
|
2100
|
+
function scandirSync(path5, optionsOrSettings) {
|
|
2101
2101
|
const settings = getSettings(optionsOrSettings);
|
|
2102
|
-
return sync.read(
|
|
2102
|
+
return sync.read(path5, settings);
|
|
2103
2103
|
}
|
|
2104
2104
|
exports2.scandirSync = scandirSync;
|
|
2105
2105
|
function getSettings(settingsOrOptions = {}) {
|
|
@@ -2703,7 +2703,7 @@ var require_settings3 = __commonJS({
|
|
|
2703
2703
|
"node_modules/@nodelib/fs.walk/out/settings.js"(exports2) {
|
|
2704
2704
|
"use strict";
|
|
2705
2705
|
Object.defineProperty(exports2, "__esModule", { value: true });
|
|
2706
|
-
var
|
|
2706
|
+
var path5 = require("path");
|
|
2707
2707
|
var fsScandir = require_out2();
|
|
2708
2708
|
var Settings = class {
|
|
2709
2709
|
constructor(_options = {}) {
|
|
@@ -2713,7 +2713,7 @@ var require_settings3 = __commonJS({
|
|
|
2713
2713
|
this.deepFilter = this._getValue(this._options.deepFilter, null);
|
|
2714
2714
|
this.entryFilter = this._getValue(this._options.entryFilter, null);
|
|
2715
2715
|
this.errorFilter = this._getValue(this._options.errorFilter, null);
|
|
2716
|
-
this.pathSegmentSeparator = this._getValue(this._options.pathSegmentSeparator,
|
|
2716
|
+
this.pathSegmentSeparator = this._getValue(this._options.pathSegmentSeparator, path5.sep);
|
|
2717
2717
|
this.fsScandirSettings = new fsScandir.Settings({
|
|
2718
2718
|
followSymbolicLinks: this._options.followSymbolicLinks,
|
|
2719
2719
|
fs: this._options.fs,
|
|
@@ -3304,7 +3304,7 @@ var require_argparse = __commonJS({
|
|
|
3304
3304
|
var util3 = require("util");
|
|
3305
3305
|
var fs2 = require("fs");
|
|
3306
3306
|
var sub = require_sub();
|
|
3307
|
-
var
|
|
3307
|
+
var path5 = require("path");
|
|
3308
3308
|
var repr = util3.inspect;
|
|
3309
3309
|
function get_argv() {
|
|
3310
3310
|
return process.argv.slice(1);
|
|
@@ -5359,7 +5359,7 @@ var require_argparse = __commonJS({
|
|
|
5359
5359
|
conflict_handler
|
|
5360
5360
|
});
|
|
5361
5361
|
if (prog === void 0) {
|
|
5362
|
-
prog =
|
|
5362
|
+
prog = path5.basename(get_argv()[0] || "");
|
|
5363
5363
|
}
|
|
5364
5364
|
this.prog = prog;
|
|
5365
5365
|
this.usage = usage;
|
|
@@ -15391,11 +15391,11 @@ var require_mime_types = __commonJS({
|
|
|
15391
15391
|
}
|
|
15392
15392
|
return exts[0];
|
|
15393
15393
|
}
|
|
15394
|
-
function lookup(
|
|
15395
|
-
if (!
|
|
15394
|
+
function lookup(path5) {
|
|
15395
|
+
if (!path5 || typeof path5 !== "string") {
|
|
15396
15396
|
return false;
|
|
15397
15397
|
}
|
|
15398
|
-
var extension2 = extname("x." +
|
|
15398
|
+
var extension2 = extname("x." + path5).toLowerCase().substr(1);
|
|
15399
15399
|
if (!extension2) {
|
|
15400
15400
|
return false;
|
|
15401
15401
|
}
|
|
@@ -15652,7 +15652,7 @@ var require_form_data = __commonJS({
|
|
|
15652
15652
|
"node_modules/form-data/lib/form_data.js"(exports2, module2) {
|
|
15653
15653
|
var CombinedStream = require_combined_stream();
|
|
15654
15654
|
var util3 = require("util");
|
|
15655
|
-
var
|
|
15655
|
+
var path5 = require("path");
|
|
15656
15656
|
var http3 = require("http");
|
|
15657
15657
|
var https3 = require("https");
|
|
15658
15658
|
var parseUrl = require("url").parse;
|
|
@@ -15779,11 +15779,11 @@ var require_form_data = __commonJS({
|
|
|
15779
15779
|
FormData3.prototype._getContentDisposition = function(value, options) {
|
|
15780
15780
|
var filename, contentDisposition;
|
|
15781
15781
|
if (typeof options.filepath === "string") {
|
|
15782
|
-
filename =
|
|
15782
|
+
filename = path5.normalize(options.filepath).replace(/\\/g, "/");
|
|
15783
15783
|
} else if (options.filename || value.name || value.path) {
|
|
15784
|
-
filename =
|
|
15784
|
+
filename = path5.basename(options.filename || value.name || value.path);
|
|
15785
15785
|
} else if (value.readable && value.hasOwnProperty("httpVersion")) {
|
|
15786
|
-
filename =
|
|
15786
|
+
filename = path5.basename(value.client._httpMessage.path || "");
|
|
15787
15787
|
}
|
|
15788
15788
|
if (filename) {
|
|
15789
15789
|
contentDisposition = 'filename="' + filename + '"';
|
|
@@ -18055,7 +18055,7 @@ var require_polyfills = __commonJS({
|
|
|
18055
18055
|
fs2.fstatSync = statFixSync(fs2.fstatSync);
|
|
18056
18056
|
fs2.lstatSync = statFixSync(fs2.lstatSync);
|
|
18057
18057
|
if (fs2.chmod && !fs2.lchmod) {
|
|
18058
|
-
fs2.lchmod = function(
|
|
18058
|
+
fs2.lchmod = function(path5, mode, cb) {
|
|
18059
18059
|
if (cb)
|
|
18060
18060
|
process.nextTick(cb);
|
|
18061
18061
|
};
|
|
@@ -18063,7 +18063,7 @@ var require_polyfills = __commonJS({
|
|
|
18063
18063
|
};
|
|
18064
18064
|
}
|
|
18065
18065
|
if (fs2.chown && !fs2.lchown) {
|
|
18066
|
-
fs2.lchown = function(
|
|
18066
|
+
fs2.lchown = function(path5, uid, gid, cb) {
|
|
18067
18067
|
if (cb)
|
|
18068
18068
|
process.nextTick(cb);
|
|
18069
18069
|
};
|
|
@@ -18134,9 +18134,9 @@ var require_polyfills = __commonJS({
|
|
|
18134
18134
|
};
|
|
18135
18135
|
}(fs2.readSync);
|
|
18136
18136
|
function patchLchmod(fs3) {
|
|
18137
|
-
fs3.lchmod = function(
|
|
18137
|
+
fs3.lchmod = function(path5, mode, callback) {
|
|
18138
18138
|
fs3.open(
|
|
18139
|
-
|
|
18139
|
+
path5,
|
|
18140
18140
|
constants.O_WRONLY | constants.O_SYMLINK,
|
|
18141
18141
|
mode,
|
|
18142
18142
|
function(err, fd) {
|
|
@@ -18154,8 +18154,8 @@ var require_polyfills = __commonJS({
|
|
|
18154
18154
|
}
|
|
18155
18155
|
);
|
|
18156
18156
|
};
|
|
18157
|
-
fs3.lchmodSync = function(
|
|
18158
|
-
var fd = fs3.openSync(
|
|
18157
|
+
fs3.lchmodSync = function(path5, mode) {
|
|
18158
|
+
var fd = fs3.openSync(path5, constants.O_WRONLY | constants.O_SYMLINK, mode);
|
|
18159
18159
|
var threw = true;
|
|
18160
18160
|
var ret;
|
|
18161
18161
|
try {
|
|
@@ -18176,8 +18176,8 @@ var require_polyfills = __commonJS({
|
|
|
18176
18176
|
}
|
|
18177
18177
|
function patchLutimes(fs3) {
|
|
18178
18178
|
if (constants.hasOwnProperty("O_SYMLINK") && fs3.futimes) {
|
|
18179
|
-
fs3.lutimes = function(
|
|
18180
|
-
fs3.open(
|
|
18179
|
+
fs3.lutimes = function(path5, at, mt, cb) {
|
|
18180
|
+
fs3.open(path5, constants.O_SYMLINK, function(er, fd) {
|
|
18181
18181
|
if (er) {
|
|
18182
18182
|
if (cb)
|
|
18183
18183
|
cb(er);
|
|
@@ -18191,8 +18191,8 @@ var require_polyfills = __commonJS({
|
|
|
18191
18191
|
});
|
|
18192
18192
|
});
|
|
18193
18193
|
};
|
|
18194
|
-
fs3.lutimesSync = function(
|
|
18195
|
-
var fd = fs3.openSync(
|
|
18194
|
+
fs3.lutimesSync = function(path5, at, mt) {
|
|
18195
|
+
var fd = fs3.openSync(path5, constants.O_SYMLINK);
|
|
18196
18196
|
var ret;
|
|
18197
18197
|
var threw = true;
|
|
18198
18198
|
try {
|
|
@@ -18328,12 +18328,12 @@ var require_legacy_streams = __commonJS({
|
|
|
18328
18328
|
ReadStream,
|
|
18329
18329
|
WriteStream
|
|
18330
18330
|
};
|
|
18331
|
-
function ReadStream(
|
|
18331
|
+
function ReadStream(path5, options) {
|
|
18332
18332
|
if (!(this instanceof ReadStream))
|
|
18333
|
-
return new ReadStream(
|
|
18333
|
+
return new ReadStream(path5, options);
|
|
18334
18334
|
Stream.call(this);
|
|
18335
18335
|
var self2 = this;
|
|
18336
|
-
this.path =
|
|
18336
|
+
this.path = path5;
|
|
18337
18337
|
this.fd = null;
|
|
18338
18338
|
this.readable = true;
|
|
18339
18339
|
this.paused = false;
|
|
@@ -18379,11 +18379,11 @@ var require_legacy_streams = __commonJS({
|
|
|
18379
18379
|
self2._read();
|
|
18380
18380
|
});
|
|
18381
18381
|
}
|
|
18382
|
-
function WriteStream(
|
|
18382
|
+
function WriteStream(path5, options) {
|
|
18383
18383
|
if (!(this instanceof WriteStream))
|
|
18384
|
-
return new WriteStream(
|
|
18384
|
+
return new WriteStream(path5, options);
|
|
18385
18385
|
Stream.call(this);
|
|
18386
|
-
this.path =
|
|
18386
|
+
this.path = path5;
|
|
18387
18387
|
this.fd = null;
|
|
18388
18388
|
this.writable = true;
|
|
18389
18389
|
this.flags = "w";
|
|
@@ -18526,14 +18526,14 @@ var require_graceful_fs = __commonJS({
|
|
|
18526
18526
|
fs3.createWriteStream = createWriteStream;
|
|
18527
18527
|
var fs$readFile = fs3.readFile;
|
|
18528
18528
|
fs3.readFile = readFile;
|
|
18529
|
-
function readFile(
|
|
18529
|
+
function readFile(path5, options, cb) {
|
|
18530
18530
|
if (typeof options === "function")
|
|
18531
18531
|
cb = options, options = null;
|
|
18532
|
-
return go$readFile(
|
|
18533
|
-
function go$readFile(
|
|
18534
|
-
return fs$readFile(
|
|
18532
|
+
return go$readFile(path5, options, cb);
|
|
18533
|
+
function go$readFile(path6, options2, cb2, startTime) {
|
|
18534
|
+
return fs$readFile(path6, options2, function(err) {
|
|
18535
18535
|
if (err && (err.code === "EMFILE" || err.code === "ENFILE"))
|
|
18536
|
-
enqueue([go$readFile, [
|
|
18536
|
+
enqueue([go$readFile, [path6, options2, cb2], err, startTime || Date.now(), Date.now()]);
|
|
18537
18537
|
else {
|
|
18538
18538
|
if (typeof cb2 === "function")
|
|
18539
18539
|
cb2.apply(this, arguments);
|
|
@@ -18543,14 +18543,14 @@ var require_graceful_fs = __commonJS({
|
|
|
18543
18543
|
}
|
|
18544
18544
|
var fs$writeFile = fs3.writeFile;
|
|
18545
18545
|
fs3.writeFile = writeFile;
|
|
18546
|
-
function writeFile(
|
|
18546
|
+
function writeFile(path5, data, options, cb) {
|
|
18547
18547
|
if (typeof options === "function")
|
|
18548
18548
|
cb = options, options = null;
|
|
18549
|
-
return go$writeFile(
|
|
18550
|
-
function go$writeFile(
|
|
18551
|
-
return fs$writeFile(
|
|
18549
|
+
return go$writeFile(path5, data, options, cb);
|
|
18550
|
+
function go$writeFile(path6, data2, options2, cb2, startTime) {
|
|
18551
|
+
return fs$writeFile(path6, data2, options2, function(err) {
|
|
18552
18552
|
if (err && (err.code === "EMFILE" || err.code === "ENFILE"))
|
|
18553
|
-
enqueue([go$writeFile, [
|
|
18553
|
+
enqueue([go$writeFile, [path6, data2, options2, cb2], err, startTime || Date.now(), Date.now()]);
|
|
18554
18554
|
else {
|
|
18555
18555
|
if (typeof cb2 === "function")
|
|
18556
18556
|
cb2.apply(this, arguments);
|
|
@@ -18561,14 +18561,14 @@ var require_graceful_fs = __commonJS({
|
|
|
18561
18561
|
var fs$appendFile = fs3.appendFile;
|
|
18562
18562
|
if (fs$appendFile)
|
|
18563
18563
|
fs3.appendFile = appendFile;
|
|
18564
|
-
function appendFile(
|
|
18564
|
+
function appendFile(path5, data, options, cb) {
|
|
18565
18565
|
if (typeof options === "function")
|
|
18566
18566
|
cb = options, options = null;
|
|
18567
|
-
return go$appendFile(
|
|
18568
|
-
function go$appendFile(
|
|
18569
|
-
return fs$appendFile(
|
|
18567
|
+
return go$appendFile(path5, data, options, cb);
|
|
18568
|
+
function go$appendFile(path6, data2, options2, cb2, startTime) {
|
|
18569
|
+
return fs$appendFile(path6, data2, options2, function(err) {
|
|
18570
18570
|
if (err && (err.code === "EMFILE" || err.code === "ENFILE"))
|
|
18571
|
-
enqueue([go$appendFile, [
|
|
18571
|
+
enqueue([go$appendFile, [path6, data2, options2, cb2], err, startTime || Date.now(), Date.now()]);
|
|
18572
18572
|
else {
|
|
18573
18573
|
if (typeof cb2 === "function")
|
|
18574
18574
|
cb2.apply(this, arguments);
|
|
@@ -18599,31 +18599,31 @@ var require_graceful_fs = __commonJS({
|
|
|
18599
18599
|
var fs$readdir = fs3.readdir;
|
|
18600
18600
|
fs3.readdir = readdir;
|
|
18601
18601
|
var noReaddirOptionVersions = /^v[0-5]\./;
|
|
18602
|
-
function readdir(
|
|
18602
|
+
function readdir(path5, options, cb) {
|
|
18603
18603
|
if (typeof options === "function")
|
|
18604
18604
|
cb = options, options = null;
|
|
18605
|
-
var go$readdir = noReaddirOptionVersions.test(process.version) ? function go$readdir2(
|
|
18606
|
-
return fs$readdir(
|
|
18607
|
-
|
|
18605
|
+
var go$readdir = noReaddirOptionVersions.test(process.version) ? function go$readdir2(path6, options2, cb2, startTime) {
|
|
18606
|
+
return fs$readdir(path6, fs$readdirCallback(
|
|
18607
|
+
path6,
|
|
18608
18608
|
options2,
|
|
18609
18609
|
cb2,
|
|
18610
18610
|
startTime
|
|
18611
18611
|
));
|
|
18612
|
-
} : function go$readdir2(
|
|
18613
|
-
return fs$readdir(
|
|
18614
|
-
|
|
18612
|
+
} : function go$readdir2(path6, options2, cb2, startTime) {
|
|
18613
|
+
return fs$readdir(path6, options2, fs$readdirCallback(
|
|
18614
|
+
path6,
|
|
18615
18615
|
options2,
|
|
18616
18616
|
cb2,
|
|
18617
18617
|
startTime
|
|
18618
18618
|
));
|
|
18619
18619
|
};
|
|
18620
|
-
return go$readdir(
|
|
18621
|
-
function fs$readdirCallback(
|
|
18620
|
+
return go$readdir(path5, options, cb);
|
|
18621
|
+
function fs$readdirCallback(path6, options2, cb2, startTime) {
|
|
18622
18622
|
return function(err, files) {
|
|
18623
18623
|
if (err && (err.code === "EMFILE" || err.code === "ENFILE"))
|
|
18624
18624
|
enqueue([
|
|
18625
18625
|
go$readdir,
|
|
18626
|
-
[
|
|
18626
|
+
[path6, options2, cb2],
|
|
18627
18627
|
err,
|
|
18628
18628
|
startTime || Date.now(),
|
|
18629
18629
|
Date.now()
|
|
@@ -18694,7 +18694,7 @@ var require_graceful_fs = __commonJS({
|
|
|
18694
18694
|
enumerable: true,
|
|
18695
18695
|
configurable: true
|
|
18696
18696
|
});
|
|
18697
|
-
function ReadStream(
|
|
18697
|
+
function ReadStream(path5, options) {
|
|
18698
18698
|
if (this instanceof ReadStream)
|
|
18699
18699
|
return fs$ReadStream.apply(this, arguments), this;
|
|
18700
18700
|
else
|
|
@@ -18714,7 +18714,7 @@ var require_graceful_fs = __commonJS({
|
|
|
18714
18714
|
}
|
|
18715
18715
|
});
|
|
18716
18716
|
}
|
|
18717
|
-
function WriteStream(
|
|
18717
|
+
function WriteStream(path5, options) {
|
|
18718
18718
|
if (this instanceof WriteStream)
|
|
18719
18719
|
return fs$WriteStream.apply(this, arguments), this;
|
|
18720
18720
|
else
|
|
@@ -18732,22 +18732,22 @@ var require_graceful_fs = __commonJS({
|
|
|
18732
18732
|
}
|
|
18733
18733
|
});
|
|
18734
18734
|
}
|
|
18735
|
-
function createReadStream(
|
|
18736
|
-
return new fs3.ReadStream(
|
|
18735
|
+
function createReadStream(path5, options) {
|
|
18736
|
+
return new fs3.ReadStream(path5, options);
|
|
18737
18737
|
}
|
|
18738
|
-
function createWriteStream(
|
|
18739
|
-
return new fs3.WriteStream(
|
|
18738
|
+
function createWriteStream(path5, options) {
|
|
18739
|
+
return new fs3.WriteStream(path5, options);
|
|
18740
18740
|
}
|
|
18741
18741
|
var fs$open = fs3.open;
|
|
18742
18742
|
fs3.open = open;
|
|
18743
|
-
function open(
|
|
18743
|
+
function open(path5, flags, mode, cb) {
|
|
18744
18744
|
if (typeof mode === "function")
|
|
18745
18745
|
cb = mode, mode = null;
|
|
18746
|
-
return go$open(
|
|
18747
|
-
function go$open(
|
|
18748
|
-
return fs$open(
|
|
18746
|
+
return go$open(path5, flags, mode, cb);
|
|
18747
|
+
function go$open(path6, flags2, mode2, cb2, startTime) {
|
|
18748
|
+
return fs$open(path6, flags2, mode2, function(err, fd) {
|
|
18749
18749
|
if (err && (err.code === "EMFILE" || err.code === "ENFILE"))
|
|
18750
|
-
enqueue([go$open, [
|
|
18750
|
+
enqueue([go$open, [path6, flags2, mode2, cb2], err, startTime || Date.now(), Date.now()]);
|
|
18751
18751
|
else {
|
|
18752
18752
|
if (typeof cb2 === "function")
|
|
18753
18753
|
cb2.apply(this, arguments);
|
|
@@ -18821,10 +18821,10 @@ var require_src2 = __commonJS({
|
|
|
18821
18821
|
var fs_1 = require("fs");
|
|
18822
18822
|
var debug_1 = __importDefault(require_src());
|
|
18823
18823
|
var log = debug_1.default("@kwsites/file-exists");
|
|
18824
|
-
function check(
|
|
18825
|
-
log(`checking %s`,
|
|
18824
|
+
function check(path5, isFile2, isDirectory) {
|
|
18825
|
+
log(`checking %s`, path5);
|
|
18826
18826
|
try {
|
|
18827
|
-
const stat = fs_1.statSync(
|
|
18827
|
+
const stat = fs_1.statSync(path5);
|
|
18828
18828
|
if (stat.isFile() && isFile2) {
|
|
18829
18829
|
log(`[OK] path represents a file`);
|
|
18830
18830
|
return true;
|
|
@@ -18844,8 +18844,8 @@ var require_src2 = __commonJS({
|
|
|
18844
18844
|
throw e;
|
|
18845
18845
|
}
|
|
18846
18846
|
}
|
|
18847
|
-
function exists2(
|
|
18848
|
-
return check(
|
|
18847
|
+
function exists2(path5, type = exports2.READABLE) {
|
|
18848
|
+
return check(path5, (type & exports2.FILE) > 0, (type & exports2.FOLDER) > 0);
|
|
18849
18849
|
}
|
|
18850
18850
|
exports2.exists = exists2;
|
|
18851
18851
|
exports2.FILE = 1;
|
|
@@ -18915,7 +18915,7 @@ var require_package = __commonJS({
|
|
|
18915
18915
|
"package.json"(exports2, module2) {
|
|
18916
18916
|
module2.exports = {
|
|
18917
18917
|
name: "braintrust",
|
|
18918
|
-
version: "0.0.
|
|
18918
|
+
version: "0.0.58",
|
|
18919
18919
|
description: "SDK for integrating Braintrust",
|
|
18920
18920
|
main: "./dist/index.js",
|
|
18921
18921
|
browser: {
|
|
@@ -18960,7 +18960,7 @@ var require_package = __commonJS({
|
|
|
18960
18960
|
},
|
|
18961
18961
|
dependencies: {
|
|
18962
18962
|
argparse: "^2.0.1",
|
|
18963
|
-
autoevals: "^0.0.
|
|
18963
|
+
autoevals: "^0.0.26",
|
|
18964
18964
|
axios: "^1.4.0",
|
|
18965
18965
|
chalk: "^4.1.2",
|
|
18966
18966
|
"cli-progress": "^3.12.0",
|
|
@@ -18982,7 +18982,7 @@ var import_chalk = __toESM(require_source());
|
|
|
18982
18982
|
var import_fs = __toESM(require("fs"));
|
|
18983
18983
|
var import_os = __toESM(require("os"));
|
|
18984
18984
|
var import_path = __toESM(require("path"));
|
|
18985
|
-
var
|
|
18985
|
+
var import_util5 = __toESM(require("util"));
|
|
18986
18986
|
var fsWalk = __toESM(require_out3());
|
|
18987
18987
|
|
|
18988
18988
|
// node_modules/minimatch/dist/mjs/index.js
|
|
@@ -20810,10 +20810,10 @@ function isVisitable(thing) {
|
|
|
20810
20810
|
function removeBrackets(key) {
|
|
20811
20811
|
return utils_default.endsWith(key, "[]") ? key.slice(0, -2) : key;
|
|
20812
20812
|
}
|
|
20813
|
-
function renderKey(
|
|
20814
|
-
if (!
|
|
20813
|
+
function renderKey(path5, key, dots) {
|
|
20814
|
+
if (!path5)
|
|
20815
20815
|
return key;
|
|
20816
|
-
return
|
|
20816
|
+
return path5.concat(key).map(function each(token, i) {
|
|
20817
20817
|
token = removeBrackets(token);
|
|
20818
20818
|
return !dots && i ? "[" + token + "]" : token;
|
|
20819
20819
|
}).join(dots ? "." : "");
|
|
@@ -20859,9 +20859,9 @@ function toFormData(obj, formData, options) {
|
|
|
20859
20859
|
}
|
|
20860
20860
|
return value;
|
|
20861
20861
|
}
|
|
20862
|
-
function defaultVisitor(value, key,
|
|
20862
|
+
function defaultVisitor(value, key, path5) {
|
|
20863
20863
|
let arr = value;
|
|
20864
|
-
if (value && !
|
|
20864
|
+
if (value && !path5 && typeof value === "object") {
|
|
20865
20865
|
if (utils_default.endsWith(key, "{}")) {
|
|
20866
20866
|
key = metaTokens ? key : key.slice(0, -2);
|
|
20867
20867
|
value = JSON.stringify(value);
|
|
@@ -20880,7 +20880,7 @@ function toFormData(obj, formData, options) {
|
|
|
20880
20880
|
if (isVisitable(value)) {
|
|
20881
20881
|
return true;
|
|
20882
20882
|
}
|
|
20883
|
-
formData.append(renderKey(
|
|
20883
|
+
formData.append(renderKey(path5, key, dots), convertValue(value));
|
|
20884
20884
|
return false;
|
|
20885
20885
|
}
|
|
20886
20886
|
const stack = [];
|
|
@@ -20889,11 +20889,11 @@ function toFormData(obj, formData, options) {
|
|
|
20889
20889
|
convertValue,
|
|
20890
20890
|
isVisitable
|
|
20891
20891
|
});
|
|
20892
|
-
function build(value,
|
|
20892
|
+
function build(value, path5) {
|
|
20893
20893
|
if (utils_default.isUndefined(value))
|
|
20894
20894
|
return;
|
|
20895
20895
|
if (stack.indexOf(value) !== -1) {
|
|
20896
|
-
throw Error("Circular reference detected in " +
|
|
20896
|
+
throw Error("Circular reference detected in " + path5.join("."));
|
|
20897
20897
|
}
|
|
20898
20898
|
stack.push(value);
|
|
20899
20899
|
utils_default.forEach(value, function each(el, key) {
|
|
@@ -20901,11 +20901,11 @@ function toFormData(obj, formData, options) {
|
|
|
20901
20901
|
formData,
|
|
20902
20902
|
el,
|
|
20903
20903
|
utils_default.isString(key) ? key.trim() : key,
|
|
20904
|
-
|
|
20904
|
+
path5,
|
|
20905
20905
|
exposedHelpers
|
|
20906
20906
|
);
|
|
20907
20907
|
if (result === true) {
|
|
20908
|
-
build(el,
|
|
20908
|
+
build(el, path5 ? path5.concat(key) : [key]);
|
|
20909
20909
|
}
|
|
20910
20910
|
});
|
|
20911
20911
|
stack.pop();
|
|
@@ -21066,7 +21066,7 @@ var node_default = {
|
|
|
21066
21066
|
// node_modules/axios/lib/helpers/toURLEncodedForm.js
|
|
21067
21067
|
function toURLEncodedForm(data, options) {
|
|
21068
21068
|
return toFormData_default(data, new node_default.classes.URLSearchParams(), Object.assign({
|
|
21069
|
-
visitor: function(value, key,
|
|
21069
|
+
visitor: function(value, key, path5, helpers) {
|
|
21070
21070
|
if (node_default.isNode && utils_default.isBuffer(value)) {
|
|
21071
21071
|
this.append(key, value.toString("base64"));
|
|
21072
21072
|
return false;
|
|
@@ -21095,10 +21095,10 @@ function arrayToObject(arr) {
|
|
|
21095
21095
|
return obj;
|
|
21096
21096
|
}
|
|
21097
21097
|
function formDataToJSON(formData) {
|
|
21098
|
-
function buildPath(
|
|
21099
|
-
let name =
|
|
21098
|
+
function buildPath(path5, value, target, index) {
|
|
21099
|
+
let name = path5[index++];
|
|
21100
21100
|
const isNumericKey = Number.isFinite(+name);
|
|
21101
|
-
const isLast = index >=
|
|
21101
|
+
const isLast = index >= path5.length;
|
|
21102
21102
|
name = !name && utils_default.isArray(target) ? target.length : name;
|
|
21103
21103
|
if (isLast) {
|
|
21104
21104
|
if (utils_default.hasOwnProp(target, name)) {
|
|
@@ -21111,7 +21111,7 @@ function formDataToJSON(formData) {
|
|
|
21111
21111
|
if (!target[name] || !utils_default.isObject(target[name])) {
|
|
21112
21112
|
target[name] = [];
|
|
21113
21113
|
}
|
|
21114
|
-
const result = buildPath(
|
|
21114
|
+
const result = buildPath(path5, value, target[name], index);
|
|
21115
21115
|
if (result && utils_default.isArray(target[name])) {
|
|
21116
21116
|
target[name] = arrayToObject(target[name]);
|
|
21117
21117
|
}
|
|
@@ -22210,9 +22210,9 @@ var http_default = isHttpAdapterSupported && function httpAdapter(config) {
|
|
|
22210
22210
|
auth = urlUsername + ":" + urlPassword;
|
|
22211
22211
|
}
|
|
22212
22212
|
auth && headers.delete("authorization");
|
|
22213
|
-
let
|
|
22213
|
+
let path5;
|
|
22214
22214
|
try {
|
|
22215
|
-
|
|
22215
|
+
path5 = buildURL(
|
|
22216
22216
|
parsed.pathname + parsed.search,
|
|
22217
22217
|
config.params,
|
|
22218
22218
|
config.paramsSerializer
|
|
@@ -22230,7 +22230,7 @@ var http_default = isHttpAdapterSupported && function httpAdapter(config) {
|
|
|
22230
22230
|
false
|
|
22231
22231
|
);
|
|
22232
22232
|
const options = {
|
|
22233
|
-
path:
|
|
22233
|
+
path: path5,
|
|
22234
22234
|
method,
|
|
22235
22235
|
headers: headers.toJSON(),
|
|
22236
22236
|
agents: { http: config.httpAgent, https: config.httpsAgent },
|
|
@@ -22453,14 +22453,14 @@ var cookies_default = node_default.isStandardBrowserEnv ? (
|
|
|
22453
22453
|
// Standard browser envs support document.cookie
|
|
22454
22454
|
function standardBrowserEnv() {
|
|
22455
22455
|
return {
|
|
22456
|
-
write: function write(name, value, expires,
|
|
22456
|
+
write: function write(name, value, expires, path5, domain, secure) {
|
|
22457
22457
|
const cookie = [];
|
|
22458
22458
|
cookie.push(name + "=" + encodeURIComponent(value));
|
|
22459
22459
|
if (utils_default.isNumber(expires)) {
|
|
22460
22460
|
cookie.push("expires=" + new Date(expires).toGMTString());
|
|
22461
22461
|
}
|
|
22462
|
-
if (utils_default.isString(
|
|
22463
|
-
cookie.push("path=" +
|
|
22462
|
+
if (utils_default.isString(path5)) {
|
|
22463
|
+
cookie.push("path=" + path5);
|
|
22464
22464
|
}
|
|
22465
22465
|
if (utils_default.isString(domain)) {
|
|
22466
22466
|
cookie.push("domain=" + domain);
|
|
@@ -23296,28 +23296,195 @@ var {
|
|
|
23296
23296
|
} = axios_default;
|
|
23297
23297
|
|
|
23298
23298
|
// src/isomorph.ts
|
|
23299
|
+
var DefaultAsyncLocalStorage = class {
|
|
23300
|
+
constructor() {
|
|
23301
|
+
}
|
|
23302
|
+
enterWith(_) {
|
|
23303
|
+
}
|
|
23304
|
+
run(_, callback) {
|
|
23305
|
+
return callback();
|
|
23306
|
+
}
|
|
23307
|
+
getStore() {
|
|
23308
|
+
return void 0;
|
|
23309
|
+
}
|
|
23310
|
+
};
|
|
23299
23311
|
var iso = {
|
|
23300
23312
|
makeAxios: (conf) => axios_default.create({
|
|
23301
23313
|
...conf
|
|
23302
23314
|
}),
|
|
23303
23315
|
getRepoStatus: async () => void 0,
|
|
23304
23316
|
getPastNAncestors: async () => [],
|
|
23305
|
-
getEnv: (_name) => void 0
|
|
23317
|
+
getEnv: (_name) => void 0,
|
|
23318
|
+
getCallerLocation: () => void 0,
|
|
23319
|
+
newAsyncLocalStorage: () => new DefaultAsyncLocalStorage()
|
|
23306
23320
|
};
|
|
23307
23321
|
var isomorph_default = iso;
|
|
23308
23322
|
|
|
23323
|
+
// src/util.ts
|
|
23324
|
+
var TRANSACTION_ID_FIELD = "_xact_id";
|
|
23325
|
+
var IS_MERGE_FIELD = "_is_merge";
|
|
23326
|
+
function runFinally(f, finallyF) {
|
|
23327
|
+
let runSyncCleanup = true;
|
|
23328
|
+
try {
|
|
23329
|
+
const ret = f();
|
|
23330
|
+
if (ret instanceof Promise) {
|
|
23331
|
+
runSyncCleanup = false;
|
|
23332
|
+
return ret.finally(finallyF);
|
|
23333
|
+
} else {
|
|
23334
|
+
return ret;
|
|
23335
|
+
}
|
|
23336
|
+
} finally {
|
|
23337
|
+
if (runSyncCleanup) {
|
|
23338
|
+
finallyF();
|
|
23339
|
+
}
|
|
23340
|
+
}
|
|
23341
|
+
}
|
|
23342
|
+
function mergeDicts(mergeInto, mergeFrom) {
|
|
23343
|
+
for (const [k, mergeFromV] of Object.entries(mergeFrom)) {
|
|
23344
|
+
const mergeIntoV = mergeInto[k];
|
|
23345
|
+
if (mergeIntoV instanceof Object && !Array.isArray(mergeIntoV) && mergeFrom instanceof Object && !Array.isArray(mergeFromV)) {
|
|
23346
|
+
mergeDicts(mergeIntoV, mergeFromV);
|
|
23347
|
+
} else {
|
|
23348
|
+
mergeInto[k] = mergeFromV;
|
|
23349
|
+
}
|
|
23350
|
+
}
|
|
23351
|
+
}
|
|
23352
|
+
|
|
23353
|
+
// src/merge_row_batch.ts
|
|
23354
|
+
function generateUniqueRowKey(row) {
|
|
23355
|
+
const coalesceEmpty = (field) => row[field] ?? "";
|
|
23356
|
+
return coalesceEmpty("experiment_id") + ":" + coalesceEmpty("dataset_id") + ":" + coalesceEmpty("prompt_session_id") + ":" + coalesceEmpty("id");
|
|
23357
|
+
}
|
|
23358
|
+
function mergeRowBatch(rows) {
|
|
23359
|
+
const out = [];
|
|
23360
|
+
const remainingRows = [];
|
|
23361
|
+
for (const row of rows) {
|
|
23362
|
+
if (row["id"] === void 0) {
|
|
23363
|
+
out.push(row);
|
|
23364
|
+
} else {
|
|
23365
|
+
remainingRows.push(row);
|
|
23366
|
+
}
|
|
23367
|
+
}
|
|
23368
|
+
const rowGroups = {};
|
|
23369
|
+
for (const row of remainingRows) {
|
|
23370
|
+
const key = generateUniqueRowKey(row);
|
|
23371
|
+
const existingRow = rowGroups[key];
|
|
23372
|
+
if (existingRow !== void 0 && row[IS_MERGE_FIELD]) {
|
|
23373
|
+
const preserveNoMerge = !existingRow[IS_MERGE_FIELD];
|
|
23374
|
+
mergeDicts(existingRow, row);
|
|
23375
|
+
if (preserveNoMerge) {
|
|
23376
|
+
delete existingRow[IS_MERGE_FIELD];
|
|
23377
|
+
}
|
|
23378
|
+
} else {
|
|
23379
|
+
rowGroups[key] = row;
|
|
23380
|
+
}
|
|
23381
|
+
}
|
|
23382
|
+
out.push(...Object.values(rowGroups));
|
|
23383
|
+
return out;
|
|
23384
|
+
}
|
|
23385
|
+
|
|
23309
23386
|
// src/logger.ts
|
|
23310
|
-
var
|
|
23311
|
-
|
|
23312
|
-
|
|
23387
|
+
var NoopSpan = class {
|
|
23388
|
+
constructor() {
|
|
23389
|
+
this.kind = "span";
|
|
23390
|
+
this.id = "";
|
|
23391
|
+
this.span_id = "";
|
|
23392
|
+
this.root_span_id = "";
|
|
23393
|
+
}
|
|
23394
|
+
log(_) {
|
|
23395
|
+
}
|
|
23396
|
+
startSpan(_0, _1) {
|
|
23397
|
+
return this;
|
|
23398
|
+
}
|
|
23399
|
+
traced(_0, callback, _1) {
|
|
23400
|
+
return callback(this);
|
|
23401
|
+
}
|
|
23402
|
+
end(args) {
|
|
23403
|
+
return args?.endTime ?? getCurrentUnixTimestamp();
|
|
23404
|
+
}
|
|
23405
|
+
close(args) {
|
|
23406
|
+
return this.end(args);
|
|
23407
|
+
}
|
|
23313
23408
|
};
|
|
23314
|
-
var
|
|
23315
|
-
var
|
|
23316
|
-
|
|
23317
|
-
|
|
23318
|
-
|
|
23319
|
-
|
|
23320
|
-
|
|
23409
|
+
var noopSpan = new NoopSpan();
|
|
23410
|
+
var BraintrustState = class {
|
|
23411
|
+
constructor() {
|
|
23412
|
+
this.id = v4_default();
|
|
23413
|
+
this.currentExperiment = isomorph_default.newAsyncLocalStorage();
|
|
23414
|
+
this.currentSpan = isomorph_default.newAsyncLocalStorage();
|
|
23415
|
+
this.apiUrl = null;
|
|
23416
|
+
this.loginToken = null;
|
|
23417
|
+
this.orgId = null;
|
|
23418
|
+
this.orgName = null;
|
|
23419
|
+
this.logUrl = null;
|
|
23420
|
+
this.loggedIn = false;
|
|
23421
|
+
this._apiConn = null;
|
|
23422
|
+
this._logConn = null;
|
|
23423
|
+
this._userInfo = null;
|
|
23424
|
+
globalThis.__inherited_braintrust_state = this;
|
|
23425
|
+
}
|
|
23426
|
+
apiConn() {
|
|
23427
|
+
if (!this._apiConn) {
|
|
23428
|
+
if (!this.apiUrl) {
|
|
23429
|
+
throw new Error("Must initialize apiUrl before requesting apiConn");
|
|
23430
|
+
}
|
|
23431
|
+
this._apiConn = new HTTPConnection(this.apiUrl);
|
|
23432
|
+
}
|
|
23433
|
+
return this._apiConn;
|
|
23434
|
+
}
|
|
23435
|
+
logConn() {
|
|
23436
|
+
if (!this._logConn) {
|
|
23437
|
+
if (!this.logUrl) {
|
|
23438
|
+
throw new Error("Must initialize logUrl before requesting logConn");
|
|
23439
|
+
}
|
|
23440
|
+
this._logConn = new HTTPConnection(this.logUrl);
|
|
23441
|
+
}
|
|
23442
|
+
return this._logConn;
|
|
23443
|
+
}
|
|
23444
|
+
async userInfo() {
|
|
23445
|
+
if (!this._userInfo) {
|
|
23446
|
+
this._userInfo = await this.logConn().get_json("ping");
|
|
23447
|
+
}
|
|
23448
|
+
return this._userInfo;
|
|
23449
|
+
}
|
|
23450
|
+
setUserInfoIfNull(info) {
|
|
23451
|
+
if (!this._userInfo) {
|
|
23452
|
+
this._userInfo = info;
|
|
23453
|
+
}
|
|
23454
|
+
}
|
|
23455
|
+
};
|
|
23456
|
+
var _state = globalThis.__inherited_braintrust_state || new BraintrustState();
|
|
23457
|
+
var _internalGetGlobalState = () => _state;
|
|
23458
|
+
var UnterminatedObjectsHandler = class {
|
|
23459
|
+
constructor() {
|
|
23460
|
+
this.unterminatedObjects = /* @__PURE__ */ new Map();
|
|
23461
|
+
process.on("exit", () => {
|
|
23462
|
+
this.warnUnterminated();
|
|
23463
|
+
});
|
|
23464
|
+
}
|
|
23465
|
+
addUnterminated(obj, createdLocation) {
|
|
23466
|
+
this.unterminatedObjects.set(obj, createdLocation);
|
|
23467
|
+
}
|
|
23468
|
+
removeUnterminated(obj) {
|
|
23469
|
+
this.unterminatedObjects.delete(obj);
|
|
23470
|
+
}
|
|
23471
|
+
warnUnterminated() {
|
|
23472
|
+
if (this.unterminatedObjects.size === 0) {
|
|
23473
|
+
return;
|
|
23474
|
+
}
|
|
23475
|
+
let warningMessage = "WARNING: Did not close the following braintrust objects. We recommend running `.close` on the listed objects, or by running them inside a callback so they are closed automatically:";
|
|
23476
|
+
this.unterminatedObjects.forEach((createdLocation, obj) => {
|
|
23477
|
+
let msg = `
|
|
23478
|
+
Object of type ${obj?.constructor?.name}`;
|
|
23479
|
+
if (createdLocation) {
|
|
23480
|
+
msg += ` created at ${JSON.stringify(createdLocation)}`;
|
|
23481
|
+
}
|
|
23482
|
+
warningMessage += msg;
|
|
23483
|
+
});
|
|
23484
|
+
console.warn(warningMessage);
|
|
23485
|
+
}
|
|
23486
|
+
};
|
|
23487
|
+
var unterminatedObjects = new UnterminatedObjectsHandler();
|
|
23321
23488
|
var HTTPConnection = class _HTTPConnection {
|
|
23322
23489
|
constructor(base_url) {
|
|
23323
23490
|
this.base_url = base_url;
|
|
@@ -23328,9 +23495,7 @@ var HTTPConnection = class _HTTPConnection {
|
|
|
23328
23495
|
async ping() {
|
|
23329
23496
|
try {
|
|
23330
23497
|
const resp = await this.get("ping");
|
|
23331
|
-
|
|
23332
|
-
_var_user_info = resp.data;
|
|
23333
|
-
}
|
|
23498
|
+
_state.setUserInfoIfNull(resp.data);
|
|
23334
23499
|
return resp.status === 200;
|
|
23335
23500
|
} catch (e) {
|
|
23336
23501
|
return false;
|
|
@@ -23355,12 +23520,12 @@ var HTTPConnection = class _HTTPConnection {
|
|
|
23355
23520
|
}
|
|
23356
23521
|
this.session = isomorph_default.makeAxios({ headers });
|
|
23357
23522
|
}
|
|
23358
|
-
async get(
|
|
23359
|
-
return await this.session.get(_urljoin(this.base_url,
|
|
23523
|
+
async get(path5, params = void 0) {
|
|
23524
|
+
return await this.session.get(_urljoin(this.base_url, path5), { params });
|
|
23360
23525
|
}
|
|
23361
|
-
async post(
|
|
23526
|
+
async post(path5, params = void 0, config = void 0) {
|
|
23362
23527
|
return await this.session.post(
|
|
23363
|
-
_urljoin(this.base_url,
|
|
23528
|
+
_urljoin(this.base_url, path5),
|
|
23364
23529
|
params,
|
|
23365
23530
|
config
|
|
23366
23531
|
);
|
|
@@ -23390,36 +23555,12 @@ var HTTPConnection = class _HTTPConnection {
|
|
|
23390
23555
|
return resp.data;
|
|
23391
23556
|
}
|
|
23392
23557
|
};
|
|
23393
|
-
var _api_conn = null;
|
|
23394
|
-
function api_conn() {
|
|
23395
|
-
if (!_api_conn) {
|
|
23396
|
-
_api_conn = new HTTPConnection(API_URL);
|
|
23397
|
-
}
|
|
23398
|
-
return _api_conn;
|
|
23399
|
-
}
|
|
23400
|
-
var _log_conn = null;
|
|
23401
|
-
function log_conn() {
|
|
23402
|
-
if (!_log_conn) {
|
|
23403
|
-
_log_conn = new HTTPConnection(LOG_URL);
|
|
23404
|
-
}
|
|
23405
|
-
return _log_conn;
|
|
23406
|
-
}
|
|
23407
|
-
var _var_user_info = null;
|
|
23408
|
-
async function _user_info() {
|
|
23409
|
-
if (_var_user_info === null) {
|
|
23410
|
-
_var_user_info = await log_conn().get_json("ping");
|
|
23411
|
-
}
|
|
23412
|
-
return _var_user_info;
|
|
23413
|
-
}
|
|
23414
|
-
function clear_cached_globals() {
|
|
23415
|
-
_api_conn = null;
|
|
23416
|
-
_log_conn = null;
|
|
23417
|
-
_var_user_info = null;
|
|
23418
|
-
}
|
|
23419
23558
|
var MaxRequestSize = 10 * 1024 * 1024;
|
|
23420
23559
|
function constructJsonArray(items) {
|
|
23421
23560
|
return `[${items.join(",")}]`;
|
|
23422
23561
|
}
|
|
23562
|
+
var DefaultBatchSize = 100;
|
|
23563
|
+
var NumRetries = 3;
|
|
23423
23564
|
var LogThread = class {
|
|
23424
23565
|
constructor() {
|
|
23425
23566
|
this.items = [];
|
|
@@ -23433,11 +23574,11 @@ var LogThread = class {
|
|
|
23433
23574
|
this.active_flush = this.flush_once();
|
|
23434
23575
|
}
|
|
23435
23576
|
}
|
|
23436
|
-
async flush_once(batchSize =
|
|
23577
|
+
async flush_once(batchSize = DefaultBatchSize) {
|
|
23437
23578
|
this.active_flush_resolved = false;
|
|
23438
|
-
const initialItems = (this.items || []).reverse();
|
|
23579
|
+
const initialItems = mergeRowBatch(this.items || []).reverse();
|
|
23439
23580
|
this.items = [];
|
|
23440
|
-
let
|
|
23581
|
+
let postPromises = [];
|
|
23441
23582
|
while (true) {
|
|
23442
23583
|
const items = [];
|
|
23443
23584
|
let itemsLen = 0;
|
|
@@ -23452,16 +23593,42 @@ var LogThread = class {
|
|
|
23452
23593
|
items.push(itemS);
|
|
23453
23594
|
itemsLen += itemS.length;
|
|
23454
23595
|
}
|
|
23455
|
-
if (items.length
|
|
23456
|
-
const resp = await log_conn().post_json(
|
|
23457
|
-
"logs",
|
|
23458
|
-
constructJsonArray(items)
|
|
23459
|
-
);
|
|
23460
|
-
ret = resp.data;
|
|
23461
|
-
} else {
|
|
23596
|
+
if (items.length === 0) {
|
|
23462
23597
|
break;
|
|
23463
23598
|
}
|
|
23599
|
+
postPromises.push(
|
|
23600
|
+
(async () => {
|
|
23601
|
+
const itemsS = constructJsonArray(items);
|
|
23602
|
+
for (let i = 0; i < NumRetries; i++) {
|
|
23603
|
+
const startTime = performance.now();
|
|
23604
|
+
try {
|
|
23605
|
+
return (await _state.logConn().post_json("logs", itemsS)).map(
|
|
23606
|
+
(res) => res.id
|
|
23607
|
+
);
|
|
23608
|
+
} catch (e) {
|
|
23609
|
+
const retryingText = i + 1 === NumRetries ? "" : " Retrying";
|
|
23610
|
+
const errMsg = (() => {
|
|
23611
|
+
if (e instanceof AxiosError2 && e.response) {
|
|
23612
|
+
return `${e.response.status}: ${JSON.stringify(
|
|
23613
|
+
e.response.data
|
|
23614
|
+
)}`;
|
|
23615
|
+
} else {
|
|
23616
|
+
return `${e}`;
|
|
23617
|
+
}
|
|
23618
|
+
})();
|
|
23619
|
+
console.warn(
|
|
23620
|
+
`log request failed. Elapsed time: ${(performance.now() - startTime) / 1e3} seconds. Payload size: ${itemsS.length}. Error: ${errMsg}.${retryingText}`
|
|
23621
|
+
);
|
|
23622
|
+
}
|
|
23623
|
+
}
|
|
23624
|
+
console.warn(
|
|
23625
|
+
`log request failed after ${NumRetries} retries. Dropping batch`
|
|
23626
|
+
);
|
|
23627
|
+
return [];
|
|
23628
|
+
})()
|
|
23629
|
+
);
|
|
23464
23630
|
}
|
|
23631
|
+
let ret = await Promise.all(postPromises);
|
|
23465
23632
|
if (this.items.length > 0) {
|
|
23466
23633
|
this.active_flush = this.flush_once();
|
|
23467
23634
|
} else {
|
|
@@ -23497,7 +23664,7 @@ async function init(project, options = {}) {
|
|
|
23497
23664
|
apiKey,
|
|
23498
23665
|
apiUrl
|
|
23499
23666
|
});
|
|
23500
|
-
|
|
23667
|
+
return await _initExperiment(project, {
|
|
23501
23668
|
experimentName: experiment,
|
|
23502
23669
|
description,
|
|
23503
23670
|
dataset,
|
|
@@ -23505,8 +23672,6 @@ async function init(project, options = {}) {
|
|
|
23505
23672
|
baseExperiment,
|
|
23506
23673
|
isPublic
|
|
23507
23674
|
});
|
|
23508
|
-
_state.current_experiment = ret;
|
|
23509
|
-
return ret;
|
|
23510
23675
|
}
|
|
23511
23676
|
async function login(options = {}) {
|
|
23512
23677
|
const {
|
|
@@ -23516,24 +23681,27 @@ async function login(options = {}) {
|
|
|
23516
23681
|
disableCache = false
|
|
23517
23682
|
} = options || {};
|
|
23518
23683
|
let { forceLogin = false } = options || {};
|
|
23519
|
-
if (apiUrl !=
|
|
23684
|
+
if (apiUrl != _state.apiUrl || apiKey !== void 0 && HTTPConnection.sanitize_token(apiKey) != _state.loginToken || orgName !== void 0 && orgName != _state.orgName) {
|
|
23520
23685
|
forceLogin = true;
|
|
23521
23686
|
}
|
|
23522
|
-
if (
|
|
23687
|
+
if (_state.loggedIn && !forceLogin) {
|
|
23523
23688
|
return;
|
|
23524
23689
|
}
|
|
23525
|
-
|
|
23526
|
-
|
|
23690
|
+
_state = new BraintrustState();
|
|
23691
|
+
_state.apiUrl = apiUrl;
|
|
23527
23692
|
let login_key_info = null;
|
|
23528
23693
|
let ping_ok = false;
|
|
23529
23694
|
let conn = null;
|
|
23530
23695
|
if (apiKey !== void 0) {
|
|
23531
|
-
const resp = await axios_default.post(
|
|
23532
|
-
|
|
23533
|
-
|
|
23696
|
+
const resp = await axios_default.post(
|
|
23697
|
+
_urljoin(_state.apiUrl, `/api/apikey/login`),
|
|
23698
|
+
{
|
|
23699
|
+
token: apiKey
|
|
23700
|
+
}
|
|
23701
|
+
);
|
|
23534
23702
|
const info = resp.data;
|
|
23535
23703
|
_check_org_info(info.org_info, orgName);
|
|
23536
|
-
conn =
|
|
23704
|
+
conn = _state.logConn();
|
|
23537
23705
|
conn.set_token(apiKey);
|
|
23538
23706
|
ping_ok = await conn.ping();
|
|
23539
23707
|
} else {
|
|
@@ -23548,9 +23716,52 @@ async function login(options = {}) {
|
|
|
23548
23716
|
await conn.get("ping");
|
|
23549
23717
|
}
|
|
23550
23718
|
conn.make_long_lived();
|
|
23551
|
-
|
|
23552
|
-
|
|
23553
|
-
|
|
23719
|
+
_state.apiConn().set_token(apiKey);
|
|
23720
|
+
_state.loginToken = conn.token;
|
|
23721
|
+
_state.loggedIn = true;
|
|
23722
|
+
}
|
|
23723
|
+
function currentExperiment() {
|
|
23724
|
+
return _state.currentExperiment.getStore();
|
|
23725
|
+
}
|
|
23726
|
+
function currentSpan() {
|
|
23727
|
+
return _state.currentSpan.getStore();
|
|
23728
|
+
}
|
|
23729
|
+
function startSpan(args) {
|
|
23730
|
+
const { name: nameOpt, ...argsRest } = args ?? {};
|
|
23731
|
+
const name = (nameOpt ?? isomorph_default.getCallerLocation()?.caller_functionname) || "root";
|
|
23732
|
+
const parentSpan = currentSpan();
|
|
23733
|
+
if (!Object.is(parentSpan, noopSpan)) {
|
|
23734
|
+
return parentSpan.startSpan(name, argsRest);
|
|
23735
|
+
}
|
|
23736
|
+
const experiment = currentExperiment();
|
|
23737
|
+
if (experiment) {
|
|
23738
|
+
return experiment.startSpan({ name, ...argsRest });
|
|
23739
|
+
}
|
|
23740
|
+
return noopSpan;
|
|
23741
|
+
}
|
|
23742
|
+
function traced(callback, args) {
|
|
23743
|
+
const span = startSpan(args);
|
|
23744
|
+
return runFinally(
|
|
23745
|
+
() => {
|
|
23746
|
+
if (args?.setCurrent ?? true) {
|
|
23747
|
+
return withCurrent(span, () => callback(span));
|
|
23748
|
+
} else {
|
|
23749
|
+
return callback(span);
|
|
23750
|
+
}
|
|
23751
|
+
},
|
|
23752
|
+
() => span.end()
|
|
23753
|
+
);
|
|
23754
|
+
}
|
|
23755
|
+
function withCurrent(object, callback) {
|
|
23756
|
+
if (object.kind === "experiment") {
|
|
23757
|
+
return _state.currentExperiment.run(object, callback);
|
|
23758
|
+
} else if (object.kind === "span") {
|
|
23759
|
+
return _state.currentSpan.run(object, callback);
|
|
23760
|
+
} else {
|
|
23761
|
+
throw new Error(
|
|
23762
|
+
`Invalid object of type ${object.constructor.name}`
|
|
23763
|
+
);
|
|
23764
|
+
}
|
|
23554
23765
|
}
|
|
23555
23766
|
function _check_org_info(org_info, org_name) {
|
|
23556
23767
|
if (org_info.length === 0) {
|
|
@@ -23558,13 +23769,13 @@ function _check_org_info(org_info, org_name) {
|
|
|
23558
23769
|
}
|
|
23559
23770
|
for (const org of org_info) {
|
|
23560
23771
|
if (org_name === void 0 || org.name === org_name) {
|
|
23561
|
-
|
|
23562
|
-
|
|
23563
|
-
|
|
23772
|
+
_state.orgId = org.id;
|
|
23773
|
+
_state.orgName = org.name;
|
|
23774
|
+
_state.logUrl = org.api_url;
|
|
23564
23775
|
break;
|
|
23565
23776
|
}
|
|
23566
23777
|
}
|
|
23567
|
-
if (
|
|
23778
|
+
if (_state.orgId === void 0) {
|
|
23568
23779
|
throw new Error(
|
|
23569
23780
|
`Organization ${org_name} not found. Must be one of ${org_info.map((x) => x.name).join(", ")}`
|
|
23570
23781
|
);
|
|
@@ -23573,6 +23784,82 @@ function _check_org_info(org_info, org_name) {
|
|
|
23573
23784
|
function _urljoin(...parts) {
|
|
23574
23785
|
return parts.map((x) => x.replace(/^\//, "")).join("/");
|
|
23575
23786
|
}
|
|
23787
|
+
function getCurrentUnixTimestamp() {
|
|
23788
|
+
return (/* @__PURE__ */ new Date()).getTime() / 1e3;
|
|
23789
|
+
}
|
|
23790
|
+
function validateAndSanitizeExperimentLogPartialArgs(event) {
|
|
23791
|
+
if (event.scores) {
|
|
23792
|
+
for (let [name, score] of Object.entries(event.scores)) {
|
|
23793
|
+
if (typeof name !== "string") {
|
|
23794
|
+
throw new Error("score names must be strings");
|
|
23795
|
+
}
|
|
23796
|
+
if (typeof score === "boolean") {
|
|
23797
|
+
score = score ? 1 : 0;
|
|
23798
|
+
event.scores[name] = score;
|
|
23799
|
+
}
|
|
23800
|
+
if (typeof score !== "number") {
|
|
23801
|
+
throw new Error("score values must be numbers");
|
|
23802
|
+
}
|
|
23803
|
+
if (score < 0 || score > 1) {
|
|
23804
|
+
throw new Error("score values must be between 0 and 1");
|
|
23805
|
+
}
|
|
23806
|
+
}
|
|
23807
|
+
}
|
|
23808
|
+
if (event.metadata) {
|
|
23809
|
+
for (const key of Object.keys(event.metadata)) {
|
|
23810
|
+
if (typeof key !== "string") {
|
|
23811
|
+
throw new Error("metadata keys must be strings");
|
|
23812
|
+
}
|
|
23813
|
+
}
|
|
23814
|
+
}
|
|
23815
|
+
if (event.metrics) {
|
|
23816
|
+
for (const key of Object.keys(event.metrics)) {
|
|
23817
|
+
if (typeof key !== "string") {
|
|
23818
|
+
throw new Error("metric keys must be strings");
|
|
23819
|
+
}
|
|
23820
|
+
}
|
|
23821
|
+
for (const forbiddenKey of [
|
|
23822
|
+
"start",
|
|
23823
|
+
"end",
|
|
23824
|
+
"caller_functionname",
|
|
23825
|
+
"caller_filename",
|
|
23826
|
+
"caller_lineno"
|
|
23827
|
+
]) {
|
|
23828
|
+
if (forbiddenKey in event.metrics) {
|
|
23829
|
+
throw new Error(`Key ${forbiddenKey} may not be specified in metrics`);
|
|
23830
|
+
}
|
|
23831
|
+
}
|
|
23832
|
+
}
|
|
23833
|
+
if ("input" in event && event.input && "inputs" in event && event.inputs) {
|
|
23834
|
+
throw new Error(
|
|
23835
|
+
"Only one of input or inputs (deprecated) can be specified. Prefer input."
|
|
23836
|
+
);
|
|
23837
|
+
}
|
|
23838
|
+
if ("inputs" in event) {
|
|
23839
|
+
const { inputs, ...rest } = event;
|
|
23840
|
+
return { input: inputs, ...rest };
|
|
23841
|
+
} else {
|
|
23842
|
+
return { ...event };
|
|
23843
|
+
}
|
|
23844
|
+
}
|
|
23845
|
+
function validateAndSanitizeExperimentLogFullArgs(event, hasDataset) {
|
|
23846
|
+
if ("input" in event && event.input && "inputs" in event && event.inputs || !("input" in event) && !("inputs" in event)) {
|
|
23847
|
+
throw new Error(
|
|
23848
|
+
"Exactly one of input or inputs (deprecated) must be specified. Prefer input."
|
|
23849
|
+
);
|
|
23850
|
+
}
|
|
23851
|
+
if (!event.scores) {
|
|
23852
|
+
throw new Error("scores must be specified");
|
|
23853
|
+
}
|
|
23854
|
+
if (hasDataset && event.datasetRecordId === void 0) {
|
|
23855
|
+
throw new Error("datasetRecordId must be specified when using a dataset");
|
|
23856
|
+
} else if (!hasDataset && event.datasetRecordId !== void 0) {
|
|
23857
|
+
throw new Error(
|
|
23858
|
+
"datasetRecordId cannot be specified when not using a dataset"
|
|
23859
|
+
);
|
|
23860
|
+
}
|
|
23861
|
+
return event;
|
|
23862
|
+
}
|
|
23576
23863
|
async function _initExperiment(projectName, {
|
|
23577
23864
|
experimentName,
|
|
23578
23865
|
description,
|
|
@@ -23588,7 +23875,7 @@ async function _initExperiment(projectName, {
|
|
|
23588
23875
|
}) {
|
|
23589
23876
|
const args = {
|
|
23590
23877
|
project_name: projectName,
|
|
23591
|
-
org_id:
|
|
23878
|
+
org_id: _state.orgId
|
|
23592
23879
|
};
|
|
23593
23880
|
if (experimentName) {
|
|
23594
23881
|
args["experiment_name"] = experimentName;
|
|
@@ -23615,10 +23902,10 @@ async function _initExperiment(projectName, {
|
|
|
23615
23902
|
if (isPublic !== void 0) {
|
|
23616
23903
|
args["public"] = isPublic;
|
|
23617
23904
|
}
|
|
23618
|
-
const response = await
|
|
23905
|
+
const response = await _state.apiConn().post_json("api/experiment/register", args);
|
|
23619
23906
|
const project = response.project;
|
|
23620
23907
|
const experiment = response.experiment;
|
|
23621
|
-
const user_id = (await
|
|
23908
|
+
const user_id = (await _state.userInfo())["id"];
|
|
23622
23909
|
return new Experiment(
|
|
23623
23910
|
project,
|
|
23624
23911
|
experiment.id,
|
|
@@ -23629,106 +23916,71 @@ async function _initExperiment(projectName, {
|
|
|
23629
23916
|
}
|
|
23630
23917
|
var Experiment = class {
|
|
23631
23918
|
constructor(project, id, name, user_id, dataset) {
|
|
23919
|
+
// For type identification.
|
|
23920
|
+
this.kind = "experiment";
|
|
23921
|
+
this.finished = false;
|
|
23632
23922
|
this.project = project;
|
|
23633
23923
|
this.id = id;
|
|
23634
23924
|
this.name = name;
|
|
23635
23925
|
this.user_id = user_id;
|
|
23636
23926
|
this.dataset = dataset;
|
|
23637
23927
|
this.logger = new LogThread();
|
|
23928
|
+
this.lastStartTime = getCurrentUnixTimestamp();
|
|
23929
|
+
unterminatedObjects.addUnterminated(this, isomorph_default.getCallerLocation());
|
|
23638
23930
|
}
|
|
23639
23931
|
/**
|
|
23640
23932
|
* Log a single event to the experiment. The event will be batched and uploaded behind the scenes.
|
|
23641
23933
|
*
|
|
23642
23934
|
* @param event The event to log.
|
|
23643
|
-
* @param event.input The arguments that uniquely define a test case (an arbitrary, JSON serializable object). Later on,
|
|
23644
|
-
*
|
|
23645
|
-
*
|
|
23646
|
-
*
|
|
23647
|
-
* @param event.
|
|
23648
|
-
*
|
|
23649
|
-
*
|
|
23650
|
-
*
|
|
23651
|
-
* @param event.
|
|
23652
|
-
*
|
|
23653
|
-
* you, since there are so many different ways to do that correctly. Instead, these values are just used to help you
|
|
23654
|
-
* navigate your experiments while digging into analyses. However, we may later use these values to re-score outputs or
|
|
23655
|
-
* fine-tune your models.
|
|
23656
|
-
* @param event.scores A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals
|
|
23657
|
-
* that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a
|
|
23658
|
-
* summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity
|
|
23659
|
-
* between the generated and grouth truth summary. The word similarity score could help you determine whether the summarization was
|
|
23660
|
-
* covering similar concepts or not. You can use these scores to help you sort, filter, and compare experiments.
|
|
23661
|
-
* @param event.metadata (Optional) a dictionary with additional data about the test example, model outputs, or just
|
|
23662
|
-
* about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the
|
|
23663
|
-
* `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any
|
|
23664
|
-
* JSON-serializable type, but its keys must be strings.
|
|
23665
|
-
* @param event.id (Optional) a unique identifier for the event. If you don't provide one, Braintrust will generate one for you.
|
|
23666
|
-
* @param event.inputs (Deprecated) the same as `input` (will be removed in a future version)
|
|
23667
|
-
* @returns The `id` of the logged event.
|
|
23935
|
+
* @param event.input: The arguments that uniquely define a test case (an arbitrary, JSON serializable object). Later on, Braintrust will use the `input` to know whether two test cases are the same between experiments, so they should not contain experiment-specific state. A simple rule of thumb is that if you run the same experiment twice, the `input` should be identical.
|
|
23936
|
+
* @param event.output: The output of your application, including post-processing (an arbitrary, JSON serializable object), that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries, the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may be multiple valid queries that answer a single question.
|
|
23937
|
+
* @param event.expected: The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to determine if your `output` value is correct or not. Braintrust currently does not compare `output` to `expected` for you, since there are so many different ways to do that correctly. Instead, these values are just used to help you navigate your experiments while digging into analyses. However, we may later use these values to re-score outputs or fine-tune your models.
|
|
23938
|
+
* @param event.scores: A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity between the generated and grouth truth summary. The word similarity score could help you determine whether the summarization was covering similar concepts or not. You can use these scores to help you sort, filter, and compare experiments.
|
|
23939
|
+
* @param event.metadata: (Optional) a dictionary with additional data about the test example, model outputs, or just about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any JSON-serializable type, but its keys must be strings.
|
|
23940
|
+
* @param event.metrics: (Optional) a dictionary of metrics to log. The following keys are populated automatically and should not be specified: "start", "end", "caller_functionname", "caller_filename", "caller_lineno".
|
|
23941
|
+
* @param event.id: (Optional) a unique identifier for the event. If you don't provide one, BrainTrust will generate one for you.
|
|
23942
|
+
* @param event.dataset_record_id: (Optional) the id of the dataset record that this event is associated with. This field is required if and only if the experiment is associated with a dataset.
|
|
23943
|
+
* @param event.inputs: (Deprecated) the same as `input` (will be removed in a future version).
|
|
23944
|
+
* :returns: The `id` of the logged event.
|
|
23668
23945
|
*/
|
|
23669
|
-
log({
|
|
23670
|
-
|
|
23671
|
-
|
|
23672
|
-
|
|
23673
|
-
|
|
23674
|
-
|
|
23675
|
-
|
|
23676
|
-
|
|
23677
|
-
|
|
23678
|
-
|
|
23679
|
-
|
|
23680
|
-
|
|
23681
|
-
|
|
23682
|
-
|
|
23683
|
-
|
|
23684
|
-
|
|
23685
|
-
|
|
23686
|
-
|
|
23687
|
-
|
|
23688
|
-
|
|
23689
|
-
|
|
23690
|
-
|
|
23691
|
-
|
|
23692
|
-
|
|
23693
|
-
|
|
23694
|
-
|
|
23695
|
-
|
|
23696
|
-
|
|
23697
|
-
|
|
23698
|
-
|
|
23699
|
-
|
|
23700
|
-
|
|
23701
|
-
|
|
23702
|
-
|
|
23703
|
-
if (metadata !== void 0) {
|
|
23704
|
-
for (const key of Object.keys(metadata)) {
|
|
23705
|
-
if (typeof key !== "string") {
|
|
23706
|
-
throw new Error("metadata keys must be strings");
|
|
23946
|
+
log(event) {
|
|
23947
|
+
this.checkNotFinished();
|
|
23948
|
+
event = validateAndSanitizeExperimentLogFullArgs(event, !!this.dataset);
|
|
23949
|
+
const span = this.startSpan({ startTime: this.lastStartTime, event });
|
|
23950
|
+
this.lastStartTime = span.end();
|
|
23951
|
+
return span.id;
|
|
23952
|
+
}
|
|
23953
|
+
/**
|
|
23954
|
+
* Create a new toplevel span. The name parameter is optional and defaults to "root".
|
|
23955
|
+
*
|
|
23956
|
+
* See `Span.startSpan` for full details.
|
|
23957
|
+
*/
|
|
23958
|
+
startSpan(args) {
|
|
23959
|
+
this.checkNotFinished();
|
|
23960
|
+
const { name, ...argsRest } = args ?? {};
|
|
23961
|
+
return new SpanImpl({
|
|
23962
|
+
experimentLogger: this.logger,
|
|
23963
|
+
name: name ?? "root",
|
|
23964
|
+
...argsRest,
|
|
23965
|
+
rootExperiment: this
|
|
23966
|
+
});
|
|
23967
|
+
}
|
|
23968
|
+
/**
|
|
23969
|
+
* Wrapper over `Experiment.startSpan`, which passes the initialized `Span` it to the given callback and ends it afterwards. See `Span.traced` for full details.
|
|
23970
|
+
*/
|
|
23971
|
+
traced(callback, args) {
|
|
23972
|
+
const { setCurrent, ...argsRest } = args ?? {};
|
|
23973
|
+
const span = this.startSpan(argsRest);
|
|
23974
|
+
return runFinally(
|
|
23975
|
+
() => {
|
|
23976
|
+
if (setCurrent ?? true) {
|
|
23977
|
+
return withCurrent(span, () => callback(span));
|
|
23978
|
+
} else {
|
|
23979
|
+
return callback(span);
|
|
23707
23980
|
}
|
|
23708
|
-
}
|
|
23709
|
-
|
|
23710
|
-
|
|
23711
|
-
throw new Error("datasetRecordId must be specified when using a dataset");
|
|
23712
|
-
} else if (!this.dataset && datasetRecordId !== void 0) {
|
|
23713
|
-
throw new Error(
|
|
23714
|
-
"datasetRecordId cannot be specified when not using a dataset"
|
|
23715
|
-
);
|
|
23716
|
-
}
|
|
23717
|
-
const args = {
|
|
23718
|
-
id: id || v4_default(),
|
|
23719
|
-
inputs: input ?? inputs,
|
|
23720
|
-
output,
|
|
23721
|
-
expected,
|
|
23722
|
-
scores,
|
|
23723
|
-
project_id: this.project.id,
|
|
23724
|
-
experiment_id: this.id,
|
|
23725
|
-
user_id: this.user_id,
|
|
23726
|
-
created: (/* @__PURE__ */ new Date()).toISOString(),
|
|
23727
|
-
dataset_record_id: datasetRecordId,
|
|
23728
|
-
metadata
|
|
23729
|
-
};
|
|
23730
|
-
this.logger.log([args]);
|
|
23731
|
-
return args.id;
|
|
23981
|
+
},
|
|
23982
|
+
() => span.end()
|
|
23983
|
+
);
|
|
23732
23984
|
}
|
|
23733
23985
|
/**
|
|
23734
23986
|
* Summarize the experiment, including the scores (compared to the closest reference experiment) and metadata.
|
|
@@ -23741,15 +23993,15 @@ var Experiment = class {
|
|
|
23741
23993
|
async summarize(options = {}) {
|
|
23742
23994
|
let { summarizeScores = true, comparisonExperimentId = void 0 } = options || {};
|
|
23743
23995
|
await this.logger.flush();
|
|
23744
|
-
const projectUrl = `${
|
|
23745
|
-
|
|
23996
|
+
const projectUrl = `${_state.apiUrl}/app/${encodeURIComponent(
|
|
23997
|
+
_state.orgName
|
|
23746
23998
|
)}/p/${encodeURIComponent(this.project.name)}`;
|
|
23747
23999
|
const experimentUrl = `${projectUrl}/${encodeURIComponent(this.name)}`;
|
|
23748
24000
|
let scores = void 0;
|
|
23749
24001
|
let comparisonExperimentName = void 0;
|
|
23750
24002
|
if (summarizeScores) {
|
|
23751
24003
|
if (comparisonExperimentId === void 0) {
|
|
23752
|
-
const conn =
|
|
24004
|
+
const conn = _state.logConn();
|
|
23753
24005
|
const resp = await conn.get("/crud/base_experiments", {
|
|
23754
24006
|
id: this.id
|
|
23755
24007
|
});
|
|
@@ -23760,7 +24012,7 @@ var Experiment = class {
|
|
|
23760
24012
|
}
|
|
23761
24013
|
}
|
|
23762
24014
|
if (comparisonExperimentId !== void 0) {
|
|
23763
|
-
scores = await
|
|
24015
|
+
scores = await _state.logConn().get_json(
|
|
23764
24016
|
"/experiment-comparison",
|
|
23765
24017
|
{
|
|
23766
24018
|
experiment_id: this.id,
|
|
@@ -23779,16 +24031,134 @@ var Experiment = class {
|
|
|
23779
24031
|
scores
|
|
23780
24032
|
};
|
|
23781
24033
|
}
|
|
24034
|
+
/**
|
|
24035
|
+
* Finish the experiment and return its id. After calling close, you may not invoke any further methods on the experiment object.
|
|
24036
|
+
*
|
|
24037
|
+
* Will be invoked automatically if the experiment is wrapped in a callback passed to `braintrust.withExperiment`.
|
|
24038
|
+
*
|
|
24039
|
+
* @returns The experiment id.
|
|
24040
|
+
*/
|
|
24041
|
+
async close() {
|
|
24042
|
+
this.checkNotFinished();
|
|
24043
|
+
await this.logger.flush();
|
|
24044
|
+
this.finished = true;
|
|
24045
|
+
unterminatedObjects.removeUnterminated(this);
|
|
24046
|
+
return this.id;
|
|
24047
|
+
}
|
|
24048
|
+
checkNotFinished() {
|
|
24049
|
+
if (this.finished) {
|
|
24050
|
+
throw new Error("Cannot invoke method on finished experiment");
|
|
24051
|
+
}
|
|
24052
|
+
}
|
|
24053
|
+
};
|
|
24054
|
+
var SpanImpl = class _SpanImpl {
|
|
24055
|
+
// root_experiment should only be specified for a root span. parent_span
|
|
24056
|
+
// should only be specified for non-root spans.
|
|
24057
|
+
constructor(args) {
|
|
24058
|
+
this.kind = "span";
|
|
24059
|
+
this.finished = false;
|
|
24060
|
+
this.experimentLogger = args.experimentLogger;
|
|
24061
|
+
const callerLocation = isomorph_default.getCallerLocation();
|
|
24062
|
+
this.internalData = {
|
|
24063
|
+
metrics: {
|
|
24064
|
+
start: args.startTime ?? getCurrentUnixTimestamp(),
|
|
24065
|
+
...callerLocation
|
|
24066
|
+
},
|
|
24067
|
+
span_attributes: { ...args.spanAttributes, name: args.name }
|
|
24068
|
+
};
|
|
24069
|
+
this.id = args.event?.id ?? v4_default();
|
|
24070
|
+
this.span_id = v4_default();
|
|
24071
|
+
if ("rootExperiment" in args) {
|
|
24072
|
+
this.root_span_id = this.span_id;
|
|
24073
|
+
this._project_id = args.rootExperiment.project.id;
|
|
24074
|
+
this._experiment_id = args.rootExperiment.id;
|
|
24075
|
+
this.internalData = Object.assign(this.internalData, {
|
|
24076
|
+
// TODO: Hopefully we can remove this.
|
|
24077
|
+
user_id: args.rootExperiment.user_id,
|
|
24078
|
+
created: (/* @__PURE__ */ new Date()).toISOString()
|
|
24079
|
+
});
|
|
24080
|
+
} else if ("parentSpan" in args) {
|
|
24081
|
+
this.root_span_id = args.parentSpan.root_span_id;
|
|
24082
|
+
this._project_id = args.parentSpan._project_id;
|
|
24083
|
+
this._experiment_id = args.parentSpan._experiment_id;
|
|
24084
|
+
this.internalData.span_parents = [args.parentSpan.span_id];
|
|
24085
|
+
} else {
|
|
24086
|
+
throw new Error("Must provide either 'rootExperiment' or 'parentSpan'");
|
|
24087
|
+
}
|
|
24088
|
+
this.isMerge = false;
|
|
24089
|
+
const { id, ...eventRest } = args.event ?? {};
|
|
24090
|
+
this.log(eventRest);
|
|
24091
|
+
this.isMerge = true;
|
|
24092
|
+
unterminatedObjects.addUnterminated(this, callerLocation);
|
|
24093
|
+
}
|
|
24094
|
+
log(event) {
|
|
24095
|
+
this.checkNotFinished();
|
|
24096
|
+
const sanitized = validateAndSanitizeExperimentLogPartialArgs(event);
|
|
24097
|
+
const record = {
|
|
24098
|
+
...sanitized,
|
|
24099
|
+
...this.internalData,
|
|
24100
|
+
id: this.id,
|
|
24101
|
+
span_id: this.span_id,
|
|
24102
|
+
root_span_id: this.root_span_id,
|
|
24103
|
+
project_id: this._project_id,
|
|
24104
|
+
experiment_id: this._experiment_id,
|
|
24105
|
+
[IS_MERGE_FIELD]: this.isMerge
|
|
24106
|
+
};
|
|
24107
|
+
this.internalData = {};
|
|
24108
|
+
this.experimentLogger.log([record]);
|
|
24109
|
+
}
|
|
24110
|
+
startSpan(name, args) {
|
|
24111
|
+
this.checkNotFinished();
|
|
24112
|
+
return new _SpanImpl({
|
|
24113
|
+
experimentLogger: this.experimentLogger,
|
|
24114
|
+
name,
|
|
24115
|
+
...args,
|
|
24116
|
+
parentSpan: this
|
|
24117
|
+
});
|
|
24118
|
+
}
|
|
24119
|
+
traced(name, callback, args) {
|
|
24120
|
+
const { setCurrent, ...argsRest } = args ?? {};
|
|
24121
|
+
const span = this.startSpan(name, argsRest);
|
|
24122
|
+
return runFinally(
|
|
24123
|
+
() => {
|
|
24124
|
+
if (setCurrent ?? true) {
|
|
24125
|
+
return withCurrent(span, () => callback(span));
|
|
24126
|
+
} else {
|
|
24127
|
+
return callback(span);
|
|
24128
|
+
}
|
|
24129
|
+
},
|
|
24130
|
+
() => span.end()
|
|
24131
|
+
);
|
|
24132
|
+
}
|
|
24133
|
+
end(args) {
|
|
24134
|
+
this.checkNotFinished();
|
|
24135
|
+
const endTime = args?.endTime ?? getCurrentUnixTimestamp();
|
|
24136
|
+
this.internalData = { metrics: { end: endTime } };
|
|
24137
|
+
this.log({});
|
|
24138
|
+
this.finished = true;
|
|
24139
|
+
unterminatedObjects.removeUnterminated(this);
|
|
24140
|
+
return endTime;
|
|
24141
|
+
}
|
|
24142
|
+
close(args) {
|
|
24143
|
+
return this.end(args);
|
|
24144
|
+
}
|
|
24145
|
+
checkNotFinished() {
|
|
24146
|
+
if (this.finished) {
|
|
24147
|
+
throw new Error("Cannot invoke method on finished span");
|
|
24148
|
+
}
|
|
24149
|
+
}
|
|
23782
24150
|
};
|
|
23783
24151
|
var Dataset = class {
|
|
23784
24152
|
constructor(project, id, name, user_id, pinnedVersion) {
|
|
23785
24153
|
this._fetchedData = void 0;
|
|
24154
|
+
this.finished = false;
|
|
23786
24155
|
this.project = project;
|
|
23787
24156
|
this.id = id;
|
|
23788
24157
|
this.name = name;
|
|
23789
24158
|
this.user_id = user_id;
|
|
23790
24159
|
this.pinnedVersion = pinnedVersion;
|
|
23791
24160
|
this.logger = new LogThread();
|
|
24161
|
+
unterminatedObjects.addUnterminated(this, isomorph_default.getCallerLocation());
|
|
23792
24162
|
}
|
|
23793
24163
|
/**
|
|
23794
24164
|
* Insert a single record to the dataset. The record will be batched and uploaded behind the scenes. If you pass in an `id`,
|
|
@@ -23810,6 +24180,7 @@ var Dataset = class {
|
|
|
23810
24180
|
metadata,
|
|
23811
24181
|
id
|
|
23812
24182
|
}) {
|
|
24183
|
+
this.checkNotFinished();
|
|
23813
24184
|
if (metadata !== void 0) {
|
|
23814
24185
|
for (const key of Object.keys(metadata)) {
|
|
23815
24186
|
if (typeof key !== "string") {
|
|
@@ -23831,6 +24202,7 @@ var Dataset = class {
|
|
|
23831
24202
|
return args.id;
|
|
23832
24203
|
}
|
|
23833
24204
|
delete(id) {
|
|
24205
|
+
this.checkNotFinished();
|
|
23834
24206
|
const user_id = this.user_id;
|
|
23835
24207
|
const args = {
|
|
23836
24208
|
id,
|
|
@@ -23850,15 +24222,16 @@ var Dataset = class {
|
|
|
23850
24222
|
* @returns A summary of the dataset.
|
|
23851
24223
|
*/
|
|
23852
24224
|
async summarize(options = {}) {
|
|
24225
|
+
this.checkNotFinished();
|
|
23853
24226
|
let { summarizeData = true } = options || {};
|
|
23854
24227
|
await this.logger.flush();
|
|
23855
|
-
const projectUrl = `${
|
|
23856
|
-
|
|
24228
|
+
const projectUrl = `${_state.apiUrl}/app/${encodeURIComponent(
|
|
24229
|
+
_state.orgName
|
|
23857
24230
|
)}/p/${encodeURIComponent(this.project.name)}`;
|
|
23858
24231
|
const datasetUrl = `${projectUrl}/d/${encodeURIComponent(this.name)}`;
|
|
23859
24232
|
let dataSummary = void 0;
|
|
23860
24233
|
if (summarizeData) {
|
|
23861
|
-
dataSummary = await
|
|
24234
|
+
dataSummary = await _state.logConn().get_json(
|
|
23862
24235
|
"dataset-summary",
|
|
23863
24236
|
{
|
|
23864
24237
|
dataset_id: this.id
|
|
@@ -23893,6 +24266,7 @@ var Dataset = class {
|
|
|
23893
24266
|
* @returns An iterator over the dataset's records.
|
|
23894
24267
|
*/
|
|
23895
24268
|
async *fetch() {
|
|
24269
|
+
this.checkNotFinished();
|
|
23896
24270
|
const records = await this.fetchedData();
|
|
23897
24271
|
for (const record of records) {
|
|
23898
24272
|
yield {
|
|
@@ -23916,11 +24290,13 @@ var Dataset = class {
|
|
|
23916
24290
|
* ```
|
|
23917
24291
|
*/
|
|
23918
24292
|
[Symbol.asyncIterator]() {
|
|
24293
|
+
this.checkNotFinished();
|
|
23919
24294
|
return this.fetch();
|
|
23920
24295
|
}
|
|
23921
24296
|
async fetchedData() {
|
|
24297
|
+
this.checkNotFinished();
|
|
23922
24298
|
if (this._fetchedData === void 0) {
|
|
23923
|
-
const resp = await
|
|
24299
|
+
const resp = await _state.logConn().get("object/dataset", {
|
|
23924
24300
|
id: this.id,
|
|
23925
24301
|
fmt: "json",
|
|
23926
24302
|
version: this.pinnedVersion
|
|
@@ -23931,9 +24307,11 @@ var Dataset = class {
|
|
|
23931
24307
|
return this._fetchedData || [];
|
|
23932
24308
|
}
|
|
23933
24309
|
clearCache() {
|
|
24310
|
+
this.checkNotFinished();
|
|
23934
24311
|
this._fetchedData = void 0;
|
|
23935
24312
|
}
|
|
23936
24313
|
async version() {
|
|
24314
|
+
this.checkNotFinished();
|
|
23937
24315
|
if (this.pinnedVersion !== void 0) {
|
|
23938
24316
|
return this.pinnedVersion;
|
|
23939
24317
|
} else {
|
|
@@ -23948,6 +24326,25 @@ var Dataset = class {
|
|
|
23948
24326
|
return maxVersion;
|
|
23949
24327
|
}
|
|
23950
24328
|
}
|
|
24329
|
+
/**
|
|
24330
|
+
* Terminate connection to the dataset and return its id. After calling close, you may not invoke any further methods on the dataset object.
|
|
24331
|
+
*
|
|
24332
|
+
* Will be invoked automatically if the dataset is bound as a context manager.
|
|
24333
|
+
*
|
|
24334
|
+
* @returns The dataset id.
|
|
24335
|
+
*/
|
|
24336
|
+
async close() {
|
|
24337
|
+
this.checkNotFinished();
|
|
24338
|
+
await this.logger.flush();
|
|
24339
|
+
this.finished = true;
|
|
24340
|
+
unterminatedObjects.removeUnterminated(this);
|
|
24341
|
+
return this.id;
|
|
24342
|
+
}
|
|
24343
|
+
checkNotFinished() {
|
|
24344
|
+
if (this.finished) {
|
|
24345
|
+
throw new Error("Cannot invoke method on finished dataset");
|
|
24346
|
+
}
|
|
24347
|
+
}
|
|
23951
24348
|
};
|
|
23952
24349
|
|
|
23953
24350
|
// src/progress.ts
|
|
@@ -24000,15 +24397,15 @@ var path2 = __toESM(require("path"));
|
|
|
24000
24397
|
|
|
24001
24398
|
// src/jest/tryRealpath.ts
|
|
24002
24399
|
var import_graceful_fs = __toESM(require_graceful_fs());
|
|
24003
|
-
function tryRealpath(
|
|
24400
|
+
function tryRealpath(path5) {
|
|
24004
24401
|
try {
|
|
24005
|
-
|
|
24402
|
+
path5 = import_graceful_fs.realpathSync.native(path5);
|
|
24006
24403
|
} catch (error2) {
|
|
24007
24404
|
if (error2.code !== "ENOENT" && error2.code !== "EISDIR") {
|
|
24008
24405
|
throw error2;
|
|
24009
24406
|
}
|
|
24010
24407
|
}
|
|
24011
|
-
return
|
|
24408
|
+
return path5;
|
|
24012
24409
|
}
|
|
24013
24410
|
|
|
24014
24411
|
// src/jest/nodeModulesPaths.ts
|
|
@@ -24085,21 +24482,21 @@ function parseFilters(filters) {
|
|
|
24085
24482
|
if (equalsIdx === -1) {
|
|
24086
24483
|
throw new Error(`Invalid filter ${f}`);
|
|
24087
24484
|
}
|
|
24088
|
-
const [
|
|
24485
|
+
const [path5, value] = [f.slice(0, equalsIdx), f.slice(equalsIdx + 1)];
|
|
24089
24486
|
let deserializedValue = deserializePlainStringAsJSON(value).value;
|
|
24090
24487
|
if (typeof deserializedValue !== "string") {
|
|
24091
24488
|
deserializedValue = value;
|
|
24092
24489
|
}
|
|
24093
24490
|
result.push({
|
|
24094
|
-
path:
|
|
24491
|
+
path: path5.split("."),
|
|
24095
24492
|
pattern: new RegExp(deserializedValue)
|
|
24096
24493
|
});
|
|
24097
24494
|
}
|
|
24098
24495
|
return result;
|
|
24099
24496
|
}
|
|
24100
24497
|
function evaluateFilter(object, filter3) {
|
|
24101
|
-
const { path:
|
|
24102
|
-
const key =
|
|
24498
|
+
const { path: path5, pattern } = filter3;
|
|
24499
|
+
const key = path5.reduce((acc, p) => acc?.[p], object);
|
|
24103
24500
|
if (key === void 0) {
|
|
24104
24501
|
return false;
|
|
24105
24502
|
}
|
|
@@ -24123,65 +24520,91 @@ async function runEvaluator(experiment, evaluator, progressReporter, filters) {
|
|
|
24123
24520
|
let output = void 0;
|
|
24124
24521
|
let error2 = void 0;
|
|
24125
24522
|
let scores = {};
|
|
24126
|
-
|
|
24127
|
-
|
|
24128
|
-
|
|
24129
|
-
|
|
24130
|
-
|
|
24131
|
-
|
|
24132
|
-
|
|
24133
|
-
|
|
24134
|
-
|
|
24135
|
-
|
|
24136
|
-
|
|
24137
|
-
|
|
24138
|
-
|
|
24139
|
-
|
|
24140
|
-
|
|
24141
|
-
|
|
24142
|
-
|
|
24143
|
-
|
|
24523
|
+
const callback = async () => {
|
|
24524
|
+
try {
|
|
24525
|
+
const meta = (o) => metadata = { ...metadata, ...o };
|
|
24526
|
+
await traced(
|
|
24527
|
+
async () => {
|
|
24528
|
+
const outputResult = evaluator.task(datum.input, {
|
|
24529
|
+
meta,
|
|
24530
|
+
span: currentSpan()
|
|
24531
|
+
});
|
|
24532
|
+
if (outputResult instanceof Promise) {
|
|
24533
|
+
output = await outputResult;
|
|
24534
|
+
} else {
|
|
24535
|
+
output = outputResult;
|
|
24536
|
+
}
|
|
24537
|
+
currentSpan().log({ input: datum.input, output });
|
|
24538
|
+
},
|
|
24539
|
+
{ name: "task" }
|
|
24540
|
+
);
|
|
24541
|
+
currentSpan().log({ output });
|
|
24542
|
+
const scoringArgs = { ...datum, metadata, output };
|
|
24543
|
+
const scoreResults = await Promise.all(
|
|
24544
|
+
evaluator.scores.map(async (score, score_idx) => {
|
|
24545
|
+
return traced(
|
|
24546
|
+
async () => {
|
|
24547
|
+
const scoreResult = score(scoringArgs);
|
|
24548
|
+
const result = scoreResult instanceof Promise ? await scoreResult : scoreResult;
|
|
24549
|
+
const {
|
|
24550
|
+
metadata: resultMetadata,
|
|
24551
|
+
name: _,
|
|
24552
|
+
...resultRest
|
|
24553
|
+
} = result;
|
|
24554
|
+
currentSpan().log({
|
|
24555
|
+
output: resultRest,
|
|
24556
|
+
metadata: resultMetadata
|
|
24557
|
+
});
|
|
24558
|
+
return result;
|
|
24559
|
+
},
|
|
24560
|
+
{
|
|
24561
|
+
name: score.name || `scorer_${score_idx}`,
|
|
24562
|
+
event: { input: scoringArgs }
|
|
24563
|
+
}
|
|
24564
|
+
);
|
|
24565
|
+
})
|
|
24566
|
+
);
|
|
24567
|
+
const scoreMetadata = {};
|
|
24568
|
+
for (const scoreResult of scoreResults) {
|
|
24569
|
+
scores[scoreResult.name] = scoreResult.score;
|
|
24570
|
+
const metadata2 = {
|
|
24571
|
+
...scoreResult.metadata
|
|
24572
|
+
};
|
|
24573
|
+
if (scoreResult.error !== void 0) {
|
|
24574
|
+
metadata2.error = scoreResult.error;
|
|
24575
|
+
}
|
|
24576
|
+
if (Object.keys(metadata2).length > 0) {
|
|
24577
|
+
scoreMetadata[scoreResult.name] = metadata2;
|
|
24144
24578
|
}
|
|
24145
|
-
})
|
|
24146
|
-
);
|
|
24147
|
-
const scoreMetadata = {};
|
|
24148
|
-
for (const scoreResult of scoreResults) {
|
|
24149
|
-
scores[scoreResult.name] = scoreResult.score;
|
|
24150
|
-
const metadata2 = {
|
|
24151
|
-
...scoreResult.metadata
|
|
24152
|
-
};
|
|
24153
|
-
if (scoreResult.error !== void 0) {
|
|
24154
|
-
metadata2.error = scoreResult.error;
|
|
24155
24579
|
}
|
|
24156
|
-
if (Object.keys(
|
|
24157
|
-
scoreMetadata
|
|
24580
|
+
if (Object.keys(scoreMetadata).length > 0) {
|
|
24581
|
+
meta({ scores: scoreMetadata });
|
|
24158
24582
|
}
|
|
24583
|
+
currentSpan().log({ scores, metadata });
|
|
24584
|
+
} catch (e) {
|
|
24585
|
+
error2 = e;
|
|
24586
|
+
} finally {
|
|
24587
|
+
progressReporter.increment(evaluator.name);
|
|
24159
24588
|
}
|
|
24160
|
-
|
|
24161
|
-
|
|
24589
|
+
return {
|
|
24590
|
+
output,
|
|
24591
|
+
metadata,
|
|
24592
|
+
scores,
|
|
24593
|
+
error: error2
|
|
24594
|
+
};
|
|
24595
|
+
};
|
|
24596
|
+
const rootSpan = experiment ? experiment.startSpan({
|
|
24597
|
+
name: "eval",
|
|
24598
|
+
event: {
|
|
24599
|
+
input: datum.input,
|
|
24600
|
+
expected: datum.expected
|
|
24162
24601
|
}
|
|
24163
|
-
}
|
|
24164
|
-
|
|
24602
|
+
}) : noopSpan;
|
|
24603
|
+
try {
|
|
24604
|
+
return await withCurrent(rootSpan, callback);
|
|
24165
24605
|
} finally {
|
|
24166
|
-
|
|
24167
|
-
}
|
|
24168
|
-
if (experiment && !error2) {
|
|
24169
|
-
experiment.log({
|
|
24170
|
-
// TODO We should rename this from inputs -> input in the logger, etc.
|
|
24171
|
-
// https://github.com/braintrustdata/braintrust/issues/217
|
|
24172
|
-
input: datum.input,
|
|
24173
|
-
metadata,
|
|
24174
|
-
expected: datum.expected,
|
|
24175
|
-
output,
|
|
24176
|
-
scores
|
|
24177
|
-
});
|
|
24606
|
+
rootSpan.end();
|
|
24178
24607
|
}
|
|
24179
|
-
return {
|
|
24180
|
-
output,
|
|
24181
|
-
metadata,
|
|
24182
|
-
scores,
|
|
24183
|
-
error: error2
|
|
24184
|
-
};
|
|
24185
24608
|
});
|
|
24186
24609
|
const results = await Promise.all(evals);
|
|
24187
24610
|
const summary = experiment ? await experiment.summarize() : null;
|
|
@@ -24194,6 +24617,7 @@ async function runEvaluator(experiment, evaluator, progressReporter, filters) {
|
|
|
24194
24617
|
// src/node.ts
|
|
24195
24618
|
var http2 = __toESM(require("http"));
|
|
24196
24619
|
var https2 = __toESM(require("https"));
|
|
24620
|
+
var import_node_async_hooks = require("node:async_hooks");
|
|
24197
24621
|
|
|
24198
24622
|
// node_modules/simple-git/dist/esm/index.js
|
|
24199
24623
|
var import_file_exists = __toESM(require_dist(), 1);
|
|
@@ -24266,8 +24690,8 @@ var __async = (__this, __arguments, generator) => {
|
|
|
24266
24690
|
step((generator = generator.apply(__this, __arguments)).next());
|
|
24267
24691
|
});
|
|
24268
24692
|
};
|
|
24269
|
-
function isPathSpec(
|
|
24270
|
-
return
|
|
24693
|
+
function isPathSpec(path5) {
|
|
24694
|
+
return path5 instanceof String && cache.has(path5);
|
|
24271
24695
|
}
|
|
24272
24696
|
function toPaths(pathSpec) {
|
|
24273
24697
|
return cache.get(pathSpec) || [];
|
|
@@ -24349,8 +24773,8 @@ function toLinesWithContent(input = "", trimmed2 = true, separator = "\n") {
|
|
|
24349
24773
|
function forEachLineWithContent(input, callback) {
|
|
24350
24774
|
return toLinesWithContent(input, true).map((line) => callback(line));
|
|
24351
24775
|
}
|
|
24352
|
-
function folderExists(
|
|
24353
|
-
return (0, import_file_exists.exists)(
|
|
24776
|
+
function folderExists(path5) {
|
|
24777
|
+
return (0, import_file_exists.exists)(path5, import_file_exists.FOLDER);
|
|
24354
24778
|
}
|
|
24355
24779
|
function append2(target, item) {
|
|
24356
24780
|
if (Array.isArray(target)) {
|
|
@@ -24713,8 +25137,8 @@ function checkIsRepoRootTask() {
|
|
|
24713
25137
|
commands,
|
|
24714
25138
|
format: "utf-8",
|
|
24715
25139
|
onError,
|
|
24716
|
-
parser(
|
|
24717
|
-
return /^\.(git)?$/.test(
|
|
25140
|
+
parser(path5) {
|
|
25141
|
+
return /^\.(git)?$/.test(path5.trim());
|
|
24718
25142
|
}
|
|
24719
25143
|
};
|
|
24720
25144
|
}
|
|
@@ -25105,11 +25529,11 @@ function parseGrep(grep) {
|
|
|
25105
25529
|
const paths = /* @__PURE__ */ new Set();
|
|
25106
25530
|
const results = {};
|
|
25107
25531
|
forEachLineWithContent(grep, (input) => {
|
|
25108
|
-
const [
|
|
25109
|
-
paths.add(
|
|
25110
|
-
(results[
|
|
25532
|
+
const [path5, line, preview] = input.split(NULL);
|
|
25533
|
+
paths.add(path5);
|
|
25534
|
+
(results[path5] = results[path5] || []).push({
|
|
25111
25535
|
line: asNumber(line),
|
|
25112
|
-
path:
|
|
25536
|
+
path: path5,
|
|
25113
25537
|
preview
|
|
25114
25538
|
});
|
|
25115
25539
|
});
|
|
@@ -25726,14 +26150,14 @@ var init_hash_object = __esm({
|
|
|
25726
26150
|
init_task();
|
|
25727
26151
|
}
|
|
25728
26152
|
});
|
|
25729
|
-
function parseInit(bare,
|
|
26153
|
+
function parseInit(bare, path5, text) {
|
|
25730
26154
|
const response = String(text).trim();
|
|
25731
26155
|
let result;
|
|
25732
26156
|
if (result = initResponseRegex.exec(response)) {
|
|
25733
|
-
return new InitSummary(bare,
|
|
26157
|
+
return new InitSummary(bare, path5, false, result[1]);
|
|
25734
26158
|
}
|
|
25735
26159
|
if (result = reInitResponseRegex.exec(response)) {
|
|
25736
|
-
return new InitSummary(bare,
|
|
26160
|
+
return new InitSummary(bare, path5, true, result[1]);
|
|
25737
26161
|
}
|
|
25738
26162
|
let gitDir = "";
|
|
25739
26163
|
const tokens = response.split(" ");
|
|
@@ -25744,7 +26168,7 @@ function parseInit(bare, path4, text) {
|
|
|
25744
26168
|
break;
|
|
25745
26169
|
}
|
|
25746
26170
|
}
|
|
25747
|
-
return new InitSummary(bare,
|
|
26171
|
+
return new InitSummary(bare, path5, /^re/i.test(response), gitDir);
|
|
25748
26172
|
}
|
|
25749
26173
|
var InitSummary;
|
|
25750
26174
|
var initResponseRegex;
|
|
@@ -25752,9 +26176,9 @@ var reInitResponseRegex;
|
|
|
25752
26176
|
var init_InitSummary = __esm({
|
|
25753
26177
|
"src/lib/responses/InitSummary.ts"() {
|
|
25754
26178
|
InitSummary = class {
|
|
25755
|
-
constructor(bare,
|
|
26179
|
+
constructor(bare, path5, existing, gitDir) {
|
|
25756
26180
|
this.bare = bare;
|
|
25757
|
-
this.path =
|
|
26181
|
+
this.path = path5;
|
|
25758
26182
|
this.existing = existing;
|
|
25759
26183
|
this.gitDir = gitDir;
|
|
25760
26184
|
}
|
|
@@ -25766,7 +26190,7 @@ var init_InitSummary = __esm({
|
|
|
25766
26190
|
function hasBareCommand(command) {
|
|
25767
26191
|
return command.includes(bareCommand);
|
|
25768
26192
|
}
|
|
25769
|
-
function initTask(bare = false,
|
|
26193
|
+
function initTask(bare = false, path5, customArgs) {
|
|
25770
26194
|
const commands = ["init", ...customArgs];
|
|
25771
26195
|
if (bare && !hasBareCommand(commands)) {
|
|
25772
26196
|
commands.splice(1, 0, bareCommand);
|
|
@@ -25775,7 +26199,7 @@ function initTask(bare = false, path4, customArgs) {
|
|
|
25775
26199
|
commands,
|
|
25776
26200
|
format: "utf-8",
|
|
25777
26201
|
parser(text) {
|
|
25778
|
-
return parseInit(commands.includes("--bare"),
|
|
26202
|
+
return parseInit(commands.includes("--bare"), path5, text);
|
|
25779
26203
|
}
|
|
25780
26204
|
};
|
|
25781
26205
|
}
|
|
@@ -26495,12 +26919,12 @@ var init_FileStatusSummary = __esm({
|
|
|
26495
26919
|
"src/lib/responses/FileStatusSummary.ts"() {
|
|
26496
26920
|
fromPathRegex = /^(.+) -> (.+)$/;
|
|
26497
26921
|
FileStatusSummary = class {
|
|
26498
|
-
constructor(
|
|
26499
|
-
this.path =
|
|
26922
|
+
constructor(path5, index, working_dir) {
|
|
26923
|
+
this.path = path5;
|
|
26500
26924
|
this.index = index;
|
|
26501
26925
|
this.working_dir = working_dir;
|
|
26502
26926
|
if (index + working_dir === "R") {
|
|
26503
|
-
const detail = fromPathRegex.exec(
|
|
26927
|
+
const detail = fromPathRegex.exec(path5) || [null, path5, path5];
|
|
26504
26928
|
this.from = detail[1] || "";
|
|
26505
26929
|
this.path = detail[2] || "";
|
|
26506
26930
|
}
|
|
@@ -26531,14 +26955,14 @@ function splitLine(result, lineStr) {
|
|
|
26531
26955
|
default:
|
|
26532
26956
|
return;
|
|
26533
26957
|
}
|
|
26534
|
-
function data(index, workingDir,
|
|
26958
|
+
function data(index, workingDir, path5) {
|
|
26535
26959
|
const raw = `${index}${workingDir}`;
|
|
26536
26960
|
const handler = parsers6.get(raw);
|
|
26537
26961
|
if (handler) {
|
|
26538
|
-
handler(result,
|
|
26962
|
+
handler(result, path5);
|
|
26539
26963
|
}
|
|
26540
26964
|
if (raw !== "##" && raw !== "!!") {
|
|
26541
|
-
result.files.push(new FileStatusSummary(
|
|
26965
|
+
result.files.push(new FileStatusSummary(path5.replace(/\0.+$/, ""), index, workingDir));
|
|
26542
26966
|
}
|
|
26543
26967
|
}
|
|
26544
26968
|
}
|
|
@@ -26783,8 +27207,8 @@ var init_simple_git_api = __esm({
|
|
|
26783
27207
|
}
|
|
26784
27208
|
return this._runTask(configurationErrorTask("Git.cwd: workingDirectory must be supplied as a string"), next);
|
|
26785
27209
|
}
|
|
26786
|
-
hashObject(
|
|
26787
|
-
return this._runTask(hashObjectTask(
|
|
27210
|
+
hashObject(path5, write) {
|
|
27211
|
+
return this._runTask(hashObjectTask(path5, write === true), trailingFunctionArgument(arguments));
|
|
26788
27212
|
}
|
|
26789
27213
|
init(bare) {
|
|
26790
27214
|
return this._runTask(initTask(bare === true, this._executor.cwd, getTrailingOptions(arguments)), trailingFunctionArgument(arguments));
|
|
@@ -27361,8 +27785,8 @@ __export(sub_module_exports, {
|
|
|
27361
27785
|
subModuleTask: () => subModuleTask,
|
|
27362
27786
|
updateSubModuleTask: () => updateSubModuleTask
|
|
27363
27787
|
});
|
|
27364
|
-
function addSubModuleTask(repo,
|
|
27365
|
-
return subModuleTask(["add", repo,
|
|
27788
|
+
function addSubModuleTask(repo, path5) {
|
|
27789
|
+
return subModuleTask(["add", repo, path5]);
|
|
27366
27790
|
}
|
|
27367
27791
|
function initSubModuleTask(customArgs) {
|
|
27368
27792
|
return subModuleTask(["init", ...customArgs]);
|
|
@@ -27628,8 +28052,8 @@ var require_git = __commonJS2({
|
|
|
27628
28052
|
}
|
|
27629
28053
|
return this._runTask(straightThroughStringTask2(command, this._trimmed), next);
|
|
27630
28054
|
};
|
|
27631
|
-
Git2.prototype.submoduleAdd = function(repo,
|
|
27632
|
-
return this._runTask(addSubModuleTask2(repo,
|
|
28055
|
+
Git2.prototype.submoduleAdd = function(repo, path5, then) {
|
|
28056
|
+
return this._runTask(addSubModuleTask2(repo, path5), trailingFunctionArgument2(arguments));
|
|
27633
28057
|
};
|
|
27634
28058
|
Git2.prototype.submoduleUpdate = function(args, then) {
|
|
27635
28059
|
return this._runTask(updateSubModuleTask2(getTrailingOptions2(arguments, true)), trailingFunctionArgument2(arguments));
|
|
@@ -28218,6 +28642,50 @@ async function getRepoStatus() {
|
|
|
28218
28642
|
};
|
|
28219
28643
|
}
|
|
28220
28644
|
|
|
28645
|
+
// src/stackutil.ts
|
|
28646
|
+
var path3 = __toESM(require("path"));
|
|
28647
|
+
function getStackTrace() {
|
|
28648
|
+
const trace = new Error().stack;
|
|
28649
|
+
if (trace === void 0) {
|
|
28650
|
+
return [];
|
|
28651
|
+
}
|
|
28652
|
+
const traceLines = trace.split("\n");
|
|
28653
|
+
const out = [];
|
|
28654
|
+
const stackFrameRegex = /at(.*)\((.*):(\d+):(\d+)\)/;
|
|
28655
|
+
for (const traceLine of traceLines.slice(1)) {
|
|
28656
|
+
const matches = traceLine.match(stackFrameRegex);
|
|
28657
|
+
if (matches === null || matches.length !== 5) {
|
|
28658
|
+
continue;
|
|
28659
|
+
}
|
|
28660
|
+
const entry = {
|
|
28661
|
+
functionName: matches[1].trim(),
|
|
28662
|
+
fileName: matches[2],
|
|
28663
|
+
lineNo: parseInt(matches[3])
|
|
28664
|
+
};
|
|
28665
|
+
if (!isNaN(entry.lineNo)) {
|
|
28666
|
+
out.push(entry);
|
|
28667
|
+
}
|
|
28668
|
+
}
|
|
28669
|
+
return out;
|
|
28670
|
+
}
|
|
28671
|
+
function getCallerLocation() {
|
|
28672
|
+
let thisDir = void 0;
|
|
28673
|
+
const entries = getStackTrace();
|
|
28674
|
+
for (const frame of entries) {
|
|
28675
|
+
if (thisDir === void 0) {
|
|
28676
|
+
thisDir = path3.dirname(frame.fileName);
|
|
28677
|
+
}
|
|
28678
|
+
if (path3.dirname(frame.fileName) !== thisDir) {
|
|
28679
|
+
return {
|
|
28680
|
+
caller_functionname: frame.functionName,
|
|
28681
|
+
caller_filename: frame.fileName,
|
|
28682
|
+
caller_lineno: frame.lineNo
|
|
28683
|
+
};
|
|
28684
|
+
}
|
|
28685
|
+
}
|
|
28686
|
+
return void 0;
|
|
28687
|
+
}
|
|
28688
|
+
|
|
28221
28689
|
// src/node.ts
|
|
28222
28690
|
function configureNode() {
|
|
28223
28691
|
isomorph_default.makeAxios = (options) => {
|
|
@@ -28232,6 +28700,8 @@ function configureNode() {
|
|
|
28232
28700
|
isomorph_default.getRepoStatus = getRepoStatus;
|
|
28233
28701
|
isomorph_default.getPastNAncestors = getPastNAncestors;
|
|
28234
28702
|
isomorph_default.getEnv = (name) => process.env[name];
|
|
28703
|
+
isomorph_default.getCallerLocation = getCallerLocation;
|
|
28704
|
+
isomorph_default.newAsyncLocalStorage = () => new import_node_async_hooks.AsyncLocalStorage();
|
|
28235
28705
|
}
|
|
28236
28706
|
|
|
28237
28707
|
// src/cli.ts
|
|
@@ -28270,6 +28740,7 @@ function evaluateBuildResults(inFile, buildResult) {
|
|
|
28270
28740
|
const moduleText = buildResult.outputFiles[0].text;
|
|
28271
28741
|
return evalWithModuleContext(inFile, () => {
|
|
28272
28742
|
globalThis._evals = {};
|
|
28743
|
+
globalThis.__inherited_braintrust_state = _internalGetGlobalState();
|
|
28273
28744
|
const __filename = inFile;
|
|
28274
28745
|
const __dirname = (0, import_path.dirname)(__filename);
|
|
28275
28746
|
new Function("require", "__filename", "__dirname", moduleText)(
|
|
@@ -28447,12 +28918,18 @@ async function runOnce(handles, opts) {
|
|
|
28447
28918
|
updateEvaluators(evaluators, buildResults, opts);
|
|
28448
28919
|
const resultPromises = Object.values(evaluators).map(async (evaluator) => {
|
|
28449
28920
|
const logger = opts.noSendLogs ? null : await initLogger(evaluator.evaluator.name);
|
|
28450
|
-
|
|
28451
|
-
|
|
28452
|
-
|
|
28453
|
-
|
|
28454
|
-
|
|
28455
|
-
|
|
28921
|
+
try {
|
|
28922
|
+
return await runEvaluator(
|
|
28923
|
+
logger,
|
|
28924
|
+
evaluator.evaluator,
|
|
28925
|
+
opts.progressReporter,
|
|
28926
|
+
opts.filters
|
|
28927
|
+
);
|
|
28928
|
+
} finally {
|
|
28929
|
+
if (logger) {
|
|
28930
|
+
await logger.close();
|
|
28931
|
+
}
|
|
28932
|
+
}
|
|
28456
28933
|
});
|
|
28457
28934
|
console.log(`Processing ${resultPromises.length} evaluators...`);
|
|
28458
28935
|
const allEvalsResults = await Promise.all(resultPromises);
|
|
@@ -28509,7 +28986,7 @@ async function collectFiles(inputPath) {
|
|
|
28509
28986
|
files.push(inputPath);
|
|
28510
28987
|
}
|
|
28511
28988
|
} else {
|
|
28512
|
-
const walked = await
|
|
28989
|
+
const walked = await import_util5.default.promisify(fsWalk.walk)(inputPath, {
|
|
28513
28990
|
deepFilter: (entry) => {
|
|
28514
28991
|
return checkMatch(entry.path, null, EXCLUDE);
|
|
28515
28992
|
},
|