braintrust 0.0.56 → 0.0.58

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -8880,11 +8880,11 @@ var require_mime_types = __commonJS({
8880
8880
  }
8881
8881
  return exts[0];
8882
8882
  }
8883
- function lookup(path) {
8884
- if (!path || typeof path !== "string") {
8883
+ function lookup(path2) {
8884
+ if (!path2 || typeof path2 !== "string") {
8885
8885
  return false;
8886
8886
  }
8887
- var extension2 = extname("x." + path).toLowerCase().substr(1);
8887
+ var extension2 = extname("x." + path2).toLowerCase().substr(1);
8888
8888
  if (!extension2) {
8889
8889
  return false;
8890
8890
  }
@@ -9141,7 +9141,7 @@ var require_form_data = __commonJS({
9141
9141
  "node_modules/form-data/lib/form_data.js"(exports, module2) {
9142
9142
  var CombinedStream = require_combined_stream();
9143
9143
  var util2 = require("util");
9144
- var path = require("path");
9144
+ var path2 = require("path");
9145
9145
  var http3 = require("http");
9146
9146
  var https3 = require("https");
9147
9147
  var parseUrl = require("url").parse;
@@ -9268,11 +9268,11 @@ var require_form_data = __commonJS({
9268
9268
  FormData3.prototype._getContentDisposition = function(value, options) {
9269
9269
  var filename, contentDisposition;
9270
9270
  if (typeof options.filepath === "string") {
9271
- filename = path.normalize(options.filepath).replace(/\\/g, "/");
9271
+ filename = path2.normalize(options.filepath).replace(/\\/g, "/");
9272
9272
  } else if (options.filename || value.name || value.path) {
9273
- filename = path.basename(options.filename || value.name || value.path);
9273
+ filename = path2.basename(options.filename || value.name || value.path);
9274
9274
  } else if (value.readable && value.hasOwnProperty("httpVersion")) {
9275
- filename = path.basename(value.client._httpMessage.path || "");
9275
+ filename = path2.basename(value.client._httpMessage.path || "");
9276
9276
  }
9277
9277
  if (filename) {
9278
9278
  contentDisposition = 'filename="' + filename + '"';
@@ -10749,10 +10749,10 @@ var require_src2 = __commonJS({
10749
10749
  var fs_1 = require("fs");
10750
10750
  var debug_1 = __importDefault(require_src());
10751
10751
  var log2 = debug_1.default("@kwsites/file-exists");
10752
- function check(path, isFile2, isDirectory) {
10753
- log2(`checking %s`, path);
10752
+ function check(path2, isFile2, isDirectory) {
10753
+ log2(`checking %s`, path2);
10754
10754
  try {
10755
- const stat = fs_1.statSync(path);
10755
+ const stat = fs_1.statSync(path2);
10756
10756
  if (stat.isFile() && isFile2) {
10757
10757
  log2(`[OK] path represents a file`);
10758
10758
  return true;
@@ -10772,8 +10772,8 @@ var require_src2 = __commonJS({
10772
10772
  throw e;
10773
10773
  }
10774
10774
  }
10775
- function exists2(path, type = exports.READABLE) {
10776
- return check(path, (type & exports.FILE) > 0, (type & exports.FOLDER) > 0);
10775
+ function exists2(path2, type = exports.READABLE) {
10776
+ return check(path2, (type & exports.FILE) > 0, (type & exports.FOLDER) > 0);
10777
10777
  }
10778
10778
  exports.exists = exists2;
10779
10779
  exports.FILE = 1;
@@ -10844,12 +10844,23 @@ __export(src_exports, {
10844
10844
  Dataset: () => Dataset,
10845
10845
  Eval: () => Eval,
10846
10846
  Experiment: () => Experiment,
10847
+ NoopSpan: () => NoopSpan,
10847
10848
  Project: () => Project,
10849
+ SpanImpl: () => SpanImpl,
10850
+ _internalGetGlobalState: () => _internalGetGlobalState,
10851
+ currentExperiment: () => currentExperiment,
10852
+ currentSpan: () => currentSpan,
10848
10853
  init: () => init,
10849
10854
  initDataset: () => initDataset,
10850
10855
  log: () => log,
10851
10856
  login: () => login,
10852
- summarize: () => summarize
10857
+ noopSpan: () => noopSpan,
10858
+ startSpan: () => startSpan,
10859
+ summarize: () => summarize,
10860
+ traced: () => traced,
10861
+ withCurrent: () => withCurrent,
10862
+ withDataset: () => withDataset,
10863
+ withExperiment: () => withExperiment
10853
10864
  });
10854
10865
  module.exports = __toCommonJS(src_exports);
10855
10866
 
@@ -11305,10 +11316,10 @@ function isVisitable(thing) {
11305
11316
  function removeBrackets(key) {
11306
11317
  return utils_default.endsWith(key, "[]") ? key.slice(0, -2) : key;
11307
11318
  }
11308
- function renderKey(path, key, dots) {
11309
- if (!path)
11319
+ function renderKey(path2, key, dots) {
11320
+ if (!path2)
11310
11321
  return key;
11311
- return path.concat(key).map(function each(token, i) {
11322
+ return path2.concat(key).map(function each(token, i) {
11312
11323
  token = removeBrackets(token);
11313
11324
  return !dots && i ? "[" + token + "]" : token;
11314
11325
  }).join(dots ? "." : "");
@@ -11354,9 +11365,9 @@ function toFormData(obj, formData, options) {
11354
11365
  }
11355
11366
  return value;
11356
11367
  }
11357
- function defaultVisitor(value, key, path) {
11368
+ function defaultVisitor(value, key, path2) {
11358
11369
  let arr = value;
11359
- if (value && !path && typeof value === "object") {
11370
+ if (value && !path2 && typeof value === "object") {
11360
11371
  if (utils_default.endsWith(key, "{}")) {
11361
11372
  key = metaTokens ? key : key.slice(0, -2);
11362
11373
  value = JSON.stringify(value);
@@ -11375,7 +11386,7 @@ function toFormData(obj, formData, options) {
11375
11386
  if (isVisitable(value)) {
11376
11387
  return true;
11377
11388
  }
11378
- formData.append(renderKey(path, key, dots), convertValue(value));
11389
+ formData.append(renderKey(path2, key, dots), convertValue(value));
11379
11390
  return false;
11380
11391
  }
11381
11392
  const stack = [];
@@ -11384,11 +11395,11 @@ function toFormData(obj, formData, options) {
11384
11395
  convertValue,
11385
11396
  isVisitable
11386
11397
  });
11387
- function build(value, path) {
11398
+ function build(value, path2) {
11388
11399
  if (utils_default.isUndefined(value))
11389
11400
  return;
11390
11401
  if (stack.indexOf(value) !== -1) {
11391
- throw Error("Circular reference detected in " + path.join("."));
11402
+ throw Error("Circular reference detected in " + path2.join("."));
11392
11403
  }
11393
11404
  stack.push(value);
11394
11405
  utils_default.forEach(value, function each(el, key) {
@@ -11396,11 +11407,11 @@ function toFormData(obj, formData, options) {
11396
11407
  formData,
11397
11408
  el,
11398
11409
  utils_default.isString(key) ? key.trim() : key,
11399
- path,
11410
+ path2,
11400
11411
  exposedHelpers
11401
11412
  );
11402
11413
  if (result === true) {
11403
- build(el, path ? path.concat(key) : [key]);
11414
+ build(el, path2 ? path2.concat(key) : [key]);
11404
11415
  }
11405
11416
  });
11406
11417
  stack.pop();
@@ -11561,7 +11572,7 @@ var node_default = {
11561
11572
  // node_modules/axios/lib/helpers/toURLEncodedForm.js
11562
11573
  function toURLEncodedForm(data, options) {
11563
11574
  return toFormData_default(data, new node_default.classes.URLSearchParams(), Object.assign({
11564
- visitor: function(value, key, path, helpers) {
11575
+ visitor: function(value, key, path2, helpers) {
11565
11576
  if (node_default.isNode && utils_default.isBuffer(value)) {
11566
11577
  this.append(key, value.toString("base64"));
11567
11578
  return false;
@@ -11590,10 +11601,10 @@ function arrayToObject(arr) {
11590
11601
  return obj;
11591
11602
  }
11592
11603
  function formDataToJSON(formData) {
11593
- function buildPath(path, value, target, index) {
11594
- let name = path[index++];
11604
+ function buildPath(path2, value, target, index) {
11605
+ let name = path2[index++];
11595
11606
  const isNumericKey = Number.isFinite(+name);
11596
- const isLast = index >= path.length;
11607
+ const isLast = index >= path2.length;
11597
11608
  name = !name && utils_default.isArray(target) ? target.length : name;
11598
11609
  if (isLast) {
11599
11610
  if (utils_default.hasOwnProp(target, name)) {
@@ -11606,7 +11617,7 @@ function formDataToJSON(formData) {
11606
11617
  if (!target[name] || !utils_default.isObject(target[name])) {
11607
11618
  target[name] = [];
11608
11619
  }
11609
- const result = buildPath(path, value, target[name], index);
11620
+ const result = buildPath(path2, value, target[name], index);
11610
11621
  if (result && utils_default.isArray(target[name])) {
11611
11622
  target[name] = arrayToObject(target[name]);
11612
11623
  }
@@ -12705,9 +12716,9 @@ var http_default = isHttpAdapterSupported && function httpAdapter(config) {
12705
12716
  auth = urlUsername + ":" + urlPassword;
12706
12717
  }
12707
12718
  auth && headers.delete("authorization");
12708
- let path;
12719
+ let path2;
12709
12720
  try {
12710
- path = buildURL(
12721
+ path2 = buildURL(
12711
12722
  parsed.pathname + parsed.search,
12712
12723
  config.params,
12713
12724
  config.paramsSerializer
@@ -12725,7 +12736,7 @@ var http_default = isHttpAdapterSupported && function httpAdapter(config) {
12725
12736
  false
12726
12737
  );
12727
12738
  const options = {
12728
- path,
12739
+ path: path2,
12729
12740
  method,
12730
12741
  headers: headers.toJSON(),
12731
12742
  agents: { http: config.httpAgent, https: config.httpsAgent },
@@ -12948,14 +12959,14 @@ var cookies_default = node_default.isStandardBrowserEnv ? (
12948
12959
  // Standard browser envs support document.cookie
12949
12960
  function standardBrowserEnv() {
12950
12961
  return {
12951
- write: function write(name, value, expires, path, domain, secure) {
12962
+ write: function write(name, value, expires, path2, domain, secure) {
12952
12963
  const cookie = [];
12953
12964
  cookie.push(name + "=" + encodeURIComponent(value));
12954
12965
  if (utils_default.isNumber(expires)) {
12955
12966
  cookie.push("expires=" + new Date(expires).toGMTString());
12956
12967
  }
12957
- if (utils_default.isString(path)) {
12958
- cookie.push("path=" + path);
12968
+ if (utils_default.isString(path2)) {
12969
+ cookie.push("path=" + path2);
12959
12970
  }
12960
12971
  if (utils_default.isString(domain)) {
12961
12972
  cookie.push("domain=" + domain);
@@ -13790,14 +13801,31 @@ var {
13790
13801
  mergeConfig: mergeConfig2
13791
13802
  } = axios_default;
13792
13803
 
13804
+ // src/node.ts
13805
+ var import_node_async_hooks = require("node:async_hooks");
13806
+
13793
13807
  // src/isomorph.ts
13808
+ var DefaultAsyncLocalStorage = class {
13809
+ constructor() {
13810
+ }
13811
+ enterWith(_) {
13812
+ }
13813
+ run(_, callback) {
13814
+ return callback();
13815
+ }
13816
+ getStore() {
13817
+ return void 0;
13818
+ }
13819
+ };
13794
13820
  var iso = {
13795
13821
  makeAxios: (conf) => axios_default.create({
13796
13822
  ...conf
13797
13823
  }),
13798
13824
  getRepoStatus: async () => void 0,
13799
13825
  getPastNAncestors: async () => [],
13800
- getEnv: (_name) => void 0
13826
+ getEnv: (_name) => void 0,
13827
+ getCallerLocation: () => void 0,
13828
+ newAsyncLocalStorage: () => new DefaultAsyncLocalStorage()
13801
13829
  };
13802
13830
  var isomorph_default = iso;
13803
13831
 
@@ -13872,8 +13900,8 @@ var __async = (__this, __arguments, generator) => {
13872
13900
  step((generator = generator.apply(__this, __arguments)).next());
13873
13901
  });
13874
13902
  };
13875
- function isPathSpec(path) {
13876
- return path instanceof String && cache.has(path);
13903
+ function isPathSpec(path2) {
13904
+ return path2 instanceof String && cache.has(path2);
13877
13905
  }
13878
13906
  function toPaths(pathSpec) {
13879
13907
  return cache.get(pathSpec) || [];
@@ -13955,8 +13983,8 @@ function toLinesWithContent(input = "", trimmed2 = true, separator = "\n") {
13955
13983
  function forEachLineWithContent(input, callback) {
13956
13984
  return toLinesWithContent(input, true).map((line) => callback(line));
13957
13985
  }
13958
- function folderExists(path) {
13959
- return (0, import_file_exists.exists)(path, import_file_exists.FOLDER);
13986
+ function folderExists(path2) {
13987
+ return (0, import_file_exists.exists)(path2, import_file_exists.FOLDER);
13960
13988
  }
13961
13989
  function append2(target, item) {
13962
13990
  if (Array.isArray(target)) {
@@ -14319,8 +14347,8 @@ function checkIsRepoRootTask() {
14319
14347
  commands,
14320
14348
  format: "utf-8",
14321
14349
  onError,
14322
- parser(path) {
14323
- return /^\.(git)?$/.test(path.trim());
14350
+ parser(path2) {
14351
+ return /^\.(git)?$/.test(path2.trim());
14324
14352
  }
14325
14353
  };
14326
14354
  }
@@ -14711,11 +14739,11 @@ function parseGrep(grep) {
14711
14739
  const paths = /* @__PURE__ */ new Set();
14712
14740
  const results = {};
14713
14741
  forEachLineWithContent(grep, (input) => {
14714
- const [path, line, preview] = input.split(NULL);
14715
- paths.add(path);
14716
- (results[path] = results[path] || []).push({
14742
+ const [path2, line, preview] = input.split(NULL);
14743
+ paths.add(path2);
14744
+ (results[path2] = results[path2] || []).push({
14717
14745
  line: asNumber(line),
14718
- path,
14746
+ path: path2,
14719
14747
  preview
14720
14748
  });
14721
14749
  });
@@ -15332,14 +15360,14 @@ var init_hash_object = __esm({
15332
15360
  init_task();
15333
15361
  }
15334
15362
  });
15335
- function parseInit(bare, path, text) {
15363
+ function parseInit(bare, path2, text) {
15336
15364
  const response = String(text).trim();
15337
15365
  let result;
15338
15366
  if (result = initResponseRegex.exec(response)) {
15339
- return new InitSummary(bare, path, false, result[1]);
15367
+ return new InitSummary(bare, path2, false, result[1]);
15340
15368
  }
15341
15369
  if (result = reInitResponseRegex.exec(response)) {
15342
- return new InitSummary(bare, path, true, result[1]);
15370
+ return new InitSummary(bare, path2, true, result[1]);
15343
15371
  }
15344
15372
  let gitDir = "";
15345
15373
  const tokens = response.split(" ");
@@ -15350,7 +15378,7 @@ function parseInit(bare, path, text) {
15350
15378
  break;
15351
15379
  }
15352
15380
  }
15353
- return new InitSummary(bare, path, /^re/i.test(response), gitDir);
15381
+ return new InitSummary(bare, path2, /^re/i.test(response), gitDir);
15354
15382
  }
15355
15383
  var InitSummary;
15356
15384
  var initResponseRegex;
@@ -15358,9 +15386,9 @@ var reInitResponseRegex;
15358
15386
  var init_InitSummary = __esm({
15359
15387
  "src/lib/responses/InitSummary.ts"() {
15360
15388
  InitSummary = class {
15361
- constructor(bare, path, existing, gitDir) {
15389
+ constructor(bare, path2, existing, gitDir) {
15362
15390
  this.bare = bare;
15363
- this.path = path;
15391
+ this.path = path2;
15364
15392
  this.existing = existing;
15365
15393
  this.gitDir = gitDir;
15366
15394
  }
@@ -15372,7 +15400,7 @@ var init_InitSummary = __esm({
15372
15400
  function hasBareCommand(command) {
15373
15401
  return command.includes(bareCommand);
15374
15402
  }
15375
- function initTask(bare = false, path, customArgs) {
15403
+ function initTask(bare = false, path2, customArgs) {
15376
15404
  const commands = ["init", ...customArgs];
15377
15405
  if (bare && !hasBareCommand(commands)) {
15378
15406
  commands.splice(1, 0, bareCommand);
@@ -15381,7 +15409,7 @@ function initTask(bare = false, path, customArgs) {
15381
15409
  commands,
15382
15410
  format: "utf-8",
15383
15411
  parser(text) {
15384
- return parseInit(commands.includes("--bare"), path, text);
15412
+ return parseInit(commands.includes("--bare"), path2, text);
15385
15413
  }
15386
15414
  };
15387
15415
  }
@@ -16101,12 +16129,12 @@ var init_FileStatusSummary = __esm({
16101
16129
  "src/lib/responses/FileStatusSummary.ts"() {
16102
16130
  fromPathRegex = /^(.+) -> (.+)$/;
16103
16131
  FileStatusSummary = class {
16104
- constructor(path, index, working_dir) {
16105
- this.path = path;
16132
+ constructor(path2, index, working_dir) {
16133
+ this.path = path2;
16106
16134
  this.index = index;
16107
16135
  this.working_dir = working_dir;
16108
16136
  if (index + working_dir === "R") {
16109
- const detail = fromPathRegex.exec(path) || [null, path, path];
16137
+ const detail = fromPathRegex.exec(path2) || [null, path2, path2];
16110
16138
  this.from = detail[1] || "";
16111
16139
  this.path = detail[2] || "";
16112
16140
  }
@@ -16137,14 +16165,14 @@ function splitLine(result, lineStr) {
16137
16165
  default:
16138
16166
  return;
16139
16167
  }
16140
- function data(index, workingDir, path) {
16168
+ function data(index, workingDir, path2) {
16141
16169
  const raw = `${index}${workingDir}`;
16142
16170
  const handler = parsers6.get(raw);
16143
16171
  if (handler) {
16144
- handler(result, path);
16172
+ handler(result, path2);
16145
16173
  }
16146
16174
  if (raw !== "##" && raw !== "!!") {
16147
- result.files.push(new FileStatusSummary(path.replace(/\0.+$/, ""), index, workingDir));
16175
+ result.files.push(new FileStatusSummary(path2.replace(/\0.+$/, ""), index, workingDir));
16148
16176
  }
16149
16177
  }
16150
16178
  }
@@ -16389,8 +16417,8 @@ var init_simple_git_api = __esm({
16389
16417
  }
16390
16418
  return this._runTask(configurationErrorTask("Git.cwd: workingDirectory must be supplied as a string"), next);
16391
16419
  }
16392
- hashObject(path, write) {
16393
- return this._runTask(hashObjectTask(path, write === true), trailingFunctionArgument(arguments));
16420
+ hashObject(path2, write) {
16421
+ return this._runTask(hashObjectTask(path2, write === true), trailingFunctionArgument(arguments));
16394
16422
  }
16395
16423
  init(bare) {
16396
16424
  return this._runTask(initTask(bare === true, this._executor.cwd, getTrailingOptions(arguments)), trailingFunctionArgument(arguments));
@@ -16967,8 +16995,8 @@ __export2(sub_module_exports, {
16967
16995
  subModuleTask: () => subModuleTask,
16968
16996
  updateSubModuleTask: () => updateSubModuleTask
16969
16997
  });
16970
- function addSubModuleTask(repo, path) {
16971
- return subModuleTask(["add", repo, path]);
16998
+ function addSubModuleTask(repo, path2) {
16999
+ return subModuleTask(["add", repo, path2]);
16972
17000
  }
16973
17001
  function initSubModuleTask(customArgs) {
16974
17002
  return subModuleTask(["init", ...customArgs]);
@@ -17234,8 +17262,8 @@ var require_git = __commonJS2({
17234
17262
  }
17235
17263
  return this._runTask(straightThroughStringTask2(command, this._trimmed), next);
17236
17264
  };
17237
- Git2.prototype.submoduleAdd = function(repo, path, then) {
17238
- return this._runTask(addSubModuleTask2(repo, path), trailingFunctionArgument2(arguments));
17265
+ Git2.prototype.submoduleAdd = function(repo, path2, then) {
17266
+ return this._runTask(addSubModuleTask2(repo, path2), trailingFunctionArgument2(arguments));
17239
17267
  };
17240
17268
  Git2.prototype.submoduleUpdate = function(args, then) {
17241
17269
  return this._runTask(updateSubModuleTask2(getTrailingOptions2(arguments, true)), trailingFunctionArgument2(arguments));
@@ -17824,6 +17852,50 @@ async function getRepoStatus() {
17824
17852
  };
17825
17853
  }
17826
17854
 
17855
+ // src/stackutil.ts
17856
+ var path = __toESM(require("path"));
17857
+ function getStackTrace() {
17858
+ const trace = new Error().stack;
17859
+ if (trace === void 0) {
17860
+ return [];
17861
+ }
17862
+ const traceLines = trace.split("\n");
17863
+ const out = [];
17864
+ const stackFrameRegex = /at(.*)\((.*):(\d+):(\d+)\)/;
17865
+ for (const traceLine of traceLines.slice(1)) {
17866
+ const matches = traceLine.match(stackFrameRegex);
17867
+ if (matches === null || matches.length !== 5) {
17868
+ continue;
17869
+ }
17870
+ const entry = {
17871
+ functionName: matches[1].trim(),
17872
+ fileName: matches[2],
17873
+ lineNo: parseInt(matches[3])
17874
+ };
17875
+ if (!isNaN(entry.lineNo)) {
17876
+ out.push(entry);
17877
+ }
17878
+ }
17879
+ return out;
17880
+ }
17881
+ function getCallerLocation() {
17882
+ let thisDir = void 0;
17883
+ const entries = getStackTrace();
17884
+ for (const frame of entries) {
17885
+ if (thisDir === void 0) {
17886
+ thisDir = path.dirname(frame.fileName);
17887
+ }
17888
+ if (path.dirname(frame.fileName) !== thisDir) {
17889
+ return {
17890
+ caller_functionname: frame.functionName,
17891
+ caller_filename: frame.fileName,
17892
+ caller_lineno: frame.lineNo
17893
+ };
17894
+ }
17895
+ }
17896
+ return void 0;
17897
+ }
17898
+
17827
17899
  // src/node.ts
17828
17900
  function configureNode() {
17829
17901
  isomorph_default.makeAxios = (options) => {
@@ -17838,6 +17910,8 @@ function configureNode() {
17838
17910
  isomorph_default.getRepoStatus = getRepoStatus;
17839
17911
  isomorph_default.getPastNAncestors = getPastNAncestors;
17840
17912
  isomorph_default.getEnv = (name) => process.env[name];
17913
+ isomorph_default.getCallerLocation = getCallerLocation;
17914
+ isomorph_default.newAsyncLocalStorage = () => new import_node_async_hooks.AsyncLocalStorage();
17841
17915
  }
17842
17916
 
17843
17917
  // node_modules/uuid/dist/esm-node/rng.js
@@ -17887,18 +17961,171 @@ function v4(options, buf, offset) {
17887
17961
  }
17888
17962
  var v4_default = v4;
17889
17963
 
17964
+ // src/util.ts
17965
+ var TRANSACTION_ID_FIELD = "_xact_id";
17966
+ var IS_MERGE_FIELD = "_is_merge";
17967
+ function runFinally(f, finallyF) {
17968
+ let runSyncCleanup = true;
17969
+ try {
17970
+ const ret = f();
17971
+ if (ret instanceof Promise) {
17972
+ runSyncCleanup = false;
17973
+ return ret.finally(finallyF);
17974
+ } else {
17975
+ return ret;
17976
+ }
17977
+ } finally {
17978
+ if (runSyncCleanup) {
17979
+ finallyF();
17980
+ }
17981
+ }
17982
+ }
17983
+ function mergeDicts(mergeInto, mergeFrom) {
17984
+ for (const [k, mergeFromV] of Object.entries(mergeFrom)) {
17985
+ const mergeIntoV = mergeInto[k];
17986
+ if (mergeIntoV instanceof Object && !Array.isArray(mergeIntoV) && mergeFrom instanceof Object && !Array.isArray(mergeFromV)) {
17987
+ mergeDicts(mergeIntoV, mergeFromV);
17988
+ } else {
17989
+ mergeInto[k] = mergeFromV;
17990
+ }
17991
+ }
17992
+ }
17993
+
17994
+ // src/merge_row_batch.ts
17995
+ function generateUniqueRowKey(row) {
17996
+ const coalesceEmpty = (field) => row[field] ?? "";
17997
+ return coalesceEmpty("experiment_id") + ":" + coalesceEmpty("dataset_id") + ":" + coalesceEmpty("prompt_session_id") + ":" + coalesceEmpty("id");
17998
+ }
17999
+ function mergeRowBatch(rows) {
18000
+ const out = [];
18001
+ const remainingRows = [];
18002
+ for (const row of rows) {
18003
+ if (row["id"] === void 0) {
18004
+ out.push(row);
18005
+ } else {
18006
+ remainingRows.push(row);
18007
+ }
18008
+ }
18009
+ const rowGroups = {};
18010
+ for (const row of remainingRows) {
18011
+ const key = generateUniqueRowKey(row);
18012
+ const existingRow = rowGroups[key];
18013
+ if (existingRow !== void 0 && row[IS_MERGE_FIELD]) {
18014
+ const preserveNoMerge = !existingRow[IS_MERGE_FIELD];
18015
+ mergeDicts(existingRow, row);
18016
+ if (preserveNoMerge) {
18017
+ delete existingRow[IS_MERGE_FIELD];
18018
+ }
18019
+ } else {
18020
+ rowGroups[key] = row;
18021
+ }
18022
+ }
18023
+ out.push(...Object.values(rowGroups));
18024
+ return out;
18025
+ }
18026
+
17890
18027
  // src/logger.ts
17891
- var _state = {
17892
- current_project: null,
17893
- current_experiment: null
18028
+ var NoopSpan = class {
18029
+ constructor() {
18030
+ this.kind = "span";
18031
+ this.id = "";
18032
+ this.span_id = "";
18033
+ this.root_span_id = "";
18034
+ }
18035
+ log(_) {
18036
+ }
18037
+ startSpan(_0, _1) {
18038
+ return this;
18039
+ }
18040
+ traced(_0, callback, _1) {
18041
+ return callback(this);
18042
+ }
18043
+ end(args) {
18044
+ return args?.endTime ?? getCurrentUnixTimestamp();
18045
+ }
18046
+ close(args) {
18047
+ return this.end(args);
18048
+ }
17894
18049
  };
17895
- var API_URL = null;
17896
- var LOGIN_TOKEN = null;
17897
- var ORG_ID = null;
17898
- var ORG_NAME = null;
17899
- var LOG_URL = null;
17900
- var LOGGED_IN = false;
17901
- var TRANSACTION_ID_FIELD = "_xact_id";
18050
+ var noopSpan = new NoopSpan();
18051
+ var BraintrustState = class {
18052
+ constructor() {
18053
+ this.id = v4_default();
18054
+ this.currentExperiment = isomorph_default.newAsyncLocalStorage();
18055
+ this.currentSpan = isomorph_default.newAsyncLocalStorage();
18056
+ this.apiUrl = null;
18057
+ this.loginToken = null;
18058
+ this.orgId = null;
18059
+ this.orgName = null;
18060
+ this.logUrl = null;
18061
+ this.loggedIn = false;
18062
+ this._apiConn = null;
18063
+ this._logConn = null;
18064
+ this._userInfo = null;
18065
+ globalThis.__inherited_braintrust_state = this;
18066
+ }
18067
+ apiConn() {
18068
+ if (!this._apiConn) {
18069
+ if (!this.apiUrl) {
18070
+ throw new Error("Must initialize apiUrl before requesting apiConn");
18071
+ }
18072
+ this._apiConn = new HTTPConnection(this.apiUrl);
18073
+ }
18074
+ return this._apiConn;
18075
+ }
18076
+ logConn() {
18077
+ if (!this._logConn) {
18078
+ if (!this.logUrl) {
18079
+ throw new Error("Must initialize logUrl before requesting logConn");
18080
+ }
18081
+ this._logConn = new HTTPConnection(this.logUrl);
18082
+ }
18083
+ return this._logConn;
18084
+ }
18085
+ async userInfo() {
18086
+ if (!this._userInfo) {
18087
+ this._userInfo = await this.logConn().get_json("ping");
18088
+ }
18089
+ return this._userInfo;
18090
+ }
18091
+ setUserInfoIfNull(info) {
18092
+ if (!this._userInfo) {
18093
+ this._userInfo = info;
18094
+ }
18095
+ }
18096
+ };
18097
+ var _state = globalThis.__inherited_braintrust_state || new BraintrustState();
18098
+ var _internalGetGlobalState = () => _state;
18099
+ var UnterminatedObjectsHandler = class {
18100
+ constructor() {
18101
+ this.unterminatedObjects = /* @__PURE__ */ new Map();
18102
+ process.on("exit", () => {
18103
+ this.warnUnterminated();
18104
+ });
18105
+ }
18106
+ addUnterminated(obj, createdLocation) {
18107
+ this.unterminatedObjects.set(obj, createdLocation);
18108
+ }
18109
+ removeUnterminated(obj) {
18110
+ this.unterminatedObjects.delete(obj);
18111
+ }
18112
+ warnUnterminated() {
18113
+ if (this.unterminatedObjects.size === 0) {
18114
+ return;
18115
+ }
18116
+ let warningMessage = "WARNING: Did not close the following braintrust objects. We recommend running `.close` on the listed objects, or by running them inside a callback so they are closed automatically:";
18117
+ this.unterminatedObjects.forEach((createdLocation, obj) => {
18118
+ let msg = `
18119
+ Object of type ${obj?.constructor?.name}`;
18120
+ if (createdLocation) {
18121
+ msg += ` created at ${JSON.stringify(createdLocation)}`;
18122
+ }
18123
+ warningMessage += msg;
18124
+ });
18125
+ console.warn(warningMessage);
18126
+ }
18127
+ };
18128
+ var unterminatedObjects = new UnterminatedObjectsHandler();
17902
18129
  var HTTPConnection = class _HTTPConnection {
17903
18130
  constructor(base_url) {
17904
18131
  this.base_url = base_url;
@@ -17909,9 +18136,7 @@ var HTTPConnection = class _HTTPConnection {
17909
18136
  async ping() {
17910
18137
  try {
17911
18138
  const resp = await this.get("ping");
17912
- if (_var_user_info === null) {
17913
- _var_user_info = resp.data;
17914
- }
18139
+ _state.setUserInfoIfNull(resp.data);
17915
18140
  return resp.status === 200;
17916
18141
  } catch (e) {
17917
18142
  return false;
@@ -17936,12 +18161,12 @@ var HTTPConnection = class _HTTPConnection {
17936
18161
  }
17937
18162
  this.session = isomorph_default.makeAxios({ headers });
17938
18163
  }
17939
- async get(path, params = void 0) {
17940
- return await this.session.get(_urljoin(this.base_url, path), { params });
18164
+ async get(path2, params = void 0) {
18165
+ return await this.session.get(_urljoin(this.base_url, path2), { params });
17941
18166
  }
17942
- async post(path, params = void 0, config = void 0) {
18167
+ async post(path2, params = void 0, config = void 0) {
17943
18168
  return await this.session.post(
17944
- _urljoin(this.base_url, path),
18169
+ _urljoin(this.base_url, path2),
17945
18170
  params,
17946
18171
  config
17947
18172
  );
@@ -17971,32 +18196,6 @@ var HTTPConnection = class _HTTPConnection {
17971
18196
  return resp.data;
17972
18197
  }
17973
18198
  };
17974
- var _api_conn = null;
17975
- function api_conn() {
17976
- if (!_api_conn) {
17977
- _api_conn = new HTTPConnection(API_URL);
17978
- }
17979
- return _api_conn;
17980
- }
17981
- var _log_conn = null;
17982
- function log_conn() {
17983
- if (!_log_conn) {
17984
- _log_conn = new HTTPConnection(LOG_URL);
17985
- }
17986
- return _log_conn;
17987
- }
17988
- var _var_user_info = null;
17989
- async function _user_info() {
17990
- if (_var_user_info === null) {
17991
- _var_user_info = await log_conn().get_json("ping");
17992
- }
17993
- return _var_user_info;
17994
- }
17995
- function clear_cached_globals() {
17996
- _api_conn = null;
17997
- _log_conn = null;
17998
- _var_user_info = null;
17999
- }
18000
18199
  var Project = class {
18001
18200
  constructor(name, id, org_id) {
18002
18201
  this.name = name;
@@ -18008,6 +18207,8 @@ var MaxRequestSize = 10 * 1024 * 1024;
18008
18207
  function constructJsonArray(items) {
18009
18208
  return `[${items.join(",")}]`;
18010
18209
  }
18210
+ var DefaultBatchSize = 100;
18211
+ var NumRetries = 3;
18011
18212
  var LogThread = class {
18012
18213
  constructor() {
18013
18214
  this.items = [];
@@ -18021,11 +18222,11 @@ var LogThread = class {
18021
18222
  this.active_flush = this.flush_once();
18022
18223
  }
18023
18224
  }
18024
- async flush_once(batchSize = 100) {
18225
+ async flush_once(batchSize = DefaultBatchSize) {
18025
18226
  this.active_flush_resolved = false;
18026
- const initialItems = (this.items || []).reverse();
18227
+ const initialItems = mergeRowBatch(this.items || []).reverse();
18027
18228
  this.items = [];
18028
- let ret = [];
18229
+ let postPromises = [];
18029
18230
  while (true) {
18030
18231
  const items = [];
18031
18232
  let itemsLen = 0;
@@ -18040,16 +18241,42 @@ var LogThread = class {
18040
18241
  items.push(itemS);
18041
18242
  itemsLen += itemS.length;
18042
18243
  }
18043
- if (items.length > 0) {
18044
- const resp = await log_conn().post_json(
18045
- "logs",
18046
- constructJsonArray(items)
18047
- );
18048
- ret = resp.data;
18049
- } else {
18244
+ if (items.length === 0) {
18050
18245
  break;
18051
18246
  }
18247
+ postPromises.push(
18248
+ (async () => {
18249
+ const itemsS = constructJsonArray(items);
18250
+ for (let i = 0; i < NumRetries; i++) {
18251
+ const startTime = performance.now();
18252
+ try {
18253
+ return (await _state.logConn().post_json("logs", itemsS)).map(
18254
+ (res) => res.id
18255
+ );
18256
+ } catch (e) {
18257
+ const retryingText = i + 1 === NumRetries ? "" : " Retrying";
18258
+ const errMsg = (() => {
18259
+ if (e instanceof AxiosError2 && e.response) {
18260
+ return `${e.response.status}: ${JSON.stringify(
18261
+ e.response.data
18262
+ )}`;
18263
+ } else {
18264
+ return `${e}`;
18265
+ }
18266
+ })();
18267
+ console.warn(
18268
+ `log request failed. Elapsed time: ${(performance.now() - startTime) / 1e3} seconds. Payload size: ${itemsS.length}. Error: ${errMsg}.${retryingText}`
18269
+ );
18270
+ }
18271
+ }
18272
+ console.warn(
18273
+ `log request failed after ${NumRetries} retries. Dropping batch`
18274
+ );
18275
+ return [];
18276
+ })()
18277
+ );
18052
18278
  }
18279
+ let ret = await Promise.all(postPromises);
18053
18280
  if (this.items.length > 0) {
18054
18281
  this.active_flush = this.flush_once();
18055
18282
  } else {
@@ -18085,7 +18312,7 @@ async function init(project, options = {}) {
18085
18312
  apiKey,
18086
18313
  apiUrl
18087
18314
  });
18088
- const ret = await _initExperiment(project, {
18315
+ return await _initExperiment(project, {
18089
18316
  experimentName: experiment,
18090
18317
  description,
18091
18318
  dataset,
@@ -18093,8 +18320,19 @@ async function init(project, options = {}) {
18093
18320
  baseExperiment,
18094
18321
  isPublic
18095
18322
  });
18096
- _state.current_experiment = ret;
18097
- return ret;
18323
+ }
18324
+ async function withExperiment(project, callback, options = {}) {
18325
+ const experiment = await init(project, options);
18326
+ return runFinally(
18327
+ () => {
18328
+ if (options.setCurrent ?? true) {
18329
+ return withCurrent(experiment, () => callback(experiment));
18330
+ } else {
18331
+ return callback(experiment);
18332
+ }
18333
+ },
18334
+ () => experiment.close()
18335
+ );
18098
18336
  }
18099
18337
  async function initDataset(project, options = {}) {
18100
18338
  const {
@@ -18118,6 +18356,13 @@ async function initDataset(project, options = {}) {
18118
18356
  version
18119
18357
  });
18120
18358
  }
18359
+ async function withDataset(project, callback, options = {}) {
18360
+ const dataset = await initDataset(project, options);
18361
+ return runFinally(
18362
+ () => callback(dataset),
18363
+ () => dataset.close()
18364
+ );
18365
+ }
18121
18366
  async function login(options = {}) {
18122
18367
  const {
18123
18368
  apiUrl = isomorph_default.getEnv("BRAINTRUST_API_URL") || "https://www.braintrustdata.com",
@@ -18126,24 +18371,27 @@ async function login(options = {}) {
18126
18371
  disableCache = false
18127
18372
  } = options || {};
18128
18373
  let { forceLogin = false } = options || {};
18129
- if (apiUrl != API_URL || apiKey !== void 0 && HTTPConnection.sanitize_token(apiKey) != LOGIN_TOKEN || orgName !== void 0 && orgName != ORG_NAME) {
18374
+ if (apiUrl != _state.apiUrl || apiKey !== void 0 && HTTPConnection.sanitize_token(apiKey) != _state.loginToken || orgName !== void 0 && orgName != _state.orgName) {
18130
18375
  forceLogin = true;
18131
18376
  }
18132
- if (LOGGED_IN && !forceLogin) {
18377
+ if (_state.loggedIn && !forceLogin) {
18133
18378
  return;
18134
18379
  }
18135
- clear_cached_globals();
18136
- API_URL = apiUrl;
18380
+ _state = new BraintrustState();
18381
+ _state.apiUrl = apiUrl;
18137
18382
  let login_key_info = null;
18138
18383
  let ping_ok = false;
18139
18384
  let conn = null;
18140
18385
  if (apiKey !== void 0) {
18141
- const resp = await axios_default.post(_urljoin(API_URL, `/api/apikey/login`), {
18142
- token: apiKey
18143
- });
18386
+ const resp = await axios_default.post(
18387
+ _urljoin(_state.apiUrl, `/api/apikey/login`),
18388
+ {
18389
+ token: apiKey
18390
+ }
18391
+ );
18144
18392
  const info = resp.data;
18145
18393
  _check_org_info(info.org_info, orgName);
18146
- conn = log_conn();
18394
+ conn = _state.logConn();
18147
18395
  conn.set_token(apiKey);
18148
18396
  ping_ok = await conn.ping();
18149
18397
  } else {
@@ -18158,21 +18406,66 @@ async function login(options = {}) {
18158
18406
  await conn.get("ping");
18159
18407
  }
18160
18408
  conn.make_long_lived();
18161
- api_conn().set_token(apiKey);
18162
- LOGIN_TOKEN = conn.token;
18163
- LOGGED_IN = true;
18409
+ _state.apiConn().set_token(apiKey);
18410
+ _state.loginToken = conn.token;
18411
+ _state.loggedIn = true;
18164
18412
  }
18165
- function log(options) {
18166
- if (!_state.current_experiment) {
18413
+ function log(event) {
18414
+ const currentExperiment2 = _state.currentExperiment.getStore();
18415
+ if (!currentExperiment2) {
18167
18416
  throw new Error("Not initialized. Please call init() first");
18168
18417
  }
18169
- return _state.current_experiment.log(options);
18418
+ return currentExperiment2.log(event);
18170
18419
  }
18171
18420
  async function summarize(options = {}) {
18172
- if (!_state.current_experiment) {
18421
+ const currentExperiment2 = _state.currentExperiment.getStore();
18422
+ if (!currentExperiment2) {
18173
18423
  throw new Error("Not initialized. Please call init() first");
18174
18424
  }
18175
- return await _state.current_experiment.summarize(options);
18425
+ return await currentExperiment2.summarize(options);
18426
+ }
18427
+ function currentExperiment() {
18428
+ return _state.currentExperiment.getStore();
18429
+ }
18430
+ function currentSpan() {
18431
+ return _state.currentSpan.getStore();
18432
+ }
18433
+ function startSpan(args) {
18434
+ const { name: nameOpt, ...argsRest } = args ?? {};
18435
+ const name = (nameOpt ?? isomorph_default.getCallerLocation()?.caller_functionname) || "root";
18436
+ const parentSpan = currentSpan();
18437
+ if (!Object.is(parentSpan, noopSpan)) {
18438
+ return parentSpan.startSpan(name, argsRest);
18439
+ }
18440
+ const experiment = currentExperiment();
18441
+ if (experiment) {
18442
+ return experiment.startSpan({ name, ...argsRest });
18443
+ }
18444
+ return noopSpan;
18445
+ }
18446
+ function traced(callback, args) {
18447
+ const span = startSpan(args);
18448
+ return runFinally(
18449
+ () => {
18450
+ if (args?.setCurrent ?? true) {
18451
+ return withCurrent(span, () => callback(span));
18452
+ } else {
18453
+ return callback(span);
18454
+ }
18455
+ },
18456
+ () => span.end()
18457
+ );
18458
+ }
18459
+ function withCurrent(object, callback) {
18460
+ if (object.kind === "experiment") {
18461
+ return _state.currentExperiment.run(object, callback);
18462
+ } else if (object.kind === "span") {
18463
+ return _state.currentSpan.run(object, callback);
18464
+ } else {
18465
+ throw new Error(
18466
+ `Invalid object of type ${object.constructor.name}`
18467
+ );
18468
+ }
18176
18469
  }
18177
18470
  function _check_org_info(org_info, org_name) {
18178
18471
  if (org_info.length === 0) {
@@ -18180,13 +18473,13 @@ function _check_org_info(org_info, org_name) {
18180
18473
  }
18181
18474
  for (const org of org_info) {
18182
18475
  if (org_name === void 0 || org.name === org_name) {
18183
- ORG_ID = org.id;
18184
- ORG_NAME = org.name;
18185
- LOG_URL = org.api_url;
18476
+ _state.orgId = org.id;
18477
+ _state.orgName = org.name;
18478
+ _state.logUrl = org.api_url;
18186
18479
  break;
18187
18480
  }
18188
18481
  }
18189
- if (ORG_ID === void 0) {
18482
+ if (_state.orgId === void 0) {
18190
18483
  throw new Error(
18191
18484
  `Organization ${org_name} not found. Must be one of ${org_info.map((x) => x.name).join(", ")}`
18192
18485
  );
@@ -18195,6 +18488,82 @@ function _check_org_info(org_info, org_name) {
18195
18488
  function _urljoin(...parts) {
18196
18489
  return parts.map((x) => x.replace(/^\//, "")).join("/");
18197
18490
  }
18491
+ function getCurrentUnixTimestamp() {
18492
+ return (/* @__PURE__ */ new Date()).getTime() / 1e3;
18493
+ }
18494
+ function validateAndSanitizeExperimentLogPartialArgs(event) {
18495
+ if (event.scores) {
18496
+ for (let [name, score] of Object.entries(event.scores)) {
18497
+ if (typeof name !== "string") {
18498
+ throw new Error("score names must be strings");
18499
+ }
18500
+ if (typeof score === "boolean") {
18501
+ score = score ? 1 : 0;
18502
+ event.scores[name] = score;
18503
+ }
18504
+ if (typeof score !== "number") {
18505
+ throw new Error("score values must be numbers");
18506
+ }
18507
+ if (score < 0 || score > 1) {
18508
+ throw new Error("score values must be between 0 and 1");
18509
+ }
18510
+ }
18511
+ }
18512
+ if (event.metadata) {
18513
+ for (const key of Object.keys(event.metadata)) {
18514
+ if (typeof key !== "string") {
18515
+ throw new Error("metadata keys must be strings");
18516
+ }
18517
+ }
18518
+ }
18519
+ if (event.metrics) {
18520
+ for (const key of Object.keys(event.metrics)) {
18521
+ if (typeof key !== "string") {
18522
+ throw new Error("metric keys must be strings");
18523
+ }
18524
+ }
18525
+ for (const forbiddenKey of [
18526
+ "start",
18527
+ "end",
18528
+ "caller_functionname",
18529
+ "caller_filename",
18530
+ "caller_lineno"
18531
+ ]) {
18532
+ if (forbiddenKey in event.metrics) {
18533
+ throw new Error(`Key ${forbiddenKey} may not be specified in metrics`);
18534
+ }
18535
+ }
18536
+ }
18537
+ if ("input" in event && event.input && "inputs" in event && event.inputs) {
18538
+ throw new Error(
18539
+ "Only one of input or inputs (deprecated) can be specified. Prefer input."
18540
+ );
18541
+ }
18542
+ if ("inputs" in event) {
18543
+ const { inputs, ...rest } = event;
18544
+ return { input: inputs, ...rest };
18545
+ } else {
18546
+ return { ...event };
18547
+ }
18548
+ }
18549
+ function validateAndSanitizeExperimentLogFullArgs(event, hasDataset) {
18550
+ if ("input" in event && event.input && "inputs" in event && event.inputs || !("input" in event) && !("inputs" in event)) {
18551
+ throw new Error(
18552
+ "Exactly one of input or inputs (deprecated) must be specified. Prefer input."
18553
+ );
18554
+ }
18555
+ if (!event.scores) {
18556
+ throw new Error("scores must be specified");
18557
+ }
18558
+ if (hasDataset && event.datasetRecordId === void 0) {
18559
+ throw new Error("datasetRecordId must be specified when using a dataset");
18560
+ } else if (!hasDataset && event.datasetRecordId !== void 0) {
18561
+ throw new Error(
18562
+ "datasetRecordId cannot be specified when not using a dataset"
18563
+ );
18564
+ }
18565
+ return event;
18566
+ }
18198
18567
  async function _initExperiment(projectName, {
18199
18568
  experimentName,
18200
18569
  description,
@@ -18210,7 +18579,7 @@ async function _initExperiment(projectName, {
18210
18579
  }) {
18211
18580
  const args = {
18212
18581
  project_name: projectName,
18213
- org_id: ORG_ID
18582
+ org_id: _state.orgId
18214
18583
  };
18215
18584
  if (experimentName) {
18216
18585
  args["experiment_name"] = experimentName;
@@ -18237,10 +18606,10 @@ async function _initExperiment(projectName, {
18237
18606
  if (isPublic !== void 0) {
18238
18607
  args["public"] = isPublic;
18239
18608
  }
18240
- const response = await api_conn().post_json("api/experiment/register", args);
18609
+ const response = await _state.apiConn().post_json("api/experiment/register", args);
18241
18610
  const project = response.project;
18242
18611
  const experiment = response.experiment;
18243
- const user_id = (await _user_info())["id"];
18612
+ const user_id = (await _state.userInfo())["id"];
18244
18613
  return new Experiment(
18245
18614
  project,
18246
18615
  experiment.id,
@@ -18251,106 +18620,71 @@ async function _initExperiment(projectName, {
18251
18620
  }
18252
18621
  var Experiment = class {
18253
18622
  constructor(project, id, name, user_id, dataset) {
18623
+ // For type identification.
18624
+ this.kind = "experiment";
18625
+ this.finished = false;
18254
18626
  this.project = project;
18255
18627
  this.id = id;
18256
18628
  this.name = name;
18257
18629
  this.user_id = user_id;
18258
18630
  this.dataset = dataset;
18259
18631
  this.logger = new LogThread();
18632
+ this.lastStartTime = getCurrentUnixTimestamp();
18633
+ unterminatedObjects.addUnterminated(this, isomorph_default.getCallerLocation());
18260
18634
  }
18261
18635
  /**
18262
18636
  * Log a single event to the experiment. The event will be batched and uploaded behind the scenes.
18263
18637
  *
18264
18638
  * @param event The event to log.
18265
- * @param event.input The arguments that uniquely define a test case (an arbitrary, JSON serializable object). Later on,
18266
- * Braintrust will use the `input` to know whether two test cases are the same between experiments, so they should
18267
- * not contain experiment-specific state. A simple rule of thumb is that if you run the same experiment twice, the
18268
- * `input` should be identical.
18269
- * @param event.output The output of your application, including post-processing (an arbitrary, JSON serializable object),
18270
- * that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries,
18271
- * the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may
18272
- * be multiple valid queries that answer a single question.
18273
- * @param event.expected The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to
18274
- * determine if your `output` value is correct or not. Braintrust currently does not compare `output` to `expected` for
18275
- * you, since there are so many different ways to do that correctly. Instead, these values are just used to help you
18276
- * navigate your experiments while digging into analyses. However, we may later use these values to re-score outputs or
18277
- * fine-tune your models.
18278
- * @param event.scores A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals
18279
- * that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a
18280
- * summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity
18281
- * between the generated and grouth truth summary. The word similarity score could help you determine whether the summarization was
18282
- * covering similar concepts or not. You can use these scores to help you sort, filter, and compare experiments.
18283
- * @param event.metadata (Optional) a dictionary with additional data about the test example, model outputs, or just
18284
- * about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the
18285
- * `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any
18286
- * JSON-serializable type, but its keys must be strings.
18287
- * @param event.id (Optional) a unique identifier for the event. If you don't provide one, Braintrust will generate one for you.
18288
- * @param event.inputs (Deprecated) the same as `input` (will be removed in a future version)
18289
- * @returns The `id` of the logged event.
18639
+ * @param event.input: The arguments that uniquely define a test case (an arbitrary, JSON serializable object). Later on, Braintrust will use the `input` to know whether two test cases are the same between experiments, so they should not contain experiment-specific state. A simple rule of thumb is that if you run the same experiment twice, the `input` should be identical.
18640
+ * @param event.output: The output of your application, including post-processing (an arbitrary, JSON serializable object), that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries, the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may be multiple valid queries that answer a single question.
18641
+ * @param event.expected: The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to determine if your `output` value is correct or not. Braintrust currently does not compare `output` to `expected` for you, since there are so many different ways to do that correctly. Instead, these values are just used to help you navigate your experiments while digging into analyses. However, we may later use these values to re-score outputs or fine-tune your models.
18642
+ * @param event.scores: A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity between the generated and grouth truth summary. The word similarity score could help you determine whether the summarization was covering similar concepts or not. You can use these scores to help you sort, filter, and compare experiments.
18643
+ * @param event.metadata: (Optional) a dictionary with additional data about the test example, model outputs, or just about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any JSON-serializable type, but its keys must be strings.
18644
+ * @param event.metrics: (Optional) a dictionary of metrics to log. The following keys are populated automatically and should not be specified: "start", "end", "caller_functionname", "caller_filename", "caller_lineno".
18645
+ * @param event.id: (Optional) a unique identifier for the event. If you don't provide one, BrainTrust will generate one for you.
18646
+ * @param event.dataset_record_id: (Optional) the id of the dataset record that this event is associated with. This field is required if and only if the experiment is associated with a dataset.
18647
+ * @param event.inputs: (Deprecated) the same as `input` (will be removed in a future version).
18648
+ * :returns: The `id` of the logged event.
18290
18649
  */
18291
- log({
18292
- input,
18293
- output,
18294
- expected,
18295
- scores,
18296
- metadata,
18297
- id,
18298
- datasetRecordId,
18299
- inputs
18300
- }) {
18301
- if (input === void 0 && inputs === void 0) {
18302
- throw new Error(
18303
- "Either input or inputs (deprecated) must be specified. Prefer input."
18304
- );
18305
- } else if (input !== void 0 && inputs !== void 0) {
18306
- throw new Error(
18307
- "Only one of input or inputs (deprecated) can be specified. Prefer input."
18308
- );
18309
- }
18310
- for (let [name, score] of Object.entries(scores)) {
18311
- if (typeof name !== "string") {
18312
- throw new Error("score names must be strings");
18313
- }
18314
- if (typeof score === "boolean") {
18315
- score = score ? 1 : 0;
18316
- scores[name] = score;
18317
- }
18318
- if (typeof score !== "number") {
18319
- throw new Error("score values must be numbers");
18320
- }
18321
- if (score < 0 || score > 1) {
18322
- throw new Error("score values must be between 0 and 1");
18323
- }
18324
- }
18325
- if (metadata !== void 0) {
18326
- for (const key of Object.keys(metadata)) {
18327
- if (typeof key !== "string") {
18328
- throw new Error("metadata keys must be strings");
18650
+ log(event) {
18651
+ this.checkNotFinished();
18652
+ event = validateAndSanitizeExperimentLogFullArgs(event, !!this.dataset);
18653
+ const span = this.startSpan({ startTime: this.lastStartTime, event });
18654
+ this.lastStartTime = span.end();
18655
+ return span.id;
18656
+ }
18657
+ /**
18658
+ * Create a new toplevel span. The name parameter is optional and defaults to "root".
18659
+ *
18660
+ * See `Span.startSpan` for full details.
18661
+ */
18662
+ startSpan(args) {
18663
+ this.checkNotFinished();
18664
+ const { name, ...argsRest } = args ?? {};
18665
+ return new SpanImpl({
18666
+ experimentLogger: this.logger,
18667
+ name: name ?? "root",
18668
+ ...argsRest,
18669
+ rootExperiment: this
18670
+ });
18671
+ }
18672
+ /**
18673
+ * Wrapper over `Experiment.startSpan`, which passes the initialized `Span` it to the given callback and ends it afterwards. See `Span.traced` for full details.
18674
+ */
18675
+ traced(callback, args) {
18676
+ const { setCurrent, ...argsRest } = args ?? {};
18677
+ const span = this.startSpan(argsRest);
18678
+ return runFinally(
18679
+ () => {
18680
+ if (setCurrent ?? true) {
18681
+ return withCurrent(span, () => callback(span));
18682
+ } else {
18683
+ return callback(span);
18329
18684
  }
18330
- }
18331
- }
18332
- if (this.dataset && datasetRecordId === void 0) {
18333
- throw new Error("datasetRecordId must be specified when using a dataset");
18334
- } else if (!this.dataset && datasetRecordId !== void 0) {
18335
- throw new Error(
18336
- "datasetRecordId cannot be specified when not using a dataset"
18337
- );
18338
- }
18339
- const args = {
18340
- id: id || v4_default(),
18341
- inputs: input ?? inputs,
18342
- output,
18343
- expected,
18344
- scores,
18345
- project_id: this.project.id,
18346
- experiment_id: this.id,
18347
- user_id: this.user_id,
18348
- created: (/* @__PURE__ */ new Date()).toISOString(),
18349
- dataset_record_id: datasetRecordId,
18350
- metadata
18351
- };
18352
- this.logger.log([args]);
18353
- return args.id;
18685
+ },
18686
+ () => span.end()
18687
+ );
18354
18688
  }
18355
18689
  /**
18356
18690
  * Summarize the experiment, including the scores (compared to the closest reference experiment) and metadata.
@@ -18363,15 +18697,15 @@ var Experiment = class {
18363
18697
  async summarize(options = {}) {
18364
18698
  let { summarizeScores = true, comparisonExperimentId = void 0 } = options || {};
18365
18699
  await this.logger.flush();
18366
- const projectUrl = `${API_URL}/app/${encodeURIComponent(
18367
- ORG_NAME
18700
+ const projectUrl = `${_state.apiUrl}/app/${encodeURIComponent(
18701
+ _state.orgName
18368
18702
  )}/p/${encodeURIComponent(this.project.name)}`;
18369
18703
  const experimentUrl = `${projectUrl}/${encodeURIComponent(this.name)}`;
18370
18704
  let scores = void 0;
18371
18705
  let comparisonExperimentName = void 0;
18372
18706
  if (summarizeScores) {
18373
18707
  if (comparisonExperimentId === void 0) {
18374
- const conn = log_conn();
18708
+ const conn = _state.logConn();
18375
18709
  const resp = await conn.get("/crud/base_experiments", {
18376
18710
  id: this.id
18377
18711
  });
@@ -18382,7 +18716,7 @@ var Experiment = class {
18382
18716
  }
18383
18717
  }
18384
18718
  if (comparisonExperimentId !== void 0) {
18385
- scores = await log_conn().get_json(
18719
+ scores = await _state.logConn().get_json(
18386
18720
  "/experiment-comparison",
18387
18721
  {
18388
18722
  experiment_id: this.id,
@@ -18401,6 +18735,122 @@ var Experiment = class {
18401
18735
  scores
18402
18736
  };
18403
18737
  }
18738
+ /**
18739
+ * Finish the experiment and return its id. After calling close, you may not invoke any further methods on the experiment object.
18740
+ *
18741
+ * Will be invoked automatically if the experiment is wrapped in a callback passed to `braintrust.withExperiment`.
18742
+ *
18743
+ * @returns The experiment id.
18744
+ */
18745
+ async close() {
18746
+ this.checkNotFinished();
18747
+ await this.logger.flush();
18748
+ this.finished = true;
18749
+ unterminatedObjects.removeUnterminated(this);
18750
+ return this.id;
18751
+ }
18752
+ checkNotFinished() {
18753
+ if (this.finished) {
18754
+ throw new Error("Cannot invoke method on finished experiment");
18755
+ }
18756
+ }
18757
+ };
18758
+ var SpanImpl = class _SpanImpl {
18759
+ // root_experiment should only be specified for a root span. parent_span
18760
+ // should only be specified for non-root spans.
18761
+ constructor(args) {
18762
+ this.kind = "span";
18763
+ this.finished = false;
18764
+ this.experimentLogger = args.experimentLogger;
18765
+ const callerLocation = isomorph_default.getCallerLocation();
18766
+ this.internalData = {
18767
+ metrics: {
18768
+ start: args.startTime ?? getCurrentUnixTimestamp(),
18769
+ ...callerLocation
18770
+ },
18771
+ span_attributes: { ...args.spanAttributes, name: args.name }
18772
+ };
18773
+ this.id = args.event?.id ?? v4_default();
18774
+ this.span_id = v4_default();
18775
+ if ("rootExperiment" in args) {
18776
+ this.root_span_id = this.span_id;
18777
+ this._project_id = args.rootExperiment.project.id;
18778
+ this._experiment_id = args.rootExperiment.id;
18779
+ this.internalData = Object.assign(this.internalData, {
18780
+ // TODO: Hopefully we can remove this.
18781
+ user_id: args.rootExperiment.user_id,
18782
+ created: (/* @__PURE__ */ new Date()).toISOString()
18783
+ });
18784
+ } else if ("parentSpan" in args) {
18785
+ this.root_span_id = args.parentSpan.root_span_id;
18786
+ this._project_id = args.parentSpan._project_id;
18787
+ this._experiment_id = args.parentSpan._experiment_id;
18788
+ this.internalData.span_parents = [args.parentSpan.span_id];
18789
+ } else {
18790
+ throw new Error("Must provide either 'rootExperiment' or 'parentSpan'");
18791
+ }
18792
+ this.isMerge = false;
18793
+ const { id, ...eventRest } = args.event ?? {};
18794
+ this.log(eventRest);
18795
+ this.isMerge = true;
18796
+ unterminatedObjects.addUnterminated(this, callerLocation);
18797
+ }
18798
+ log(event) {
18799
+ this.checkNotFinished();
18800
+ const sanitized = validateAndSanitizeExperimentLogPartialArgs(event);
18801
+ const record = {
18802
+ ...sanitized,
18803
+ ...this.internalData,
18804
+ id: this.id,
18805
+ span_id: this.span_id,
18806
+ root_span_id: this.root_span_id,
18807
+ project_id: this._project_id,
18808
+ experiment_id: this._experiment_id,
18809
+ [IS_MERGE_FIELD]: this.isMerge
18810
+ };
18811
+ this.internalData = {};
18812
+ this.experimentLogger.log([record]);
18813
+ }
18814
+ startSpan(name, args) {
18815
+ this.checkNotFinished();
18816
+ return new _SpanImpl({
18817
+ experimentLogger: this.experimentLogger,
18818
+ name,
18819
+ ...args,
18820
+ parentSpan: this
18821
+ });
18822
+ }
18823
+ traced(name, callback, args) {
18824
+ const { setCurrent, ...argsRest } = args ?? {};
18825
+ const span = this.startSpan(name, argsRest);
18826
+ return runFinally(
18827
+ () => {
18828
+ if (setCurrent ?? true) {
18829
+ return withCurrent(span, () => callback(span));
18830
+ } else {
18831
+ return callback(span);
18832
+ }
18833
+ },
18834
+ () => span.end()
18835
+ );
18836
+ }
18837
+ end(args) {
18838
+ this.checkNotFinished();
18839
+ const endTime = args?.endTime ?? getCurrentUnixTimestamp();
18840
+ this.internalData = { metrics: { end: endTime } };
18841
+ this.log({});
18842
+ this.finished = true;
18843
+ unterminatedObjects.removeUnterminated(this);
18844
+ return endTime;
18845
+ }
18846
+ close(args) {
18847
+ return this.end(args);
18848
+ }
18849
+ checkNotFinished() {
18850
+ if (this.finished) {
18851
+ throw new Error("Cannot invoke method on finished span");
18852
+ }
18853
+ }
18404
18854
  };
18405
18855
  async function _initDataset(project_name, {
18406
18856
  name,
@@ -18408,26 +18858,28 @@ async function _initDataset(project_name, {
18408
18858
  version
18409
18859
  } = {}) {
18410
18860
  const args = {
18411
- org_id: ORG_ID,
18861
+ org_id: _state.orgId,
18412
18862
  project_name,
18413
18863
  dataset_name: name,
18414
18864
  description
18415
18865
  };
18416
- const response = await api_conn().post_json("api/dataset/register", args);
18866
+ const response = await _state.apiConn().post_json("api/dataset/register", args);
18417
18867
  const project = response.project;
18418
18868
  const dataset = response.dataset;
18419
- const user_id = (await _user_info())["id"];
18869
+ const user_id = (await _state.userInfo())["id"];
18420
18870
  return new Dataset(project, dataset.id, dataset.name, user_id, version);
18421
18871
  }
18422
18872
  var Dataset = class {
18423
18873
  constructor(project, id, name, user_id, pinnedVersion) {
18424
18874
  this._fetchedData = void 0;
18875
+ this.finished = false;
18425
18876
  this.project = project;
18426
18877
  this.id = id;
18427
18878
  this.name = name;
18428
18879
  this.user_id = user_id;
18429
18880
  this.pinnedVersion = pinnedVersion;
18430
18881
  this.logger = new LogThread();
18882
+ unterminatedObjects.addUnterminated(this, isomorph_default.getCallerLocation());
18431
18883
  }
18432
18884
  /**
18433
18885
  * Insert a single record to the dataset. The record will be batched and uploaded behind the scenes. If you pass in an `id`,
@@ -18449,6 +18901,7 @@ var Dataset = class {
18449
18901
  metadata,
18450
18902
  id
18451
18903
  }) {
18904
+ this.checkNotFinished();
18452
18905
  if (metadata !== void 0) {
18453
18906
  for (const key of Object.keys(metadata)) {
18454
18907
  if (typeof key !== "string") {
@@ -18470,6 +18923,7 @@ var Dataset = class {
18470
18923
  return args.id;
18471
18924
  }
18472
18925
  delete(id) {
18926
+ this.checkNotFinished();
18473
18927
  const user_id = this.user_id;
18474
18928
  const args = {
18475
18929
  id,
@@ -18489,15 +18943,16 @@ var Dataset = class {
18489
18943
  * @returns A summary of the dataset.
18490
18944
  */
18491
18945
  async summarize(options = {}) {
18946
+ this.checkNotFinished();
18492
18947
  let { summarizeData = true } = options || {};
18493
18948
  await this.logger.flush();
18494
- const projectUrl = `${API_URL}/app/${encodeURIComponent(
18495
- ORG_NAME
18949
+ const projectUrl = `${_state.apiUrl}/app/${encodeURIComponent(
18950
+ _state.orgName
18496
18951
  )}/p/${encodeURIComponent(this.project.name)}`;
18497
18952
  const datasetUrl = `${projectUrl}/d/${encodeURIComponent(this.name)}`;
18498
18953
  let dataSummary = void 0;
18499
18954
  if (summarizeData) {
18500
- dataSummary = await log_conn().get_json(
18955
+ dataSummary = await _state.logConn().get_json(
18501
18956
  "dataset-summary",
18502
18957
  {
18503
18958
  dataset_id: this.id
@@ -18532,6 +18987,7 @@ var Dataset = class {
18532
18987
  * @returns An iterator over the dataset's records.
18533
18988
  */
18534
18989
  async *fetch() {
18990
+ this.checkNotFinished();
18535
18991
  const records = await this.fetchedData();
18536
18992
  for (const record of records) {
18537
18993
  yield {
@@ -18555,11 +19011,13 @@ var Dataset = class {
18555
19011
  * ```
18556
19012
  */
18557
19013
  [Symbol.asyncIterator]() {
19014
+ this.checkNotFinished();
18558
19015
  return this.fetch();
18559
19016
  }
18560
19017
  async fetchedData() {
19018
+ this.checkNotFinished();
18561
19019
  if (this._fetchedData === void 0) {
18562
- const resp = await log_conn().get("object/dataset", {
19020
+ const resp = await _state.logConn().get("object/dataset", {
18563
19021
  id: this.id,
18564
19022
  fmt: "json",
18565
19023
  version: this.pinnedVersion
@@ -18570,9 +19028,11 @@ var Dataset = class {
18570
19028
  return this._fetchedData || [];
18571
19029
  }
18572
19030
  clearCache() {
19031
+ this.checkNotFinished();
18573
19032
  this._fetchedData = void 0;
18574
19033
  }
18575
19034
  async version() {
19035
+ this.checkNotFinished();
18576
19036
  if (this.pinnedVersion !== void 0) {
18577
19037
  return this.pinnedVersion;
18578
19038
  } else {
@@ -18587,6 +19047,25 @@ var Dataset = class {
18587
19047
  return maxVersion;
18588
19048
  }
18589
19049
  }
19050
+ /**
19051
+ * Terminate connection to the dataset and return its id. After calling close, you may not invoke any further methods on the dataset object.
19052
+ *
19053
+ * Will be invoked automatically if the dataset is bound as a context manager.
19054
+ *
19055
+ * @returns The dataset id.
19056
+ */
19057
+ async close() {
19058
+ this.checkNotFinished();
19059
+ await this.logger.flush();
19060
+ this.finished = true;
19061
+ unterminatedObjects.removeUnterminated(this);
19062
+ return this.id;
19063
+ }
19064
+ checkNotFinished() {
19065
+ if (this.finished) {
19066
+ throw new Error("Cannot invoke method on finished dataset");
19067
+ }
19068
+ }
18590
19069
  };
18591
19070
 
18592
19071
  // src/framework.ts
@@ -18605,12 +19084,23 @@ configureNode();
18605
19084
  Dataset,
18606
19085
  Eval,
18607
19086
  Experiment,
19087
+ NoopSpan,
18608
19088
  Project,
19089
+ SpanImpl,
19090
+ _internalGetGlobalState,
19091
+ currentExperiment,
19092
+ currentSpan,
18609
19093
  init,
18610
19094
  initDataset,
18611
19095
  log,
18612
19096
  login,
18613
- summarize
19097
+ noopSpan,
19098
+ startSpan,
19099
+ summarize,
19100
+ traced,
19101
+ withCurrent,
19102
+ withDataset,
19103
+ withExperiment
18614
19104
  });
18615
19105
  /*! Bundled license information:
18616
19106