braintrust 0.0.85 → 0.0.87

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/cli.js CHANGED
@@ -595,9 +595,9 @@ var require_reusify = __commonJS({
595
595
  }
596
596
  });
597
597
 
598
- // ../../node_modules/.pnpm/fastq@1.15.0/node_modules/fastq/queue.js
598
+ // ../../node_modules/.pnpm/fastq@1.16.0/node_modules/fastq/queue.js
599
599
  var require_queue = __commonJS({
600
- "../../node_modules/.pnpm/fastq@1.15.0/node_modules/fastq/queue.js"(exports2, module2) {
600
+ "../../node_modules/.pnpm/fastq@1.16.0/node_modules/fastq/queue.js"(exports2, module2) {
601
601
  "use strict";
602
602
  var reusify = require_reusify();
603
603
  function fastqueue(context2, worker, concurrency) {
@@ -696,6 +696,7 @@ var require_queue = __commonJS({
696
696
  current.release = release;
697
697
  current.value = value;
698
698
  current.callback = done || noop;
699
+ current.errorHandler = errorHandler;
699
700
  if (_running === self.concurrency || self.paused) {
700
701
  if (queueHead) {
701
702
  current.next = queueHead;
@@ -9064,7 +9065,7 @@ var require_package = __commonJS({
9064
9065
  "package.json"(exports2, module2) {
9065
9066
  module2.exports = {
9066
9067
  name: "braintrust",
9067
- version: "0.0.85",
9068
+ version: "0.0.87",
9068
9069
  description: "SDK for integrating Braintrust",
9069
9070
  main: "./dist/index.js",
9070
9071
  browser: {
@@ -10507,7 +10508,7 @@ var v4_default = v4;
10507
10508
  // src/cli.ts
10508
10509
  var import_pluralize2 = __toESM(require_pluralize());
10509
10510
 
10510
- // ../../node_modules/.pnpm/@braintrust+core@0.0.7/node_modules/@braintrust/core/dist/index.mjs
10511
+ // ../core/js/dist/index.mjs
10511
10512
  var TRANSACTION_ID_FIELD = "_xact_id";
10512
10513
  var IS_MERGE_FIELD = "_is_merge";
10513
10514
  function mergeDicts(mergeInto, mergeFrom) {
@@ -10619,12 +10620,12 @@ var NoopSpan = class {
10619
10620
  }
10620
10621
  log(_) {
10621
10622
  }
10622
- startSpan(_0, _1) {
10623
- return this;
10624
- }
10625
- traced(_0, callback, _1) {
10623
+ traced(callback, _1) {
10626
10624
  return callback(this);
10627
10625
  }
10626
+ startSpan(_1) {
10627
+ return this;
10628
+ }
10628
10629
  end(args) {
10629
10630
  return args?.endTime ?? getCurrentUnixTimestamp();
10630
10631
  }
@@ -10632,16 +10633,25 @@ var NoopSpan = class {
10632
10633
  return this.end(args);
10633
10634
  }
10634
10635
  };
10635
- var noopSpan = new NoopSpan();
10636
+ var NOOP_SPAN = new NoopSpan();
10636
10637
  var BraintrustState = class {
10637
10638
  constructor() {
10639
+ this.apiUrl = null;
10640
+ this.loginToken = null;
10641
+ this.orgId = null;
10642
+ this.orgName = null;
10643
+ this.logUrl = null;
10644
+ this.loggedIn = false;
10645
+ this._apiConn = null;
10646
+ this._logConn = null;
10638
10647
  this.id = v4_default();
10639
- this.currentExperiment = isomorph_default.newAsyncLocalStorage();
10640
- this.currentLogger = isomorph_default.newAsyncLocalStorage();
10648
+ this.currentExperiment = void 0;
10649
+ this.currentLogger = void 0;
10641
10650
  this.currentSpan = isomorph_default.newAsyncLocalStorage();
10642
- if (this.currentSpan.enterWith) {
10643
- this.currentSpan.enterWith(noopSpan);
10644
- }
10651
+ this.resetLoginInfo();
10652
+ globalThis.__inherited_braintrust_state = this;
10653
+ }
10654
+ resetLoginInfo() {
10645
10655
  this.apiUrl = null;
10646
10656
  this.loginToken = null;
10647
10657
  this.orgId = null;
@@ -10650,7 +10660,6 @@ var BraintrustState = class {
10650
10660
  this.loggedIn = false;
10651
10661
  this._apiConn = null;
10652
10662
  this._logConn = null;
10653
- globalThis.__inherited_braintrust_state = this;
10654
10663
  }
10655
10664
  apiConn() {
10656
10665
  if (!this._apiConn) {
@@ -10679,36 +10688,6 @@ function _internalSetInitialState() {
10679
10688
  _state = globalThis.__inherited_braintrust_state || new BraintrustState();
10680
10689
  }
10681
10690
  var _internalGetGlobalState = () => _state;
10682
- var UnterminatedObjectsHandler = class {
10683
- constructor() {
10684
- this.unterminatedObjects = /* @__PURE__ */ new Map();
10685
- isomorph_default.processOn("exit", () => {
10686
- this.warnUnterminated();
10687
- });
10688
- }
10689
- addUnterminated(obj, createdLocation) {
10690
- this.unterminatedObjects.set(obj, createdLocation);
10691
- }
10692
- removeUnterminated(obj) {
10693
- this.unterminatedObjects.delete(obj);
10694
- }
10695
- warnUnterminated() {
10696
- if (this.unterminatedObjects.size === 0) {
10697
- return;
10698
- }
10699
- let warningMessage = "WARNING: Did not close the following braintrust objects. We recommend running `.close` on the listed objects, or by running them inside a callback so they are closed automatically:";
10700
- this.unterminatedObjects.forEach((createdLocation, obj) => {
10701
- let msg = `
10702
- Object of type ${obj?.constructor?.name}`;
10703
- if (createdLocation) {
10704
- msg += ` created at ${JSON.stringify(createdLocation)}`;
10705
- }
10706
- warningMessage += msg;
10707
- });
10708
- console.warn(warningMessage);
10709
- }
10710
- };
10711
- var unterminatedObjects = new UnterminatedObjectsHandler();
10712
10691
  var FailedHTTPResponse = class extends Error {
10713
10692
  constructor(status, text, data = null) {
10714
10693
  super(`${status}: ${text}`);
@@ -10825,10 +10804,11 @@ function now() {
10825
10804
  return (/* @__PURE__ */ new Date()).getTime();
10826
10805
  }
10827
10806
  var BackgroundLogger = class {
10828
- constructor() {
10807
+ constructor(logConn) {
10829
10808
  this.items = [];
10830
10809
  this.active_flush = Promise.resolve([]);
10831
10810
  this.active_flush_resolved = true;
10811
+ this.logConn = logConn;
10832
10812
  isomorph_default.processOn("beforeExit", async () => {
10833
10813
  await this.flush();
10834
10814
  });
@@ -10842,8 +10822,9 @@ var BackgroundLogger = class {
10842
10822
  }
10843
10823
  async flush_once(batchSize = DefaultBatchSize) {
10844
10824
  this.active_flush_resolved = false;
10845
- const allItems = mergeRowBatch(this.items || []).reverse();
10825
+ const itemPromises = this.items;
10846
10826
  this.items = [];
10827
+ const allItems = mergeRowBatch(await Promise.all(itemPromises)).reverse();
10847
10828
  let postPromises = [];
10848
10829
  while (true) {
10849
10830
  const items = [];
@@ -10868,7 +10849,7 @@ var BackgroundLogger = class {
10868
10849
  for (let i = 0; i < NumRetries; i++) {
10869
10850
  const startTime = now();
10870
10851
  try {
10871
- return (await _state.logConn().post_json("logs", itemsS)).map(
10852
+ return (await (await this.logConn).post_json("logs", itemsS)).map(
10872
10853
  (res) => res.id
10873
10854
  );
10874
10855
  } catch (e) {
@@ -10909,7 +10890,7 @@ var BackgroundLogger = class {
10909
10890
  }
10910
10891
  }
10911
10892
  };
10912
- async function init(project, options = {}) {
10893
+ function init(project, options = {}) {
10913
10894
  const {
10914
10895
  experiment,
10915
10896
  description,
@@ -10920,40 +10901,90 @@ async function init(project, options = {}) {
10920
10901
  apiUrl,
10921
10902
  apiKey,
10922
10903
  orgName,
10923
- disableCache,
10924
10904
  metadata
10925
10905
  } = options || {};
10926
- await login({
10927
- orgName,
10928
- disableCache,
10929
- apiKey,
10930
- apiUrl
10931
- });
10932
- return await _initExperiment(project, {
10933
- experimentName: experiment,
10934
- description,
10935
- dataset,
10936
- update,
10937
- baseExperiment,
10938
- isPublic,
10939
- metadata
10940
- });
10906
+ const lazyMetadata = (async () => {
10907
+ await login({
10908
+ orgName,
10909
+ apiKey,
10910
+ apiUrl
10911
+ });
10912
+ const args = {
10913
+ project_name: project,
10914
+ org_id: _state.orgId
10915
+ };
10916
+ if (experiment) {
10917
+ args["experiment_name"] = experiment;
10918
+ }
10919
+ if (description) {
10920
+ args["description"] = description;
10921
+ }
10922
+ if (update) {
10923
+ args["update"] = update;
10924
+ }
10925
+ const repoStatus = await isomorph_default.getRepoStatus();
10926
+ if (repoStatus) {
10927
+ args["repo_info"] = repoStatus;
10928
+ }
10929
+ if (baseExperiment) {
10930
+ args["base_experiment"] = baseExperiment;
10931
+ } else {
10932
+ args["ancestor_commits"] = await isomorph_default.getPastNAncestors();
10933
+ }
10934
+ if (dataset !== void 0) {
10935
+ args["dataset_id"] = dataset.id;
10936
+ args["dataset_version"] = await dataset.version();
10937
+ }
10938
+ if (isPublic !== void 0) {
10939
+ args["public"] = isPublic;
10940
+ }
10941
+ if (metadata) {
10942
+ args["metadata"] = metadata;
10943
+ }
10944
+ let response = null;
10945
+ while (true) {
10946
+ try {
10947
+ response = await _state.apiConn().post_json("api/experiment/register", args);
10948
+ break;
10949
+ } catch (e) {
10950
+ if (args["base_experiment"] && `${"data" in e && e.data}`.includes("base experiment")) {
10951
+ console.warn(`Base experiment ${args["base_experiment"]} not found.`);
10952
+ delete args["base_experiment"];
10953
+ } else {
10954
+ throw e;
10955
+ }
10956
+ }
10957
+ }
10958
+ return {
10959
+ project: {
10960
+ id: response.project.id,
10961
+ name: response.project.name,
10962
+ fullInfo: response.project
10963
+ },
10964
+ experiment: {
10965
+ id: response.experiment.id,
10966
+ name: response.experiment.name,
10967
+ fullInfo: response.experiment
10968
+ }
10969
+ };
10970
+ })();
10971
+ const ret = new Experiment(lazyMetadata, dataset);
10972
+ if (options.setCurrent ?? true) {
10973
+ _state.currentExperiment = ret;
10974
+ }
10975
+ return ret;
10941
10976
  }
10942
10977
  async function login(options = {}) {
10943
10978
  const {
10944
10979
  apiUrl = isomorph_default.getEnv("BRAINTRUST_API_URL") || "https://www.braintrustdata.com",
10945
10980
  apiKey = isomorph_default.getEnv("BRAINTRUST_API_KEY"),
10946
- orgName = isomorph_default.getEnv("BRAINTRUST_ORG_NAME"),
10947
- disableCache = false
10981
+ orgName = isomorph_default.getEnv("BRAINTRUST_ORG_NAME")
10948
10982
  } = options || {};
10949
10983
  let { forceLogin = false } = options || {};
10950
- if (apiUrl != _state.apiUrl || apiKey !== void 0 && HTTPConnection.sanitize_token(apiKey) != _state.loginToken || orgName !== void 0 && orgName != _state.orgName) {
10951
- forceLogin = true;
10952
- }
10953
10984
  if (_state.loggedIn && !forceLogin) {
10954
10985
  return;
10955
10986
  }
10956
- _state = new BraintrustState();
10987
+ _state.resetLoginInfo();
10957
10988
  _state.apiUrl = apiUrl;
10958
10989
  let conn = null;
10959
10990
  if (apiKey !== void 0) {
@@ -10985,64 +11016,8 @@ async function login(options = {}) {
10985
11016
  _state.loginToken = conn.token;
10986
11017
  _state.loggedIn = true;
10987
11018
  }
10988
- function currentExperiment() {
10989
- return _state.currentExperiment.getStore();
10990
- }
10991
- function currentLogger() {
10992
- return _state.currentLogger.getStore();
10993
- }
10994
- function currentSpan() {
10995
- return _state.currentSpan.getStore() ?? noopSpan;
10996
- }
10997
- function startSpan(args) {
10998
- const { name: nameOpt, ...argsRest } = args ?? {};
10999
- const name = (nameOpt ?? isomorph_default.getCallerLocation()?.caller_functionname) || "root";
11000
- const parentSpan = currentSpan();
11001
- if (!parentSpan) {
11002
- throw new Error(
11003
- "Cannot call startSpan() from outside a trace. Please wrap this code in a traced() callback."
11004
- );
11005
- }
11006
- if (!Object.is(parentSpan, noopSpan)) {
11007
- return parentSpan.startSpan(name, argsRest);
11008
- }
11009
- const experiment = currentExperiment();
11010
- if (experiment) {
11011
- return experiment.startSpan({ name, ...argsRest });
11012
- }
11013
- const logger = currentLogger();
11014
- if (logger) {
11015
- throw new Error(
11016
- "Cannot start a span within a logger from startSpan(). Use logger.startSpan() instead."
11017
- );
11018
- }
11019
- return noopSpan;
11020
- }
11021
- function traced(callback, args) {
11022
- const span = startSpan(args);
11023
- return runFinally(
11024
- () => {
11025
- if (args?.setCurrent ?? true) {
11026
- return withCurrent(span, () => callback(span));
11027
- } else {
11028
- return callback(span);
11029
- }
11030
- },
11031
- () => span.end()
11032
- );
11033
- }
11034
- function withCurrent(object, callback) {
11035
- if (object.kind === "experiment") {
11036
- return _state.currentExperiment.run(object, callback);
11037
- } else if (object.kind === "logger") {
11038
- return _state.currentLogger.run(object, callback);
11039
- } else if (object.kind === "span") {
11040
- return _state.currentSpan.run(object, callback);
11041
- } else {
11042
- throw new Error(
11043
- `Invalid object of type ${object.constructor.name}`
11044
- );
11045
- }
11019
+ function withCurrent(span, callback) {
11020
+ return _state.currentSpan.run(span, () => callback(span));
11046
11021
  }
11047
11022
  function _check_org_info(org_info, org_name) {
11048
11023
  if (org_info.length === 0) {
@@ -11130,83 +11105,34 @@ function validateAndSanitizeExperimentLogFullArgs(event, hasDataset) {
11130
11105
  }
11131
11106
  return event;
11132
11107
  }
11133
- async function _initExperiment(projectName, {
11134
- experimentName,
11135
- description,
11136
- dataset,
11137
- update,
11138
- baseExperiment,
11139
- isPublic,
11140
- metadata
11141
- } = {
11142
- experimentName: void 0,
11143
- description: void 0,
11144
- baseExperiment: void 0,
11145
- isPublic: false,
11146
- metadata: void 0
11147
- }) {
11148
- const args = {
11149
- project_name: projectName,
11150
- org_id: _state.orgId
11151
- };
11152
- if (experimentName) {
11153
- args["experiment_name"] = experimentName;
11154
- }
11155
- if (description) {
11156
- args["description"] = description;
11157
- }
11158
- if (update) {
11159
- args["update"] = update;
11160
- }
11161
- const repoStatus = await isomorph_default.getRepoStatus();
11162
- if (repoStatus) {
11163
- args["repo_info"] = repoStatus;
11164
- }
11165
- if (baseExperiment) {
11166
- args["base_experiment"] = baseExperiment;
11167
- } else {
11168
- args["ancestor_commits"] = await isomorph_default.getPastNAncestors();
11169
- }
11170
- if (dataset !== void 0) {
11171
- args["dataset_id"] = dataset.id;
11172
- args["dataset_version"] = await dataset.version();
11173
- }
11174
- if (isPublic !== void 0) {
11175
- args["public"] = isPublic;
11176
- }
11177
- if (metadata) {
11178
- args["metadata"] = metadata;
11179
- }
11180
- let response = null;
11181
- while (true) {
11182
- try {
11183
- response = await _state.apiConn().post_json("api/experiment/register", args);
11184
- break;
11185
- } catch (e) {
11186
- if (args["base_experiment"] && `${"data" in e && e.data}`.includes("base experiment")) {
11187
- console.warn(`Base experiment ${args["base_experiment"]} not found.`);
11188
- delete args["base_experiment"];
11189
- } else {
11190
- throw e;
11191
- }
11192
- }
11193
- }
11194
- const project = response.project;
11195
- const experiment = response.experiment;
11196
- return new Experiment(project, experiment.id, experiment.name, dataset);
11197
- }
11198
11108
  var Experiment = class {
11199
- constructor(project, id, name, dataset) {
11109
+ constructor(lazyMetadata, dataset) {
11200
11110
  // For type identification.
11201
11111
  this.kind = "experiment";
11202
- this.finished = false;
11203
- this.project = project;
11204
- this.id = id;
11205
- this.name = name;
11112
+ this.lazyMetadata = lazyMetadata;
11206
11113
  this.dataset = dataset;
11207
- this.bgLogger = new BackgroundLogger();
11114
+ const logConn = this.getState().then((state) => state.logConn());
11115
+ this.bgLogger = new BackgroundLogger(logConn);
11208
11116
  this.lastStartTime = getCurrentUnixTimestamp();
11209
- unterminatedObjects.addUnterminated(this, isomorph_default.getCallerLocation());
11117
+ }
11118
+ get id() {
11119
+ return (async () => {
11120
+ return (await this.lazyMetadata).experiment.id;
11121
+ })();
11122
+ }
11123
+ get name() {
11124
+ return (async () => {
11125
+ return (await this.lazyMetadata).experiment.name;
11126
+ })();
11127
+ }
11128
+ get project() {
11129
+ return (async () => {
11130
+ return (await this.lazyMetadata).project;
11131
+ })();
11132
+ }
11133
+ async getState() {
11134
+ await this.lazyMetadata;
11135
+ return _state;
11210
11136
  }
11211
11137
  /**
11212
11138
  * Log a single event to the experiment. The event will be batched and uploaded behind the scenes.
@@ -11224,29 +11150,15 @@ var Experiment = class {
11224
11150
  * :returns: The `id` of the logged event.
11225
11151
  */
11226
11152
  log(event) {
11227
- this.checkNotFinished();
11228
11153
  event = validateAndSanitizeExperimentLogFullArgs(event, !!this.dataset);
11229
11154
  const span = this.startSpan({ startTime: this.lastStartTime, event });
11230
11155
  this.lastStartTime = span.end();
11231
11156
  return span.id;
11232
11157
  }
11233
11158
  /**
11234
- * Create a new toplevel span. The name parameter is optional and defaults to "root".
11159
+ * Create a new toplevel span underneath the experiment. The name defaults to "root".
11235
11160
  *
11236
- * See `Span.startSpan` for full details.
11237
- */
11238
- startSpan(args) {
11239
- this.checkNotFinished();
11240
- const { name, ...argsRest } = args ?? {};
11241
- return new SpanImpl({
11242
- bgLogger: this.bgLogger,
11243
- name: name ?? "root",
11244
- ...argsRest,
11245
- rootExperiment: this
11246
- });
11247
- }
11248
- /**
11249
- * Wrapper over `Experiment.startSpan`, which passes the initialized `Span` it to the given callback and ends it afterwards. See `Span.traced` for full details.
11161
+ * See `Span.traced` for full details.
11250
11162
  */
11251
11163
  traced(callback, args) {
11252
11164
  const { setCurrent, ...argsRest } = args ?? {};
@@ -11254,7 +11166,7 @@ var Experiment = class {
11254
11166
  return runFinally(
11255
11167
  () => {
11256
11168
  if (setCurrent ?? true) {
11257
- return withCurrent(span, () => callback(span));
11169
+ return withCurrent(span, callback);
11258
11170
  } else {
11259
11171
  return callback(span);
11260
11172
  }
@@ -11262,6 +11174,25 @@ var Experiment = class {
11262
11174
  () => span.end()
11263
11175
  );
11264
11176
  }
11177
+ /**
11178
+ * Lower-level alternative to `traced`, which does not automatically end the span or mark it as current.
11179
+ *
11180
+ * See `traced` for full details.
11181
+ */
11182
+ startSpan(args) {
11183
+ const { name, ...argsRest } = args ?? {};
11184
+ const parentIds = (async () => ({
11185
+ kind: "experiment",
11186
+ project_id: (await this.project).id,
11187
+ experiment_id: await this.id
11188
+ }))();
11189
+ return new SpanImpl({
11190
+ parentIds,
11191
+ bgLogger: this.bgLogger,
11192
+ name: name ?? "root",
11193
+ ...argsRest
11194
+ });
11195
+ }
11265
11196
  /**
11266
11197
  * Summarize the experiment, including the scores (compared to the closest reference experiment) and metadata.
11267
11198
  *
@@ -11273,18 +11204,21 @@ var Experiment = class {
11273
11204
  async summarize(options = {}) {
11274
11205
  let { summarizeScores = true, comparisonExperimentId = void 0 } = options || {};
11275
11206
  await this.bgLogger.flush();
11276
- const projectUrl = `${_state.apiUrl}/app/${encodeURIComponent(
11277
- _state.orgName
11278
- )}/p/${encodeURIComponent(this.project.name)}`;
11279
- const experimentUrl = `${projectUrl}/${encodeURIComponent(this.name)}`;
11207
+ const state = await this.getState();
11208
+ const projectUrl = `${state.apiUrl}/app/${encodeURIComponent(
11209
+ state.orgName
11210
+ )}/p/${encodeURIComponent((await this.project).name)}`;
11211
+ const experimentUrl = `${projectUrl}/${encodeURIComponent(
11212
+ await this.name
11213
+ )}`;
11280
11214
  let scores = void 0;
11281
11215
  let metrics = void 0;
11282
11216
  let comparisonExperimentName = void 0;
11283
11217
  if (summarizeScores) {
11284
11218
  if (comparisonExperimentId === void 0) {
11285
- const conn = _state.logConn();
11219
+ const conn = state.logConn();
11286
11220
  const resp = await conn.get("/crud/base_experiments", {
11287
- id: this.id
11221
+ id: await this.id
11288
11222
  });
11289
11223
  const base_experiments = await resp.json();
11290
11224
  if (base_experiments.length > 0) {
@@ -11293,10 +11227,10 @@ var Experiment = class {
11293
11227
  }
11294
11228
  }
11295
11229
  if (comparisonExperimentId !== void 0) {
11296
- const results = await _state.logConn().get_json(
11230
+ const results = await state.logConn().get_json(
11297
11231
  "/experiment-comparison2",
11298
11232
  {
11299
- experiment_id: this.id,
11233
+ experiment_id: await this.id,
11300
11234
  base_experiment_id: comparisonExperimentId
11301
11235
  },
11302
11236
  3
@@ -11306,8 +11240,8 @@ var Experiment = class {
11306
11240
  }
11307
11241
  }
11308
11242
  return {
11309
- projectName: this.project.name,
11310
- experimentName: this.name,
11243
+ projectName: (await this.project).name,
11244
+ experimentName: await this.name,
11311
11245
  projectUrl,
11312
11246
  experimentUrl,
11313
11247
  comparisonExperimentName,
@@ -11316,118 +11250,99 @@ var Experiment = class {
11316
11250
  };
11317
11251
  }
11318
11252
  /**
11319
- * Finish the experiment and return its id. After calling close, you may not invoke any further methods on the experiment object.
11320
- *
11321
- * Will be invoked automatically if the experiment is wrapped in a callback passed to `braintrust.withExperiment`.
11322
- *
11323
- * @returns The experiment id.
11253
+ * Flush any pending rows to the server.
11254
+ */
11255
+ async flush() {
11256
+ return await this.bgLogger.flush();
11257
+ }
11258
+ /**
11259
+ * This function is deprecated. You can simply remove it from your code.
11324
11260
  */
11325
11261
  async close() {
11326
- this.checkNotFinished();
11327
- await this.bgLogger.flush();
11328
- this.finished = true;
11329
- unterminatedObjects.removeUnterminated(this);
11262
+ console.warn(
11263
+ "close is deprecated and will be removed in a future version of braintrust. It is now a no-op and can be removed"
11264
+ );
11330
11265
  return this.id;
11331
11266
  }
11332
- checkNotFinished() {
11333
- if (this.finished) {
11334
- throw new Error("Cannot invoke method on finished experiment");
11335
- }
11336
- }
11337
11267
  };
11338
11268
  var SpanImpl = class _SpanImpl {
11339
11269
  // root_experiment should only be specified for a root span. parent_span
11340
11270
  // should only be specified for non-root spans.
11341
11271
  constructor(args) {
11342
11272
  this.kind = "span";
11343
- this.finished = false;
11344
11273
  this.loggedEndTime = void 0;
11345
11274
  this.bgLogger = args.bgLogger;
11346
11275
  const callerLocation = isomorph_default.getCallerLocation();
11276
+ const name = (() => {
11277
+ if (args.name)
11278
+ return args.name;
11279
+ if (callerLocation) {
11280
+ const pathComponents = callerLocation.caller_filename.split("/");
11281
+ const filename = pathComponents[pathComponents.length - 1];
11282
+ return [callerLocation.caller_functionname].concat(
11283
+ filename ? [`${filename}:${callerLocation.caller_lineno}`] : []
11284
+ ).join(":");
11285
+ }
11286
+ return "subspan";
11287
+ })();
11347
11288
  this.internalData = {
11348
11289
  metrics: {
11349
11290
  start: args.startTime ?? getCurrentUnixTimestamp(),
11350
11291
  ...callerLocation
11351
11292
  },
11352
- span_attributes: { ...args.spanAttributes, name: args.name },
11293
+ span_attributes: { ...args.spanAttributes, name },
11353
11294
  created: (/* @__PURE__ */ new Date()).toISOString()
11354
11295
  };
11296
+ this.parentIds = args.parentIds;
11355
11297
  const id = args.event?.id ?? v4_default();
11356
11298
  const span_id = v4_default();
11357
- if ("rootExperiment" in args) {
11358
- this._object_info = {
11359
- id,
11360
- span_id,
11361
- root_span_id: span_id,
11362
- project_id: args.rootExperiment.project.id,
11363
- experiment_id: args.rootExperiment.id
11364
- };
11365
- } else if ("rootProject" in args) {
11366
- this._object_info = {
11367
- id,
11368
- span_id,
11369
- root_span_id: span_id,
11370
- org_id: _state.orgId,
11371
- project_id: args.rootProject.id,
11372
- log_id: "g"
11373
- };
11374
- } else if ("parentSpan" in args) {
11375
- this._object_info = {
11376
- ...args.parentSpan._object_info,
11377
- id,
11378
- span_id
11379
- };
11380
- this.internalData.span_parents = [args.parentSpan.span_id];
11381
- } else {
11382
- throw new Error("Must provide either 'rootExperiment' or 'parentSpan'");
11299
+ this.rowIds = {
11300
+ id,
11301
+ span_id,
11302
+ root_span_id: args.parentSpanInfo?.root_span_id ?? span_id
11303
+ };
11304
+ if (args.parentSpanInfo) {
11305
+ this.internalData.span_parents = [args.parentSpanInfo.span_id];
11383
11306
  }
11384
11307
  this.isMerge = false;
11385
11308
  const { id: _id, ...eventRest } = args.event ?? {};
11386
11309
  this.log(eventRest);
11387
11310
  this.isMerge = true;
11388
- unterminatedObjects.addUnterminated(this, callerLocation);
11389
11311
  }
11390
11312
  get id() {
11391
- return this._object_info.id;
11313
+ return this.rowIds.id;
11392
11314
  }
11393
11315
  get span_id() {
11394
- return this._object_info.span_id;
11316
+ return this.rowIds.span_id;
11395
11317
  }
11396
11318
  get root_span_id() {
11397
- return this._object_info.root_span_id;
11319
+ return this.rowIds.root_span_id;
11398
11320
  }
11399
11321
  log(event) {
11400
- this.checkNotFinished();
11401
11322
  const sanitized = validateAndSanitizeExperimentLogPartialArgs(event);
11402
11323
  const sanitizedAndInternalData = { ...this.internalData };
11403
11324
  mergeDicts(sanitizedAndInternalData, sanitized);
11404
- const record = {
11405
- ...sanitizedAndInternalData,
11406
- ...this._object_info,
11407
- [IS_MERGE_FIELD]: this.isMerge
11408
- };
11409
- if (record.metrics?.end) {
11410
- this.loggedEndTime = record.metrics?.end;
11411
- }
11412
11325
  this.internalData = {};
11326
+ if (sanitizedAndInternalData.metrics?.end) {
11327
+ this.loggedEndTime = sanitizedAndInternalData.metrics?.end;
11328
+ }
11329
+ const record = (async () => {
11330
+ return {
11331
+ ...sanitizedAndInternalData,
11332
+ ...this.rowIds,
11333
+ ...await this.parentIds,
11334
+ [IS_MERGE_FIELD]: this.isMerge
11335
+ };
11336
+ })();
11413
11337
  this.bgLogger.log([record]);
11414
11338
  }
11415
- startSpan(name, args) {
11416
- this.checkNotFinished();
11417
- return new _SpanImpl({
11418
- bgLogger: this.bgLogger,
11419
- name,
11420
- ...args,
11421
- parentSpan: this
11422
- });
11423
- }
11424
- traced(name, callback, args) {
11339
+ traced(callback, args) {
11425
11340
  const { setCurrent, ...argsRest } = args ?? {};
11426
- const span = this.startSpan(name, argsRest);
11341
+ const span = this.startSpan(argsRest);
11427
11342
  return runFinally(
11428
11343
  () => {
11429
11344
  if (setCurrent ?? true) {
11430
- return withCurrent(span, () => callback(span));
11345
+ return withCurrent(span, callback);
11431
11346
  } else {
11432
11347
  return callback(span);
11433
11348
  }
@@ -11435,8 +11350,18 @@ var SpanImpl = class _SpanImpl {
11435
11350
  () => span.end()
11436
11351
  );
11437
11352
  }
11353
+ startSpan(args) {
11354
+ return new _SpanImpl({
11355
+ parentIds: this.parentIds,
11356
+ bgLogger: this.bgLogger,
11357
+ parentSpanInfo: {
11358
+ span_id: this.rowIds.span_id,
11359
+ root_span_id: this.rowIds.root_span_id
11360
+ },
11361
+ ...args
11362
+ });
11363
+ }
11438
11364
  end(args) {
11439
- this.checkNotFinished();
11440
11365
  let endTime;
11441
11366
  if (!this.loggedEndTime) {
11442
11367
  endTime = args?.endTime ?? getCurrentUnixTimestamp();
@@ -11445,29 +11370,38 @@ var SpanImpl = class _SpanImpl {
11445
11370
  endTime = this.loggedEndTime;
11446
11371
  }
11447
11372
  this.log({});
11448
- this.finished = true;
11449
- unterminatedObjects.removeUnterminated(this);
11450
11373
  return endTime;
11451
11374
  }
11452
11375
  close(args) {
11453
11376
  return this.end(args);
11454
11377
  }
11455
- checkNotFinished() {
11456
- if (this.finished) {
11457
- throw new Error("Cannot invoke method on finished span");
11458
- }
11459
- }
11460
11378
  };
11461
11379
  var Dataset = class {
11462
- constructor(project, id, name, pinnedVersion) {
11380
+ constructor(lazyMetadata, pinnedVersion) {
11463
11381
  this._fetchedData = void 0;
11464
- this.finished = false;
11465
- this.project = project;
11466
- this.id = id;
11467
- this.name = name;
11382
+ this.lazyMetadata = lazyMetadata;
11468
11383
  this.pinnedVersion = pinnedVersion;
11469
- this.logger = new BackgroundLogger();
11470
- unterminatedObjects.addUnterminated(this, isomorph_default.getCallerLocation());
11384
+ const logConn = this.getState().then((state) => state.logConn());
11385
+ this.bgLogger = new BackgroundLogger(logConn);
11386
+ }
11387
+ get id() {
11388
+ return (async () => {
11389
+ return (await this.lazyMetadata).dataset.id;
11390
+ })();
11391
+ }
11392
+ get name() {
11393
+ return (async () => {
11394
+ return (await this.lazyMetadata).dataset.name;
11395
+ })();
11396
+ }
11397
+ get project() {
11398
+ return (async () => {
11399
+ return (await this.lazyMetadata).project;
11400
+ })();
11401
+ }
11402
+ async getState() {
11403
+ await this.lazyMetadata;
11404
+ return _state;
11471
11405
  }
11472
11406
  /**
11473
11407
  * Insert a single record to the dataset. The record will be batched and uploaded behind the scenes. If you pass in an `id`,
@@ -11489,7 +11423,6 @@ var Dataset = class {
11489
11423
  metadata,
11490
11424
  id
11491
11425
  }) {
11492
- this.checkNotFinished();
11493
11426
  if (metadata !== void 0) {
11494
11427
  for (const key of Object.keys(metadata)) {
11495
11428
  if (typeof key !== "string") {
@@ -11497,29 +11430,29 @@ var Dataset = class {
11497
11430
  }
11498
11431
  }
11499
11432
  }
11500
- const args = {
11501
- id: id || v4_default(),
11433
+ const rowId = id || v4_default();
11434
+ const args = (async () => ({
11435
+ id: rowId,
11502
11436
  inputs: input,
11503
11437
  output,
11504
- project_id: this.project.id,
11505
- dataset_id: this.id,
11438
+ project_id: (await this.project).id,
11439
+ dataset_id: await this.id,
11506
11440
  created: (/* @__PURE__ */ new Date()).toISOString(),
11507
11441
  metadata
11508
- };
11509
- this.logger.log([args]);
11510
- return args.id;
11442
+ }))();
11443
+ this.bgLogger.log([args]);
11444
+ return rowId;
11511
11445
  }
11512
11446
  delete(id) {
11513
- this.checkNotFinished();
11514
- const args = {
11447
+ const args = (async () => ({
11515
11448
  id,
11516
- project_id: this.project.id,
11517
- dataset_id: this.id,
11449
+ project_id: (await this.project).id,
11450
+ dataset_id: await this.id,
11518
11451
  created: (/* @__PURE__ */ new Date()).toISOString(),
11519
11452
  _object_delete: true
11520
- };
11521
- this.logger.log([args]);
11522
- return args.id;
11453
+ }))();
11454
+ this.bgLogger.log([args]);
11455
+ return id;
11523
11456
  }
11524
11457
  /**
11525
11458
  * Summarize the dataset, including high level metrics about its size and other metadata.
@@ -11528,26 +11461,26 @@ var Dataset = class {
11528
11461
  * @returns A summary of the dataset.
11529
11462
  */
11530
11463
  async summarize(options = {}) {
11531
- this.checkNotFinished();
11532
11464
  let { summarizeData = true } = options || {};
11533
- await this.logger.flush();
11534
- const projectUrl = `${_state.apiUrl}/app/${encodeURIComponent(
11535
- _state.orgName
11536
- )}/p/${encodeURIComponent(this.project.name)}`;
11537
- const datasetUrl = `${projectUrl}/d/${encodeURIComponent(this.name)}`;
11465
+ await this.bgLogger.flush();
11466
+ const state = await this.getState();
11467
+ const projectUrl = `${state.apiUrl}/app/${encodeURIComponent(
11468
+ state.orgName
11469
+ )}/p/${encodeURIComponent((await this.project).name)}`;
11470
+ const datasetUrl = `${projectUrl}/d/${encodeURIComponent(await this.name)}`;
11538
11471
  let dataSummary = void 0;
11539
11472
  if (summarizeData) {
11540
- dataSummary = await _state.logConn().get_json(
11473
+ dataSummary = await state.logConn().get_json(
11541
11474
  "dataset-summary",
11542
11475
  {
11543
- dataset_id: this.id
11476
+ dataset_id: await this.id
11544
11477
  },
11545
11478
  3
11546
11479
  );
11547
11480
  }
11548
11481
  return {
11549
- projectName: this.project.name,
11550
- datasetName: this.name,
11482
+ projectName: (await this.project).name,
11483
+ datasetName: await this.name,
11551
11484
  projectUrl,
11552
11485
  datasetUrl,
11553
11486
  dataSummary
@@ -11572,7 +11505,6 @@ var Dataset = class {
11572
11505
  * @returns An iterator over the dataset's records.
11573
11506
  */
11574
11507
  async *fetch() {
11575
- this.checkNotFinished();
11576
11508
  const records = await this.fetchedData();
11577
11509
  for (const record of records) {
11578
11510
  yield {
@@ -11596,14 +11528,13 @@ var Dataset = class {
11596
11528
  * ```
11597
11529
  */
11598
11530
  [Symbol.asyncIterator]() {
11599
- this.checkNotFinished();
11600
11531
  return this.fetch();
11601
11532
  }
11602
11533
  async fetchedData() {
11603
- this.checkNotFinished();
11604
11534
  if (this._fetchedData === void 0) {
11605
- const resp = await _state.logConn().get("object/dataset", {
11606
- id: this.id,
11535
+ const state = await this.getState();
11536
+ const resp = await state.logConn().get("object/dataset", {
11537
+ id: await this.id,
11607
11538
  fmt: "json",
11608
11539
  version: this.pinnedVersion
11609
11540
  });
@@ -11613,11 +11544,9 @@ var Dataset = class {
11613
11544
  return this._fetchedData || [];
11614
11545
  }
11615
11546
  clearCache() {
11616
- this.checkNotFinished();
11617
11547
  this._fetchedData = void 0;
11618
11548
  }
11619
11549
  async version() {
11620
- this.checkNotFinished();
11621
11550
  if (this.pinnedVersion !== void 0) {
11622
11551
  return this.pinnedVersion;
11623
11552
  } else {
@@ -11633,24 +11562,20 @@ var Dataset = class {
11633
11562
  }
11634
11563
  }
11635
11564
  /**
11636
- * Terminate connection to the dataset and return its id. After calling close, you may not invoke any further methods on the dataset object.
11637
- *
11638
- * Will be invoked automatically if the dataset is bound as a context manager.
11639
- *
11640
- * @returns The dataset id.
11565
+ * Flush any pending rows to the server.
11566
+ */
11567
+ async flush() {
11568
+ return await this.bgLogger.flush();
11569
+ }
11570
+ /**
11571
+ * This function is deprecated. You can simply remove it from your code.
11641
11572
  */
11642
11573
  async close() {
11643
- this.checkNotFinished();
11644
- await this.logger.flush();
11645
- this.finished = true;
11646
- unterminatedObjects.removeUnterminated(this);
11574
+ console.warn(
11575
+ "close is deprecated and will be removed in a future version of braintrust. It is now a no-op and can be removed"
11576
+ );
11647
11577
  return this.id;
11648
11578
  }
11649
- checkNotFinished() {
11650
- if (this.finished) {
11651
- throw new Error("Cannot invoke method on finished dataset");
11652
- }
11653
- }
11654
11579
  };
11655
11580
 
11656
11581
  // src/progress.ts
@@ -11830,30 +11755,27 @@ async function runEvaluator(experiment, evaluator, progressReporter, filters) {
11830
11755
  let output = void 0;
11831
11756
  let error2 = void 0;
11832
11757
  let scores = {};
11833
- const callback = async () => {
11758
+ const callback = async (rootSpan) => {
11834
11759
  try {
11835
11760
  const meta = (o) => metadata = { ...metadata, ...o };
11836
- await traced(
11837
- async () => {
11838
- const outputResult = evaluator.task(datum.input, {
11839
- meta,
11840
- span: currentSpan()
11841
- });
11761
+ await rootSpan.traced(
11762
+ async (span) => {
11763
+ const outputResult = evaluator.task(datum.input, { meta, span });
11842
11764
  if (outputResult instanceof Promise) {
11843
11765
  output = await outputResult;
11844
11766
  } else {
11845
11767
  output = outputResult;
11846
11768
  }
11847
- currentSpan().log({ input: datum.input, output });
11769
+ span.log({ input: datum.input, output });
11848
11770
  },
11849
11771
  { name: "task" }
11850
11772
  );
11851
- currentSpan().log({ output });
11773
+ rootSpan.log({ output });
11852
11774
  const scoringArgs = { ...datum, metadata, output };
11853
11775
  const scoreResults = await Promise.all(
11854
11776
  evaluator.scores.map(async (score, score_idx) => {
11855
- return traced(
11856
- async () => {
11777
+ return rootSpan.traced(
11778
+ async (span) => {
11857
11779
  const scoreResult = score(scoringArgs);
11858
11780
  const result = scoreResult instanceof Promise ? await scoreResult : scoreResult;
11859
11781
  const {
@@ -11861,7 +11783,7 @@ async function runEvaluator(experiment, evaluator, progressReporter, filters) {
11861
11783
  name: _,
11862
11784
  ...resultRest
11863
11785
  } = result;
11864
- currentSpan().log({
11786
+ span.log({
11865
11787
  output: resultRest,
11866
11788
  metadata: resultMetadata
11867
11789
  });
@@ -11890,7 +11812,7 @@ async function runEvaluator(experiment, evaluator, progressReporter, filters) {
11890
11812
  if (Object.keys(scoreMetadata).length > 0) {
11891
11813
  meta({ scores: scoreMetadata });
11892
11814
  }
11893
- currentSpan().log({ scores, metadata });
11815
+ rootSpan.log({ scores, metadata });
11894
11816
  } catch (e) {
11895
11817
  error2 = e;
11896
11818
  } finally {
@@ -11903,17 +11825,16 @@ async function runEvaluator(experiment, evaluator, progressReporter, filters) {
11903
11825
  error: error2
11904
11826
  };
11905
11827
  };
11906
- const rootSpan = experiment ? experiment.startSpan({
11907
- name: "eval",
11908
- event: {
11909
- input: datum.input,
11910
- expected: datum.expected
11911
- }
11912
- }) : noopSpan;
11913
- try {
11914
- return await withCurrent(rootSpan, callback);
11915
- } finally {
11916
- rootSpan.end();
11828
+ if (!experiment) {
11829
+ return await callback(NOOP_SPAN);
11830
+ } else {
11831
+ return await experiment.traced(callback, {
11832
+ name: "eval",
11833
+ event: {
11834
+ input: datum.input,
11835
+ expected: datum.expected
11836
+ }
11837
+ });
11917
11838
  }
11918
11839
  });
11919
11840
  const results = await Promise.all(evals);
@@ -15920,10 +15841,14 @@ var simpleGit = gitInstanceFactory;
15920
15841
  // src/gitutil.ts
15921
15842
  var COMMON_BASE_BRANCHES = ["main", "master", "develop"];
15922
15843
  async function currentRepo() {
15923
- const git = simpleGit();
15924
- if (await git.checkIsRepo()) {
15925
- return git;
15926
- } else {
15844
+ try {
15845
+ const git = simpleGit();
15846
+ if (await git.checkIsRepo()) {
15847
+ return git;
15848
+ } else {
15849
+ return null;
15850
+ }
15851
+ } catch (e) {
15927
15852
  return null;
15928
15853
  }
15929
15854
  }
@@ -16309,7 +16234,7 @@ async function runOnce(handles, opts) {
16309
16234
  );
16310
16235
  } finally {
16311
16236
  if (logger) {
16312
- await logger.close();
16237
+ await logger.flush();
16313
16238
  }
16314
16239
  }
16315
16240
  });