braintrust 0.0.92 → 0.0.94

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -7681,7 +7681,9 @@ async function getBaseBranchAncestor(remote = void 0) {
7681
7681
  if (git === null) {
7682
7682
  throw new Error("Not in a git repo");
7683
7683
  }
7684
- const { remote: remoteName, branch: baseBranch } = await getBaseBranch(remote);
7684
+ const { remote: remoteName, branch: baseBranch } = await getBaseBranch(
7685
+ remote
7686
+ );
7685
7687
  const isDirty = (await git.diffSummary()).files.length > 0;
7686
7688
  const head = isDirty ? "HEAD" : "HEAD^";
7687
7689
  try {
@@ -7999,6 +8001,21 @@ function getCurrentUnixTimestamp() {
7999
8001
  function isEmpty(a) {
8000
8002
  return a === void 0 || a === null;
8001
8003
  }
8004
+ var LazyValue = class {
8005
+ constructor(callable) {
8006
+ this.value = {
8007
+ hasComputed: false
8008
+ };
8009
+ this.callable = callable;
8010
+ }
8011
+ async get() {
8012
+ if (this.value.hasComputed) {
8013
+ return this.value.val;
8014
+ }
8015
+ this.value = { hasComputed: true, val: await this.callable() };
8016
+ return this.value.val;
8017
+ }
8018
+ };
8002
8019
 
8003
8020
  // src/logger.ts
8004
8021
  var NoopSpan = class {
@@ -8028,7 +8045,7 @@ var NoopSpan = class {
8028
8045
  var NOOP_SPAN = new NoopSpan();
8029
8046
  var BraintrustState = class {
8030
8047
  constructor() {
8031
- this.apiUrl = null;
8048
+ this.appUrl = null;
8032
8049
  this.loginToken = null;
8033
8050
  this.orgId = null;
8034
8051
  this.orgName = null;
@@ -8044,7 +8061,7 @@ var BraintrustState = class {
8044
8061
  globalThis.__inherited_braintrust_state = this;
8045
8062
  }
8046
8063
  resetLoginInfo() {
8047
- this.apiUrl = null;
8064
+ this.appUrl = null;
8048
8065
  this.loginToken = null;
8049
8066
  this.orgId = null;
8050
8067
  this.orgName = null;
@@ -8056,10 +8073,10 @@ var BraintrustState = class {
8056
8073
  }
8057
8074
  apiConn() {
8058
8075
  if (!this._apiConn) {
8059
- if (!this.apiUrl) {
8060
- throw new Error("Must initialize apiUrl before requesting apiConn");
8076
+ if (!this.appUrl) {
8077
+ throw new Error("Must initialize appUrl before requesting apiConn");
8061
8078
  }
8062
- this._apiConn = new HTTPConnection(this.apiUrl);
8079
+ this._apiConn = new HTTPConnection(this.appUrl);
8063
8080
  }
8064
8081
  return this._apiConn;
8065
8082
  }
@@ -8213,25 +8230,25 @@ function logFeedbackImpl(bgLogger, parentIds, {
8213
8230
  updateEvent = Object.fromEntries(
8214
8231
  Object.entries(updateEvent).filter(([_, v]) => !isEmpty(v))
8215
8232
  );
8216
- const trueParentIds = (async () => {
8217
- const { kind, ...ids } = await parentIds;
8233
+ const trueParentIds = new LazyValue(async () => {
8234
+ const { kind, ...ids } = await parentIds.get();
8218
8235
  return ids;
8219
- })();
8236
+ });
8220
8237
  if (Object.keys(updateEvent).length > 0) {
8221
- const record = (async () => {
8238
+ const record = new LazyValue(async () => {
8222
8239
  return {
8223
8240
  id,
8224
8241
  ...updateEvent,
8225
- ...await trueParentIds,
8242
+ ...await trueParentIds.get(),
8226
8243
  [AUDIT_SOURCE_FIELD]: source,
8227
8244
  [AUDIT_METADATA_FIELD]: metadata,
8228
8245
  [IS_MERGE_FIELD]: true
8229
8246
  };
8230
- })();
8247
+ });
8231
8248
  bgLogger.log([record]);
8232
8249
  }
8233
8250
  if (!isEmpty(comment)) {
8234
- const record = (async () => {
8251
+ const record = new LazyValue(async () => {
8235
8252
  return {
8236
8253
  id: v4_default(),
8237
8254
  created: (/* @__PURE__ */ new Date()).toISOString(),
@@ -8243,11 +8260,11 @@ function logFeedbackImpl(bgLogger, parentIds, {
8243
8260
  comment: {
8244
8261
  text: comment
8245
8262
  },
8246
- ...await trueParentIds,
8263
+ ...await trueParentIds.get(),
8247
8264
  [AUDIT_SOURCE_FIELD]: source,
8248
8265
  [AUDIT_METADATA_FIELD]: metadata
8249
8266
  };
8250
- })();
8267
+ });
8251
8268
  bgLogger.log([record]);
8252
8269
  }
8253
8270
  }
@@ -8257,32 +8274,34 @@ var Logger = class {
8257
8274
  this.kind = "logger";
8258
8275
  this.lazyMetadata = lazyMetadata;
8259
8276
  this.logOptions = logOptions;
8260
- const logConn = this.getState().then((state) => state.logConn());
8277
+ const logConn = new LazyValue(
8278
+ () => this.getState().then((state) => state.logConn())
8279
+ );
8261
8280
  this.bgLogger = new BackgroundLogger(logConn);
8262
8281
  this.lastStartTime = getCurrentUnixTimestamp();
8263
8282
  }
8264
8283
  get org_id() {
8265
8284
  return (async () => {
8266
- return (await this.lazyMetadata).org_id;
8285
+ return (await this.lazyMetadata.get()).org_id;
8267
8286
  })();
8268
8287
  }
8269
8288
  get project() {
8270
8289
  return (async () => {
8271
- return (await this.lazyMetadata).project;
8290
+ return (await this.lazyMetadata.get()).project;
8272
8291
  })();
8273
8292
  }
8274
8293
  async getState() {
8275
- await this.lazyMetadata;
8294
+ await this.lazyMetadata.get();
8276
8295
  return _state;
8277
8296
  }
8278
8297
  /**
8279
8298
  * Log a single event. The event will be batched and uploaded behind the scenes if `logOptions.asyncFlush` is true.
8280
8299
  *
8281
8300
  * @param event The event to log.
8282
- * @param event.input: The arguments that uniquely define a user input (an arbitrary, JSON serializable object).
8283
- * @param event.output: The output of your application, including post-processing (an arbitrary, JSON serializable object), that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries, the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may be multiple valid queries that answer a single question.
8284
- * @param event.expected: The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to determine if your `output` value is correct or not. Braintrust currently does not compare `output` to `expected` for you, since there are so many different ways to do that correctly. Instead, these values are just used to help you navigate while digging into analyses. However, we may later use these values to re-score outputs or fine-tune your models.
8285
- * @param event.scores: A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity between the generated and grouth truth summary. The word similarity score could help you determine whether the summarization was covering similar concepts or not. You can use these scores to help you sort, filter, and compare logs.
8301
+ * @param event.input: (Optional) the arguments that uniquely define a user input (an arbitrary, JSON serializable object).
8302
+ * @param event.output: (Optional) the output of your application, including post-processing (an arbitrary, JSON serializable object), that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries, the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may be multiple valid queries that answer a single question.
8303
+ * @param event.expected: (Optional) the ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to determine if your `output` value is correct or not. Braintrust currently does not compare `output` to `expected` for you, since there are so many different ways to do that correctly. Instead, these values are just used to help you navigate while digging into analyses. However, we may later use these values to re-score outputs or fine-tune your models.
8304
+ * @param event.scores: (Optional) a dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity between the generated and grouth truth summary. The word similarity score could help you determine whether the summarization was covering similar concepts or not. You can use these scores to help you sort, filter, and compare logs.
8286
8305
  * @param event.metadata: (Optional) a dictionary with additional data about the test example, model outputs, or just about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any JSON-serializable type, but its keys must be strings.
8287
8306
  * @param event.metrics: (Optional) a dictionary of metrics to log. The following keys are populated automatically: "start", "end".
8288
8307
  * @param event.id: (Optional) a unique identifier for the event. If you don't provide one, BrainTrust will generate one for you.
@@ -8338,14 +8357,16 @@ var Logger = class {
8338
8357
  };
8339
8358
  }
8340
8359
  /**
8341
- * Lower-level alternative to `traced`, which does not automatically end the span or mark it as current.
8360
+ * Lower-level alternative to `traced`. This allows you to start a span yourself, and can be useful in situations
8361
+ * where you cannot use callbacks. However, spans started with `startSpan` will not be marked as the "current span",
8362
+ * so `currentSpan()` and `traced()` will be no-ops. If you want to mark a span as current, use `traced` instead.
8342
8363
  *
8343
8364
  * See `traced` for full details.
8344
8365
  */
8345
8366
  startSpan(args) {
8346
8367
  const { name, ...argsRest } = args ?? {};
8347
8368
  return new SpanImpl({
8348
- parentIds: this.lazyParentIds(),
8369
+ parentIds: new LazyValue(() => this.lazyParentIds()),
8349
8370
  bgLogger: this.bgLogger,
8350
8371
  name: name ?? "root",
8351
8372
  ...argsRest
@@ -8363,7 +8384,11 @@ var Logger = class {
8363
8384
  * @param event.source (Optional) the source of the feedback. Must be one of "external" (default), "app", or "api".
8364
8385
  */
8365
8386
  logFeedback(event) {
8366
- logFeedbackImpl(this.bgLogger, this.lazyParentIds(), event);
8387
+ logFeedbackImpl(
8388
+ this.bgLogger,
8389
+ new LazyValue(() => this.lazyParentIds()),
8390
+ event
8391
+ );
8367
8392
  }
8368
8393
  /*
8369
8394
  * Flush any pending logs to the server.
@@ -8413,9 +8438,20 @@ var BackgroundLogger = class {
8413
8438
  }
8414
8439
  async flush_once(batchSize = DefaultBatchSize) {
8415
8440
  this.active_flush_resolved = false;
8416
- const itemPromises = this.items;
8441
+ const itemLazyValues = this.items;
8417
8442
  this.items = [];
8418
- const allItems = mergeRowBatch(await Promise.all(itemPromises)).reverse();
8443
+ const allItems = await (async () => {
8444
+ try {
8445
+ const itemPromises = itemLazyValues.map((x) => x.get());
8446
+ return mergeRowBatch(await Promise.all(itemPromises)).reverse();
8447
+ } catch (e) {
8448
+ console.warn(
8449
+ "Encountered error when constructing records to flush:\n",
8450
+ e
8451
+ );
8452
+ return [];
8453
+ }
8454
+ })();
8419
8455
  let postPromises = [];
8420
8456
  while (true) {
8421
8457
  const items = [];
@@ -8440,9 +8476,7 @@ var BackgroundLogger = class {
8440
8476
  for (let i = 0; i < NumRetries; i++) {
8441
8477
  const startTime = now();
8442
8478
  try {
8443
- return (await (await this.logConn).post_json("logs", itemsS)).map(
8444
- (res) => res.id
8445
- );
8479
+ return (await (await this.logConn.get()).post_json("logs", itemsS)).map((res) => res.id);
8446
8480
  } catch (e) {
8447
8481
  const retryingText = i + 1 === NumRetries ? "" : " Retrying";
8448
8482
  const errMsg = (() => {
@@ -8489,88 +8523,92 @@ function init(project, options = {}) {
8489
8523
  baseExperiment,
8490
8524
  isPublic,
8491
8525
  update,
8492
- apiUrl,
8526
+ appUrl,
8493
8527
  apiKey,
8494
8528
  orgName,
8495
8529
  metadata,
8496
8530
  gitMetadataSettings
8497
8531
  } = options || {};
8498
- const lazyMetadata = (async () => {
8499
- await login({
8500
- orgName,
8501
- apiKey,
8502
- apiUrl
8503
- });
8504
- const args = {
8505
- project_name: project,
8506
- org_id: _state.orgId
8507
- };
8508
- if (experiment) {
8509
- args["experiment_name"] = experiment;
8510
- }
8511
- if (description) {
8512
- args["description"] = description;
8513
- }
8514
- if (update) {
8515
- args["update"] = update;
8516
- }
8517
- let mergedGitMetadataSettings = {
8518
- ..._state.gitMetadataSettings || {
8519
- collect: "all"
8532
+ const lazyMetadata = new LazyValue(
8533
+ async () => {
8534
+ await login({
8535
+ orgName,
8536
+ apiKey,
8537
+ appUrl
8538
+ });
8539
+ const args = {
8540
+ project_name: project,
8541
+ org_id: _state.orgId
8542
+ };
8543
+ if (experiment) {
8544
+ args["experiment_name"] = experiment;
8520
8545
  }
8521
- };
8522
- if (gitMetadataSettings) {
8523
- mergedGitMetadataSettings = mergeGitMetadataSettings(
8524
- mergedGitMetadataSettings,
8525
- gitMetadataSettings
8526
- );
8527
- }
8528
- const repoStatus2 = await isomorph_default.getRepoStatus(gitMetadataSettings);
8529
- if (repoStatus2) {
8530
- args["repo_info"] = repoStatus2;
8531
- }
8532
- if (baseExperiment) {
8533
- args["base_experiment"] = baseExperiment;
8534
- } else {
8535
- args["ancestor_commits"] = await isomorph_default.getPastNAncestors();
8536
- }
8537
- if (dataset !== void 0) {
8538
- args["dataset_id"] = dataset.id;
8539
- args["dataset_version"] = await dataset.version();
8540
- }
8541
- if (isPublic !== void 0) {
8542
- args["public"] = isPublic;
8543
- }
8544
- if (metadata) {
8545
- args["metadata"] = metadata;
8546
- }
8547
- let response = null;
8548
- while (true) {
8549
- try {
8550
- response = await _state.apiConn().post_json("api/experiment/register", args);
8551
- break;
8552
- } catch (e) {
8553
- if (args["base_experiment"] && `${"data" in e && e.data}`.includes("base experiment")) {
8554
- console.warn(`Base experiment ${args["base_experiment"]} not found.`);
8555
- delete args["base_experiment"];
8556
- } else {
8557
- throw e;
8546
+ if (description) {
8547
+ args["description"] = description;
8548
+ }
8549
+ if (update) {
8550
+ args["update"] = update;
8551
+ }
8552
+ let mergedGitMetadataSettings = {
8553
+ ..._state.gitMetadataSettings || {
8554
+ collect: "all"
8558
8555
  }
8556
+ };
8557
+ if (gitMetadataSettings) {
8558
+ mergedGitMetadataSettings = mergeGitMetadataSettings(
8559
+ mergedGitMetadataSettings,
8560
+ gitMetadataSettings
8561
+ );
8559
8562
  }
8560
- }
8561
- return {
8562
- project: {
8563
- id: response.project.id,
8564
- name: response.project.name,
8565
- fullInfo: response.project
8566
- },
8567
- experiment: {
8568
- id: response.experiment.id,
8569
- name: response.experiment.name,
8570
- fullInfo: response.experiment
8563
+ const repoStatus2 = await isomorph_default.getRepoStatus(gitMetadataSettings);
8564
+ if (repoStatus2) {
8565
+ args["repo_info"] = repoStatus2;
8571
8566
  }
8572
- };
8573
- })();
8567
+ if (baseExperiment) {
8568
+ args["base_experiment"] = baseExperiment;
8569
+ } else {
8570
+ args["ancestor_commits"] = await isomorph_default.getPastNAncestors();
8571
+ }
8572
+ if (dataset !== void 0) {
8573
+ args["dataset_id"] = dataset.id;
8574
+ args["dataset_version"] = await dataset.version();
8575
+ }
8576
+ if (isPublic !== void 0) {
8577
+ args["public"] = isPublic;
8578
+ }
8579
+ if (metadata) {
8580
+ args["metadata"] = metadata;
8581
+ }
8582
+ let response = null;
8583
+ while (true) {
8584
+ try {
8585
+ response = await _state.apiConn().post_json("api/experiment/register", args);
8586
+ break;
8587
+ } catch (e) {
8588
+ if (args["base_experiment"] && `${"data" in e && e.data}`.includes("base experiment")) {
8589
+ console.warn(
8590
+ `Base experiment ${args["base_experiment"]} not found.`
8591
+ );
8592
+ delete args["base_experiment"];
8593
+ } else {
8594
+ throw e;
8595
+ }
8596
+ }
8597
+ }
8598
+ return {
8599
+ project: {
8600
+ id: response.project.id,
8601
+ name: response.project.name,
8602
+ fullInfo: response.project
8603
+ },
8604
+ experiment: {
8605
+ id: response.experiment.id,
8606
+ name: response.experiment.name,
8607
+ fullInfo: response.experiment
8608
+ }
8609
+ };
8610
+ }
8611
+ );
8574
8612
  const ret = new Experiment(lazyMetadata, dataset);
8575
8613
  if (options.setCurrent ?? true) {
8576
8614
  _state.currentExperiment = ret;
@@ -8592,33 +8630,35 @@ function withLogger(callback, options = {}) {
8592
8630
  return callback(logger);
8593
8631
  }
8594
8632
  function initDataset(project, options = {}) {
8595
- const { dataset, description, version, apiUrl, apiKey, orgName } = options || {};
8596
- const lazyMetadata = (async () => {
8597
- await login({
8598
- orgName,
8599
- apiKey,
8600
- apiUrl
8601
- });
8602
- const args = {
8603
- org_id: _state.orgId,
8604
- project_name: project,
8605
- dataset_name: dataset,
8606
- description
8607
- };
8608
- const response = await _state.apiConn().post_json("api/dataset/register", args);
8609
- return {
8610
- project: {
8611
- id: response.project.id,
8612
- name: response.project.name,
8613
- fullInfo: response.project
8614
- },
8615
- dataset: {
8616
- id: response.dataset.id,
8617
- name: response.dataset.name,
8618
- fullInfo: response.dataset
8619
- }
8620
- };
8621
- })();
8633
+ const { dataset, description, version, appUrl, apiKey, orgName } = options || {};
8634
+ const lazyMetadata = new LazyValue(
8635
+ async () => {
8636
+ await login({
8637
+ orgName,
8638
+ apiKey,
8639
+ appUrl
8640
+ });
8641
+ const args = {
8642
+ org_id: _state.orgId,
8643
+ project_name: project,
8644
+ dataset_name: dataset,
8645
+ description
8646
+ };
8647
+ const response = await _state.apiConn().post_json("api/dataset/register", args);
8648
+ return {
8649
+ project: {
8650
+ id: response.project.id,
8651
+ name: response.project.name,
8652
+ fullInfo: response.project
8653
+ },
8654
+ dataset: {
8655
+ id: response.dataset.id,
8656
+ name: response.dataset.name,
8657
+ fullInfo: response.dataset
8658
+ }
8659
+ };
8660
+ }
8661
+ );
8622
8662
  return new Dataset(lazyMetadata, version);
8623
8663
  }
8624
8664
  function withDataset(project, callback, options = {}) {
@@ -8633,51 +8673,53 @@ function initLogger(options = {}) {
8633
8673
  projectName,
8634
8674
  projectId,
8635
8675
  asyncFlush,
8636
- apiUrl,
8676
+ appUrl,
8637
8677
  apiKey,
8638
8678
  orgName,
8639
8679
  forceLogin
8640
8680
  } = options || {};
8641
- const lazyMetadata = (async () => {
8642
- await login({
8643
- orgName,
8644
- apiKey,
8645
- apiUrl,
8646
- forceLogin
8647
- });
8648
- const org_id = _state.orgId;
8649
- if (projectId === void 0) {
8650
- const response = await _state.apiConn().post_json("api/project/register", {
8651
- project_name: projectName || GLOBAL_PROJECT,
8652
- org_id
8653
- });
8654
- return {
8655
- org_id,
8656
- project: {
8657
- id: response.project.id,
8658
- name: response.project.name,
8659
- fullInfo: response.project
8660
- }
8661
- };
8662
- } else if (projectName === void 0) {
8663
- const response = await _state.apiConn().get_json("api/project", {
8664
- id: projectId
8681
+ const lazyMetadata = new LazyValue(
8682
+ async () => {
8683
+ await login({
8684
+ orgName,
8685
+ apiKey,
8686
+ appUrl,
8687
+ forceLogin
8665
8688
  });
8666
- return {
8667
- org_id,
8668
- project: {
8669
- id: projectId,
8670
- name: response.name,
8671
- fullInfo: response.project
8672
- }
8673
- };
8674
- } else {
8675
- return {
8676
- org_id,
8677
- project: { id: projectId, name: projectName, fullInfo: {} }
8678
- };
8689
+ const org_id = _state.orgId;
8690
+ if (projectId === void 0) {
8691
+ const response = await _state.apiConn().post_json("api/project/register", {
8692
+ project_name: projectName || GLOBAL_PROJECT,
8693
+ org_id
8694
+ });
8695
+ return {
8696
+ org_id,
8697
+ project: {
8698
+ id: response.project.id,
8699
+ name: response.project.name,
8700
+ fullInfo: response.project
8701
+ }
8702
+ };
8703
+ } else if (projectName === void 0) {
8704
+ const response = await _state.apiConn().get_json("api/project", {
8705
+ id: projectId
8706
+ });
8707
+ return {
8708
+ org_id,
8709
+ project: {
8710
+ id: projectId,
8711
+ name: response.name,
8712
+ fullInfo: response.project
8713
+ }
8714
+ };
8715
+ } else {
8716
+ return {
8717
+ org_id,
8718
+ project: { id: projectId, name: projectName, fullInfo: {} }
8719
+ };
8720
+ }
8679
8721
  }
8680
- })();
8722
+ );
8681
8723
  const ret = new Logger(lazyMetadata, {
8682
8724
  asyncFlush
8683
8725
  });
@@ -8688,7 +8730,7 @@ function initLogger(options = {}) {
8688
8730
  }
8689
8731
  async function login(options = {}) {
8690
8732
  const {
8691
- apiUrl = isomorph_default.getEnv("BRAINTRUST_API_URL") || "https://www.braintrustdata.com",
8733
+ appUrl = isomorph_default.getEnv("BRAINTRUST_APP_URL") || "https://www.braintrustdata.com",
8692
8734
  apiKey = isomorph_default.getEnv("BRAINTRUST_API_KEY"),
8693
8735
  orgName = isomorph_default.getEnv("BRAINTRUST_ORG_NAME")
8694
8736
  } = options || {};
@@ -8697,11 +8739,11 @@ async function login(options = {}) {
8697
8739
  return;
8698
8740
  }
8699
8741
  _state.resetLoginInfo();
8700
- _state.apiUrl = apiUrl;
8742
+ _state.appUrl = appUrl;
8701
8743
  let conn = null;
8702
8744
  if (apiKey !== void 0) {
8703
8745
  const resp = await checkResponse(
8704
- await fetch(_urljoin(_state.apiUrl, `/api/apikey/login`), {
8746
+ await fetch(_urljoin(_state.appUrl, `/api/apikey/login`), {
8705
8747
  method: "POST",
8706
8748
  headers: {
8707
8749
  "Content-Type": "application/json"
@@ -8818,7 +8860,7 @@ function _check_org_info(org_info, org_name) {
8818
8860
  if (org_name === void 0 || org.name === org_name) {
8819
8861
  _state.orgId = org.id;
8820
8862
  _state.orgName = org.name;
8821
- _state.logUrl = isomorph_default.getEnv("BRAINTRUST_LOG_URL") ?? org.api_url;
8863
+ _state.logUrl = isomorph_default.getEnv("BRAINTRUST_API_URL") ?? org.api_url;
8822
8864
  _state.gitMetadataSettings = org.git_metadata || void 0;
8823
8865
  break;
8824
8866
  }
@@ -8888,6 +8930,9 @@ function validateAndSanitizeExperimentLogFullArgs(event, hasDataset) {
8888
8930
  "Exactly one of input or inputs (deprecated) must be specified. Prefer input."
8889
8931
  );
8890
8932
  }
8933
+ if (!event.output) {
8934
+ throw new Error("output must be specified");
8935
+ }
8891
8936
  if (!event.scores) {
8892
8937
  throw new Error("scores must be specified");
8893
8938
  }
@@ -8906,27 +8951,29 @@ var Experiment = class {
8906
8951
  this.kind = "experiment";
8907
8952
  this.lazyMetadata = lazyMetadata;
8908
8953
  this.dataset = dataset;
8909
- const logConn = this.getState().then((state) => state.logConn());
8954
+ const logConn = new LazyValue(
8955
+ () => this.getState().then((state) => state.logConn())
8956
+ );
8910
8957
  this.bgLogger = new BackgroundLogger(logConn);
8911
8958
  this.lastStartTime = getCurrentUnixTimestamp();
8912
8959
  }
8913
8960
  get id() {
8914
8961
  return (async () => {
8915
- return (await this.lazyMetadata).experiment.id;
8962
+ return (await this.lazyMetadata.get()).experiment.id;
8916
8963
  })();
8917
8964
  }
8918
8965
  get name() {
8919
8966
  return (async () => {
8920
- return (await this.lazyMetadata).experiment.name;
8967
+ return (await this.lazyMetadata.get()).experiment.name;
8921
8968
  })();
8922
8969
  }
8923
8970
  get project() {
8924
8971
  return (async () => {
8925
- return (await this.lazyMetadata).project;
8972
+ return (await this.lazyMetadata.get()).project;
8926
8973
  })();
8927
8974
  }
8928
8975
  async getState() {
8929
- await this.lazyMetadata;
8976
+ await this.lazyMetadata.get();
8930
8977
  return _state;
8931
8978
  }
8932
8979
  /**
@@ -8935,7 +8982,7 @@ var Experiment = class {
8935
8982
  * @param event The event to log.
8936
8983
  * @param event.input: The arguments that uniquely define a test case (an arbitrary, JSON serializable object). Later on, Braintrust will use the `input` to know whether two test cases are the same between experiments, so they should not contain experiment-specific state. A simple rule of thumb is that if you run the same experiment twice, the `input` should be identical.
8937
8984
  * @param event.output: The output of your application, including post-processing (an arbitrary, JSON serializable object), that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries, the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may be multiple valid queries that answer a single question.
8938
- * @param event.expected: The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to determine if your `output` value is correct or not. Braintrust currently does not compare `output` to `expected` for you, since there are so many different ways to do that correctly. Instead, these values are just used to help you navigate your experiments while digging into analyses. However, we may later use these values to re-score outputs or fine-tune your models.
8985
+ * @param event.expected: (Optional) The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to determine if your `output` value is correct or not. Braintrust currently does not compare `output` to `expected` for you, since there are so many different ways to do that correctly. Instead, these values are just used to help you navigate your experiments while digging into analyses. However, we may later use these values to re-score outputs or fine-tune your models.
8939
8986
  * @param event.scores: A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity between the generated and grouth truth summary. The word similarity score could help you determine whether the summarization was covering similar concepts or not. You can use these scores to help you sort, filter, and compare experiments.
8940
8987
  * @param event.metadata: (Optional) a dictionary with additional data about the test example, model outputs, or just about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any JSON-serializable type, but its keys must be strings.
8941
8988
  * @param event.metrics: (Optional) a dictionary of metrics to log. The following keys are populated automatically: "start", "end".
@@ -8977,14 +9024,16 @@ var Experiment = class {
8977
9024
  };
8978
9025
  }
8979
9026
  /**
8980
- * Lower-level alternative to `traced`, which does not automatically end the span or mark it as current.
9027
+ * Lower-level alternative to `traced`. This allows you to start a span yourself, and can be useful in situations
9028
+ * where you cannot use callbacks. However, spans started with `startSpan` will not be marked as the "current span",
9029
+ * so `currentSpan()` and `traced()` will be no-ops. If you want to mark a span as current, use `traced` instead.
8981
9030
  *
8982
9031
  * See `traced` for full details.
8983
9032
  */
8984
9033
  startSpan(args) {
8985
9034
  const { name, ...argsRest } = args ?? {};
8986
9035
  return new SpanImpl({
8987
- parentIds: this.lazyParentIds(),
9036
+ parentIds: new LazyValue(() => this.lazyParentIds()),
8988
9037
  bgLogger: this.bgLogger,
8989
9038
  name: name ?? "root",
8990
9039
  ...argsRest
@@ -9002,7 +9051,7 @@ var Experiment = class {
9002
9051
  let { summarizeScores = true, comparisonExperimentId = void 0 } = options || {};
9003
9052
  await this.bgLogger.flush();
9004
9053
  const state = await this.getState();
9005
- const projectUrl = `${state.apiUrl}/app/${encodeURIComponent(
9054
+ const projectUrl = `${state.appUrl}/app/${encodeURIComponent(
9006
9055
  state.orgName
9007
9056
  )}/p/${encodeURIComponent((await this.project).name)}`;
9008
9057
  const experimentUrl = `${projectUrl}/${encodeURIComponent(
@@ -9058,7 +9107,11 @@ var Experiment = class {
9058
9107
  * @param event.source (Optional) the source of the feedback. Must be one of "external" (default), "app", or "api".
9059
9108
  */
9060
9109
  logFeedback(event) {
9061
- logFeedbackImpl(this.bgLogger, this.lazyParentIds(), event);
9110
+ logFeedbackImpl(
9111
+ this.bgLogger,
9112
+ new LazyValue(() => this.lazyParentIds()),
9113
+ event
9114
+ );
9062
9115
  }
9063
9116
  /**
9064
9117
  * Flush any pending rows to the server.
@@ -9139,18 +9192,18 @@ var SpanImpl = class _SpanImpl {
9139
9192
  if (sanitizedAndInternalData.metrics?.end) {
9140
9193
  this.loggedEndTime = sanitizedAndInternalData.metrics?.end;
9141
9194
  }
9142
- const parentIds = (async () => {
9143
- const { kind, ...ids } = await this.parentIds;
9195
+ const parentIds = new LazyValue(async () => {
9196
+ const { kind, ...ids } = await this.parentIds.get();
9144
9197
  return ids;
9145
- })();
9146
- const record = (async () => {
9198
+ });
9199
+ const record = new LazyValue(async () => {
9147
9200
  return {
9148
9201
  ...sanitizedAndInternalData,
9149
9202
  ...this.rowIds,
9150
- ...await parentIds,
9203
+ ...await parentIds.get(),
9151
9204
  [IS_MERGE_FIELD]: this.isMerge
9152
9205
  };
9153
- })();
9206
+ });
9154
9207
  this.bgLogger.log([record]);
9155
9208
  }
9156
9209
  logFeedback(event) {
@@ -9204,26 +9257,28 @@ var Dataset = class {
9204
9257
  this._fetchedData = void 0;
9205
9258
  this.lazyMetadata = lazyMetadata;
9206
9259
  this.pinnedVersion = pinnedVersion;
9207
- const logConn = this.getState().then((state) => state.logConn());
9260
+ const logConn = new LazyValue(
9261
+ () => this.getState().then((state) => state.logConn())
9262
+ );
9208
9263
  this.bgLogger = new BackgroundLogger(logConn);
9209
9264
  }
9210
9265
  get id() {
9211
9266
  return (async () => {
9212
- return (await this.lazyMetadata).dataset.id;
9267
+ return (await this.lazyMetadata.get()).dataset.id;
9213
9268
  })();
9214
9269
  }
9215
9270
  get name() {
9216
9271
  return (async () => {
9217
- return (await this.lazyMetadata).dataset.name;
9272
+ return (await this.lazyMetadata.get()).dataset.name;
9218
9273
  })();
9219
9274
  }
9220
9275
  get project() {
9221
9276
  return (async () => {
9222
- return (await this.lazyMetadata).project;
9277
+ return (await this.lazyMetadata.get()).project;
9223
9278
  })();
9224
9279
  }
9225
9280
  async getState() {
9226
- await this.lazyMetadata;
9281
+ await this.lazyMetadata.get();
9227
9282
  return _state;
9228
9283
  }
9229
9284
  /**
@@ -9254,7 +9309,7 @@ var Dataset = class {
9254
9309
  }
9255
9310
  }
9256
9311
  const rowId = id || v4_default();
9257
- const args = (async () => ({
9312
+ const args = new LazyValue(async () => ({
9258
9313
  id: rowId,
9259
9314
  inputs: input,
9260
9315
  output,
@@ -9262,18 +9317,18 @@ var Dataset = class {
9262
9317
  dataset_id: await this.id,
9263
9318
  created: (/* @__PURE__ */ new Date()).toISOString(),
9264
9319
  metadata
9265
- }))();
9320
+ }));
9266
9321
  this.bgLogger.log([args]);
9267
9322
  return rowId;
9268
9323
  }
9269
9324
  delete(id) {
9270
- const args = (async () => ({
9325
+ const args = new LazyValue(async () => ({
9271
9326
  id,
9272
9327
  project_id: (await this.project).id,
9273
9328
  dataset_id: await this.id,
9274
9329
  created: (/* @__PURE__ */ new Date()).toISOString(),
9275
9330
  _object_delete: true
9276
- }))();
9331
+ }));
9277
9332
  this.bgLogger.log([args]);
9278
9333
  return id;
9279
9334
  }
@@ -9287,7 +9342,7 @@ var Dataset = class {
9287
9342
  let { summarizeData = true } = options || {};
9288
9343
  await this.bgLogger.flush();
9289
9344
  const state = await this.getState();
9290
- const projectUrl = `${state.apiUrl}/app/${encodeURIComponent(
9345
+ const projectUrl = `${state.appUrl}/app/${encodeURIComponent(
9291
9346
  state.orgName
9292
9347
  )}/p/${encodeURIComponent((await this.project).name)}`;
9293
9348
  const datasetUrl = `${projectUrl}/d/${encodeURIComponent(await this.name)}`;