braintrust 0.0.97 → 0.0.99

package/dist/index.js

@@ -3648,7 +3648,6 @@ var require_pluralize = __commonJS({
 var src_exports = {};
 __export(src_exports, {
   BaseExperiment: () => BaseExperiment,
-  Dataset: () => Dataset,
   Eval: () => Eval,
   Experiment: () => Experiment,
   Logger: () => Logger,
@@ -3664,6 +3663,7 @@ __export(src_exports, {
   getSpanParentObject: () => getSpanParentObject,
   init: () => init,
   initDataset: () => initDataset,
+  initExperiment: () => initExperiment,
   initLogger: () => initLogger,
   log: () => log,
   login: () => login,
@@ -3695,7 +3695,7 @@ var DefaultAsyncLocalStorage = class {
   }
 };
 var iso = {
-  getRepoStatus: async (_settings) => void 0,
+  getRepoInfo: async (_settings) => void 0,
   getPastNAncestors: async () => [],
   getEnv: (_name) => void 0,
   getCallerLocation: () => void 0,
@@ -7734,11 +7734,11 @@ function truncateToByteLimit(s, byteLimit = 65536) {
   const truncated = encoded.subarray(0, byteLimit);
   return new TextDecoder().decode(truncated);
 }
-async function getRepoStatus(settings) {
+async function getRepoInfo(settings) {
   if (settings && settings.collect === "none") {
     return void 0;
   }
-  const repo = await repoStatus();
+  const repo = await repoInfo();
   if (!repo || !settings || settings.collect === "all") {
     return repo;
   }
@@ -7748,7 +7748,7 @@ async function getRepoStatus(settings) {
   });
   return sanitized;
 }
-async function repoStatus() {
+async function repoInfo() {
   const git = await currentRepo();
   if (git === null) {
     return void 0;
@@ -7890,9 +7890,10 @@ function v4(options, buf, offset) {
 }
 var v4_default = v4;
 
-// ../core/js/dist/index.mjs
+// ../core/js/dist/main/index.mjs
 var TRANSACTION_ID_FIELD = "_xact_id";
 var IS_MERGE_FIELD = "_is_merge";
+var MERGE_PATHS_FIELD = "_merge_paths";
 var AUDIT_SOURCE_FIELD = "_audit_source";
 var AUDIT_METADATA_FIELD = "_audit_metadata";
 var VALID_SOURCES = ["app", "api", "external"];
@@ -7951,6 +7952,54 @@ function mergeRowBatch(rows) {
   out.push(...Object.values(rowGroups));
   return out;
 }
+var DEFAULT_IS_LEGACY_DATASET = true;
+function ensureDatasetRecord(r, legacy) {
+  if (legacy) {
+    return ensureLegacyDatasetRecord(r);
+  } else {
+    return ensureNewDatasetRecord(r);
+  }
+}
+function ensureLegacyDatasetRecord(r) {
+  if ("output" in r) {
+    return r;
+  }
+  const row = {
+    ...r,
+    output: r.expected
+  };
+  delete row.expected;
+  return row;
+}
+function ensureNewDatasetRecord(r) {
+  if ("expected" in r) {
+    return r;
+  }
+  const row = {
+    ...r,
+    expected: r.output
+  };
+  delete row.output;
+  return row;
+}
+function makeLegacyEvent(e) {
+  if (!("dataset_id" in e) || !("expected" in e)) {
+    return e;
+  }
+  const event = {
+    ...e,
+    output: e.expected
+  };
+  delete event.expected;
+  if (MERGE_PATHS_FIELD in event) {
+    for (const path2 of event[MERGE_PATHS_FIELD] || []) {
+      if (path2.length > 0 && path2[0] === "expected") {
+        path2[0] = "output";
+      }
+    }
+  }
+  return event;
+}
 var SpanTypeAttribute = /* @__PURE__ */ ((SpanTypeAttribute2) => {
   SpanTypeAttribute2["LLM"] = "llm";
   SpanTypeAttribute2["SCORE"] = "score";
@@ -8004,10 +8053,11 @@ function isEmpty(a) {
   return a === void 0 || a === null;
 }
 var LazyValue = class {
+  callable;
+  value = {
+    hasComputed: false
+  };
   constructor(callable) {
-    this.value = {
-      hasComputed: false
-    };
     this.callable = callable;
   }
   async get() {
@@ -8021,8 +8071,11 @@ var LazyValue = class {
 
 // src/logger.ts
 var NoopSpan = class {
+  id;
+  span_id;
+  root_span_id;
+  kind = "span";
   constructor() {
-    this.kind = "span";
     this.id = "";
     this.span_id = "";
     this.root_span_id = "";
@@ -8046,15 +8099,22 @@ var NoopSpan = class {
 };
 var NOOP_SPAN = new NoopSpan();
 var BraintrustState = class {
+  id;
+  currentExperiment;
+  // Note: the value of IsAsyncFlush doesn't really matter here, since we
+  // (safely) dynamically cast it whenever retrieving the logger.
+  currentLogger;
+  currentSpan;
+  appUrl = null;
+  loginToken = null;
+  orgId = null;
+  orgName = null;
+  logUrl = null;
+  loggedIn = false;
+  gitMetadataSettings;
+  _apiConn = null;
+  _logConn = null;
   constructor() {
-    this.appUrl = null;
-    this.loginToken = null;
-    this.orgId = null;
-    this.orgName = null;
-    this.logUrl = null;
-    this.loggedIn = false;
-    this._apiConn = null;
-    this._logConn = null;
     this.id = v4_default();
     this.currentExperiment = void 0;
     this.currentLogger = void 0;
@@ -8101,6 +8161,9 @@ function _internalSetInitialState() {
 }
 var _internalGetGlobalState = () => _state;
 var FailedHTTPResponse = class extends Error {
+  status;
+  text;
+  data;
   constructor(status, text, data = null) {
     super(`${status}: ${text}`);
     this.status = status;
@@ -8120,6 +8183,9 @@ async function checkResponse(resp) {
   }
 }
 var HTTPConnection = class _HTTPConnection {
+  base_url;
+  token;
+  headers;
   constructor(base_url) {
     this.base_url = base_url;
     this.token = null;
@@ -8271,9 +8337,13 @@ function logFeedbackImpl(bgLogger, parentIds, {
   }
 }
 var Logger = class {
+  lazyMetadata;
+  logOptions;
+  bgLogger;
+  lastStartTime;
+  // For type identification.
+  kind = "logger";
   constructor(lazyMetadata, logOptions = {}) {
-    // For type identification.
-    this.kind = "logger";
     this.lazyMetadata = lazyMetadata;
     this.logOptions = logOptions;
     const logConn = new LazyValue(
@@ -8307,9 +8377,19 @@ var Logger = class {
    * @param event.metadata: (Optional) a dictionary with additional data about the test example, model outputs, or just about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any JSON-serializable type, but its keys must be strings.
    * @param event.metrics: (Optional) a dictionary of metrics to log. The following keys are populated automatically: "start", "end".
    * @param event.id: (Optional) a unique identifier for the event. If you don't provide one, BrainTrust will generate one for you.
+   * @param options Additional logging options
+   * @param options.allowLogConcurrentWithActiveSpan in rare cases where you need to log at the top level separately from an active span on the logger, set this to true.
    * :returns: The `id` of the logged event.
    */
-  log(event) {
+  log(event, options) {
+    if (!options?.allowLogConcurrentWithActiveSpan) {
+      const checkCurrentSpan = currentSpan();
+      if (checkCurrentSpan instanceof SpanImpl && checkCurrentSpan.parentObject === this) {
+        throw new Error(
+          "Cannot run toplevel Logger.log method while there is an active span. To log to the span, use Span.log"
+        );
+      }
+    }
     const span = this.startSpan({ startTime: this.lastStartTime, event });
     this.lastStartTime = span.end();
     const ret = span.id;
@@ -8368,6 +8448,7 @@ var Logger = class {
   startSpan(args) {
     const { name, ...argsRest } = args ?? {};
     return new SpanImpl({
+      parentObject: this,
       parentIds: new LazyValue(() => this.lazyParentIds()),
       bgLogger: this.bgLogger,
       name: name ?? "root",
@@ -8416,16 +8497,20 @@ var MaxRequestSize = 6 * 1024 * 1024;
 function constructJsonArray(items) {
   return `[${items.join(",")}]`;
 }
+function constructLogs3Data(items) {
+  return `{"rows": ${constructJsonArray(items)}, "api_version": 2}`;
+}
 var DefaultBatchSize = 100;
 var NumRetries = 3;
 function now() {
   return (/* @__PURE__ */ new Date()).getTime();
 }
 var BackgroundLogger = class {
+  logConn;
+  items = [];
+  active_flush = Promise.resolve([]);
+  active_flush_resolved = true;
   constructor(logConn) {
-    this.items = [];
-    this.active_flush = Promise.resolve([]);
-    this.active_flush_resolved = true;
     this.logConn = logConn;
     isomorph_default.processOn("beforeExit", async () => {
       await this.flush();
@@ -8474,11 +8559,20 @@ var BackgroundLogger = class {
       }
       postPromises.push(
         (async () => {
-          const itemsS = constructJsonArray(items);
+          const dataStr = constructLogs3Data(items);
           for (let i = 0; i < NumRetries; i++) {
             const startTime = now();
             try {
-              return (await (await this.logConn.get()).post_json("logs", itemsS)).map((res) => res.id);
+              try {
+                return (await (await this.logConn.get()).post_json("logs3", dataStr)).ids.map((res) => res.id);
+              } catch (e) {
+                const legacyDataS = constructJsonArray(
+                  items.map(
+                    (r) => JSON.stringify(makeLegacyEvent(JSON.parse(r)))
+                  )
+                );
+                return (await (await this.logConn.get()).post_json("logs", legacyDataS)).map((res) => res.id);
+              }
             } catch (e) {
               const retryingText = i + 1 === NumRetries ? "" : " Retrying";
               const errMsg = (() => {
@@ -8489,7 +8583,7 @@ var BackgroundLogger = class {
                 }
               })();
               console.warn(
-                `log request failed. Elapsed time: ${(now() - startTime) / 1e3} seconds. Payload size: ${itemsS.length}. Error: ${errMsg}.${retryingText}`
+                `log request failed. Elapsed time: ${(now() - startTime) / 1e3} seconds. Payload size: ${dataStr.length}. Error: ${errMsg}.${retryingText}`
               );
             }
           }
@@ -8517,8 +8611,21 @@ var BackgroundLogger = class {
     }
   }
 };
-function init(project, options = {}) {
+function init(projectOrOptions, optionalOptions) {
+  const options = (() => {
+    if (typeof projectOrOptions === "string") {
+      return { ...optionalOptions, project: projectOrOptions };
+    } else {
+      if (optionalOptions !== void 0) {
+        throw new Error(
+          "Cannot specify options struct as both parameters. Must call either init(project, options) or init(options)."
+        );
+      }
+      return projectOrOptions;
+    }
+  })();
   const {
+    project,
     experiment,
     description,
     dataset,
@@ -8530,47 +8637,55 @@ function init(project, options = {}) {
     apiKey,
     orgName,
     metadata,
-    gitMetadataSettings
-  } = options || {};
+    gitMetadataSettings,
+    projectId,
+    baseExperimentId,
+    repoInfo: repoInfo2
+  } = options;
   if (open && update) {
     throw new Error("Cannot open and update an experiment at the same time");
   }
   if (open || update) {
     if (isEmpty(experiment)) {
       const action = open ? "open" : "update";
-      throw new Error(`Cannot ${action} an experiment without specifying its name`);
+      throw new Error(
+        `Cannot ${action} an experiment without specifying its name`
+      );
     }
-    const lazyMetadata2 = new LazyValue(async () => {
-      await login({
-        orgName,
-        apiKey,
-        appUrl
-      });
-      const args = {
-        project_name: project,
-        org_name: _state.orgName,
-        experiment_name: experiment
-      };
-      const response = await _state.apiConn().post_json("api/experiment/get", args);
-      if (response.length === 0) {
-        throw new Error(
-          `Experiment ${experiment} not found in project ${project}.`
-        );
-      }
-      const info = response[0];
-      return {
-        project: {
-          id: info.project_id,
-          name: "",
-          fullInfo: {}
-        },
-        experiment: {
-          id: info.id,
-          name: info.name,
-          fullInfo: info
+    const lazyMetadata2 = new LazyValue(
+      async () => {
+        await login({
+          orgName,
+          apiKey,
+          appUrl
+        });
+        const args = {
+          project_name: project,
+          project_id: projectId,
+          org_name: _state.orgName,
+          experiment_name: experiment
+        };
+        const response = await _state.apiConn().post_json("api/experiment/get", args);
+        if (response.length === 0) {
+          throw new Error(
+            `Experiment ${experiment} not found in project ${projectId ?? project}.`
+          );
         }
-      };
-    });
+        const info = response[0];
+        return {
+          project: {
+            id: info.project_id,
+            name: "",
+            fullInfo: {}
+          },
+          experiment: {
+            id: info.id,
+            name: info.name,
+            fullInfo: info
+          }
+        };
+      }
+    );
     if (open) {
       return new ReadonlyExperiment(
         lazyMetadata2
@@ -8592,6 +8707,7 @@ function init(project, options = {}) {
       });
       const args = {
         project_name: project,
+        project_id: projectId,
         org_id: _state.orgId
       };
       if (experiment) {
@@ -8600,22 +8716,29 @@ function init(project, options = {}) {
       if (description) {
         args["description"] = description;
       }
-      let mergedGitMetadataSettings = {
-        ..._state.gitMetadataSettings || {
-          collect: "all"
+      const repoInfoArg = await (async () => {
+        if (repoInfo2) {
+          return repoInfo2;
         }
-      };
-      if (gitMetadataSettings) {
-        mergedGitMetadataSettings = mergeGitMetadataSettings(
-          mergedGitMetadataSettings,
-          gitMetadataSettings
-        );
-      }
-      const repoStatus2 = await isomorph_default.getRepoStatus(gitMetadataSettings);
-      if (repoStatus2) {
-        args["repo_info"] = repoStatus2;
+        let mergedGitMetadataSettings = {
+          ..._state.gitMetadataSettings || {
+            collect: "all"
+          }
+        };
+        if (gitMetadataSettings) {
+          mergedGitMetadataSettings = mergeGitMetadataSettings(
+            mergedGitMetadataSettings,
+            gitMetadataSettings
+          );
+        }
+        return await isomorph_default.getRepoInfo(mergedGitMetadataSettings);
+      })();
+      if (repoInfoArg) {
+        args["repo_info"] = repoInfoArg;
       }
-      if (baseExperiment) {
+      if (baseExperimentId) {
+        args["base_exp_id"] = baseExperimentId;
+      } else if (baseExperiment) {
         args["base_experiment"] = baseExperiment;
       } else {
         args["ancestor_commits"] = await isomorph_default.getPastNAncestors();
@@ -8666,6 +8789,21 @@ function init(project, options = {}) {
   }
   return ret;
 }
+function initExperiment(projectOrOptions, optionalOptions) {
+  const options = (() => {
+    if (typeof projectOrOptions === "string") {
+      return { ...optionalOptions, project: projectOrOptions };
+    } else {
+      if (optionalOptions !== void 0) {
+        throw new Error(
+          "Cannot specify options struct as both parameters. Must call either init(project, options) or init(options)."
+        );
+      }
+      return projectOrOptions;
+    }
+  })();
+  return init(options);
+}
 function withExperiment(project, callback, options = {}) {
   console.warn(
     "withExperiment is deprecated and will be removed in a future version of braintrust. Simply create the experiment with `init`."
@@ -8680,8 +8818,30 @@ function withLogger(callback, options = {}) {
   const logger = initLogger(options);
   return callback(logger);
 }
-function initDataset(project, options = {}) {
-  const { dataset, description, version, appUrl, apiKey, orgName } = options || {};
+function initDataset(projectOrOptions, optionalOptions) {
+  const options = (() => {
+    if (typeof projectOrOptions === "string") {
+      return { ...optionalOptions, project: projectOrOptions };
+    } else {
+      if (optionalOptions !== void 0) {
+        throw new Error(
+          "Cannot specify options struct as both parameters. Must call either initDataset(project, options) or initDataset(options)."
+        );
+      }
+      return projectOrOptions;
+    }
+  })();
+  const {
+    project,
+    dataset,
+    description,
+    version,
+    appUrl,
+    apiKey,
+    orgName,
+    projectId,
+    useOutput: legacy
+  } = options;
   const lazyMetadata = new LazyValue(
     async () => {
       await login({
@@ -8692,6 +8852,7 @@ function initDataset(project, options = {}) {
       const args = {
         org_id: _state.orgId,
         project_name: project,
+        project_id: projectId,
         dataset_name: dataset,
         description
       };
@@ -8710,7 +8871,7 @@ function initDataset(project, options = {}) {
       };
     }
   );
-  return new Dataset(lazyMetadata, version);
+  return new Dataset(lazyMetadata, version, legacy);
 }
 function withDataset(project, callback, options = {}) {
   console.warn(
@@ -8780,15 +8941,30 @@ function initLogger(options = {}) {
   return ret;
 }
 async function login(options = {}) {
+  let { forceLogin = false } = options || {};
+  if (_state.loggedIn && !forceLogin) {
+    let checkUpdatedParam2 = function(varname, arg, orig) {
+      if (!isEmpty(arg) && !isEmpty(orig) && arg !== orig) {
+        throw new Error(
+          `Re-logging in with different ${varname} (${arg}) than original (${orig}). To force re-login, pass \`forceLogin: true\``
+        );
+      }
+    };
+    var checkUpdatedParam = checkUpdatedParam2;
+    checkUpdatedParam2("appUrl", options.appUrl, _state.appUrl);
+    checkUpdatedParam2(
+      "apiKey",
+      options.apiKey ? HTTPConnection.sanitize_token(options.apiKey) : void 0,
+      _state.loginToken
+    );
+    checkUpdatedParam2("orgName", options.orgName, _state.orgName);
+    return;
+  }
   const {
     appUrl = isomorph_default.getEnv("BRAINTRUST_APP_URL") || "https://www.braintrustdata.com",
     apiKey = isomorph_default.getEnv("BRAINTRUST_API_KEY"),
     orgName = isomorph_default.getEnv("BRAINTRUST_ORG_NAME")
   } = options || {};
-  let { forceLogin = false } = options || {};
-  if (_state.loggedIn && !forceLogin) {
-    return;
-  }
   _state.resetLoginInfo();
   _state.appUrl = appUrl;
   let conn = null;
@@ -8997,11 +9173,12 @@ function validateAndSanitizeExperimentLogFullArgs(event, hasDataset) {
   return event;
 }
 var ObjectFetcher = class {
-  constructor(objectType, pinnedVersion) {
+  constructor(objectType, pinnedVersion, mutateRecord) {
     this.objectType = objectType;
     this.pinnedVersion = pinnedVersion;
-    this._fetchedData = void 0;
+    this.mutateRecord = mutateRecord;
   }
+  _fetchedData = void 0;
   get id() {
     throw new Error("ObjectFetcher subclasses must have an 'id' attribute");
   }
@@ -9020,12 +9197,24 @@ var ObjectFetcher = class {
   async fetchedData() {
     if (this._fetchedData === void 0) {
       const state = await this.getState();
-      const resp = await state.logConn().get(`object/${this.objectType}`, {
-        id: await this.id,
-        fmt: "json2",
-        version: this.pinnedVersion
-      });
-      this._fetchedData = await resp.json();
+      let data = void 0;
+      try {
+        const resp = await state.logConn().get(`object3/${this.objectType}`, {
+          id: await this.id,
+          fmt: "json2",
+          version: this.pinnedVersion,
+          api_version: "2"
+        });
+        data = await resp.json();
+      } catch (e) {
+        const resp = await state.logConn().get(`object/${this.objectType}`, {
+          id: await this.id,
+          fmt: "json2",
+          version: this.pinnedVersion
+        });
+        data = await resp.json();
+      }
+      this._fetchedData = this.mutateRecord ? data?.map(this.mutateRecord) : data;
     }
     return this._fetchedData || [];
   }
@@ -9049,10 +9238,14 @@ var ObjectFetcher = class {
   }
 };
 var Experiment = class extends ObjectFetcher {
+  lazyMetadata;
+  dataset;
+  bgLogger;
+  lastStartTime;
+  // For type identification.
+  kind = "experiment";
   constructor(lazyMetadata, dataset) {
     super("experiment", void 0);
-    // For type identification.
-    this.kind = "experiment";
     this.lazyMetadata = lazyMetadata;
     this.dataset = dataset;
     const logConn = new LazyValue(
@@ -9093,9 +9286,19 @@ var Experiment = class extends ObjectFetcher {
    * @param event.id: (Optional) a unique identifier for the event. If you don't provide one, BrainTrust will generate one for you.
    * @param event.dataset_record_id: (Optional) the id of the dataset record that this event is associated with. This field is required if and only if the experiment is associated with a dataset.
    * @param event.inputs: (Deprecated) the same as `input` (will be removed in a future version).
+   * @param options Additional logging options
+   * @param options.allowLogConcurrentWithActiveSpan in rare cases where you need to log at the top level separately from an active span on the experiment, set this to true.
    * :returns: The `id` of the logged event.
    */
-  log(event) {
+  log(event, options) {
+    if (!options?.allowLogConcurrentWithActiveSpan) {
+      const checkCurrentSpan = currentSpan();
+      if (checkCurrentSpan instanceof SpanImpl && checkCurrentSpan.parentObject === this) {
+        throw new Error(
+          "Cannot run toplevel Experiment.log method while there is an active span. To log to the span, use Span.log"
+        );
+      }
+    }
     event = validateAndSanitizeExperimentLogFullArgs(event, !!this.dataset);
     const span = this.startSpan({ startTime: this.lastStartTime, event });
     this.lastStartTime = span.end();
@@ -9137,6 +9340,7 @@ var Experiment = class extends ObjectFetcher {
   startSpan(args) {
     const { name, ...argsRest } = args ?? {};
     return new SpanImpl({
+      parentObject: this,
       parentIds: new LazyValue(() => this.lazyParentIds()),
       bgLogger: this.bgLogger,
       name: name ?? "root",
@@ -9274,20 +9478,38 @@ var ReadonlyExperiment = class extends ObjectFetcher {
       if (record.root_span_id !== record.span_id) {
         continue;
       }
-      const { output, expected } = record;
-      yield {
-        input: record.input,
-        expected: expected ?? output
-      };
+      const { output, expected: expectedRecord } = record;
+      const expected = expectedRecord ?? output;
+      if (isEmpty(expected)) {
+        yield {
+          input: record.input
+        };
+      } else {
+        yield {
+          input: record.input,
+          expected
+        };
+      }
     }
   }
 };
 var executionCounter = 0;
 var SpanImpl = class _SpanImpl {
+  bgLogger;
+  // `internalData` contains fields that are not part of the "user-sanitized"
+  // set of fields which we want to log in just one of the span rows.
+  internalData;
+  isMerge;
+  loggedEndTime;
+  // For internal use only.
+  parentObject;
+  // These fields are logged to every span row.
+  parentIds;
+  rowIds;
+  kind = "span";
   // root_experiment should only be specified for a root span. parent_span
   // should only be specified for non-root spans.
   constructor(args) {
-    this.kind = "span";
     this.loggedEndTime = void 0;
     this.bgLogger = args.bgLogger;
     const callerLocation = isomorph_default.getCallerLocation();
@@ -9315,6 +9537,7 @@ var SpanImpl = class _SpanImpl {
       },
       created: (/* @__PURE__ */ new Date()).toISOString()
     };
+    this.parentObject = args.parentObject;
     this.parentIds = args.parentIds;
     const id = args.event?.id ?? v4_default();
     const span_id = v4_default();
@@ -9386,6 +9609,7 @@ var SpanImpl = class _SpanImpl {
   }
   startSpan(args) {
     return new _SpanImpl({
+      parentObject: this.parentObject,
       parentIds: this.parentIds,
       bgLogger: this.bgLogger,
       parentSpanInfo: {
@@ -9411,8 +9635,20 @@ var SpanImpl = class _SpanImpl {
   }
 };
 var Dataset = class extends ObjectFetcher {
-  constructor(lazyMetadata, pinnedVersion) {
-    super("dataset", pinnedVersion);
+  lazyMetadata;
+  bgLogger;
+  constructor(lazyMetadata, pinnedVersion, legacy) {
+    const isLegacyDataset = legacy ?? DEFAULT_IS_LEGACY_DATASET;
+    if (isLegacyDataset) {
+      console.warn(
+        `Records will be fetched from this dataset in the legacy format, with the "expected" field renamed to "output". Please update your code to use "expected", and use \`braintrust.initDataset()\` with \`{ useOutput: false }\`, which will become the default in a future version of Braintrust.`
+      );
+    }
+    super(
+      "dataset",
+      pinnedVersion,
+      (r) => ensureDatasetRecord(r, isLegacyDataset)
+    );
     this.lazyMetadata = lazyMetadata;
     const logConn = new LazyValue(
       () => this.getState().then((state) => state.logConn())
@@ -9444,19 +9680,21 @@ var Dataset = class extends ObjectFetcher {
    *
    * @param event The event to log.
    * @param event.input The argument that uniquely define an input case (an arbitrary, JSON serializable object).
-   * @param event.output The output of your application, including post-processing (an arbitrary, JSON serializable object).
+   * @param event.expected The output of your application, including post-processing (an arbitrary, JSON serializable object).
    * @param event.metadata (Optional) a dictionary with additional data about the test example, model outputs, or just
    * about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the
    * `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any
    * JSON-serializable type, but its keys must be strings.
    * @param event.id (Optional) a unique identifier for the event. If you don't provide one, Braintrust will generate one for you.
+   * @param event.output: (Deprecated) The output of your application. Use `expected` instead.
    * @returns The `id` of the logged record.
    */
   insert({
     input,
-    output,
+    expected,
     metadata,
-    id
+    id,
+    output
   }) {
     if (metadata !== void 0) {
       for (const key of Object.keys(metadata)) {
@@ -9465,11 +9703,16 @@ var Dataset = class extends ObjectFetcher {
         }
       }
     }
+    if (expected && output) {
+      throw new Error(
+        "Only one of expected or output (deprecated) can be specified. Prefer expected."
+      );
+    }
     const rowId = id || v4_default();
     const args = new LazyValue(async () => ({
       id: rowId,
-      inputs: input,
-      output,
+      input,
+      expected: expected === void 0 ? output : expected,
       project_id: (await this.project).id,
       dataset_id: await this.id,
       created: (/* @__PURE__ */ new Date()).toISOString(),
@@ -9540,7 +9783,7 @@ var Dataset = class extends ObjectFetcher {
 
 // src/node.ts
 function configureNode() {
-  isomorph_default.getRepoStatus = getRepoStatus;
+  isomorph_default.getRepoInfo = getRepoInfo;
   isomorph_default.getPastNAncestors = getPastNAncestors;
   isomorph_default.getEnv = (name) => process.env[name];
   isomorph_default.getCallerLocation = getCallerLocation;
@@ -9565,8 +9808,9 @@ function fitNameToSpaces(name, length) {
   return padded.substring(0, length - 3) + "...";
 }
 var BarProgressReporter = class {
+  multiBar;
+  bars = {};
   constructor() {
-    this.bars = {};
     this.multiBar = new cliProgress.MultiBar(
       {
         clearOnComplete: false,
@@ -9602,7 +9846,7 @@ function makeEvalName(projectName, experimentName) {
   }
   return out;
 }
-function initExperiment(projectName, options = {}) {
+function initExperiment2(projectName, options = {}) {
   return init(projectName, {
     ...options,
     setCurrent: false
@@ -9610,9 +9854,9 @@ function initExperiment(projectName, options = {}) {
 }
 globalThis._evals = {};
 async function Eval(name, evaluator) {
-  const evalName = makeEvalName(name, evaluator.experimentName);
+  let evalName = makeEvalName(name, evaluator.experimentName);
   if (_evals[evalName]) {
-    throw new Error(`Evaluator ${evalName} already exists`);
+    evalName = `${evalName}_${Object.keys(_evals).length}`;
   }
   if (globalThis._lazy_load) {
     _evals[evalName] = { evalName, projectName: name, ...evaluator };
@@ -9628,7 +9872,7 @@ async function Eval(name, evaluator) {
   }
   const progressReporter = new BarProgressReporter();
   try {
-    const experiment = initExperiment(name, {
+    const experiment = initExperiment2(name, {
       experiment: evaluator.experimentName,
       metadata: evaluator.metadata,
       isPublic: evaluator.isPublic
@@ -9668,6 +9912,9 @@ function evaluateFilter(object, filter) {
   }
   return pattern.test(serializeJSONWithPlainString(key));
 }
+function scorerName(scorer, scorer_idx) {
+  return scorer.name || `scorer_${scorer_idx}`;
+}
 async function runEvaluator(experiment, evaluator, progressReporter, filters) {
   if (typeof evaluator.data === "string") {
     throw new Error("Unimplemented: string data paths");
@@ -9690,7 +9937,7 @@ async function runEvaluator(experiment, evaluator, progressReporter, filters) {
       }
       name = baseExperiment.name;
     }
-    dataResult = initExperiment(evaluator.projectName, {
+    dataResult = initExperiment2(evaluator.projectName, {
       experiment: name,
       open: true
     }).asDataset();
@@ -9711,11 +9958,13 @@ async function runEvaluator(experiment, evaluator, progressReporter, filters) {
   );
   progressReporter.start(evaluator.evalName, data.length);
   const evals = data.map(async (datum) => {
-    let metadata = { ...datum.metadata };
-    let output = void 0;
-    let error2 = void 0;
-    let scores = {};
     const callback = async (rootSpan) => {
+      let metadata = {
+        ..."metadata" in datum ? datum.metadata : {}
+      };
+      let output = void 0;
+      let error2 = void 0;
+      let scores = {};
       try {
         const meta = (o) => metadata = { ...metadata, ...o };
         await rootSpan.traced(
@@ -9732,42 +9981,55 @@ async function runEvaluator(experiment, evaluator, progressReporter, filters) {
         );
         rootSpan.log({ output });
         const scoringArgs = { ...datum, metadata, output };
+        const scorerNames = evaluator.scores.map(scorerName);
         const scoreResults = await Promise.all(
           evaluator.scores.map(async (score, score_idx) => {
-            return rootSpan.traced(
-              async (span) => {
-                const scoreResult = score(scoringArgs);
-                const result = scoreResult instanceof Promise ? await scoreResult : scoreResult;
-                const {
-                  metadata: resultMetadata,
-                  name: _,
-                  ...resultRest
-                } = result;
-                span.log({
-                  output: resultRest,
-                  metadata: resultMetadata
-                });
-                return result;
-              },
-              {
-                name: score.name || `scorer_${score_idx}`,
-                spanAttributes: {
-                  type: SpanTypeAttribute.SCORE
+            try {
+              const result = await rootSpan.traced(
+                async (span) => {
+                  const scoreResult = score(scoringArgs);
+                  const result2 = scoreResult instanceof Promise ? await scoreResult : scoreResult;
+                  const {
+                    metadata: resultMetadata,
+                    name: _,
+                    ...resultRest
+                  } = result2;
+                  span.log({
+                    output: resultRest,
+                    metadata: resultMetadata
+                  });
+                  return result2;
                 },
-                event: { input: scoringArgs }
-              }
-            );
+                {
+                  name: scorerNames[score_idx],
+                  spanAttributes: {
+                    type: SpanTypeAttribute.SCORE
+                  },
+                  event: { input: scoringArgs }
+                }
+              );
+              return { kind: "score", value: result };
+            } catch (e) {
+              return { kind: "error", value: e };
+            }
           })
         );
+        const passingScorersAndResults = [];
+        const failingScorersAndResults = [];
+        scoreResults.forEach((result, i) => {
+          const name = scorerNames[i];
+          if (result.kind === "score") {
+            passingScorersAndResults.push({ name, score: result.value });
+          } else {
+            failingScorersAndResults.push({ name, error: result.value });
+          }
+        });
         const scoreMetadata = {};
-        for (const scoreResult of scoreResults) {
+        for (const { score: scoreResult } of passingScorersAndResults) {
           scores[scoreResult.name] = scoreResult.score;
           const metadata2 = {
             ...scoreResult.metadata
           };
-          if (scoreResult.error !== void 0) {
-            metadata2.error = scoreResult.error;
-          }
           if (Object.keys(metadata2).length > 0) {
             scoreMetadata[scoreResult.name] = metadata2;
           }
@@ -9776,6 +10038,21 @@ async function runEvaluator(experiment, evaluator, progressReporter, filters) {
           meta({ scores: scoreMetadata });
         }
         rootSpan.log({ scores, metadata });
+        if (failingScorersAndResults.length) {
+          const scorerErrors = Object.fromEntries(
+            failingScorersAndResults.map(({ name, error: error3 }) => [
+              name,
+              error3 instanceof Error ? error3.stack : `${error3}`
+            ])
+          );
+          metadata["scorer_errors"] = scorerErrors;
+          const names = Object.keys(scorerErrors).join(", ");
+          const errors = failingScorersAndResults.map((item) => item.error);
+          throw new AggregateError(
+            errors,
+            `Found exceptions for the following scorers: ${names}`
+          );
+        }
       } catch (e) {
         error2 = e;
       } finally {
@@ -9798,7 +10075,7 @@ async function runEvaluator(experiment, evaluator, progressReporter, filters) {
         },
         event: {
           input: datum.input,
-          expected: datum.expected
+          expected: "expected" in datum ? datum.expected : void 0
         }
       });
     }
@@ -9854,7 +10131,8 @@ function reportEvaluatorResult(evaluatorName, evaluatorResult, {
     if (!verbose && !jsonl) {
       console.error(warning("Add --verbose to see full stack traces."));
     }
-  } else if (summary) {
+  }
+  if (summary) {
     console.log(jsonl ? JSON.stringify(summary) : summary);
   } else {
     const scoresByName = {};
@@ -10082,6 +10360,9 @@ function wrapEmbeddings(create) {
   };
 }
 var WrapperStream = class {
+  span;
+  iter;
+  startTime;
   constructor(span, startTime, iter) {
     this.span = span;
     this.iter = iter;
@@ -10118,7 +10399,6 @@ configureNode();
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   BaseExperiment,
-  Dataset,
   Eval,
   Experiment,
   Logger,
@@ -10134,6 +10414,7 @@ configureNode();
   getSpanParentObject,
   init,
   initDataset,
+  initExperiment,
   initLogger,
   log,
   login,