braintrust 0.0.93 → 0.0.95

package/dist/cli.js

@@ -9065,7 +9065,7 @@ var require_package = __commonJS({
   "package.json"(exports2, module2) {
     module2.exports = {
       name: "braintrust",
-      version: "0.0.93",
+      version: "0.0.95",
       description: "SDK for integrating Braintrust",
       main: "./dist/index.js",
       browser: {
@@ -9108,7 +9108,7 @@ var require_package = __commonJS({
         typescript: "^5.3.3"
       },
       dependencies: {
-        "@braintrust/core": "^0.0.13",
+        "@braintrust/core": "^0.0.14",
         argparse: "^2.0.1",
         chalk: "^4.1.2",
         "cli-progress": "^3.12.0",
@@ -9128,7 +9128,7 @@ var esbuild = __toESM(require("esbuild"));
 var import_fs = __toESM(require("fs"));
 var import_os = __toESM(require("os"));
 var import_path = __toESM(require("path"));
-var import_util2 = __toESM(require("util"));
+var import_util3 = __toESM(require("util"));
 var fsWalk = __toESM(require_out3());
 
 // ../../node_modules/.pnpm/minimatch@9.0.3/node_modules/minimatch/dist/mjs/index.js
@@ -10644,6 +10644,21 @@ function getCurrentUnixTimestamp() {
 function isEmpty(a) {
   return a === void 0 || a === null;
 }
+var LazyValue = class {
+  constructor(callable) {
+    this.value = {
+      hasComputed: false
+    };
+    this.callable = callable;
+  }
+  async get() {
+    if (this.value.hasComputed) {
+      return this.value.val;
+    }
+    this.value = { hasComputed: true, val: await this.callable() };
+    return this.value.val;
+  }
+};
 
 // src/logger.ts
 var NoopSpan = class {
@@ -10858,25 +10873,25 @@ function logFeedbackImpl(bgLogger, parentIds, {
   updateEvent = Object.fromEntries(
     Object.entries(updateEvent).filter(([_, v]) => !isEmpty(v))
   );
-  const trueParentIds = (async () => {
-    const { kind, ...ids } = await parentIds;
+  const trueParentIds = new LazyValue(async () => {
+    const { kind, ...ids } = await parentIds.get();
     return ids;
-  })();
+  });
   if (Object.keys(updateEvent).length > 0) {
-    const record = (async () => {
+    const record = new LazyValue(async () => {
       return {
         id,
         ...updateEvent,
-        ...await trueParentIds,
+        ...await trueParentIds.get(),
         [AUDIT_SOURCE_FIELD]: source,
         [AUDIT_METADATA_FIELD]: metadata,
         [IS_MERGE_FIELD]: true
       };
-    })();
+    });
     bgLogger.log([record]);
   }
   if (!isEmpty(comment)) {
-    const record = (async () => {
+    const record = new LazyValue(async () => {
       return {
         id: v4_default(),
         created: (/* @__PURE__ */ new Date()).toISOString(),
@@ -10888,11 +10903,11 @@ function logFeedbackImpl(bgLogger, parentIds, {
         comment: {
           text: comment
         },
-        ...await trueParentIds,
+        ...await trueParentIds.get(),
         [AUDIT_SOURCE_FIELD]: source,
         [AUDIT_METADATA_FIELD]: metadata
       };
-    })();
+    });
     bgLogger.log([record]);
   }
 }
@@ -10924,9 +10939,20 @@ var BackgroundLogger = class {
   }
   async flush_once(batchSize = DefaultBatchSize) {
     this.active_flush_resolved = false;
-    const itemPromises = this.items;
+    const itemLazyValues = this.items;
     this.items = [];
-    const allItems = mergeRowBatch(await Promise.all(itemPromises)).reverse();
+    const allItems = await (async () => {
+      try {
+        const itemPromises = itemLazyValues.map((x) => x.get());
+        return mergeRowBatch(await Promise.all(itemPromises)).reverse();
+      } catch (e) {
+        console.warn(
+          "Encountered error when constructing records to flush:\n",
+          e
+        );
+        return [];
+      }
+    })();
     let postPromises = [];
     while (true) {
       const items = [];
@@ -10951,9 +10977,7 @@ var BackgroundLogger = class {
           for (let i = 0; i < NumRetries; i++) {
             const startTime = now();
             try {
-              return (await (await this.logConn).post_json("logs", itemsS)).map(
-                (res) => res.id
-              );
+              return (await (await this.logConn.get()).post_json("logs", itemsS)).map((res) => res.id);
             } catch (e) {
               const retryingText = i + 1 === NumRetries ? "" : " Retrying";
               const errMsg = (() => {
@@ -10999,6 +11023,7 @@ function init(project, options = {}) {
     dataset,
     baseExperiment,
     isPublic,
+    open,
     update,
     appUrl,
     apiKey,
@@ -11006,82 +11031,121 @@ function init(project, options = {}) {
     metadata,
     gitMetadataSettings
   } = options || {};
-  const lazyMetadata = (async () => {
-    await login({
-      orgName,
-      apiKey,
-      appUrl
-    });
-    const args = {
-      project_name: project,
-      org_id: _state.orgId
-    };
-    if (experiment) {
-      args["experiment_name"] = experiment;
-    }
-    if (description) {
-      args["description"] = description;
+  if (open) {
+    if (isEmpty(experiment)) {
+      throw new Error("Cannot open an experiment without specifying its name");
     }
     if (update) {
-      args["update"] = update;
+      throw new Error("Cannot open and update an experiment at the same time");
     }
-    let mergedGitMetadataSettings = {
-      ..._state.gitMetadataSettings || {
-        collect: "all"
+    const lazyMetadata2 = new LazyValue(async () => {
+      await login({
+        orgName,
+        apiKey,
+        appUrl
+      });
+      const args = {
+        project_name: project,
+        org_name: _state.orgName,
+        experiment_name: experiment
+      };
+      const response = await _state.apiConn().post_json("api/experiment/get", args);
+      if (response.length === 0) {
+        throw new Error(
+          `Experiment ${experiment} not found in project ${project}.`
+        );
       }
-    };
-    if (gitMetadataSettings) {
-      mergedGitMetadataSettings = mergeGitMetadataSettings(
-        mergedGitMetadataSettings,
-        gitMetadataSettings
-      );
-    }
-    const repoStatus2 = await isomorph_default.getRepoStatus(gitMetadataSettings);
-    if (repoStatus2) {
-      args["repo_info"] = repoStatus2;
-    }
-    if (baseExperiment) {
-      args["base_experiment"] = baseExperiment;
-    } else {
-      args["ancestor_commits"] = await isomorph_default.getPastNAncestors();
-    }
-    if (dataset !== void 0) {
-      args["dataset_id"] = dataset.id;
-      args["dataset_version"] = await dataset.version();
-    }
-    if (isPublic !== void 0) {
-      args["public"] = isPublic;
-    }
-    if (metadata) {
-      args["metadata"] = metadata;
-    }
-    let response = null;
-    while (true) {
-      try {
-        response = await _state.apiConn().post_json("api/experiment/register", args);
-        break;
-      } catch (e) {
-        if (args["base_experiment"] && `${"data" in e && e.data}`.includes("base experiment")) {
-          console.warn(`Base experiment ${args["base_experiment"]} not found.`);
-          delete args["base_experiment"];
-        } else {
-          throw e;
+      const info = response[0];
+      return {
+        id: info.id,
+        name: info.name,
+        fullInfo: info
+      };
+    });
+    return new ReadonlyExperiment(
+      lazyMetadata2
+    );
+  }
+  const lazyMetadata = new LazyValue(
+    async () => {
+      await login({
+        orgName,
+        apiKey,
+        appUrl
+      });
+      const args = {
+        project_name: project,
+        org_id: _state.orgId
+      };
+      if (experiment) {
+        args["experiment_name"] = experiment;
+      }
+      if (description) {
+        args["description"] = description;
+      }
+      if (update) {
+        args["update"] = update;
+      }
+      let mergedGitMetadataSettings = {
+        ..._state.gitMetadataSettings || {
+          collect: "all"
         }
+      };
+      if (gitMetadataSettings) {
+        mergedGitMetadataSettings = mergeGitMetadataSettings(
+          mergedGitMetadataSettings,
+          gitMetadataSettings
+        );
       }
-    }
-    return {
-      project: {
-        id: response.project.id,
-        name: response.project.name,
-        fullInfo: response.project
-      },
-      experiment: {
-        id: response.experiment.id,
-        name: response.experiment.name,
-        fullInfo: response.experiment
+      const repoStatus2 = await isomorph_default.getRepoStatus(gitMetadataSettings);
+      if (repoStatus2) {
+        args["repo_info"] = repoStatus2;
       }
-    };
-  })();
+      if (baseExperiment) {
+        args["base_experiment"] = baseExperiment;
+      } else {
+        args["ancestor_commits"] = await isomorph_default.getPastNAncestors();
+      }
+      if (dataset !== void 0) {
+        args["dataset_id"] = await dataset.id;
+        args["dataset_version"] = await dataset.version();
+      }
+      if (isPublic !== void 0) {
+        args["public"] = isPublic;
+      }
+      if (metadata) {
+        args["metadata"] = metadata;
+      }
+      let response = null;
+      while (true) {
+        try {
+          response = await _state.apiConn().post_json("api/experiment/register", args);
+          break;
+        } catch (e) {
+          if (args["base_experiment"] && `${"data" in e && e.data}`.includes("base experiment")) {
+            console.warn(
+              `Base experiment ${args["base_experiment"]} not found.`
+            );
+            delete args["base_experiment"];
+          } else {
+            throw e;
+          }
+        }
+      }
+      return {
+        project: {
+          id: response.project.id,
+          name: response.project.name,
+          fullInfo: response.project
+        },
+        experiment: {
+          id: response.experiment.id,
+          name: response.experiment.name,
+          fullInfo: response.experiment
+        }
+      };
+    }
+  );
   const ret = new Experiment(lazyMetadata, dataset);
   if (options.setCurrent ?? true) {
     _state.currentExperiment = ret;
@@ -11206,15 +11270,15 @@ function validateAndSanitizeExperimentLogPartialArgs(event) {
   }
 }
 function validateAndSanitizeExperimentLogFullArgs(event, hasDataset) {
-  if ("input" in event && event.input && "inputs" in event && event.inputs || !("input" in event) && !("inputs" in event)) {
+  if ("input" in event && !isEmpty(event.input) && "inputs" in event && !isEmpty(event.inputs) || !("input" in event) && !("inputs" in event)) {
     throw new Error(
       "Exactly one of input or inputs (deprecated) must be specified. Prefer input."
     );
   }
-  if (!event.output) {
+  if (isEmpty(event.output)) {
     throw new Error("output must be specified");
   }
-  if (!event.scores) {
+  if (isEmpty(event.scores)) {
     throw new Error("scores must be specified");
   }
   if (hasDataset && event.datasetRecordId === void 0) {
@@ -11226,33 +11290,88 @@ function validateAndSanitizeExperimentLogFullArgs(event, hasDataset) {
   }
   return event;
 }
-var Experiment = class {
+var ObjectFetcher = class {
+  constructor(objectType, pinnedVersion) {
+    this.objectType = objectType;
+    this.pinnedVersion = pinnedVersion;
+    this._fetchedData = void 0;
+  }
+  get id() {
+    throw new Error("ObjectFetcher subclasses must have an 'id' attribute");
+  }
+  async getState() {
+    throw new Error("ObjectFetcher subclasses must have a 'getState' method");
+  }
+  async *fetch() {
+    const records = await this.fetchedData();
+    for (const record of records) {
+      yield record;
+    }
+  }
+  [Symbol.iterator]() {
+    return this.fetch();
+  }
+  async fetchedData() {
+    if (this._fetchedData === void 0) {
+      const state = await this.getState();
+      const resp = await state.logConn().get(`object/${this.objectType}`, {
+        id: await this.id,
+        fmt: "json2",
+        version: this.pinnedVersion
+      });
+      this._fetchedData = await resp.json();
+    }
+    return this._fetchedData || [];
+  }
+  clearCache() {
+    this._fetchedData = void 0;
+  }
+  async version() {
+    if (this.pinnedVersion !== void 0) {
+      return this.pinnedVersion;
+    } else {
+      const fetchedData = await this.fetchedData();
+      let maxVersion = void 0;
+      for (const record of fetchedData) {
+        const xactId = record[TRANSACTION_ID_FIELD];
+        if (maxVersion === void 0 || (xactId ?? xactId > maxVersion)) {
+          maxVersion = xactId;
+        }
+      }
+      return maxVersion;
+    }
+  }
+};
+var Experiment = class extends ObjectFetcher {
   constructor(lazyMetadata, dataset) {
+    super("experiment", void 0);
     // For type identification.
     this.kind = "experiment";
     this.lazyMetadata = lazyMetadata;
     this.dataset = dataset;
-    const logConn = this.getState().then((state) => state.logConn());
+    const logConn = new LazyValue(
+      () => this.getState().then((state) => state.logConn())
+    );
     this.bgLogger = new BackgroundLogger(logConn);
     this.lastStartTime = getCurrentUnixTimestamp();
   }
   get id() {
     return (async () => {
-      return (await this.lazyMetadata).experiment.id;
+      return (await this.lazyMetadata.get()).experiment.id;
     })();
   }
   get name() {
     return (async () => {
-      return (await this.lazyMetadata).experiment.name;
+      return (await this.lazyMetadata.get()).experiment.name;
     })();
   }
   get project() {
     return (async () => {
-      return (await this.lazyMetadata).project;
+      return (await this.lazyMetadata.get()).project;
     })();
   }
   async getState() {
-    await this.lazyMetadata;
+    await this.lazyMetadata.get();
     return _state;
   }
   /**
@@ -11312,12 +11431,32 @@ var Experiment = class {
   startSpan(args) {
     const { name, ...argsRest } = args ?? {};
     return new SpanImpl({
-      parentIds: this.lazyParentIds(),
+      parentIds: new LazyValue(() => this.lazyParentIds()),
       bgLogger: this.bgLogger,
       name: name ?? "root",
       ...argsRest
     });
   }
+  async fetchBaseExperiment() {
+    const state = await this.getState();
+    const conn = state.apiConn();
+    try {
+      const resp = await conn.post("/api/base_experiment/get_id", {
+        id: await this.id
+      });
+      const base = await resp.json();
+      return {
+        id: base["base_exp_id"],
+        name: base["base_exp_name"]
+      };
+    } catch (e) {
+      if (e instanceof FailedHTTPResponse && e.status === 400) {
+        return null;
+      } else {
+        throw e;
+      }
+    }
+  }
   /**
    * Summarize the experiment, including the scores (compared to the closest reference experiment) and metadata.
    *
@@ -11341,14 +11480,10 @@ var Experiment = class {
     let comparisonExperimentName = void 0;
     if (summarizeScores) {
       if (comparisonExperimentId === void 0) {
-        const conn = state.logConn();
-        const resp = await conn.get("/crud/base_experiments", {
-          id: await this.id
-        });
-        const base_experiments = await resp.json();
-        if (base_experiments.length > 0) {
-          comparisonExperimentId = base_experiments[0]["base_exp_id"];
-          comparisonExperimentName = base_experiments[0]["base_exp_name"];
+        const baseExperiment = await this.fetchBaseExperiment();
+        if (baseExperiment !== null) {
+          comparisonExperimentId = baseExperiment.id;
+          comparisonExperimentName = baseExperiment.name;
         }
       }
       if (comparisonExperimentId !== void 0) {
@@ -11386,7 +11521,11 @@ var Experiment = class {
    * @param event.source (Optional) the source of the feedback. Must be one of "external" (default), "app", or "api".
    */
   logFeedback(event) {
-    logFeedbackImpl(this.bgLogger, this.lazyParentIds(), event);
+    logFeedbackImpl(
+      this.bgLogger,
+      new LazyValue(() => this.lazyParentIds()),
+      event
+    );
   }
   /**
    * Flush any pending rows to the server.
@@ -11404,6 +11543,40 @@ var Experiment = class {
     return this.id;
   }
 };
+var ReadonlyExperiment = class extends ObjectFetcher {
+  constructor(lazyMetadata) {
+    super("experiment", void 0);
+    this.lazyMetadata = lazyMetadata;
+  }
+  get id() {
+    return (async () => {
+      return (await this.lazyMetadata.get()).id;
+    })();
+  }
+  get name() {
+    return (async () => {
+      return (await this.lazyMetadata.get()).name;
+    })();
+  }
+  async getState() {
+    await this.lazyMetadata.get();
+    return _state;
+  }
+  async *asDataset() {
+    const records = this.fetch();
+    for await (const record of records) {
+      if (record.root_span_id !== record.span_id) {
+        continue;
+      }
+      const { output, expected } = record;
+      yield {
+        input: record.input,
+        expected: expected ?? output
+      };
+    }
+  }
+};
+var executionCounter = 0;
 var SpanImpl = class _SpanImpl {
   // root_experiment should only be specified for a root span. parent_span
   // should only be specified for non-root spans.
@@ -11429,7 +11602,11 @@ var SpanImpl = class _SpanImpl {
         start: args.startTime ?? getCurrentUnixTimestamp()
       },
       context: { ...callerLocation },
-      span_attributes: { ...args.spanAttributes, name },
+      span_attributes: {
+        ...args.spanAttributes,
+        name,
+        exec_counter: executionCounter++
+      },
       created: (/* @__PURE__ */ new Date()).toISOString()
     };
     this.parentIds = args.parentIds;
@@ -11467,18 +11644,18 @@ var SpanImpl = class _SpanImpl {
     if (sanitizedAndInternalData.metrics?.end) {
       this.loggedEndTime = sanitizedAndInternalData.metrics?.end;
     }
-    const parentIds = (async () => {
-      const { kind, ...ids } = await this.parentIds;
+    const parentIds = new LazyValue(async () => {
+      const { kind, ...ids } = await this.parentIds.get();
       return ids;
-    })();
-    const record = (async () => {
+    });
+    const record = new LazyValue(async () => {
       return {
         ...sanitizedAndInternalData,
         ...this.rowIds,
-        ...await parentIds,
+        ...await parentIds.get(),
         [IS_MERGE_FIELD]: this.isMerge
       };
-    })();
+    });
     this.bgLogger.log([record]);
   }
   logFeedback(event) {
@@ -11527,207 +11704,6 @@ var SpanImpl = class _SpanImpl {
     return this.end(args);
   }
 };
-var Dataset = class {
-  constructor(lazyMetadata, pinnedVersion) {
-    this._fetchedData = void 0;
-    this.lazyMetadata = lazyMetadata;
-    this.pinnedVersion = pinnedVersion;
-    const logConn = this.getState().then((state) => state.logConn());
-    this.bgLogger = new BackgroundLogger(logConn);
-  }
-  get id() {
-    return (async () => {
-      return (await this.lazyMetadata).dataset.id;
-    })();
-  }
-  get name() {
-    return (async () => {
-      return (await this.lazyMetadata).dataset.name;
-    })();
-  }
-  get project() {
-    return (async () => {
-      return (await this.lazyMetadata).project;
-    })();
-  }
-  async getState() {
-    await this.lazyMetadata;
-    return _state;
-  }
-  /**
-   * Insert a single record to the dataset. The record will be batched and uploaded behind the scenes. If you pass in an `id`,
-   * and a record with that `id` already exists, it will be overwritten (upsert).
-   *
-   * @param event The event to log.
-   * @param event.input The argument that uniquely define an input case (an arbitrary, JSON serializable object).
-   * @param event.output The output of your application, including post-processing (an arbitrary, JSON serializable object).
-   * @param event.metadata (Optional) a dictionary with additional data about the test example, model outputs, or just
-   * about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the
-   * `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any
-   * JSON-serializable type, but its keys must be strings.
-   * @param event.id (Optional) a unique identifier for the event. If you don't provide one, Braintrust will generate one for you.
-   * @returns The `id` of the logged record.
-   */
-  insert({
-    input,
-    output,
-    metadata,
-    id
-  }) {
-    if (metadata !== void 0) {
-      for (const key of Object.keys(metadata)) {
-        if (typeof key !== "string") {
-          throw new Error("metadata keys must be strings");
-        }
-      }
-    }
-    const rowId = id || v4_default();
-    const args = (async () => ({
-      id: rowId,
-      inputs: input,
-      output,
-      project_id: (await this.project).id,
-      dataset_id: await this.id,
-      created: (/* @__PURE__ */ new Date()).toISOString(),
-      metadata
-    }))();
-    this.bgLogger.log([args]);
-    return rowId;
-  }
-  delete(id) {
-    const args = (async () => ({
-      id,
-      project_id: (await this.project).id,
-      dataset_id: await this.id,
-      created: (/* @__PURE__ */ new Date()).toISOString(),
-      _object_delete: true
-    }))();
-    this.bgLogger.log([args]);
-    return id;
-  }
-  /**
-   * Summarize the dataset, including high level metrics about its size and other metadata.
-   * @param summarizeData Whether to summarize the data. If false, only the metadata will be returned.
-   * @returns `DatasetSummary`
-   * @returns A summary of the dataset.
-   */
-  async summarize(options = {}) {
-    let { summarizeData = true } = options || {};
-    await this.bgLogger.flush();
-    const state = await this.getState();
-    const projectUrl = `${state.appUrl}/app/${encodeURIComponent(
-      state.orgName
-    )}/p/${encodeURIComponent((await this.project).name)}`;
-    const datasetUrl = `${projectUrl}/d/${encodeURIComponent(await this.name)}`;
-    let dataSummary = void 0;
-    if (summarizeData) {
-      dataSummary = await state.logConn().get_json(
-        "dataset-summary",
-        {
-          dataset_id: await this.id
-        },
-        3
-      );
-    }
-    return {
-      projectName: (await this.project).name,
-      datasetName: await this.name,
-      projectUrl,
-      datasetUrl,
-      dataSummary
-    };
-  }
-  /**
-   * Fetch all records in the dataset.
-   *
-   * @example
-   * ```
-   * // Use an async iterator to fetch all records in the dataset.
-   * for await (const record of dataset.fetch()) {
-   *  console.log(record);
-   * }
-   *
-   * // You can also iterate over the dataset directly.
-   * for await (const record of dataset) {
-   *  console.log(record);
-   * }
-   * ```
-   *
-   * @returns An iterator over the dataset's records.
-   */
-  async *fetch() {
-    const records = await this.fetchedData();
-    for (const record of records) {
-      yield {
-        id: record.id,
-        input: record.input && JSON.parse(record.input),
-        output: record.input && JSON.parse(record.output),
-        metadata: record.metadata && JSON.parse(record.metadata)
-      };
-    }
-    this.clearCache();
-  }
-  /**
-   * Fetch all records in the dataset.
-   *
-   * @example
-   * ```
-   * // Use an async iterator to fetch all records in the dataset.
-   * for await (const record of dataset) {
-   *  console.log(record);
-   * }
-   * ```
-   */
-  [Symbol.asyncIterator]() {
-    return this.fetch();
-  }
-  async fetchedData() {
-    if (this._fetchedData === void 0) {
-      const state = await this.getState();
-      const resp = await state.logConn().get("object/dataset", {
-        id: await this.id,
-        fmt: "json",
-        version: this.pinnedVersion
-      });
-      const text = await resp.text();
-      this._fetchedData = text.split("\n").filter((x) => x.trim() !== "").map((x) => JSON.parse(x));
-    }
-    return this._fetchedData || [];
-  }
-  clearCache() {
-    this._fetchedData = void 0;
-  }
-  async version() {
-    if (this.pinnedVersion !== void 0) {
-      return this.pinnedVersion;
-    } else {
-      const fetchedData = await this.fetchedData();
-      let maxVersion = void 0;
-      for (const record of fetchedData) {
-        const xactId = record[TRANSACTION_ID_FIELD];
-        if (maxVersion === void 0 || (xactId ?? xactId > maxVersion)) {
-          maxVersion = xactId;
-        }
-      }
-      return maxVersion;
-    }
-  }
-  /**
-   * Flush any pending rows to the server.
-   */
-  async flush() {
-    return await this.bgLogger.flush();
-  }
-  /**
-   * This function is deprecated. You can simply remove it from your code.
-   */
-  async close() {
-    console.warn(
-      "close is deprecated and will be removed in a future version of braintrust. It is now a no-op and can be removed"
-    );
-    return this.id;
-  }
-};
 
 // src/progress.ts
 var cliProgress = __toESM(require_cli_progress());
@@ -11844,6 +11820,12 @@ var GlobalPaths = findGlobalPaths();
 // src/framework.ts
 var import_chalk = __toESM(require_source());
 var import_pluralize = __toESM(require_pluralize());
+function initExperiment(projectName, options = {}) {
+  return init(projectName, {
+    ...options,
+    setCurrent: false
+  });
+}
 globalThis._evals = {};
 function serializeJSONWithPlainString(v) {
   if (typeof v === "string") {
@@ -11890,10 +11872,37 @@ async function runEvaluator(experiment, evaluator, progressReporter, filters) {
   if (typeof evaluator.data === "string") {
     throw new Error("Unimplemented: string data paths");
   }
-  const dataResult = evaluator.data();
-  let data = null;
+  let dataResult = typeof evaluator.data === "function" ? evaluator.data() : evaluator.data;
+  if ("_type" in dataResult) {
+    if (dataResult._type !== "BaseExperiment") {
+      throw new Error("Invalid _type");
+    }
+    if (!experiment) {
+      throw new Error(
+        "Cannot use BaseExperiment() without connecting to Braintrust (you most likely set --no-send-logs)"
+      );
+    }
+    let name = dataResult.name;
+    if (isEmpty(name)) {
+      const baseExperiment = await experiment.fetchBaseExperiment();
+      if (!baseExperiment) {
+        throw new Error("BaseExperiment() failed to fetch base experiment");
+      }
+      name = baseExperiment.name;
+    }
+    dataResult = initExperiment(evaluator.projectName, {
+      experiment: name,
+      open: true
+    }).asDataset();
+  }
+  let data = [];
   if (dataResult instanceof Promise) {
     data = await dataResult;
+  } else if (Symbol.asyncIterator in dataResult) {
+    data = [];
+    for await (const d of dataResult) {
+      data.push(d);
+    }
   } else {
     data = dataResult;
   }
@@ -16054,7 +16063,9 @@ async function getBaseBranchAncestor(remote = void 0) {
   if (git === null) {
     throw new Error("Not in a git repo");
   }
-  const { remote: remoteName, branch: baseBranch } = await getBaseBranch(remote);
+  const { remote: remoteName, branch: baseBranch } = await getBaseBranch(
+    remote
+  );
   const isDirty = (await git.diffSummary()).files.length > 0;
   const head = isDirty ? "HEAD" : "HEAD^";
   try {
@@ -16474,7 +16485,7 @@ async function collectFiles(inputPath) {
       files.push(inputPath);
     }
   } else {
-    const walked = await import_util2.default.promisify(fsWalk.walk)(inputPath, {
+    const walked = await import_util3.default.promisify(fsWalk.walk)(inputPath, {
       deepFilter: (entry) => {
         return checkMatch(entry.path, null, EXCLUDE);
       },