braintrust 0.0.94 → 0.0.96

package/dist/cli.js

@@ -9065,7 +9065,7 @@ var require_package = __commonJS({
   "package.json"(exports2, module2) {
     module2.exports = {
       name: "braintrust",
-      version: "0.0.94",
+      version: "0.0.96",
       description: "SDK for integrating Braintrust",
       main: "./dist/index.js",
       browser: {
@@ -9108,7 +9108,7 @@ var require_package = __commonJS({
         typescript: "^5.3.3"
       },
       dependencies: {
-        "@braintrust/core": "^0.0.13",
+        "@braintrust/core": "^0.0.14",
         argparse: "^2.0.1",
         chalk: "^4.1.2",
         "cli-progress": "^3.12.0",
@@ -9128,7 +9128,7 @@ var esbuild = __toESM(require("esbuild"));
 var import_fs = __toESM(require("fs"));
 var import_os = __toESM(require("os"));
 var import_path = __toESM(require("path"));
-var import_util2 = __toESM(require("util"));
+var import_util3 = __toESM(require("util"));
 var fsWalk = __toESM(require_out3());
 
 // ../../node_modules/.pnpm/minimatch@9.0.3/node_modules/minimatch/dist/mjs/index.js
@@ -11023,6 +11023,7 @@ function init(project, options = {}) {
     dataset,
     baseExperiment,
     isPublic,
+    open,
     update,
     appUrl,
     apiKey,
@@ -11030,6 +11031,41 @@ function init(project, options = {}) {
     metadata,
     gitMetadataSettings
   } = options || {};
+  if (open) {
+    if (isEmpty(experiment)) {
+      throw new Error("Cannot open an experiment without specifying its name");
+    }
+    if (update) {
+      throw new Error("Cannot open and update an experiment at the same time");
+    }
+    const lazyMetadata2 = new LazyValue(async () => {
+      await login({
+        orgName,
+        apiKey,
+        appUrl
+      });
+      const args = {
+        project_name: project,
+        org_name: _state.orgName,
+        experiment_name: experiment
+      };
+      const response = await _state.apiConn().post_json("api/experiment/get", args);
+      if (response.length === 0) {
+        throw new Error(
+          `Experiment ${experiment} not found in project ${project}.`
+        );
+      }
+      const info = response[0];
+      return {
+        id: info.id,
+        name: info.name,
+        fullInfo: info
+      };
+    });
+    return new ReadonlyExperiment(
+      lazyMetadata2
+    );
+  }
   const lazyMetadata = new LazyValue(
     async () => {
       await login({
@@ -11071,7 +11107,7 @@ function init(project, options = {}) {
         args["ancestor_commits"] = await isomorph_default.getPastNAncestors();
       }
       if (dataset !== void 0) {
-        args["dataset_id"] = dataset.id;
+        args["dataset_id"] = await dataset.id;
         args["dataset_version"] = await dataset.version();
       }
       if (isPublic !== void 0) {
@@ -11234,15 +11270,15 @@ function validateAndSanitizeExperimentLogPartialArgs(event) {
   }
 }
 function validateAndSanitizeExperimentLogFullArgs(event, hasDataset) {
-  if ("input" in event && event.input && "inputs" in event && event.inputs || !("input" in event) && !("inputs" in event)) {
+  if ("input" in event && !isEmpty(event.input) && "inputs" in event && !isEmpty(event.inputs) || !("input" in event) && !("inputs" in event)) {
     throw new Error(
       "Exactly one of input or inputs (deprecated) must be specified. Prefer input."
     );
   }
-  if (!event.output) {
+  if (isEmpty(event.output)) {
     throw new Error("output must be specified");
   }
-  if (!event.scores) {
+  if (isEmpty(event.scores)) {
     throw new Error("scores must be specified");
   }
   if (hasDataset && event.datasetRecordId === void 0) {
@@ -11254,8 +11290,61 @@ function validateAndSanitizeExperimentLogFullArgs(event, hasDataset) {
   }
   return event;
 }
-var Experiment = class {
+var ObjectFetcher = class {
+  constructor(objectType, pinnedVersion) {
+    this.objectType = objectType;
+    this.pinnedVersion = pinnedVersion;
+    this._fetchedData = void 0;
+  }
+  get id() {
+    throw new Error("ObjectFetcher subclasses must have an 'id' attribute");
+  }
+  async getState() {
+    throw new Error("ObjectFetcher subclasses must have a 'getState' method");
+  }
+  async *fetch() {
+    const records = await this.fetchedData();
+    for (const record of records) {
+      yield record;
+    }
+  }
+  [Symbol.iterator]() {
+    return this.fetch();
+  }
+  async fetchedData() {
+    if (this._fetchedData === void 0) {
+      const state = await this.getState();
+      const resp = await state.logConn().get(`object/${this.objectType}`, {
+        id: await this.id,
+        fmt: "json2",
+        version: this.pinnedVersion
+      });
+      this._fetchedData = await resp.json();
+    }
+    return this._fetchedData || [];
+  }
+  clearCache() {
+    this._fetchedData = void 0;
+  }
+  async version() {
+    if (this.pinnedVersion !== void 0) {
+      return this.pinnedVersion;
+    } else {
+      const fetchedData = await this.fetchedData();
+      let maxVersion = void 0;
+      for (const record of fetchedData) {
+        const xactId = record[TRANSACTION_ID_FIELD];
+        if (maxVersion === void 0 || (xactId ?? xactId > maxVersion)) {
+          maxVersion = xactId;
+        }
+      }
+      return maxVersion;
+    }
+  }
+};
+var Experiment = class extends ObjectFetcher {
   constructor(lazyMetadata, dataset) {
+    super("experiment", void 0);
     // For type identification.
     this.kind = "experiment";
     this.lazyMetadata = lazyMetadata;
@@ -11348,6 +11437,26 @@ var Experiment = class {
       ...argsRest
     });
   }
+  async fetchBaseExperiment() {
+    const state = await this.getState();
+    const conn = state.apiConn();
+    try {
+      const resp = await conn.post("/api/base_experiment/get_id", {
+        id: await this.id
+      });
+      const base = await resp.json();
+      return {
+        id: base["base_exp_id"],
+        name: base["base_exp_name"]
+      };
+    } catch (e) {
+      if (e instanceof FailedHTTPResponse && e.status === 400) {
+        return null;
+      } else {
+        throw e;
+      }
+    }
+  }
   /**
    * Summarize the experiment, including the scores (compared to the closest reference experiment) and metadata.
    *
@@ -11371,14 +11480,10 @@ var Experiment = class {
     let comparisonExperimentName = void 0;
     if (summarizeScores) {
       if (comparisonExperimentId === void 0) {
-        const conn = state.logConn();
-        const resp = await conn.get("/crud/base_experiments", {
-          id: await this.id
-        });
-        const base_experiments = await resp.json();
-        if (base_experiments.length > 0) {
-          comparisonExperimentId = base_experiments[0]["base_exp_id"];
-          comparisonExperimentName = base_experiments[0]["base_exp_name"];
+        const baseExperiment = await this.fetchBaseExperiment();
+        if (baseExperiment !== null) {
+          comparisonExperimentId = baseExperiment.id;
+          comparisonExperimentName = baseExperiment.name;
         }
       }
       if (comparisonExperimentId !== void 0) {
@@ -11438,6 +11543,40 @@ var Experiment = class {
     return this.id;
   }
 };
+var ReadonlyExperiment = class extends ObjectFetcher {
+  constructor(lazyMetadata) {
+    super("experiment", void 0);
+    this.lazyMetadata = lazyMetadata;
+  }
+  get id() {
+    return (async () => {
+      return (await this.lazyMetadata.get()).id;
+    })();
+  }
+  get name() {
+    return (async () => {
+      return (await this.lazyMetadata.get()).name;
+    })();
+  }
+  async getState() {
+    await this.lazyMetadata.get();
+    return _state;
+  }
+  async *asDataset() {
+    const records = this.fetch();
+    for await (const record of records) {
+      if (record.root_span_id !== record.span_id) {
+        continue;
+      }
+      const { output, expected } = record;
+      yield {
+        input: record.input,
+        expected: expected ?? output
+      };
+    }
+  }
+};
+var executionCounter = 0;
 var SpanImpl = class _SpanImpl {
   // root_experiment should only be specified for a root span. parent_span
   // should only be specified for non-root spans.
@@ -11463,7 +11602,11 @@ var SpanImpl = class _SpanImpl {
         start: args.startTime ?? getCurrentUnixTimestamp()
       },
       context: { ...callerLocation },
-      span_attributes: { ...args.spanAttributes, name },
+      span_attributes: {
+        ...args.spanAttributes,
+        name,
+        exec_counter: executionCounter++
+      },
       created: (/* @__PURE__ */ new Date()).toISOString()
     };
     this.parentIds = args.parentIds;
@@ -11561,209 +11704,6 @@ var SpanImpl = class _SpanImpl {
     return this.end(args);
   }
 };
-var Dataset = class {
-  constructor(lazyMetadata, pinnedVersion) {
-    this._fetchedData = void 0;
-    this.lazyMetadata = lazyMetadata;
-    this.pinnedVersion = pinnedVersion;
-    const logConn = new LazyValue(
-      () => this.getState().then((state) => state.logConn())
-    );
-    this.bgLogger = new BackgroundLogger(logConn);
-  }
-  get id() {
-    return (async () => {
-      return (await this.lazyMetadata.get()).dataset.id;
-    })();
-  }
-  get name() {
-    return (async () => {
-      return (await this.lazyMetadata.get()).dataset.name;
-    })();
-  }
-  get project() {
-    return (async () => {
-      return (await this.lazyMetadata.get()).project;
-    })();
-  }
-  async getState() {
-    await this.lazyMetadata.get();
-    return _state;
-  }
-  /**
-   * Insert a single record to the dataset. The record will be batched and uploaded behind the scenes. If you pass in an `id`,
-   * and a record with that `id` already exists, it will be overwritten (upsert).
-   *
-   * @param event The event to log.
-   * @param event.input The argument that uniquely define an input case (an arbitrary, JSON serializable object).
-   * @param event.output The output of your application, including post-processing (an arbitrary, JSON serializable object).
-   * @param event.metadata (Optional) a dictionary with additional data about the test example, model outputs, or just
-   * about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the
-   * `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any
-   * JSON-serializable type, but its keys must be strings.
-   * @param event.id (Optional) a unique identifier for the event. If you don't provide one, Braintrust will generate one for you.
-   * @returns The `id` of the logged record.
-   */
-  insert({
-    input,
-    output,
-    metadata,
-    id
-  }) {
-    if (metadata !== void 0) {
-      for (const key of Object.keys(metadata)) {
-        if (typeof key !== "string") {
-          throw new Error("metadata keys must be strings");
-        }
-      }
-    }
-    const rowId = id || v4_default();
-    const args = new LazyValue(async () => ({
-      id: rowId,
-      inputs: input,
-      output,
-      project_id: (await this.project).id,
-      dataset_id: await this.id,
-      created: (/* @__PURE__ */ new Date()).toISOString(),
-      metadata
-    }));
-    this.bgLogger.log([args]);
-    return rowId;
-  }
-  delete(id) {
-    const args = new LazyValue(async () => ({
-      id,
-      project_id: (await this.project).id,
-      dataset_id: await this.id,
-      created: (/* @__PURE__ */ new Date()).toISOString(),
-      _object_delete: true
-    }));
-    this.bgLogger.log([args]);
-    return id;
-  }
-  /**
-   * Summarize the dataset, including high level metrics about its size and other metadata.
-   * @param summarizeData Whether to summarize the data. If false, only the metadata will be returned.
-   * @returns `DatasetSummary`
-   * @returns A summary of the dataset.
-   */
-  async summarize(options = {}) {
-    let { summarizeData = true } = options || {};
-    await this.bgLogger.flush();
-    const state = await this.getState();
-    const projectUrl = `${state.appUrl}/app/${encodeURIComponent(
-      state.orgName
-    )}/p/${encodeURIComponent((await this.project).name)}`;
-    const datasetUrl = `${projectUrl}/d/${encodeURIComponent(await this.name)}`;
-    let dataSummary = void 0;
-    if (summarizeData) {
-      dataSummary = await state.logConn().get_json(
-        "dataset-summary",
-        {
-          dataset_id: await this.id
-        },
-        3
-      );
-    }
-    return {
-      projectName: (await this.project).name,
-      datasetName: await this.name,
-      projectUrl,
-      datasetUrl,
-      dataSummary
-    };
-  }
-  /**
-   * Fetch all records in the dataset.
-   *
-   * @example
-   * ```
-   * // Use an async iterator to fetch all records in the dataset.
-   * for await (const record of dataset.fetch()) {
-   *  console.log(record);
-   * }
-   *
-   * // You can also iterate over the dataset directly.
-   * for await (const record of dataset) {
-   *  console.log(record);
-   * }
-   * ```
-   *
-   * @returns An iterator over the dataset's records.
-   */
-  async *fetch() {
-    const records = await this.fetchedData();
-    for (const record of records) {
-      yield {
-        id: record.id,
-        input: record.input && JSON.parse(record.input),
-        output: record.input && JSON.parse(record.output),
-        metadata: record.metadata && JSON.parse(record.metadata)
-      };
-    }
-    this.clearCache();
-  }
-  /**
-   * Fetch all records in the dataset.
-   *
-   * @example
-   * ```
-   * // Use an async iterator to fetch all records in the dataset.
-   * for await (const record of dataset) {
-   *  console.log(record);
-   * }
-   * ```
-   */
-  [Symbol.asyncIterator]() {
-    return this.fetch();
-  }
-  async fetchedData() {
-    if (this._fetchedData === void 0) {
-      const state = await this.getState();
-      const resp = await state.logConn().get("object/dataset", {
-        id: await this.id,
-        fmt: "json",
-        version: this.pinnedVersion
-      });
-      const text = await resp.text();
-      this._fetchedData = text.split("\n").filter((x) => x.trim() !== "").map((x) => JSON.parse(x));
-    }
-    return this._fetchedData || [];
-  }
-  clearCache() {
-    this._fetchedData = void 0;
-  }
-  async version() {
-    if (this.pinnedVersion !== void 0) {
-      return this.pinnedVersion;
-    } else {
-      const fetchedData = await this.fetchedData();
-      let maxVersion = void 0;
-      for (const record of fetchedData) {
-        const xactId = record[TRANSACTION_ID_FIELD];
-        if (maxVersion === void 0 || (xactId ?? xactId > maxVersion)) {
-          maxVersion = xactId;
-        }
-      }
-      return maxVersion;
-    }
-  }
-  /**
-   * Flush any pending rows to the server.
-   */
-  async flush() {
-    return await this.bgLogger.flush();
-  }
-  /**
-   * This function is deprecated. You can simply remove it from your code.
-   */
-  async close() {
-    console.warn(
-      "close is deprecated and will be removed in a future version of braintrust. It is now a no-op and can be removed"
-    );
-    return this.id;
-  }
-};
 
 // src/progress.ts
 var cliProgress = __toESM(require_cli_progress());
@@ -11880,6 +11820,12 @@ var GlobalPaths = findGlobalPaths();
 // src/framework.ts
 var import_chalk = __toESM(require_source());
 var import_pluralize = __toESM(require_pluralize());
+function initExperiment(projectName, options = {}) {
+  return init(projectName, {
+    ...options,
+    setCurrent: false
+  });
+}
 globalThis._evals = {};
 function serializeJSONWithPlainString(v) {
   if (typeof v === "string") {
@@ -11926,18 +11872,45 @@ async function runEvaluator(experiment, evaluator, progressReporter, filters) {
   if (typeof evaluator.data === "string") {
     throw new Error("Unimplemented: string data paths");
   }
-  const dataResult = evaluator.data();
-  let data = null;
+  let dataResult = typeof evaluator.data === "function" ? evaluator.data() : evaluator.data;
+  if ("_type" in dataResult) {
+    if (dataResult._type !== "BaseExperiment") {
+      throw new Error("Invalid _type");
+    }
+    if (!experiment) {
+      throw new Error(
+        "Cannot use BaseExperiment() without connecting to Braintrust (you most likely set --no-send-logs)"
+      );
+    }
+    let name = dataResult.name;
+    if (isEmpty(name)) {
+      const baseExperiment = await experiment.fetchBaseExperiment();
+      if (!baseExperiment) {
+        throw new Error("BaseExperiment() failed to fetch base experiment");
+      }
+      name = baseExperiment.name;
+    }
+    dataResult = initExperiment(evaluator.projectName, {
+      experiment: name,
+      open: true
+    }).asDataset();
+  }
+  let data = [];
   if (dataResult instanceof Promise) {
     data = await dataResult;
+  } else if (Symbol.asyncIterator in dataResult) {
+    data = [];
+    for await (const d of dataResult) {
+      data.push(d);
+    }
   } else {
     data = dataResult;
   }
-  data = data.filter((d) => filters.every((f) => evaluateFilter(d, f)));
-  progressReporter.start(evaluator.evalName, data.length);
-  const evals = data.flatMap(
+  data = data.filter((d) => filters.every((f) => evaluateFilter(d, f))).flatMap(
     (datum) => [...Array(evaluator.trialCount ?? 1).keys()].map(() => datum)
-  ).map(async (datum) => {
+  );
+  progressReporter.start(evaluator.evalName, data.length);
+  const evals = data.map(async (datum) => {
     let metadata = { ...datum.metadata };
     let output = void 0;
     let error2 = void 0;
@@ -16512,7 +16485,7 @@ async function collectFiles(inputPath) {
       files.push(inputPath);
     }
   } else {
-    const walked = await import_util2.default.promisify(fsWalk.walk)(inputPath, {
+    const walked = await import_util3.default.promisify(fsWalk.walk)(inputPath, {
       deepFilter: (entry) => {
         return checkMatch(entry.path, null, EXCLUDE);
       },

package/dist/framework.d.ts

@@ -1,13 +1,26 @@
 import chalk from "chalk";
-import { Experiment, ExperimentSummary, Metadata, Span } from "./logger";
+import { Experiment, ExperimentSummary, Metadata, Span, EvalCase } from "./logger";
 import { Score } from "@braintrust/core";
 import { ProgressReporter } from "./progress";
-export interface EvalCase<Input, Expected> {
-    input: Input;
-    expected?: Expected;
-    metadata?: Metadata;
-}
-export type EvalData<Input, Expected> = (() => EvalCase<Input, Expected>[]) | (() => Promise<EvalCase<Input, Expected>[]>);
+export type BaseExperiment<Input, Expected> = {
+    _type: "BaseExperiment";
+    _phantom?: [Input, Expected];
+    name?: string;
+};
+/**
+ * Use this to specify that the dataset should actually be the data from a previous (base) experiment.
+ * If you do not specify a name, Braintrust will automatically figure out the best base experiment to
+ * use based on your git history (or fall back to timestamps).
+ *
+ * @param options
+ * @param options.name The name of the base experiment to use. If unspecified, Braintrust will automatically figure out the best base
+ * using your git history (or fall back to timestamps).
+ * @returns
+ */
+export declare function BaseExperiment<Input = unknown, Expected = unknown>(options?: {
+    name?: string;
+}): BaseExperiment<Input, Expected>;
+export type EvalData<Input, Expected> = EvalCase<Input, Expected>[] | (() => EvalCase<Input, Expected>[]) | (() => Promise<EvalCase<Input, Expected>[]>) | AsyncGenerator<EvalCase<Input, Expected>> | BaseExperiment<Input, Expected> | (() => BaseExperiment<Input, Expected>);
 export type EvalTask<Input, Output> = ((input: Input, hooks: EvalHooks) => Promise<Output>) | ((input: Input, hooks: EvalHooks) => Output);
 export interface EvalHooks {
     meta: (info: Record<string, unknown>) => void;
@@ -60,7 +73,7 @@ declare global {
     var _evals: EvaluatorFile;
     var _lazy_load: boolean;
 }
-export declare function Eval<Input, Output, Expected>(name: string, evaluator: Evaluator<Input, Output, Expected>): Promise<void | ExperimentSummary>;
+export declare function Eval<Input, Output, Expected>(name: string, evaluator: Evaluator<Input, Output, Expected>): Promise<ExperimentSummary>;
 export declare function getLoadedEvals(): EvaluatorFile;
 export interface Filter {
     path: string[];

package/dist/index.d.ts

@@ -43,5 +43,5 @@
  * @module braintrust
  */
 export * from "./logger";
-export { Evaluator, EvalTask, Eval, EvalScorerArgs } from "./framework";
+export { BaseExperiment, Evaluator, EvalTask, Eval, EvalScorerArgs, } from "./framework";
 export * from "./oai";