braintrust 0.0.94 → 0.0.95

package/dist/index.js

@@ -3647,12 +3647,14 @@ var require_pluralize = __commonJS({
 // src/index.ts
 var src_exports = {};
 __export(src_exports, {
+  BaseExperiment: () => BaseExperiment,
   Dataset: () => Dataset,
   Eval: () => Eval,
   Experiment: () => Experiment,
   Logger: () => Logger,
   NOOP_SPAN: () => NOOP_SPAN,
   NoopSpan: () => NoopSpan,
+  ReadonlyExperiment: () => ReadonlyExperiment,
   SpanImpl: () => SpanImpl,
   _internalGetGlobalState: () => _internalGetGlobalState,
   _internalSetInitialState: () => _internalSetInitialState,
@@ -8522,6 +8524,7 @@ function init(project, options = {}) {
     dataset,
     baseExperiment,
     isPublic,
+    open,
     update,
     appUrl,
     apiKey,
@@ -8529,6 +8532,41 @@ function init(project, options = {}) {
     metadata,
     gitMetadataSettings
   } = options || {};
+  if (open) {
+    if (isEmpty(experiment)) {
+      throw new Error("Cannot open an experiment without specifying its name");
+    }
+    if (update) {
+      throw new Error("Cannot open and update an experiment at the same time");
+    }
+    const lazyMetadata2 = new LazyValue(async () => {
+      await login({
+        orgName,
+        apiKey,
+        appUrl
+      });
+      const args = {
+        project_name: project,
+        org_name: _state.orgName,
+        experiment_name: experiment
+      };
+      const response = await _state.apiConn().post_json("api/experiment/get", args);
+      if (response.length === 0) {
+        throw new Error(
+          `Experiment ${experiment} not found in project ${project}.`
+        );
+      }
+      const info = response[0];
+      return {
+        id: info.id,
+        name: info.name,
+        fullInfo: info
+      };
+    });
+    return new ReadonlyExperiment(
+      lazyMetadata2
+    );
+  }
   const lazyMetadata = new LazyValue(
     async () => {
       await login({
@@ -8570,7 +8608,7 @@ function init(project, options = {}) {
         args["ancestor_commits"] = await isomorph_default.getPastNAncestors();
       }
       if (dataset !== void 0) {
-        args["dataset_id"] = dataset.id;
+        args["dataset_id"] = await dataset.id;
         args["dataset_version"] = await dataset.version();
       }
       if (isPublic !== void 0) {
@@ -8925,15 +8963,15 @@ function validateAndSanitizeExperimentLogPartialArgs(event) {
   }
 }
 function validateAndSanitizeExperimentLogFullArgs(event, hasDataset) {
-  if ("input" in event && event.input && "inputs" in event && event.inputs || !("input" in event) && !("inputs" in event)) {
+  if ("input" in event && !isEmpty(event.input) && "inputs" in event && !isEmpty(event.inputs) || !("input" in event) && !("inputs" in event)) {
     throw new Error(
       "Exactly one of input or inputs (deprecated) must be specified. Prefer input."
     );
   }
-  if (!event.output) {
+  if (isEmpty(event.output)) {
     throw new Error("output must be specified");
   }
-  if (!event.scores) {
+  if (isEmpty(event.scores)) {
     throw new Error("scores must be specified");
   }
   if (hasDataset && event.datasetRecordId === void 0) {
@@ -8945,8 +8983,61 @@ function validateAndSanitizeExperimentLogFullArgs(event, hasDataset) {
   }
   return event;
 }
-var Experiment = class {
+var ObjectFetcher = class {
+  constructor(objectType, pinnedVersion) {
+    this.objectType = objectType;
+    this.pinnedVersion = pinnedVersion;
+    this._fetchedData = void 0;
+  }
+  get id() {
+    throw new Error("ObjectFetcher subclasses must have an 'id' attribute");
+  }
+  async getState() {
+    throw new Error("ObjectFetcher subclasses must have a 'getState' method");
+  }
+  async *fetch() {
+    const records = await this.fetchedData();
+    for (const record of records) {
+      yield record;
+    }
+  }
+  [Symbol.iterator]() {
+    return this.fetch();
+  }
+  async fetchedData() {
+    if (this._fetchedData === void 0) {
+      const state = await this.getState();
+      const resp = await state.logConn().get(`object/${this.objectType}`, {
+        id: await this.id,
+        fmt: "json2",
+        version: this.pinnedVersion
+      });
+      this._fetchedData = await resp.json();
+    }
+    return this._fetchedData || [];
+  }
+  clearCache() {
+    this._fetchedData = void 0;
+  }
+  async version() {
+    if (this.pinnedVersion !== void 0) {
+      return this.pinnedVersion;
+    } else {
+      const fetchedData = await this.fetchedData();
+      let maxVersion = void 0;
+      for (const record of fetchedData) {
+        const xactId = record[TRANSACTION_ID_FIELD];
+        if (maxVersion === void 0 || (xactId ?? xactId > maxVersion)) {
+          maxVersion = xactId;
+        }
+      }
+      return maxVersion;
+    }
+  }
+};
+var Experiment = class extends ObjectFetcher {
   constructor(lazyMetadata, dataset) {
+    super("experiment", void 0);
     // For type identification.
     this.kind = "experiment";
     this.lazyMetadata = lazyMetadata;
@@ -9039,6 +9130,26 @@ var Experiment = class {
       ...argsRest
     });
   }
+  async fetchBaseExperiment() {
+    const state = await this.getState();
+    const conn = state.apiConn();
+    try {
+      const resp = await conn.post("/api/base_experiment/get_id", {
+        id: await this.id
+      });
+      const base = await resp.json();
+      return {
+        id: base["base_exp_id"],
+        name: base["base_exp_name"]
+      };
+    } catch (e) {
+      if (e instanceof FailedHTTPResponse && e.status === 400) {
+        return null;
+      } else {
+        throw e;
+      }
+    }
+  }
   /**
    * Summarize the experiment, including the scores (compared to the closest reference experiment) and metadata.
    *
@@ -9062,14 +9173,10 @@ var Experiment = class {
     let comparisonExperimentName = void 0;
     if (summarizeScores) {
       if (comparisonExperimentId === void 0) {
-        const conn = state.logConn();
-        const resp = await conn.get("/crud/base_experiments", {
-          id: await this.id
-        });
-        const base_experiments = await resp.json();
-        if (base_experiments.length > 0) {
-          comparisonExperimentId = base_experiments[0]["base_exp_id"];
-          comparisonExperimentName = base_experiments[0]["base_exp_name"];
+        const baseExperiment = await this.fetchBaseExperiment();
+        if (baseExperiment !== null) {
+          comparisonExperimentId = baseExperiment.id;
+          comparisonExperimentName = baseExperiment.name;
         }
       }
       if (comparisonExperimentId !== void 0) {
@@ -9129,6 +9236,40 @@ var Experiment = class {
     return this.id;
   }
 };
+var ReadonlyExperiment = class extends ObjectFetcher {
+  constructor(lazyMetadata) {
+    super("experiment", void 0);
+    this.lazyMetadata = lazyMetadata;
+  }
+  get id() {
+    return (async () => {
+      return (await this.lazyMetadata.get()).id;
+    })();
+  }
+  get name() {
+    return (async () => {
+      return (await this.lazyMetadata.get()).name;
+    })();
+  }
+  async getState() {
+    await this.lazyMetadata.get();
+    return _state;
+  }
+  async *asDataset() {
+    const records = this.fetch();
+    for await (const record of records) {
+      if (record.root_span_id !== record.span_id) {
+        continue;
+      }
+      const { output, expected } = record;
+      yield {
+        input: record.input,
+        expected: expected ?? output
+      };
+    }
+  }
+};
+var executionCounter = 0;
 var SpanImpl = class _SpanImpl {
   // root_experiment should only be specified for a root span. parent_span
   // should only be specified for non-root spans.
@@ -9154,7 +9295,11 @@ var SpanImpl = class _SpanImpl {
         start: args.startTime ?? getCurrentUnixTimestamp()
       },
       context: { ...callerLocation },
-      span_attributes: { ...args.spanAttributes, name },
+      span_attributes: {
+        ...args.spanAttributes,
+        name,
+        exec_counter: executionCounter++
+      },
       created: (/* @__PURE__ */ new Date()).toISOString()
     };
     this.parentIds = args.parentIds;
@@ -9252,11 +9397,10 @@ var SpanImpl = class _SpanImpl {
     return this.end(args);
   }
 };
-var Dataset = class {
+var Dataset = class extends ObjectFetcher {
   constructor(lazyMetadata, pinnedVersion) {
-    this._fetchedData = void 0;
+    super("dataset", pinnedVersion);
     this.lazyMetadata = lazyMetadata;
-    this.pinnedVersion = pinnedVersion;
     const logConn = new LazyValue(
       () => this.getState().then((state) => state.logConn())
     );
@@ -9364,81 +9508,6 @@ var Dataset = class {
       dataSummary
     };
   }
-  /**
-   * Fetch all records in the dataset.
-   *
-   * @example
-   * ```
-   * // Use an async iterator to fetch all records in the dataset.
-   * for await (const record of dataset.fetch()) {
-   *  console.log(record);
-   * }
-   *
-   * // You can also iterate over the dataset directly.
-   * for await (const record of dataset) {
-   *  console.log(record);
-   * }
-   * ```
-   *
-   * @returns An iterator over the dataset's records.
-   */
-  async *fetch() {
-    const records = await this.fetchedData();
-    for (const record of records) {
-      yield {
-        id: record.id,
-        input: record.input && JSON.parse(record.input),
-        output: record.input && JSON.parse(record.output),
-        metadata: record.metadata && JSON.parse(record.metadata)
-      };
-    }
-    this.clearCache();
-  }
-  /**
-   * Fetch all records in the dataset.
-   *
-   * @example
-   * ```
-   * // Use an async iterator to fetch all records in the dataset.
-   * for await (const record of dataset) {
-   *  console.log(record);
-   * }
-   * ```
-   */
-  [Symbol.asyncIterator]() {
-    return this.fetch();
-  }
-  async fetchedData() {
-    if (this._fetchedData === void 0) {
-      const state = await this.getState();
-      const resp = await state.logConn().get("object/dataset", {
-        id: await this.id,
-        fmt: "json",
-        version: this.pinnedVersion
-      });
-      const text = await resp.text();
-      this._fetchedData = text.split("\n").filter((x) => x.trim() !== "").map((x) => JSON.parse(x));
-    }
-    return this._fetchedData || [];
-  }
-  clearCache() {
-    this._fetchedData = void 0;
-  }
-  async version() {
-    if (this.pinnedVersion !== void 0) {
-      return this.pinnedVersion;
-    } else {
-      const fetchedData = await this.fetchedData();
-      let maxVersion = void 0;
-      for (const record of fetchedData) {
-        const xactId = record[TRANSACTION_ID_FIELD];
-        if (maxVersion === void 0 || (xactId ?? xactId > maxVersion)) {
-          maxVersion = xactId;
-        }
-      }
-      return maxVersion;
-    }
-  }
   /**
    * Flush any pending rows to the server.
    */
@@ -9510,6 +9579,9 @@ var BarProgressReporter = class {
 
 // src/framework.ts
 var import_pluralize = __toESM(require_pluralize());
+function BaseExperiment(options = {}) {
+  return { _type: "BaseExperiment", ...options };
+}
 function makeEvalName(projectName, experimentName) {
   let out = projectName;
   if (experimentName) {
@@ -9517,6 +9589,12 @@ function makeEvalName(projectName, experimentName) {
   }
   return out;
 }
+function initExperiment(projectName, options = {}) {
+  return init(projectName, {
+    ...options,
+    setCurrent: false
+  });
+}
 globalThis._evals = {};
 async function Eval(name, evaluator) {
   const evalName = makeEvalName(name, evaluator.experimentName);
@@ -9525,15 +9603,22 @@ async function Eval(name, evaluator) {
   }
   if (globalThis._lazy_load) {
     _evals[evalName] = { evalName, projectName: name, ...evaluator };
-    return;
+    return {
+      projectName: "_lazy_load",
+      experimentName: "_lazy_load",
+      projectUrl: "",
+      experimentUrl: "",
+      comparisonExperimentName: "",
+      scores: {},
+      metrics: {}
+    };
   }
   const progressReporter = new BarProgressReporter();
   try {
-    const experiment = init(name, {
+    const experiment = initExperiment(name, {
       experiment: evaluator.experimentName,
       metadata: evaluator.metadata,
-      isPublic: evaluator.isPublic,
-      setCurrent: false
+      isPublic: evaluator.isPublic
     });
     try {
       const ret = await runEvaluator(
@@ -9574,10 +9659,37 @@ async function runEvaluator(experiment, evaluator, progressReporter, filters) {
   if (typeof evaluator.data === "string") {
     throw new Error("Unimplemented: string data paths");
   }
-  const dataResult = evaluator.data();
-  let data = null;
+  let dataResult = typeof evaluator.data === "function" ? evaluator.data() : evaluator.data;
+  if ("_type" in dataResult) {
+    if (dataResult._type !== "BaseExperiment") {
+      throw new Error("Invalid _type");
+    }
+    if (!experiment) {
+      throw new Error(
+        "Cannot use BaseExperiment() without connecting to Braintrust (you most likely set --no-send-logs)"
+      );
+    }
+    let name = dataResult.name;
+    if (isEmpty(name)) {
+      const baseExperiment = await experiment.fetchBaseExperiment();
+      if (!baseExperiment) {
+        throw new Error("BaseExperiment() failed to fetch base experiment");
+      }
+      name = baseExperiment.name;
+    }
+    dataResult = initExperiment(evaluator.projectName, {
+      experiment: name,
+      open: true
+    }).asDataset();
+  }
+  let data = [];
   if (dataResult instanceof Promise) {
     data = await dataResult;
+  } else if (Symbol.asyncIterator in dataResult) {
+    data = [];
+    for await (const d of dataResult) {
+      data.push(d);
+    }
   } else {
     data = dataResult;
   }
@@ -9990,12 +10102,14 @@ var WrapperStream = class {
 configureNode();
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
+  BaseExperiment,
   Dataset,
   Eval,
   Experiment,
   Logger,
   NOOP_SPAN,
   NoopSpan,
+  ReadonlyExperiment,
   SpanImpl,
   _internalGetGlobalState,
   _internalSetInitialState,

package/dist/logger.d.ts

@@ -1,5 +1,5 @@
 /// <reference lib="dom" />
-import { IS_MERGE_FIELD, PARENT_ID_FIELD, Source, AUDIT_SOURCE_FIELD, AUDIT_METADATA_FIELD, GitMetadataSettings } from "@braintrust/core";
+import { TRANSACTION_ID_FIELD, IS_MERGE_FIELD, PARENT_ID_FIELD, Source, AUDIT_SOURCE_FIELD, AUDIT_METADATA_FIELD, GitMetadataSettings, TransactionId } from "@braintrust/core";
 import { IsoAsyncLocalStorage } from "./isomorph";
 import { LazyValue } from "./util";
 export type Metadata = Record<string, unknown>;
@@ -311,7 +311,10 @@ declare class BackgroundLogger {
     flush_once(batchSize?: number): Promise<string[]>;
     flush(): Promise<void>;
 }
-export type InitOptions = {
+type InitOpenOption<IsOpen extends boolean> = {
+    open?: IsOpen;
+};
+export type InitOptions<IsOpen extends boolean> = {
     experiment?: string;
     description?: string;
     dataset?: Dataset;
@@ -324,7 +327,8 @@ export type InitOptions = {
     metadata?: Metadata;
     gitMetadataSettings?: GitMetadataSettings;
     setCurrent?: boolean;
-};
+} & InitOpenOption<IsOpen>;
+type InitializedExperiment<IsOpen extends boolean | undefined> = IsOpen extends true ? ReadonlyExperiment : Experiment;
 /**
  * Log in, and then initialize a new experiment in a specified project. If the project does not exist, it will be created.
  *
@@ -348,13 +352,14 @@ export type InitOptions = {
  * JSON-serializable type, but its keys must be strings.
  * @param options.gitMetadataSettings (Optional) Settings for collecting git metadata. By default, will collect all git metadata fields allowed in org-level settings.
  * @param setCurrent If true (the default), set the global current-experiment to the newly-created one.
+ * @param options.open If the experiment already exists, open it in read-only mode.
  * @returns The newly created Experiment.
  */
-export declare function init(project: string, options?: Readonly<InitOptions>): Experiment;
+export declare function init<IsOpen extends boolean = false>(project: string, options?: Readonly<InitOptions<IsOpen>>): InitializedExperiment<IsOpen>;
 /**
  * This function is deprecated. Use `init` instead.
  */
-export declare function withExperiment<R>(project: string, callback: (experiment: Experiment) => R, options?: Readonly<InitOptions & SetCurrentArg>): R;
+export declare function withExperiment<R>(project: string, callback: (experiment: Experiment) => R, options?: Readonly<InitOptions<false> & SetCurrentArg>): R;
 /**
  * This function is deprecated. Use `initLogger` instead.
  */
@@ -486,6 +491,27 @@ export declare function traced<IsAsyncFlush extends boolean = false, R = void>(c
  * See `traced` for full details.
  */
 export declare function startSpan<IsAsyncFlush extends boolean = false>(args?: StartSpanArgs & AsyncFlushArg<IsAsyncFlush>): Span;
+export type WithTransactionId<R> = R & {
+    [TRANSACTION_ID_FIELD]: TransactionId;
+};
+declare class ObjectFetcher<RecordType> {
+    private objectType;
+    private pinnedVersion;
+    private _fetchedData;
+    constructor(objectType: "dataset" | "experiment", pinnedVersion: string | undefined);
+    get id(): Promise<string>;
+    protected getState(): Promise<BraintrustState>;
+    fetch(): AsyncGenerator<WithTransactionId<RecordType>>;
+    [Symbol.iterator](): AsyncGenerator<WithTransactionId<RecordType>, any, unknown>;
+    fetchedData(): Promise<WithTransactionId<RecordType>[]>;
+    clearCache(): void;
+    version(): Promise<string | bigint | undefined>;
+}
+export interface EvalCase<Input, Expected> {
+    input: Input;
+    expected?: Expected;
+    metadata?: Metadata;
+}
 /**
  * An experiment is a collection of logged events, such as model inputs and outputs, which represent
  * a snapshot of your application at a particular point in time. An experiment is meant to capture more
@@ -498,7 +524,7 @@ export declare function startSpan<IsAsyncFlush extends boolean = false>(args?: S
  *
  * You should not create `Experiment` objects directly. Instead, use the `braintrust.init()` method.
  */
-export declare class Experiment {
+export declare class Experiment extends ObjectFetcher<ExperimentEvent> {
     private readonly lazyMetadata;
     readonly dataset?: Dataset;
     private bgLogger;
@@ -508,7 +534,7 @@ export declare class Experiment {
     get id(): Promise<string>;
     get name(): Promise<string>;
     get project(): Promise<ObjectMetadata>;
-    private getState;
+    protected getState(): Promise<BraintrustState>;
     /**
      * Log a single event to the experiment. The event will be batched and uploaded behind the scenes.
      *
@@ -540,6 +566,10 @@ export declare class Experiment {
      * See `traced` for full details.
      */
     startSpan(args?: StartSpanArgs): Span;
+    fetchBaseExperiment(): Promise<{
+        id: any;
+        name: any;
+    } | null>;
     /**
      * Summarize the experiment, including the scores (compared to the closest reference experiment) and metadata.
      *
@@ -573,6 +603,17 @@ export declare class Experiment {
      */
     close(): Promise<string>;
 }
+/**
+ * A read-only view of an experiment, initialized by passing `open: true` to `init()`.
+ */
+export declare class ReadonlyExperiment extends ObjectFetcher<ExperimentEvent> {
+    private readonly lazyMetadata;
+    constructor(lazyMetadata: LazyValue<ObjectMetadata>);
+    get id(): Promise<string>;
+    get name(): Promise<string>;
+    protected getState(): Promise<BraintrustState>;
+    asDataset<Input = unknown, Expected = unknown>(): AsyncGenerator<EvalCase<Input, Expected>>;
+}
 interface ParentExperimentIds {
     kind: "experiment";
     project_id: string;
@@ -625,16 +666,14 @@ export declare class SpanImpl implements Span {
  *
  * You should not create `Dataset` objects directly. Instead, use the `braintrust.initDataset()` method.
  */
-export declare class Dataset {
+export declare class Dataset extends ObjectFetcher<DatasetRecord> {
     private readonly lazyMetadata;
-    private pinnedVersion?;
-    private _fetchedData?;
     private bgLogger;
     constructor(lazyMetadata: LazyValue<ProjectDatasetMetadata>, pinnedVersion?: string);
     get id(): Promise<string>;
     get name(): Promise<string>;
     get project(): Promise<ObjectMetadata>;
-    private getState;
+    protected getState(): Promise<BraintrustState>;
     /**
      * Insert a single record to the dataset. The record will be batched and uploaded behind the scenes. If you pass in an `id`,
      * and a record with that `id` already exists, it will be overwritten (upsert).
@@ -665,40 +704,6 @@ export declare class Dataset {
     summarize(options?: {
         readonly summarizeData?: boolean;
     }): Promise<DatasetSummary>;
-    /**
-     * Fetch all records in the dataset.
-     *
-     * @example
-     * ```
-     * // Use an async iterator to fetch all records in the dataset.
-     * for await (const record of dataset.fetch()) {
-     *  console.log(record);
-     * }
-     *
-     * // You can also iterate over the dataset directly.
-     * for await (const record of dataset) {
-     *  console.log(record);
-     * }
-     * ```
-     *
-     * @returns An iterator over the dataset's records.
-     */
-    fetch(): AsyncGenerator<DatasetRecord>;
-    /**
-     * Fetch all records in the dataset.
-     *
-     * @example
-     * ```
-     * // Use an async iterator to fetch all records in the dataset.
-     * for await (const record of dataset) {
-     *  console.log(record);
-     * }
-     * ```
-     */
-    [Symbol.asyncIterator](): AsyncGenerator<DatasetRecord, any, unknown>;
-    fetchedData(): Promise<any[]>;
-    clearCache(): void;
-    version(): Promise<any>;
     /**
      * Flush any pending rows to the server.
      */