braintrust 0.0.96 → 0.0.98

package/dist/browser.js

@@ -12,7 +12,7 @@ var DefaultAsyncLocalStorage = class {
   }
 };
 var iso = {
-  getRepoStatus: async (_settings) => void 0,
+  getRepoInfo: async (_settings) => void 0,
   getPastNAncestors: async () => [],
   getEnv: (_name) => void 0,
   getCallerLocation: () => void 0,
@@ -70,9 +70,10 @@ function v4(options, buf, offset) {
 }
 var v4_default = v4;
 
-// ../core/js/dist/index.mjs
+// ../core/js/dist/main/index.mjs
 var TRANSACTION_ID_FIELD = "_xact_id";
 var IS_MERGE_FIELD = "_is_merge";
+var MERGE_PATHS_FIELD = "_merge_paths";
 var AUDIT_SOURCE_FIELD = "_audit_source";
 var AUDIT_METADATA_FIELD = "_audit_metadata";
 var VALID_SOURCES = ["app", "api", "external"];
@@ -131,6 +132,54 @@ function mergeRowBatch(rows) {
   out.push(...Object.values(rowGroups));
   return out;
 }
+var DEFAULT_IS_LEGACY_DATASET = true;
+function ensureDatasetRecord(r, legacy) {
+  if (legacy) {
+    return ensureLegacyDatasetRecord(r);
+  } else {
+    return ensureNewDatasetRecord(r);
+  }
+}
+function ensureLegacyDatasetRecord(r) {
+  if ("output" in r) {
+    return r;
+  }
+  const row = {
+    ...r,
+    output: r.expected
+  };
+  delete row.expected;
+  return row;
+}
+function ensureNewDatasetRecord(r) {
+  if ("expected" in r) {
+    return r;
+  }
+  const row = {
+    ...r,
+    expected: r.output
+  };
+  delete row.output;
+  return row;
+}
+function makeLegacyEvent(e) {
+  if (!("dataset_id" in e) || !("expected" in e)) {
+    return e;
+  }
+  const event = {
+    ...e,
+    output: e.expected
+  };
+  delete event.expected;
+  if (MERGE_PATHS_FIELD in event) {
+    for (const path of event[MERGE_PATHS_FIELD] || []) {
+      if (path.length > 0 && path[0] === "expected") {
+        path[0] = "output";
+      }
+    }
+  }
+  return event;
+}
 var SpanTypeAttribute = /* @__PURE__ */ ((SpanTypeAttribute2) => {
   SpanTypeAttribute2["LLM"] = "llm";
   SpanTypeAttribute2["SCORE"] = "score";
@@ -596,6 +645,9 @@ var MaxRequestSize = 6 * 1024 * 1024;
 function constructJsonArray(items) {
   return `[${items.join(",")}]`;
 }
+function constructLogs3Data(items) {
+  return `{"rows": ${constructJsonArray(items)}, "api_version": 2}`;
+}
 var DefaultBatchSize = 100;
 var NumRetries = 3;
 function now() {
@@ -654,11 +706,20 @@ var BackgroundLogger = class {
       }
       postPromises.push(
         (async () => {
-          const itemsS = constructJsonArray(items);
+          const dataS = constructLogs3Data(items);
           for (let i = 0; i < NumRetries; i++) {
             const startTime = now();
             try {
-              return (await (await this.logConn.get()).post_json("logs", itemsS)).map((res) => res.id);
+              try {
+                return (await (await this.logConn.get()).post_json("logs3", dataS)).ids.map((res) => res.id);
+              } catch (e) {
+                const legacyDataS = constructJsonArray(
+                  items.map(
+                    (r) => JSON.stringify(makeLegacyEvent(JSON.parse(r)))
+                  )
+                );
+                return (await (await this.logConn.get()).post_json("logs", legacyDataS)).map((res) => res.id);
+              }
             } catch (e) {
               const retryingText = i + 1 === NumRetries ? "" : " Retrying";
               const errMsg = (() => {
@@ -669,7 +730,7 @@ var BackgroundLogger = class {
                 }
               })();
               console.warn(
-                `log request failed. Elapsed time: ${(now() - startTime) / 1e3} seconds. Payload size: ${itemsS.length}. Error: ${errMsg}.${retryingText}`
+                `log request failed. Elapsed time: ${(now() - startTime) / 1e3} seconds. Payload size: ${dataS.length}. Error: ${errMsg}.${retryingText}`
               );
             }
           }
@@ -697,8 +758,21 @@ var BackgroundLogger = class {
     }
   }
 };
-function init(project, options = {}) {
+function init(projectOrOptions, optionalOptions) {
+  const options = (() => {
+    if (typeof projectOrOptions === "string") {
+      return { ...optionalOptions, project: projectOrOptions };
+    } else {
+      if (optionalOptions !== void 0) {
+        throw new Error(
+          "Cannot specify options struct as both parameters. Must call either init(project, options) or init(options)."
+        );
+      }
+      return projectOrOptions;
+    }
+  })();
   const {
+    project,
     experiment,
     description,
     dataset,
@@ -710,42 +784,66 @@ function init(project, options = {}) {
     apiKey,
     orgName,
     metadata,
-    gitMetadataSettings
-  } = options || {};
-  if (open) {
+    gitMetadataSettings,
+    projectId,
+    baseExperimentId,
+    repoInfo
+  } = options;
+  if (open && update) {
+    throw new Error("Cannot open and update an experiment at the same time");
+  }
+  if (open || update) {
     if (isEmpty(experiment)) {
-      throw new Error("Cannot open an experiment without specifying its name");
-    }
-    if (update) {
-      throw new Error("Cannot open and update an experiment at the same time");
+      const action = open ? "open" : "update";
+      throw new Error(
+        `Cannot ${action} an experiment without specifying its name`
+      );
     }
-    const lazyMetadata2 = new LazyValue(async () => {
-      await login({
-        orgName,
-        apiKey,
-        appUrl
-      });
-      const args = {
-        project_name: project,
-        org_name: _state.orgName,
-        experiment_name: experiment
-      };
-      const response = await _state.apiConn().post_json("api/experiment/get", args);
-      if (response.length === 0) {
-        throw new Error(
-          `Experiment ${experiment} not found in project ${project}.`
-        );
+    const lazyMetadata2 = new LazyValue(
+      async () => {
+        await login({
+          orgName,
+          apiKey,
+          appUrl
+        });
+        const args = {
+          project_name: project,
+          project_id: projectId,
+          org_name: _state.orgName,
+          experiment_name: experiment
+        };
+        const response = await _state.apiConn().post_json("api/experiment/get", args);
+        if (response.length === 0) {
+          throw new Error(
+            `Experiment ${experiment} not found in project ${projectId ?? project}.`
+          );
+        }
+        const info = response[0];
+        return {
+          project: {
+            id: info.project_id,
+            name: "",
+            fullInfo: {}
+          },
+          experiment: {
+            id: info.id,
+            name: info.name,
+            fullInfo: info
+          }
+        };
       }
-      const info = response[0];
-      return {
-        id: info.id,
-        name: info.name,
-        fullInfo: info
-      };
-    });
-    return new ReadonlyExperiment(
-      lazyMetadata2
     );
+    if (open) {
+      return new ReadonlyExperiment(
+        lazyMetadata2
+      );
+    } else {
+      const ret2 = new Experiment(lazyMetadata2, dataset);
+      if (options.setCurrent ?? true) {
+        _state.currentExperiment = ret2;
+      }
+      return ret2;
+    }
   }
   const lazyMetadata = new LazyValue(
     async () => {
@@ -756,6 +854,7 @@ function init(project, options = {}) {
       });
       const args = {
         project_name: project,
+        project_id: projectId,
         org_id: _state.orgId
       };
       if (experiment) {
@@ -764,25 +863,29 @@ function init(project, options = {}) {
       if (description) {
         args["description"] = description;
       }
-      if (update) {
-        args["update"] = update;
-      }
-      let mergedGitMetadataSettings = {
-        ..._state.gitMetadataSettings || {
-          collect: "all"
+      const repoInfoArg = await (async () => {
+        if (repoInfo) {
+          return repoInfo;
         }
-      };
-      if (gitMetadataSettings) {
-        mergedGitMetadataSettings = mergeGitMetadataSettings(
-          mergedGitMetadataSettings,
-          gitMetadataSettings
-        );
-      }
-      const repoStatus = await isomorph_default.getRepoStatus(gitMetadataSettings);
-      if (repoStatus) {
-        args["repo_info"] = repoStatus;
+        let mergedGitMetadataSettings = {
+          ..._state.gitMetadataSettings || {
+            collect: "all"
+          }
+        };
+        if (gitMetadataSettings) {
+          mergedGitMetadataSettings = mergeGitMetadataSettings(
+            mergedGitMetadataSettings,
+            gitMetadataSettings
+          );
+        }
+        return await isomorph_default.getRepoInfo(mergedGitMetadataSettings);
+      })();
+      if (repoInfoArg) {
+        args["repo_info"] = repoInfoArg;
       }
-      if (baseExperiment) {
+      if (baseExperimentId) {
+        args["base_exp_id"] = baseExperimentId;
+      } else if (baseExperiment) {
         args["base_experiment"] = baseExperiment;
       } else {
         args["ancestor_commits"] = await isomorph_default.getPastNAncestors();
@@ -833,6 +936,21 @@ function init(project, options = {}) {
   }
   return ret;
 }
+function initExperiment(projectOrOptions, optionalOptions) {
+  const options = (() => {
+    if (typeof projectOrOptions === "string") {
+      return { ...optionalOptions, project: projectOrOptions };
+    } else {
+      if (optionalOptions !== void 0) {
+        throw new Error(
+          "Cannot specify options struct as both parameters. Must call either init(project, options) or init(options)."
+        );
+      }
+      return projectOrOptions;
+    }
+  })();
+  return init(options);
+}
 function withExperiment(project, callback, options = {}) {
   console.warn(
     "withExperiment is deprecated and will be removed in a future version of braintrust. Simply create the experiment with `init`."
@@ -847,8 +965,30 @@ function withLogger(callback, options = {}) {
   const logger = initLogger(options);
   return callback(logger);
 }
-function initDataset(project, options = {}) {
-  const { dataset, description, version, appUrl, apiKey, orgName } = options || {};
+function initDataset(projectOrOptions, optionalOptions) {
+  const options = (() => {
+    if (typeof projectOrOptions === "string") {
+      return { ...optionalOptions, project: projectOrOptions };
+    } else {
+      if (optionalOptions !== void 0) {
+        throw new Error(
+          "Cannot specify options struct as both parameters. Must call either initDataset(project, options) or initDataset(options)."
+        );
+      }
+      return projectOrOptions;
+    }
+  })();
+  const {
+    project,
+    dataset,
+    description,
+    version,
+    appUrl,
+    apiKey,
+    orgName,
+    projectId,
+    useOutput: legacy
+  } = options;
   const lazyMetadata = new LazyValue(
     async () => {
       await login({
@@ -859,6 +999,7 @@ function initDataset(project, options = {}) {
       const args = {
         org_id: _state.orgId,
         project_name: project,
+        project_id: projectId,
         dataset_name: dataset,
         description
       };
@@ -877,7 +1018,7 @@ function initDataset(project, options = {}) {
       };
     }
   );
-  return new Dataset(lazyMetadata, version);
+  return new Dataset(lazyMetadata, version, legacy);
 }
 function withDataset(project, callback, options = {}) {
   console.warn(
@@ -947,15 +1088,27 @@ function initLogger(options = {}) {
   return ret;
 }
 async function login(options = {}) {
+  let { forceLogin = false } = options || {};
+  if (_state.loggedIn && !forceLogin) {
+    let checkUpdatedParam2 = function(varname, arg, orig) {
+      if (!isEmpty(arg) && !isEmpty(orig) && arg !== orig) {
+        throw new Error(
+          `Re-logging in with different ${varname} (${arg}) than original (${orig}). To force re-login, pass \`forceLogin: true\``
+        );
+      }
+    };
+    var checkUpdatedParam = checkUpdatedParam2;
+    ;
+    checkUpdatedParam2("appUrl", options.appUrl, _state.appUrl);
+    checkUpdatedParam2("apiKey", options.apiKey ? HTTPConnection.sanitize_token(options.apiKey) : void 0, _state.loginToken);
+    checkUpdatedParam2("orgName", options.orgName, _state.orgName);
+    return;
+  }
   const {
     appUrl = isomorph_default.getEnv("BRAINTRUST_APP_URL") || "https://www.braintrustdata.com",
     apiKey = isomorph_default.getEnv("BRAINTRUST_API_KEY"),
     orgName = isomorph_default.getEnv("BRAINTRUST_ORG_NAME")
   } = options || {};
-  let { forceLogin = false } = options || {};
-  if (_state.loggedIn && !forceLogin) {
-    return;
-  }
   _state.resetLoginInfo();
   _state.appUrl = appUrl;
   let conn = null;
@@ -1164,9 +1317,10 @@ function validateAndSanitizeExperimentLogFullArgs(event, hasDataset) {
   return event;
 }
 var ObjectFetcher = class {
-  constructor(objectType, pinnedVersion) {
+  constructor(objectType, pinnedVersion, mutateRecord) {
     this.objectType = objectType;
     this.pinnedVersion = pinnedVersion;
+    this.mutateRecord = mutateRecord;
     this._fetchedData = void 0;
   }
   get id() {
@@ -1181,18 +1335,30 @@ var ObjectFetcher = class {
       yield record;
     }
   }
-  [Symbol.iterator]() {
+  [Symbol.asyncIterator]() {
     return this.fetch();
   }
   async fetchedData() {
     if (this._fetchedData === void 0) {
       const state = await this.getState();
-      const resp = await state.logConn().get(`object/${this.objectType}`, {
-        id: await this.id,
-        fmt: "json2",
-        version: this.pinnedVersion
-      });
-      this._fetchedData = await resp.json();
+      let data = void 0;
+      try {
+        const resp = await state.logConn().get(`object3/${this.objectType}`, {
+          id: await this.id,
+          fmt: "json2",
+          version: this.pinnedVersion,
+          api_version: "2"
+        });
+        data = await resp.json();
+      } catch (e) {
+        const resp = await state.logConn().get(`object/${this.objectType}`, {
+          id: await this.id,
+          fmt: "json2",
+          version: this.pinnedVersion
+        });
+        data = await resp.json();
+      }
+      this._fetchedData = this.mutateRecord ? data?.map(this.mutateRecord) : data;
     }
     return this._fetchedData || [];
   }
@@ -1423,12 +1589,12 @@ var ReadonlyExperiment = class extends ObjectFetcher {
   }
   get id() {
     return (async () => {
-      return (await this.lazyMetadata.get()).id;
+      return (await this.lazyMetadata.get()).experiment.id;
     })();
   }
   get name() {
     return (async () => {
-      return (await this.lazyMetadata.get()).name;
+      return (await this.lazyMetadata.get()).experiment.name;
     })();
   }
   async getState() {
@@ -1441,11 +1607,18 @@ var ReadonlyExperiment = class extends ObjectFetcher {
       if (record.root_span_id !== record.span_id) {
         continue;
       }
-      const { output, expected } = record;
-      yield {
-        input: record.input,
-        expected: expected ?? output
-      };
+      const { output, expected: expectedRecord } = record;
+      const expected = expectedRecord ?? output;
+      if (isEmpty(expected)) {
+        yield {
+          input: record.input
+        };
+      } else {
+        yield {
+          input: record.input,
+          expected
+        };
+      }
     }
   }
 };
@@ -1578,8 +1751,18 @@ var SpanImpl = class _SpanImpl {
   }
 };
 var Dataset = class extends ObjectFetcher {
-  constructor(lazyMetadata, pinnedVersion) {
-    super("dataset", pinnedVersion);
+  constructor(lazyMetadata, pinnedVersion, legacy) {
+    const isLegacyDataset = legacy ?? DEFAULT_IS_LEGACY_DATASET;
+    if (isLegacyDataset) {
+      console.warn(
+        `Records will be fetched from this dataset in the legacy format, with the "expected" field renamed to "output". Please update your code to use "expected", and use \`braintrust.initDataset()\` with \`{ useOutput: false }\`, which will become the default in a future version of Braintrust.`
+      );
+    }
+    super(
+      "dataset",
+      pinnedVersion,
+      (r) => ensureDatasetRecord(r, isLegacyDataset)
+    );
     this.lazyMetadata = lazyMetadata;
     const logConn = new LazyValue(
       () => this.getState().then((state) => state.logConn())
@@ -1611,19 +1794,21 @@ var Dataset = class extends ObjectFetcher {
    *
    * @param event The event to log.
    * @param event.input The argument that uniquely define an input case (an arbitrary, JSON serializable object).
-   * @param event.output The output of your application, including post-processing (an arbitrary, JSON serializable object).
+   * @param event.expected The output of your application, including post-processing (an arbitrary, JSON serializable object).
    * @param event.metadata (Optional) a dictionary with additional data about the test example, model outputs, or just
    * about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the
    * `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any
    * JSON-serializable type, but its keys must be strings.
    * @param event.id (Optional) a unique identifier for the event. If you don't provide one, Braintrust will generate one for you.
+   * @param event.output: (Deprecated) The output of your application. Use `expected` instead.
    * @returns The `id` of the logged record.
    */
   insert({
     input,
-    output,
+    expected,
     metadata,
-    id
+    id,
+    output
   }) {
     if (metadata !== void 0) {
       for (const key of Object.keys(metadata)) {
@@ -1632,11 +1817,16 @@ var Dataset = class extends ObjectFetcher {
         }
       }
     }
+    if (expected && output) {
+      throw new Error(
+        "Only one of expected or output (deprecated) can be specified. Prefer expected."
+      );
+    }
     const rowId = id || v4_default();
     const args = new LazyValue(async () => ({
       id: rowId,
-      inputs: input,
-      output,
+      input,
+      expected: expected === void 0 ? output : expected,
       project_id: (await this.project).id,
       dataset_id: await this.id,
       created: (/* @__PURE__ */ new Date()).toISOString(),
@@ -1955,7 +2145,6 @@ var WrapperStream = class {
 // src/browser.ts
 configureBrowser();
 export {
-  Dataset,
   Experiment,
   Logger,
   NOOP_SPAN,
@@ -1970,6 +2159,7 @@ export {
   getSpanParentObject,
   init,
   initDataset,
+  initExperiment,
   initLogger,
   log,
   login,