braintrust 0.0.96 → 0.0.98

package/dist/index.js

@@ -3648,7 +3648,6 @@ var require_pluralize = __commonJS({
 var src_exports = {};
 __export(src_exports, {
   BaseExperiment: () => BaseExperiment,
-  Dataset: () => Dataset,
   Eval: () => Eval,
   Experiment: () => Experiment,
   Logger: () => Logger,
@@ -3664,6 +3663,7 @@ __export(src_exports, {
   getSpanParentObject: () => getSpanParentObject,
   init: () => init,
   initDataset: () => initDataset,
+  initExperiment: () => initExperiment,
   initLogger: () => initLogger,
   log: () => log,
   login: () => login,
@@ -3695,7 +3695,7 @@ var DefaultAsyncLocalStorage = class {
   }
 };
 var iso = {
-  getRepoStatus: async (_settings) => void 0,
+  getRepoInfo: async (_settings) => void 0,
   getPastNAncestors: async () => [],
   getEnv: (_name) => void 0,
   getCallerLocation: () => void 0,
@@ -7734,11 +7734,11 @@ function truncateToByteLimit(s, byteLimit = 65536) {
   const truncated = encoded.subarray(0, byteLimit);
   return new TextDecoder().decode(truncated);
 }
-async function getRepoStatus(settings) {
+async function getRepoInfo(settings) {
   if (settings && settings.collect === "none") {
     return void 0;
   }
-  const repo = await repoStatus();
+  const repo = await repoInfo();
   if (!repo || !settings || settings.collect === "all") {
     return repo;
   }
@@ -7748,7 +7748,7 @@ async function getRepoStatus(settings) {
   });
   return sanitized;
 }
-async function repoStatus() {
+async function repoInfo() {
   const git = await currentRepo();
   if (git === null) {
     return void 0;
@@ -7890,9 +7890,10 @@ function v4(options, buf, offset) {
 }
 var v4_default = v4;
 
-// ../core/js/dist/index.mjs
+// ../core/js/dist/main/index.mjs
 var TRANSACTION_ID_FIELD = "_xact_id";
 var IS_MERGE_FIELD = "_is_merge";
+var MERGE_PATHS_FIELD = "_merge_paths";
 var AUDIT_SOURCE_FIELD = "_audit_source";
 var AUDIT_METADATA_FIELD = "_audit_metadata";
 var VALID_SOURCES = ["app", "api", "external"];
@@ -7951,6 +7952,54 @@ function mergeRowBatch(rows) {
   out.push(...Object.values(rowGroups));
   return out;
 }
+var DEFAULT_IS_LEGACY_DATASET = true;
+function ensureDatasetRecord(r, legacy) {
+  if (legacy) {
+    return ensureLegacyDatasetRecord(r);
+  } else {
+    return ensureNewDatasetRecord(r);
+  }
+}
+function ensureLegacyDatasetRecord(r) {
+  if ("output" in r) {
+    return r;
+  }
+  const row = {
+    ...r,
+    output: r.expected
+  };
+  delete row.expected;
+  return row;
+}
+function ensureNewDatasetRecord(r) {
+  if ("expected" in r) {
+    return r;
+  }
+  const row = {
+    ...r,
+    expected: r.output
+  };
+  delete row.output;
+  return row;
+}
+function makeLegacyEvent(e) {
+  if (!("dataset_id" in e) || !("expected" in e)) {
+    return e;
+  }
+  const event = {
+    ...e,
+    output: e.expected
+  };
+  delete event.expected;
+  if (MERGE_PATHS_FIELD in event) {
+    for (const path2 of event[MERGE_PATHS_FIELD] || []) {
+      if (path2.length > 0 && path2[0] === "expected") {
+        path2[0] = "output";
+      }
+    }
+  }
+  return event;
+}
 var SpanTypeAttribute = /* @__PURE__ */ ((SpanTypeAttribute2) => {
   SpanTypeAttribute2["LLM"] = "llm";
   SpanTypeAttribute2["SCORE"] = "score";
@@ -8416,6 +8465,9 @@ var MaxRequestSize = 6 * 1024 * 1024;
 function constructJsonArray(items) {
   return `[${items.join(",")}]`;
 }
+function constructLogs3Data(items) {
+  return `{"rows": ${constructJsonArray(items)}, "api_version": 2}`;
+}
 var DefaultBatchSize = 100;
 var NumRetries = 3;
 function now() {
@@ -8474,11 +8526,20 @@ var BackgroundLogger = class {
       }
       postPromises.push(
         (async () => {
-          const itemsS = constructJsonArray(items);
+          const dataS = constructLogs3Data(items);
           for (let i = 0; i < NumRetries; i++) {
             const startTime = now();
             try {
-              return (await (await this.logConn.get()).post_json("logs", itemsS)).map((res) => res.id);
+              try {
+                return (await (await this.logConn.get()).post_json("logs3", dataS)).ids.map((res) => res.id);
+              } catch (e) {
+                const legacyDataS = constructJsonArray(
+                  items.map(
+                    (r) => JSON.stringify(makeLegacyEvent(JSON.parse(r)))
+                  )
+                );
+                return (await (await this.logConn.get()).post_json("logs", legacyDataS)).map((res) => res.id);
+              }
             } catch (e) {
               const retryingText = i + 1 === NumRetries ? "" : " Retrying";
               const errMsg = (() => {
@@ -8489,7 +8550,7 @@ var BackgroundLogger = class {
                 }
               })();
               console.warn(
-                `log request failed. Elapsed time: ${(now() - startTime) / 1e3} seconds. Payload size: ${itemsS.length}. Error: ${errMsg}.${retryingText}`
+                `log request failed. Elapsed time: ${(now() - startTime) / 1e3} seconds. Payload size: ${dataS.length}. Error: ${errMsg}.${retryingText}`
               );
             }
           }
@@ -8517,8 +8578,21 @@ var BackgroundLogger = class {
     }
   }
 };
-function init(project, options = {}) {
+function init(projectOrOptions, optionalOptions) {
+  const options = (() => {
+    if (typeof projectOrOptions === "string") {
+      return { ...optionalOptions, project: projectOrOptions };
+    } else {
+      if (optionalOptions !== void 0) {
+        throw new Error(
+          "Cannot specify options struct as both parameters. Must call either init(project, options) or init(options)."
+        );
+      }
+      return projectOrOptions;
+    }
+  })();
   const {
+    project,
     experiment,
     description,
     dataset,
@@ -8530,42 +8604,66 @@ function init(project, options = {}) {
     apiKey,
     orgName,
     metadata,
-    gitMetadataSettings
-  } = options || {};
-  if (open) {
+    gitMetadataSettings,
+    projectId,
+    baseExperimentId,
+    repoInfo: repoInfo2
+  } = options;
+  if (open && update) {
+    throw new Error("Cannot open and update an experiment at the same time");
+  }
+  if (open || update) {
     if (isEmpty(experiment)) {
-      throw new Error("Cannot open an experiment without specifying its name");
-    }
-    if (update) {
-      throw new Error("Cannot open and update an experiment at the same time");
+      const action = open ? "open" : "update";
+      throw new Error(
+        `Cannot ${action} an experiment without specifying its name`
+      );
     }
-    const lazyMetadata2 = new LazyValue(async () => {
-      await login({
-        orgName,
-        apiKey,
-        appUrl
-      });
-      const args = {
-        project_name: project,
-        org_name: _state.orgName,
-        experiment_name: experiment
-      };
-      const response = await _state.apiConn().post_json("api/experiment/get", args);
-      if (response.length === 0) {
-        throw new Error(
-          `Experiment ${experiment} not found in project ${project}.`
-        );
+    const lazyMetadata2 = new LazyValue(
+      async () => {
+        await login({
+          orgName,
+          apiKey,
+          appUrl
+        });
+        const args = {
+          project_name: project,
+          project_id: projectId,
+          org_name: _state.orgName,
+          experiment_name: experiment
+        };
+        const response = await _state.apiConn().post_json("api/experiment/get", args);
+        if (response.length === 0) {
+          throw new Error(
+            `Experiment ${experiment} not found in project ${projectId ?? project}.`
+          );
+        }
+        const info = response[0];
+        return {
+          project: {
+            id: info.project_id,
+            name: "",
+            fullInfo: {}
+          },
+          experiment: {
+            id: info.id,
+            name: info.name,
+            fullInfo: info
+          }
+        };
       }
-      const info = response[0];
-      return {
-        id: info.id,
-        name: info.name,
-        fullInfo: info
-      };
-    });
-    return new ReadonlyExperiment(
-      lazyMetadata2
     );
+    if (open) {
+      return new ReadonlyExperiment(
+        lazyMetadata2
+      );
+    } else {
+      const ret2 = new Experiment(lazyMetadata2, dataset);
+      if (options.setCurrent ?? true) {
+        _state.currentExperiment = ret2;
+      }
+      return ret2;
+    }
   }
   const lazyMetadata = new LazyValue(
     async () => {
@@ -8576,6 +8674,7 @@ function init(project, options = {}) {
       });
       const args = {
         project_name: project,
+        project_id: projectId,
         org_id: _state.orgId
       };
       if (experiment) {
@@ -8584,25 +8683,29 @@ function init(project, options = {}) {
       if (description) {
         args["description"] = description;
       }
-      if (update) {
-        args["update"] = update;
-      }
-      let mergedGitMetadataSettings = {
-        ..._state.gitMetadataSettings || {
-          collect: "all"
+      const repoInfoArg = await (async () => {
+        if (repoInfo2) {
+          return repoInfo2;
         }
-      };
-      if (gitMetadataSettings) {
-        mergedGitMetadataSettings = mergeGitMetadataSettings(
-          mergedGitMetadataSettings,
-          gitMetadataSettings
-        );
-      }
-      const repoStatus2 = await isomorph_default.getRepoStatus(gitMetadataSettings);
-      if (repoStatus2) {
-        args["repo_info"] = repoStatus2;
+        let mergedGitMetadataSettings = {
+          ..._state.gitMetadataSettings || {
+            collect: "all"
+          }
+        };
+        if (gitMetadataSettings) {
+          mergedGitMetadataSettings = mergeGitMetadataSettings(
+            mergedGitMetadataSettings,
+            gitMetadataSettings
+          );
+        }
+        return await isomorph_default.getRepoInfo(mergedGitMetadataSettings);
+      })();
+      if (repoInfoArg) {
+        args["repo_info"] = repoInfoArg;
       }
-      if (baseExperiment) {
+      if (baseExperimentId) {
+        args["base_exp_id"] = baseExperimentId;
+      } else if (baseExperiment) {
         args["base_experiment"] = baseExperiment;
       } else {
         args["ancestor_commits"] = await isomorph_default.getPastNAncestors();
@@ -8653,6 +8756,21 @@ function init(project, options = {}) {
   }
   return ret;
 }
+function initExperiment(projectOrOptions, optionalOptions) {
+  const options = (() => {
+    if (typeof projectOrOptions === "string") {
+      return { ...optionalOptions, project: projectOrOptions };
+    } else {
+      if (optionalOptions !== void 0) {
+        throw new Error(
+          "Cannot specify options struct as both parameters. Must call either init(project, options) or init(options)."
+        );
+      }
+      return projectOrOptions;
+    }
+  })();
+  return init(options);
+}
 function withExperiment(project, callback, options = {}) {
   console.warn(
     "withExperiment is deprecated and will be removed in a future version of braintrust. Simply create the experiment with `init`."
@@ -8667,8 +8785,30 @@ function withLogger(callback, options = {}) {
   const logger = initLogger(options);
   return callback(logger);
 }
-function initDataset(project, options = {}) {
-  const { dataset, description, version, appUrl, apiKey, orgName } = options || {};
+function initDataset(projectOrOptions, optionalOptions) {
+  const options = (() => {
+    if (typeof projectOrOptions === "string") {
+      return { ...optionalOptions, project: projectOrOptions };
+    } else {
+      if (optionalOptions !== void 0) {
+        throw new Error(
+          "Cannot specify options struct as both parameters. Must call either initDataset(project, options) or initDataset(options)."
+        );
+      }
+      return projectOrOptions;
+    }
+  })();
+  const {
+    project,
+    dataset,
+    description,
+    version,
+    appUrl,
+    apiKey,
+    orgName,
+    projectId,
+    useOutput: legacy
+  } = options;
   const lazyMetadata = new LazyValue(
     async () => {
       await login({
@@ -8679,6 +8819,7 @@ function initDataset(project, options = {}) {
       const args = {
         org_id: _state.orgId,
         project_name: project,
+        project_id: projectId,
         dataset_name: dataset,
         description
       };
@@ -8697,7 +8838,7 @@ function initDataset(project, options = {}) {
       };
     }
   );
-  return new Dataset(lazyMetadata, version);
+  return new Dataset(lazyMetadata, version, legacy);
 }
 function withDataset(project, callback, options = {}) {
   console.warn(
@@ -8767,15 +8908,27 @@ function initLogger(options = {}) {
   return ret;
 }
 async function login(options = {}) {
+  let { forceLogin = false } = options || {};
+  if (_state.loggedIn && !forceLogin) {
+    let checkUpdatedParam2 = function(varname, arg, orig) {
+      if (!isEmpty(arg) && !isEmpty(orig) && arg !== orig) {
+        throw new Error(
+          `Re-logging in with different ${varname} (${arg}) than original (${orig}). To force re-login, pass \`forceLogin: true\``
+        );
+      }
+    };
+    var checkUpdatedParam = checkUpdatedParam2;
+    ;
+    checkUpdatedParam2("appUrl", options.appUrl, _state.appUrl);
+    checkUpdatedParam2("apiKey", options.apiKey ? HTTPConnection.sanitize_token(options.apiKey) : void 0, _state.loginToken);
+    checkUpdatedParam2("orgName", options.orgName, _state.orgName);
+    return;
+  }
   const {
     appUrl = isomorph_default.getEnv("BRAINTRUST_APP_URL") || "https://www.braintrustdata.com",
     apiKey = isomorph_default.getEnv("BRAINTRUST_API_KEY"),
     orgName = isomorph_default.getEnv("BRAINTRUST_ORG_NAME")
   } = options || {};
-  let { forceLogin = false } = options || {};
-  if (_state.loggedIn && !forceLogin) {
-    return;
-  }
   _state.resetLoginInfo();
   _state.appUrl = appUrl;
   let conn = null;
@@ -8984,9 +9137,10 @@ function validateAndSanitizeExperimentLogFullArgs(event, hasDataset) {
   return event;
 }
 var ObjectFetcher = class {
-  constructor(objectType, pinnedVersion) {
+  constructor(objectType, pinnedVersion, mutateRecord) {
     this.objectType = objectType;
     this.pinnedVersion = pinnedVersion;
+    this.mutateRecord = mutateRecord;
     this._fetchedData = void 0;
   }
   get id() {
@@ -9001,18 +9155,30 @@ var ObjectFetcher = class {
       yield record;
     }
   }
-  [Symbol.iterator]() {
+  [Symbol.asyncIterator]() {
     return this.fetch();
   }
   async fetchedData() {
     if (this._fetchedData === void 0) {
       const state = await this.getState();
-      const resp = await state.logConn().get(`object/${this.objectType}`, {
-        id: await this.id,
-        fmt: "json2",
-        version: this.pinnedVersion
-      });
-      this._fetchedData = await resp.json();
+      let data = void 0;
+      try {
+        const resp = await state.logConn().get(`object3/${this.objectType}`, {
+          id: await this.id,
+          fmt: "json2",
+          version: this.pinnedVersion,
+          api_version: "2"
+        });
+        data = await resp.json();
+      } catch (e) {
+        const resp = await state.logConn().get(`object/${this.objectType}`, {
+          id: await this.id,
+          fmt: "json2",
+          version: this.pinnedVersion
+        });
+        data = await resp.json();
+      }
+      this._fetchedData = this.mutateRecord ? data?.map(this.mutateRecord) : data;
     }
     return this._fetchedData || [];
   }
@@ -9243,12 +9409,12 @@ var ReadonlyExperiment = class extends ObjectFetcher {
   }
   get id() {
     return (async () => {
-      return (await this.lazyMetadata.get()).id;
+      return (await this.lazyMetadata.get()).experiment.id;
     })();
   }
   get name() {
     return (async () => {
-      return (await this.lazyMetadata.get()).name;
+      return (await this.lazyMetadata.get()).experiment.name;
     })();
   }
   async getState() {
@@ -9261,11 +9427,18 @@ var ReadonlyExperiment = class extends ObjectFetcher {
       if (record.root_span_id !== record.span_id) {
         continue;
       }
-      const { output, expected } = record;
-      yield {
-        input: record.input,
-        expected: expected ?? output
-      };
+      const { output, expected: expectedRecord } = record;
+      const expected = expectedRecord ?? output;
+      if (isEmpty(expected)) {
+        yield {
+          input: record.input
+        };
+      } else {
+        yield {
+          input: record.input,
+          expected
+        };
+      }
     }
   }
 };
@@ -9398,8 +9571,18 @@ var SpanImpl = class _SpanImpl {
   }
 };
 var Dataset = class extends ObjectFetcher {
-  constructor(lazyMetadata, pinnedVersion) {
-    super("dataset", pinnedVersion);
+  constructor(lazyMetadata, pinnedVersion, legacy) {
+    const isLegacyDataset = legacy ?? DEFAULT_IS_LEGACY_DATASET;
+    if (isLegacyDataset) {
+      console.warn(
+        `Records will be fetched from this dataset in the legacy format, with the "expected" field renamed to "output". Please update your code to use "expected", and use \`braintrust.initDataset()\` with \`{ useOutput: false }\`, which will become the default in a future version of Braintrust.`
+      );
+    }
+    super(
+      "dataset",
+      pinnedVersion,
+      (r) => ensureDatasetRecord(r, isLegacyDataset)
+    );
     this.lazyMetadata = lazyMetadata;
     const logConn = new LazyValue(
       () => this.getState().then((state) => state.logConn())
@@ -9431,19 +9614,21 @@ var Dataset = class extends ObjectFetcher {
    *
    * @param event The event to log.
    * @param event.input The argument that uniquely define an input case (an arbitrary, JSON serializable object).
-   * @param event.output The output of your application, including post-processing (an arbitrary, JSON serializable object).
+   * @param event.expected The output of your application, including post-processing (an arbitrary, JSON serializable object).
    * @param event.metadata (Optional) a dictionary with additional data about the test example, model outputs, or just
    * about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the
    * `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any
    * JSON-serializable type, but its keys must be strings.
    * @param event.id (Optional) a unique identifier for the event. If you don't provide one, Braintrust will generate one for you.
+   * @param event.output: (Deprecated) The output of your application. Use `expected` instead.
    * @returns The `id` of the logged record.
    */
   insert({
     input,
-    output,
+    expected,
     metadata,
-    id
+    id,
+    output
   }) {
     if (metadata !== void 0) {
       for (const key of Object.keys(metadata)) {
@@ -9452,11 +9637,16 @@ var Dataset = class extends ObjectFetcher {
         }
       }
     }
+    if (expected && output) {
+      throw new Error(
+        "Only one of expected or output (deprecated) can be specified. Prefer expected."
+      );
+    }
     const rowId = id || v4_default();
     const args = new LazyValue(async () => ({
       id: rowId,
-      inputs: input,
-      output,
+      input,
+      expected: expected === void 0 ? output : expected,
       project_id: (await this.project).id,
       dataset_id: await this.id,
       created: (/* @__PURE__ */ new Date()).toISOString(),
@@ -9527,7 +9717,7 @@ var Dataset = class extends ObjectFetcher {
 
 // src/node.ts
 function configureNode() {
-  isomorph_default.getRepoStatus = getRepoStatus;
+  isomorph_default.getRepoInfo = getRepoInfo;
   isomorph_default.getPastNAncestors = getPastNAncestors;
   isomorph_default.getEnv = (name) => process.env[name];
   isomorph_default.getCallerLocation = getCallerLocation;
@@ -9589,7 +9779,7 @@ function makeEvalName(projectName, experimentName) {
   }
   return out;
 }
-function initExperiment(projectName, options = {}) {
+function initExperiment2(projectName, options = {}) {
   return init(projectName, {
     ...options,
     setCurrent: false
@@ -9615,7 +9805,7 @@ async function Eval(name, evaluator) {
   }
   const progressReporter = new BarProgressReporter();
   try {
-    const experiment = initExperiment(name, {
+    const experiment = initExperiment2(name, {
       experiment: evaluator.experimentName,
       metadata: evaluator.metadata,
       isPublic: evaluator.isPublic
@@ -9677,7 +9867,7 @@ async function runEvaluator(experiment, evaluator, progressReporter, filters) {
       }
       name = baseExperiment.name;
     }
-    dataResult = initExperiment(evaluator.projectName, {
+    dataResult = initExperiment2(evaluator.projectName, {
       experiment: name,
       open: true
     }).asDataset();
@@ -9698,7 +9888,7 @@ async function runEvaluator(experiment, evaluator, progressReporter, filters) {
   );
   progressReporter.start(evaluator.evalName, data.length);
   const evals = data.map(async (datum) => {
-    let metadata = { ...datum.metadata };
+    let metadata = { ..."metadata" in datum ? datum.metadata : {} };
     let output = void 0;
     let error2 = void 0;
     let scores = {};
@@ -9785,7 +9975,7 @@ async function runEvaluator(experiment, evaluator, progressReporter, filters) {
         },
         event: {
           input: datum.input,
-          expected: datum.expected
+          expected: "expected" in datum ? datum.expected : void 0
         }
       });
     }
@@ -10105,7 +10295,6 @@ configureNode();
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   BaseExperiment,
-  Dataset,
   Eval,
   Experiment,
   Logger,
@@ -10121,6 +10310,7 @@ configureNode();
   getSpanParentObject,
   init,
   initDataset,
+  initExperiment,
   initLogger,
   log,
   login,

package/dist/isomorph.d.ts

@@ -1,4 +1,4 @@
-import { GitMetadataSettings, RepoStatus } from "@braintrust/core";
+import { GitMetadataSettings, RepoInfo } from "@braintrust/core";
 export interface CallerLocation {
     caller_functionname: string;
     caller_filename: string;
@@ -10,7 +10,7 @@ export interface IsoAsyncLocalStorage<T> {
     getStore(): T | undefined;
 }
 export interface Common {
-    getRepoStatus: (settings?: GitMetadataSettings) => Promise<RepoStatus | undefined>;
+    getRepoInfo: (settings?: GitMetadataSettings) => Promise<RepoInfo | undefined>;
     getPastNAncestors: () => Promise<string[]>;
     getEnv: (name: string) => string | undefined;
     getCallerLocation: () => CallerLocation | undefined;