braintrust 0.0.96 → 0.0.98

package/dist/cli.js

@@ -9065,7 +9065,7 @@ var require_package = __commonJS({
   "package.json"(exports2, module2) {
     module2.exports = {
       name: "braintrust",
-      version: "0.0.96",
+      version: "0.0.98",
       description: "SDK for integrating Braintrust",
       main: "./dist/index.js",
       browser: {
@@ -9108,7 +9108,7 @@ var require_package = __commonJS({
         typescript: "^5.3.3"
       },
       dependencies: {
-        "@braintrust/core": "^0.0.14",
+        "@braintrust/core": "^0.0.16",
         argparse: "^2.0.1",
         chalk: "^4.1.2",
         "cli-progress": "^3.12.0",
@@ -10508,9 +10508,10 @@ var v4_default = v4;
 // src/cli.ts
 var import_pluralize2 = __toESM(require_pluralize());
 
-// ../core/js/dist/index.mjs
+// ../core/js/dist/main/index.mjs
 var TRANSACTION_ID_FIELD = "_xact_id";
 var IS_MERGE_FIELD = "_is_merge";
+var MERGE_PATHS_FIELD = "_merge_paths";
 var AUDIT_SOURCE_FIELD = "_audit_source";
 var AUDIT_METADATA_FIELD = "_audit_metadata";
 var VALID_SOURCES = ["app", "api", "external"];
@@ -10569,6 +10570,24 @@ function mergeRowBatch(rows) {
   out.push(...Object.values(rowGroups));
   return out;
 }
+function makeLegacyEvent(e) {
+  if (!("dataset_id" in e) || !("expected" in e)) {
+    return e;
+  }
+  const event = {
+    ...e,
+    output: e.expected
+  };
+  delete event.expected;
+  if (MERGE_PATHS_FIELD in event) {
+    for (const path5 of event[MERGE_PATHS_FIELD] || []) {
+      if (path5.length > 0 && path5[0] === "expected") {
+        path5[0] = "output";
+      }
+    }
+  }
+  return event;
+}
 var SpanTypeAttribute = /* @__PURE__ */ ((SpanTypeAttribute2) => {
   SpanTypeAttribute2["LLM"] = "llm";
   SpanTypeAttribute2["SCORE"] = "score";
@@ -10611,7 +10630,7 @@ var DefaultAsyncLocalStorage = class {
   }
 };
 var iso = {
-  getRepoStatus: async (_settings) => void 0,
+  getRepoInfo: async (_settings) => void 0,
   getPastNAncestors: async () => [],
   getEnv: (_name) => void 0,
   getCallerLocation: () => void 0,
@@ -10915,6 +10934,9 @@ var MaxRequestSize = 6 * 1024 * 1024;
 function constructJsonArray(items) {
   return `[${items.join(",")}]`;
 }
+function constructLogs3Data(items) {
+  return `{"rows": ${constructJsonArray(items)}, "api_version": 2}`;
+}
 var DefaultBatchSize = 100;
 var NumRetries = 3;
 function now() {
@@ -10973,11 +10995,20 @@ var BackgroundLogger = class {
       }
       postPromises.push(
         (async () => {
-          const itemsS = constructJsonArray(items);
+          const dataS = constructLogs3Data(items);
           for (let i = 0; i < NumRetries; i++) {
             const startTime = now();
             try {
-              return (await (await this.logConn.get()).post_json("logs", itemsS)).map((res) => res.id);
+              try {
+                return (await (await this.logConn.get()).post_json("logs3", dataS)).ids.map((res) => res.id);
+              } catch (e) {
+                const legacyDataS = constructJsonArray(
+                  items.map(
+                    (r) => JSON.stringify(makeLegacyEvent(JSON.parse(r)))
+                  )
+                );
+                return (await (await this.logConn.get()).post_json("logs", legacyDataS)).map((res) => res.id);
+              }
             } catch (e) {
               const retryingText = i + 1 === NumRetries ? "" : " Retrying";
               const errMsg = (() => {
@@ -10988,7 +11019,7 @@ var BackgroundLogger = class {
                 }
               })();
               console.warn(
-                `log request failed. Elapsed time: ${(now() - startTime) / 1e3} seconds. Payload size: ${itemsS.length}. Error: ${errMsg}.${retryingText}`
+                `log request failed. Elapsed time: ${(now() - startTime) / 1e3} seconds. Payload size: ${dataS.length}. Error: ${errMsg}.${retryingText}`
               );
             }
           }
@@ -11016,8 +11047,21 @@ var BackgroundLogger = class {
     }
   }
 };
-function init(project, options = {}) {
+function init(projectOrOptions, optionalOptions) {
+  const options = (() => {
+    if (typeof projectOrOptions === "string") {
+      return { ...optionalOptions, project: projectOrOptions };
+    } else {
+      if (optionalOptions !== void 0) {
+        throw new Error(
+          "Cannot specify options struct as both parameters. Must call either init(project, options) or init(options)."
+        );
+      }
+      return projectOrOptions;
+    }
+  })();
   const {
+    project,
     experiment,
     description,
     dataset,
@@ -11029,42 +11073,66 @@ function init(project, options = {}) {
     apiKey,
     orgName,
     metadata,
-    gitMetadataSettings
-  } = options || {};
-  if (open) {
+    gitMetadataSettings,
+    projectId,
+    baseExperimentId,
+    repoInfo: repoInfo2
+  } = options;
+  if (open && update) {
+    throw new Error("Cannot open and update an experiment at the same time");
+  }
+  if (open || update) {
     if (isEmpty(experiment)) {
-      throw new Error("Cannot open an experiment without specifying its name");
-    }
-    if (update) {
-      throw new Error("Cannot open and update an experiment at the same time");
+      const action = open ? "open" : "update";
+      throw new Error(
+        `Cannot ${action} an experiment without specifying its name`
+      );
     }
-    const lazyMetadata2 = new LazyValue(async () => {
-      await login({
-        orgName,
-        apiKey,
-        appUrl
-      });
-      const args = {
-        project_name: project,
-        org_name: _state.orgName,
-        experiment_name: experiment
-      };
-      const response = await _state.apiConn().post_json("api/experiment/get", args);
-      if (response.length === 0) {
-        throw new Error(
-          `Experiment ${experiment} not found in project ${project}.`
-        );
+    const lazyMetadata2 = new LazyValue(
+      async () => {
+        await login({
+          orgName,
+          apiKey,
+          appUrl
+        });
+        const args = {
+          project_name: project,
+          project_id: projectId,
+          org_name: _state.orgName,
+          experiment_name: experiment
+        };
+        const response = await _state.apiConn().post_json("api/experiment/get", args);
+        if (response.length === 0) {
+          throw new Error(
+            `Experiment ${experiment} not found in project ${projectId ?? project}.`
+          );
+        }
+        const info = response[0];
+        return {
+          project: {
+            id: info.project_id,
+            name: "",
+            fullInfo: {}
+          },
+          experiment: {
+            id: info.id,
+            name: info.name,
+            fullInfo: info
+          }
+        };
       }
-      const info = response[0];
-      return {
-        id: info.id,
-        name: info.name,
-        fullInfo: info
-      };
-    });
-    return new ReadonlyExperiment(
-      lazyMetadata2
     );
+    if (open) {
+      return new ReadonlyExperiment(
+        lazyMetadata2
+      );
+    } else {
+      const ret2 = new Experiment(lazyMetadata2, dataset);
+      if (options.setCurrent ?? true) {
+        _state.currentExperiment = ret2;
+      }
+      return ret2;
+    }
   }
   const lazyMetadata = new LazyValue(
     async () => {
@@ -11075,6 +11143,7 @@ function init(project, options = {}) {
       });
       const args = {
         project_name: project,
+        project_id: projectId,
         org_id: _state.orgId
       };
       if (experiment) {
@@ -11083,25 +11152,29 @@ function init(project, options = {}) {
       if (description) {
         args["description"] = description;
       }
-      if (update) {
-        args["update"] = update;
-      }
-      let mergedGitMetadataSettings = {
-        ..._state.gitMetadataSettings || {
-          collect: "all"
+      const repoInfoArg = await (async () => {
+        if (repoInfo2) {
+          return repoInfo2;
         }
-      };
-      if (gitMetadataSettings) {
-        mergedGitMetadataSettings = mergeGitMetadataSettings(
-          mergedGitMetadataSettings,
-          gitMetadataSettings
-        );
-      }
-      const repoStatus2 = await isomorph_default.getRepoStatus(gitMetadataSettings);
-      if (repoStatus2) {
-        args["repo_info"] = repoStatus2;
+        let mergedGitMetadataSettings = {
+          ..._state.gitMetadataSettings || {
+            collect: "all"
+          }
+        };
+        if (gitMetadataSettings) {
+          mergedGitMetadataSettings = mergeGitMetadataSettings(
+            mergedGitMetadataSettings,
+            gitMetadataSettings
+          );
+        }
+        return await isomorph_default.getRepoInfo(mergedGitMetadataSettings);
+      })();
+      if (repoInfoArg) {
+        args["repo_info"] = repoInfoArg;
       }
-      if (baseExperiment) {
+      if (baseExperimentId) {
+        args["base_exp_id"] = baseExperimentId;
+      } else if (baseExperiment) {
         args["base_experiment"] = baseExperiment;
       } else {
         args["ancestor_commits"] = await isomorph_default.getPastNAncestors();
@@ -11153,15 +11226,27 @@ function init(project, options = {}) {
   return ret;
 }
 async function login(options = {}) {
+  let { forceLogin = false } = options || {};
+  if (_state.loggedIn && !forceLogin) {
+    let checkUpdatedParam2 = function(varname, arg, orig) {
+      if (!isEmpty(arg) && !isEmpty(orig) && arg !== orig) {
+        throw new Error(
+          `Re-logging in with different ${varname} (${arg}) than original (${orig}). To force re-login, pass \`forceLogin: true\``
+        );
+      }
+    };
+    var checkUpdatedParam = checkUpdatedParam2;
+    ;
+    checkUpdatedParam2("appUrl", options.appUrl, _state.appUrl);
+    checkUpdatedParam2("apiKey", options.apiKey ? HTTPConnection.sanitize_token(options.apiKey) : void 0, _state.loginToken);
+    checkUpdatedParam2("orgName", options.orgName, _state.orgName);
+    return;
+  }
   const {
     appUrl = isomorph_default.getEnv("BRAINTRUST_APP_URL") || "https://www.braintrustdata.com",
     apiKey = isomorph_default.getEnv("BRAINTRUST_API_KEY"),
     orgName = isomorph_default.getEnv("BRAINTRUST_ORG_NAME")
   } = options || {};
-  let { forceLogin = false } = options || {};
-  if (_state.loggedIn && !forceLogin) {
-    return;
-  }
   _state.resetLoginInfo();
   _state.appUrl = appUrl;
   let conn = null;
@@ -11291,9 +11376,10 @@ function validateAndSanitizeExperimentLogFullArgs(event, hasDataset) {
   return event;
 }
 var ObjectFetcher = class {
-  constructor(objectType, pinnedVersion) {
+  constructor(objectType, pinnedVersion, mutateRecord) {
     this.objectType = objectType;
     this.pinnedVersion = pinnedVersion;
+    this.mutateRecord = mutateRecord;
     this._fetchedData = void 0;
   }
   get id() {
@@ -11308,18 +11394,30 @@ var ObjectFetcher = class {
       yield record;
     }
   }
-  [Symbol.iterator]() {
+  [Symbol.asyncIterator]() {
     return this.fetch();
   }
   async fetchedData() {
     if (this._fetchedData === void 0) {
       const state = await this.getState();
-      const resp = await state.logConn().get(`object/${this.objectType}`, {
-        id: await this.id,
-        fmt: "json2",
-        version: this.pinnedVersion
-      });
-      this._fetchedData = await resp.json();
+      let data = void 0;
+      try {
+        const resp = await state.logConn().get(`object3/${this.objectType}`, {
+          id: await this.id,
+          fmt: "json2",
+          version: this.pinnedVersion,
+          api_version: "2"
+        });
+        data = await resp.json();
+      } catch (e) {
+        const resp = await state.logConn().get(`object/${this.objectType}`, {
+          id: await this.id,
+          fmt: "json2",
+          version: this.pinnedVersion
+        });
+        data = await resp.json();
+      }
+      this._fetchedData = this.mutateRecord ? data?.map(this.mutateRecord) : data;
     }
     return this._fetchedData || [];
   }
@@ -11550,12 +11648,12 @@ var ReadonlyExperiment = class extends ObjectFetcher {
   }
   get id() {
     return (async () => {
-      return (await this.lazyMetadata.get()).id;
+      return (await this.lazyMetadata.get()).experiment.id;
     })();
   }
   get name() {
     return (async () => {
-      return (await this.lazyMetadata.get()).name;
+      return (await this.lazyMetadata.get()).experiment.name;
     })();
   }
   async getState() {
@@ -11568,11 +11666,18 @@ var ReadonlyExperiment = class extends ObjectFetcher {
       if (record.root_span_id !== record.span_id) {
         continue;
       }
-      const { output, expected } = record;
-      yield {
-        input: record.input,
-        expected: expected ?? output
-      };
+      const { output, expected: expectedRecord } = record;
+      const expected = expectedRecord ?? output;
+      if (isEmpty(expected)) {
+        yield {
+          input: record.input
+        };
+      } else {
+        yield {
+          input: record.input,
+          expected
+        };
+      }
     }
   }
 };
@@ -11911,7 +12016,7 @@ async function runEvaluator(experiment, evaluator, progressReporter, filters) {
   );
   progressReporter.start(evaluator.evalName, data.length);
   const evals = data.map(async (datum) => {
-    let metadata = { ...datum.metadata };
+    let metadata = { ..."metadata" in datum ? datum.metadata : {} };
     let output = void 0;
     let error2 = void 0;
     let scores = {};
@@ -11998,7 +12103,7 @@ async function runEvaluator(experiment, evaluator, progressReporter, filters) {
         },
         event: {
           input: datum.input,
-          expected: datum.expected
+          expected: "expected" in datum ? datum.expected : void 0
         }
       });
     }
@@ -16114,11 +16219,11 @@ function truncateToByteLimit(s, byteLimit = 65536) {
   const truncated = encoded.subarray(0, byteLimit);
   return new TextDecoder().decode(truncated);
 }
-async function getRepoStatus(settings) {
+async function getRepoInfo(settings) {
   if (settings && settings.collect === "none") {
     return void 0;
   }
-  const repo = await repoStatus();
+  const repo = await repoInfo();
   if (!repo || !settings || settings.collect === "all") {
     return repo;
   }
@@ -16128,7 +16233,7 @@ async function getRepoStatus(settings) {
   });
   return sanitized;
 }
-async function repoStatus() {
+async function repoInfo() {
   const git = await currentRepo();
   if (git === null) {
     return void 0;
@@ -16225,7 +16330,7 @@ function getCallerLocation() {
 
 // src/node.ts
 function configureNode() {
-  isomorph_default.getRepoStatus = getRepoStatus;
+  isomorph_default.getRepoInfo = getRepoInfo;
   isomorph_default.getPastNAncestors = getPastNAncestors;
   isomorph_default.getEnv = (name) => process.env[name];
   isomorph_default.getCallerLocation = getCallerLocation;

package/dist/framework.d.ts

@@ -1,10 +1,10 @@
 import chalk from "chalk";
-import { Experiment, ExperimentSummary, Metadata, Span, EvalCase } from "./logger";
+import { Experiment, ExperimentSummary, Span, EvalCase, BaseMetadata, DefaultMetadataType } from "./logger";
 import { Score } from "@braintrust/core";
 import { ProgressReporter } from "./progress";
-export type BaseExperiment<Input, Expected> = {
+export type BaseExperiment<Input, Expected, Metadata extends BaseMetadata = DefaultMetadataType> = {
     _type: "BaseExperiment";
-    _phantom?: [Input, Expected];
+    _phantom?: [Input, Expected, Metadata];
     name?: string;
 };
 /**
@@ -17,24 +17,24 @@ export type BaseExperiment<Input, Expected> = {
  * using your git history (or fall back to timestamps).
  * @returns
  */
-export declare function BaseExperiment<Input = unknown, Expected = unknown>(options?: {
+export declare function BaseExperiment<Input = unknown, Expected = unknown, Metadata extends BaseMetadata = DefaultMetadataType>(options?: {
     name?: string;
-}): BaseExperiment<Input, Expected>;
-export type EvalData<Input, Expected> = EvalCase<Input, Expected>[] | (() => EvalCase<Input, Expected>[]) | (() => Promise<EvalCase<Input, Expected>[]>) | AsyncGenerator<EvalCase<Input, Expected>> | BaseExperiment<Input, Expected> | (() => BaseExperiment<Input, Expected>);
+}): BaseExperiment<Input, Expected, Metadata>;
+export type EvalData<Input, Expected, Metadata extends BaseMetadata = DefaultMetadataType> = EvalCase<Input, Expected, Metadata>[] | (() => EvalCase<Input, Expected, Metadata>[]) | (() => Promise<EvalCase<Input, Expected, Metadata>[]>) | AsyncGenerator<EvalCase<Input, Expected, Metadata>> | AsyncIterable<EvalCase<Input, Expected, Metadata>> | BaseExperiment<Input, Expected, Metadata> | (() => BaseExperiment<Input, Expected, Metadata>);
 export type EvalTask<Input, Output> = ((input: Input, hooks: EvalHooks) => Promise<Output>) | ((input: Input, hooks: EvalHooks) => Output);
 export interface EvalHooks {
     meta: (info: Record<string, unknown>) => void;
     span: Span;
 }
-export type EvalScorerArgs<Input, Output, Expected> = EvalCase<Input, Expected> & {
+export type EvalScorerArgs<Input, Output, Expected, Metadata extends BaseMetadata = DefaultMetadataType> = EvalCase<Input, Expected, Metadata> & {
     output: Output;
 };
-export type EvalScorer<Input, Output, Expected> = (args: EvalScorerArgs<Input, Output, Expected>) => Score | Promise<Score>;
-export interface Evaluator<Input, Output, Expected> {
+export type EvalScorer<Input, Output, Expected, Metadata extends BaseMetadata = DefaultMetadataType> = (args: EvalScorerArgs<Input, Output, Expected, Metadata>) => Score | Promise<Score>;
+export interface Evaluator<Input, Output, Expected, Metadata extends BaseMetadata = DefaultMetadataType> {
     /**
      * A function that returns a list of inputs, expected outputs, and metadata.
      */
-    data: EvalData<Input, Expected>;
+    data: EvalData<Input, Expected, Metadata>;
     /**
      * A function that takes an input and returns an output.
      */
@@ -42,7 +42,7 @@ export interface Evaluator<Input, Output, Expected> {
     /**
      * A set of functions that take an input, output, and expected value and return a score.
      */
-    scores: EvalScorer<Input, Output, Expected>[];
+    scores: EvalScorer<Input, Output, Expected, Metadata>[];
     /**
      * An optional name for the experiment.
      */
@@ -56,24 +56,24 @@ export interface Evaluator<Input, Output, Expected> {
     /**
      * Optional additional metadata for the experiment.
      */
-    metadata?: Metadata;
+    metadata?: Record<string, unknown>;
     /**
      * Whether the experiment should be public. Defaults to false.
      */
     isPublic?: boolean;
 }
-export type EvaluatorDef<Input, Output, Expected> = {
+export type EvaluatorDef<Input, Output, Expected, Metadata extends BaseMetadata = DefaultMetadataType> = {
     projectName: string;
     evalName: string;
-} & Evaluator<Input, Output, Expected>;
+} & Evaluator<Input, Output, Expected, Metadata>;
 export type EvaluatorFile = {
-    [evalName: string]: EvaluatorDef<any, any, any>;
+    [evalName: string]: EvaluatorDef<any, any, any, any>;
 };
 declare global {
     var _evals: EvaluatorFile;
     var _lazy_load: boolean;
 }
-export declare function Eval<Input, Output, Expected>(name: string, evaluator: Evaluator<Input, Output, Expected>): Promise<ExperimentSummary>;
+export declare function Eval<Input, Output, Expected, Metadata extends BaseMetadata = DefaultMetadataType>(name: string, evaluator: Evaluator<Input, Output, Expected, Metadata>): Promise<ExperimentSummary>;
 export declare function getLoadedEvals(): EvaluatorFile;
 export interface Filter {
     path: string[];
@@ -88,10 +88,10 @@ export declare function deserializePlainStringAsJSON(s: string): {
     error: unknown;
 };
 export declare function parseFilters(filters: string[]): Filter[];
-export declare function runEvaluator(experiment: Experiment | null, evaluator: EvaluatorDef<unknown, unknown, unknown>, progressReporter: ProgressReporter, filters: Filter[]): Promise<{
+export declare function runEvaluator(experiment: Experiment | null, evaluator: EvaluatorDef<any, any, any | void, any | void>, progressReporter: ProgressReporter, filters: Filter[]): Promise<{
     results: {
         output: any;
-        metadata: Metadata;
+        metadata: object;
         scores: Record<string, number | null>;
         error: unknown;
     }[];

package/dist/gitutil.d.ts

@@ -1,7 +1,7 @@
-import { GitMetadataSettings, RepoStatus } from "@braintrust/core";
+import { GitMetadataSettings, RepoInfo } from "@braintrust/core";
 /**
  * Information about the current HEAD of the repo.
  */
 export declare function currentRepo(): Promise<import("simple-git").SimpleGit | null>;
 export declare function getPastNAncestors(n?: number, remote?: string | undefined): Promise<string[]>;
-export declare function getRepoStatus(settings?: GitMetadataSettings): Promise<RepoStatus | undefined>;
+export declare function getRepoInfo(settings?: GitMetadataSettings): Promise<RepoInfo | undefined>;

package/dist/index.d.ts

@@ -45,3 +45,4 @@
 export * from "./logger";
 export { BaseExperiment, Evaluator, EvalTask, Eval, EvalScorerArgs, } from "./framework";
 export * from "./oai";
+export { ParentExperimentIds, ParentProjectLogIds, IdField, InputField, InputsField, OtherExperimentLogFields, ExperimentLogPartialArgs, ExperimentLogFullArgs, LogFeedbackFullArgs, LogCommentFullArgs, CommentEvent, DatasetRecord, } from "@braintrust/core";