braintrust 0.0.80 → 0.0.81

package/dist/framework.d.ts

@@ -1,8 +1,7 @@
 import chalk from "chalk";
-import { Experiment, ExperimentSummary, InitOptions, Span } from "./logger";
+import { Experiment, ExperimentSummary, Metadata, Span } from "./logger";
 import { Score } from "@braintrust/core";
 import { ProgressReporter } from "./progress";
-export type Metadata = Record<string, unknown>;
 export interface EvalCase<Input, Expected> {
     input: Input;
     expected?: Expected;
@@ -18,30 +17,25 @@ export type EvalScorerArgs<Input, Output, Expected> = EvalCase<Input, Expected>
     output: Output;
 };
 export type EvalScorer<Input, Output, Expected> = (args: EvalScorerArgs<Input, Output, Expected>) => Score | Promise<Score>;
-/**
- * Additional metadata for the eval definition, such as experiment name.
- */
-export interface EvalMetadata {
-    experimentName?: string;
-}
-export declare function evalMetadataToInitOptions(metadata: EvalMetadata | undefined): InitOptions;
 /**
  * An evaluator is a collection of functions that can be used to evaluate a model.
  * It consists of:
  * - `data`, a function that returns a list of inputs, expected outputs, and metadata
  * - `task`, a function that takes an input and returns an output
  * - `scores`, a set of functions that take an input, output, and expected value and return a score
- * - `metadata`, optional additional metadata for the eval definition, such as experiment name.
+ * - `experimentName`, an optional name for the experiment.
  * - `trialCount`, the number of times to run the evaluator per input. This is useful for evaluating applications that
  *   have non-deterministic behavior and gives you both a stronger aggregate measure and a sense of the
  *   variance in the results.
+ * - `metadata`, optional additional metadata for the experiment.
  */
 export interface Evaluator<Input, Output, Expected> {
     data: EvalData<Input, Expected>;
     task: EvalTask<Input, Output>;
     scores: EvalScorer<Input, Output, Expected>[];
-    metadata?: EvalMetadata;
+    experimentName?: string;
     trialCount?: number;
+    metadata?: Metadata;
 }
 export type EvaluatorDef<Input, Output, Expected> = {
     projectName: string;

package/dist/index.d.ts

@@ -1,5 +1,6 @@
 /**
  * An isomorphic JS library for logging data to Braintrust. `braintrust` is distributed as a [library on NPM](https://www.npmjs.com/package/braintrust).
+ * It is also open source and available on [GitHub](https://github.com/braintrustdata/braintrust-sdk/tree/main/js).
  *
  * ### Quickstart
  *
@@ -42,5 +43,5 @@
  * @module braintrust
  */
 export * from "./logger";
-export { Evaluator, EvalTask, Eval, EvalMetadata, EvalScorerArgs, } from "./framework";
+export { Evaluator, EvalTask, Eval, EvalScorerArgs } from "./framework";
 export * from "./oai";

package/dist/index.js

@@ -3703,7 +3703,7 @@ var iso = {
 };
 var isomorph_default = iso;
 
-// ../../node_modules/.pnpm/simple-git@3.19.1/node_modules/simple-git/dist/esm/index.js
+// ../../node_modules/.pnpm/simple-git@3.21.0/node_modules/simple-git/dist/esm/index.js
 var import_file_exists = __toESM(require_dist(), 1);
 var import_debug = __toESM(require_src(), 1);
 var import_child_process = require("child_process");
@@ -3774,6 +3774,11 @@ var __async = (__this, __arguments, generator) => {
     step((generator = generator.apply(__this, __arguments)).next());
   });
 };
+function pathspec(...paths) {
+  const key = new String(paths);
+  cache.set(key, paths);
+  return key;
+}
 function isPathSpec(path2) {
   return path2 instanceof String && cache.has(path2);
 }
@@ -5222,6 +5227,19 @@ var init_commit = __esm({
     init_task();
   }
 });
+function first_commit_default() {
+  return {
+    firstCommit() {
+      return this._runTask(straightThroughStringTask(["rev-list", "--max-parents=0", "HEAD"], true), trailingFunctionArgument(arguments));
+    }
+  };
+}
+var init_first_commit = __esm({
+  "src/lib/tasks/first-commit.ts"() {
+    init_utils();
+    init_task();
+  }
+});
 function hashObjectTask(filePath, write) {
   const commands = ["hash-object", filePath];
   if (write) {
@@ -5555,7 +5573,7 @@ function parseLogOptions(opt = {}, customArgs = []) {
     suffix.push(`${opt.from || ""}${rangeOperator}${opt.to || ""}`);
   }
   if (filterString(opt.file)) {
-    suffix.push("--follow", opt.file);
+    command.push("--follow", pathspec(opt.file));
   }
   appendTaskOptions(userOptions(opt), command);
   return {
@@ -5592,6 +5610,7 @@ var excludeOptions;
 var init_log = __esm({
   "src/lib/tasks/log.ts"() {
     init_log_format();
+    init_pathspec();
     init_parse_list_log_summary();
     init_utils();
     init_task();
@@ -6251,6 +6270,7 @@ var init_simple_git_api = __esm({
     init_checkout();
     init_commit();
     init_config();
+    init_first_commit();
     init_grep();
     init_hash_object();
     init_init();
@@ -6324,7 +6344,7 @@ var init_simple_git_api = __esm({
         return this._runTask(statusTask(getTrailingOptions(arguments)), trailingFunctionArgument(arguments));
       }
     };
-    Object.assign(SimpleGitApi.prototype, checkout_default(), commit_default(), config_default(), grep_default(), log_default(), show_default(), version_default());
+    Object.assign(SimpleGitApi.prototype, checkout_default(), commit_default(), config_default(), first_commit_default(), grep_default(), log_default(), show_default(), version_default());
   }
 });
 var scheduler_exports = {};
@@ -7793,7 +7813,7 @@ function getCallerLocation() {
   return void 0;
 }
 
-// ../../node_modules/.pnpm/uuid@9.0.0/node_modules/uuid/dist/esm-node/rng.js
+// ../../node_modules/.pnpm/uuid@9.0.1/node_modules/uuid/dist/esm-node/rng.js
 var import_crypto = __toESM(require("crypto"));
 var rnds8Pool = new Uint8Array(256);
 var poolPtr = rnds8Pool.length;
@@ -7805,22 +7825,22 @@ function rng() {
   return rnds8Pool.slice(poolPtr, poolPtr += 16);
 }
 
-// ../../node_modules/.pnpm/uuid@9.0.0/node_modules/uuid/dist/esm-node/stringify.js
+// ../../node_modules/.pnpm/uuid@9.0.1/node_modules/uuid/dist/esm-node/stringify.js
 var byteToHex = [];
 for (let i = 0; i < 256; ++i) {
   byteToHex.push((i + 256).toString(16).slice(1));
 }
 function unsafeStringify(arr, offset = 0) {
-  return (byteToHex[arr[offset + 0]] + byteToHex[arr[offset + 1]] + byteToHex[arr[offset + 2]] + byteToHex[arr[offset + 3]] + "-" + byteToHex[arr[offset + 4]] + byteToHex[arr[offset + 5]] + "-" + byteToHex[arr[offset + 6]] + byteToHex[arr[offset + 7]] + "-" + byteToHex[arr[offset + 8]] + byteToHex[arr[offset + 9]] + "-" + byteToHex[arr[offset + 10]] + byteToHex[arr[offset + 11]] + byteToHex[arr[offset + 12]] + byteToHex[arr[offset + 13]] + byteToHex[arr[offset + 14]] + byteToHex[arr[offset + 15]]).toLowerCase();
+  return byteToHex[arr[offset + 0]] + byteToHex[arr[offset + 1]] + byteToHex[arr[offset + 2]] + byteToHex[arr[offset + 3]] + "-" + byteToHex[arr[offset + 4]] + byteToHex[arr[offset + 5]] + "-" + byteToHex[arr[offset + 6]] + byteToHex[arr[offset + 7]] + "-" + byteToHex[arr[offset + 8]] + byteToHex[arr[offset + 9]] + "-" + byteToHex[arr[offset + 10]] + byteToHex[arr[offset + 11]] + byteToHex[arr[offset + 12]] + byteToHex[arr[offset + 13]] + byteToHex[arr[offset + 14]] + byteToHex[arr[offset + 15]];
 }
 
-// ../../node_modules/.pnpm/uuid@9.0.0/node_modules/uuid/dist/esm-node/native.js
+// ../../node_modules/.pnpm/uuid@9.0.1/node_modules/uuid/dist/esm-node/native.js
 var import_crypto2 = __toESM(require("crypto"));
 var native_default = {
   randomUUID: import_crypto2.default.randomUUID
 };
 
-// ../../node_modules/.pnpm/uuid@9.0.0/node_modules/uuid/dist/esm-node/v4.js
+// ../../node_modules/.pnpm/uuid@9.0.1/node_modules/uuid/dist/esm-node/v4.js
 function v4(options, buf, offset) {
   if (native_default.randomUUID && !buf && !options) {
     return native_default.randomUUID();
@@ -7840,58 +7860,40 @@ function v4(options, buf, offset) {
 }
 var v4_default = v4;
 
-// src/util.ts
+// ../core/js/dist/index.mjs
 var TRANSACTION_ID_FIELD = "_xact_id";
 var IS_MERGE_FIELD = "_is_merge";
-var GLOBAL_PROJECT = "Global";
-function runFinally(f, finallyF) {
-  let runSyncCleanup = true;
-  try {
-    const ret = f();
-    if (ret instanceof Promise) {
-      runSyncCleanup = false;
-      return ret.finally(finallyF);
-    } else {
-      return ret;
-    }
-  } finally {
-    if (runSyncCleanup) {
-      finallyF();
-    }
-  }
-}
 function mergeDicts(mergeInto, mergeFrom) {
   for (const [k, mergeFromV] of Object.entries(mergeFrom)) {
     const mergeIntoV = mergeInto[k];
     if (mergeIntoV instanceof Object && !Array.isArray(mergeIntoV) && mergeFrom instanceof Object && !Array.isArray(mergeFromV)) {
-      mergeDicts(mergeIntoV, mergeFromV);
+      mergeDicts(
+        mergeIntoV,
+        mergeFromV
+      );
     } else {
       mergeInto[k] = mergeFromV;
     }
   }
 }
-function getCurrentUnixTimestamp() {
-  return (/* @__PURE__ */ new Date()).getTime() / 1e3;
-}
-
-// src/merge_row_batch.ts
-var DATA_OBJECT_KEYS = [
-  "org_id",
-  "project_id",
-  "experiment_id",
-  "dataset_id",
-  "prompt_session_id",
-  "log_id"
-];
-function generateUniqueRowKey(row) {
-  const coalesceEmpty = (field) => row[field] ?? "";
-  return DATA_OBJECT_KEYS.concat(["id"]).map(coalesceEmpty).join(":");
+function generateMergedRowKey(row) {
+  return JSON.stringify(
+    [
+      "org_id",
+      "project_id",
+      "experiment_id",
+      "dataset_id",
+      "prompt_session_id",
+      "log_id",
+      "id"
+    ].map((k) => row[k])
+  );
 }
 function mergeRowBatch(rows) {
   const out = [];
   const remainingRows = [];
   for (const row of rows) {
-    if (row["id"] === void 0) {
+    if (row.id === void 0) {
       out.push(row);
     } else {
       remainingRows.push(row);
@@ -7899,7 +7901,7 @@ function mergeRowBatch(rows) {
   }
   const rowGroups = {};
   for (const row of remainingRows) {
-    const key = generateUniqueRowKey(row);
+    const key = generateMergedRowKey(row);
     const existingRow = rowGroups[key];
     if (existingRow !== void 0 && row[IS_MERGE_FIELD]) {
       const preserveNoMerge = !existingRow[IS_MERGE_FIELD];
@@ -7915,6 +7917,28 @@ function mergeRowBatch(rows) {
   return out;
 }
 
+// src/util.ts
+var GLOBAL_PROJECT = "Global";
+function runFinally(f, finallyF) {
+  let runSyncCleanup = true;
+  try {
+    const ret = f();
+    if (ret instanceof Promise) {
+      runSyncCleanup = false;
+      return ret.finally(finallyF);
+    } else {
+      return ret;
+    }
+  } finally {
+    if (runSyncCleanup) {
+      finallyF();
+    }
+  }
+}
+function getCurrentUnixTimestamp() {
+  return (/* @__PURE__ */ new Date()).getTime() / 1e3;
+}
+
 // src/logger.ts
 var NoopSpan = class {
   constructor() {
@@ -8325,7 +8349,8 @@ async function init(project, options = {}) {
     apiUrl,
     apiKey,
     orgName,
-    disableCache
+    disableCache,
+    metadata
   } = options || {};
   await login({
     orgName,
@@ -8339,7 +8364,8 @@ async function init(project, options = {}) {
     dataset,
     update,
     baseExperiment,
-    isPublic
+    isPublic,
+    metadata
   });
 }
 async function withExperiment(project, callback, options = {}) {
@@ -8637,12 +8663,14 @@ async function _initExperiment(projectName, {
   dataset,
   update,
   baseExperiment,
-  isPublic
+  isPublic,
+  metadata
 } = {
   experimentName: void 0,
   description: void 0,
   baseExperiment: void 0,
-  isPublic: false
+  isPublic: false,
+  metadata: void 0
 }) {
   const args = {
     project_name: projectName,
@@ -8673,6 +8701,9 @@ async function _initExperiment(projectName, {
   if (isPublic !== void 0) {
     args["public"] = isPublic;
   }
+  if (metadata) {
+    args["metadata"] = metadata;
+  }
   let response = null;
   while (true) {
     try {
@@ -9219,19 +9250,16 @@ var BarProgressReporter = class {
 
 // src/framework.ts
 var import_pluralize = __toESM(require_pluralize());
-function evalMetadataToInitOptions(metadata) {
-  return { experiment: metadata?.experimentName };
-}
-function makeEvalName(projectName, metadata) {
+function makeEvalName(projectName, experimentName) {
   let out = projectName;
-  if (metadata?.experimentName) {
-    out += ` [experimentName=${metadata.experimentName}]`;
+  if (experimentName) {
+    out += ` [experimentName=${experimentName}]`;
   }
   return out;
 }
 globalThis._evals = {};
 async function Eval(name, evaluator) {
-  const evalName = makeEvalName(name, evaluator.metadata);
+  const evalName = makeEvalName(name, evaluator.experimentName);
   if (_evals[evalName]) {
     throw new Error(`Evaluator ${evalName} already exists`);
   }
@@ -9241,7 +9269,6 @@ async function Eval(name, evaluator) {
   }
   const progressReporter = new BarProgressReporter();
   try {
-    const { metadata } = evaluator;
     return await withExperiment(
       name,
       async (experiment) => {
@@ -9258,7 +9285,10 @@ async function Eval(name, evaluator) {
         reportEvaluatorResult(name, ret, true);
         return ret.summary;
       },
-      evalMetadataToInitOptions(metadata)
+      {
+        experiment: evaluator.experimentName,
+        metadata: evaluator.metadata
+      }
     );
   } finally {
     progressReporter.stop();

package/dist/logger.d.ts

@@ -1,6 +1,7 @@
 /// <reference lib="dom" />
+import { IS_MERGE_FIELD } from "@braintrust/core";
 import { IsoAsyncLocalStorage } from "./isomorph";
-import { IS_MERGE_FIELD } from "./util";
+export type Metadata = Record<string, unknown>;
 export type SetCurrentArg = {
     setCurrent?: boolean;
 };
@@ -249,6 +250,7 @@ export type InitOptions = {
     apiKey?: string;
     orgName?: string;
     disableCache?: boolean;
+    metadata?: Metadata;
 };
 /**
  * Log in, and then initialize a new experiment in a specified project. If the project does not exist, it will be created.
@@ -270,6 +272,10 @@ export type InitOptions = {
  * key is specified, will prompt the user to login.
  * @param options.orgName (Optional) The name of a specific organization to connect to. This is useful if you belong to multiple.
  * @param options.disableCache Do not use cached login information.
+ * @param options.metadata (Optional) A dictionary with additional data about the test example, model outputs, or just
+ * about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the
+ * `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any
+ * JSON-serializable type, but its keys must be strings.
  * @returns The newly created Experiment.
  */
 export declare function init(project: string, options?: Readonly<InitOptions>): Promise<Experiment>;