braintrust 1.0.2 → 1.0.3

package/dist/cli.js

@@ -37,8 +37,8 @@ var require_async = __commonJS({
     "use strict";
     Object.defineProperty(exports2, "__esModule", { value: true });
     exports2.read = void 0;
-    function read(path9, settings, callback) {
-      settings.fs.lstat(path9, (lstatError, lstat) => {
+    function read(path8, settings, callback) {
+      settings.fs.lstat(path8, (lstatError, lstat) => {
         if (lstatError !== null) {
           callFailureCallback(callback, lstatError);
           return;
@@ -47,7 +47,7 @@ var require_async = __commonJS({
           callSuccessCallback(callback, lstat);
           return;
         }
-        settings.fs.stat(path9, (statError, stat2) => {
+        settings.fs.stat(path8, (statError, stat2) => {
           if (statError !== null) {
             if (settings.throwErrorOnBrokenSymbolicLink) {
               callFailureCallback(callback, statError);
@@ -79,13 +79,13 @@ var require_sync = __commonJS({
     "use strict";
     Object.defineProperty(exports2, "__esModule", { value: true });
     exports2.read = void 0;
-    function read(path9, settings) {
-      const lstat = settings.fs.lstatSync(path9);
+    function read(path8, settings) {
+      const lstat = settings.fs.lstatSync(path8);
       if (!lstat.isSymbolicLink() || !settings.followSymbolicLink) {
         return lstat;
       }
       try {
-        const stat2 = settings.fs.statSync(path9);
+        const stat2 = settings.fs.statSync(path8);
         if (settings.markSymbolicLink) {
           stat2.isSymbolicLink = () => true;
         }
@@ -156,17 +156,17 @@ var require_out = __commonJS({
     var sync = require_sync();
     var settings_1 = require_settings();
     exports2.Settings = settings_1.default;
-    function stat2(path9, optionsOrSettingsOrCallback, callback) {
+    function stat2(path8, optionsOrSettingsOrCallback, callback) {
       if (typeof optionsOrSettingsOrCallback === "function") {
-        async.read(path9, getSettings(), optionsOrSettingsOrCallback);
+        async.read(path8, getSettings(), optionsOrSettingsOrCallback);
         return;
       }
-      async.read(path9, getSettings(optionsOrSettingsOrCallback), callback);
+      async.read(path8, getSettings(optionsOrSettingsOrCallback), callback);
     }
     exports2.stat = stat2;
-    function statSync2(path9, optionsOrSettings) {
+    function statSync2(path8, optionsOrSettings) {
       const settings = getSettings(optionsOrSettings);
-      return sync.read(path9, settings);
+      return sync.read(path8, settings);
     }
     exports2.statSync = statSync2;
     function getSettings(settingsOrOptions = {}) {
@@ -384,16 +384,16 @@ var require_async2 = __commonJS({
           return;
         }
         const tasks = names.map((name) => {
-          const path9 = common.joinPathSegments(directory, name, settings.pathSegmentSeparator);
+          const path8 = common.joinPathSegments(directory, name, settings.pathSegmentSeparator);
           return (done) => {
-            fsStat.stat(path9, settings.fsStatSettings, (error2, stats) => {
+            fsStat.stat(path8, settings.fsStatSettings, (error2, stats) => {
               if (error2 !== null) {
                 done(error2);
                 return;
               }
               const entry = {
                 name,
-                path: path9,
+                path: path8,
                 dirent: utils.fs.createDirentFromStats(name, stats)
               };
               if (settings.stats) {
@@ -511,7 +511,7 @@ var require_settings2 = __commonJS({
   "../node_modules/@nodelib/fs.scandir/out/settings.js"(exports2) {
     "use strict";
     Object.defineProperty(exports2, "__esModule", { value: true });
-    var path9 = require("path");
+    var path8 = require("path");
     var fsStat = require_out();
     var fs6 = require_fs3();
     var Settings = class {
@@ -519,7 +519,7 @@ var require_settings2 = __commonJS({
         this._options = _options;
         this.followSymbolicLinks = this._getValue(this._options.followSymbolicLinks, false);
         this.fs = fs6.createFileSystemAdapter(this._options.fs);
-        this.pathSegmentSeparator = this._getValue(this._options.pathSegmentSeparator, path9.sep);
+        this.pathSegmentSeparator = this._getValue(this._options.pathSegmentSeparator, path8.sep);
         this.stats = this._getValue(this._options.stats, false);
         this.throwErrorOnBrokenSymbolicLink = this._getValue(this._options.throwErrorOnBrokenSymbolicLink, true);
         this.fsStatSettings = new fsStat.Settings({
@@ -546,17 +546,17 @@ var require_out2 = __commonJS({
     var sync = require_sync2();
     var settings_1 = require_settings2();
     exports2.Settings = settings_1.default;
-    function scandir(path9, optionsOrSettingsOrCallback, callback) {
+    function scandir(path8, optionsOrSettingsOrCallback, callback) {
       if (typeof optionsOrSettingsOrCallback === "function") {
-        async.read(path9, getSettings(), optionsOrSettingsOrCallback);
+        async.read(path8, getSettings(), optionsOrSettingsOrCallback);
         return;
       }
-      async.read(path9, getSettings(optionsOrSettingsOrCallback), callback);
+      async.read(path8, getSettings(optionsOrSettingsOrCallback), callback);
     }
     exports2.scandir = scandir;
-    function scandirSync(path9, optionsOrSettings) {
+    function scandirSync(path8, optionsOrSettings) {
       const settings = getSettings(optionsOrSettings);
-      return sync.read(path9, settings);
+      return sync.read(path8, settings);
     }
     exports2.scandirSync = scandirSync;
     function getSettings(settingsOrOptions = {}) {
@@ -1180,7 +1180,7 @@ var require_settings3 = __commonJS({
   "../node_modules/@nodelib/fs.walk/out/settings.js"(exports2) {
     "use strict";
     Object.defineProperty(exports2, "__esModule", { value: true });
-    var path9 = require("path");
+    var path8 = require("path");
     var fsScandir = require_out2();
     var Settings = class {
       constructor(_options = {}) {
@@ -1190,7 +1190,7 @@ var require_settings3 = __commonJS({
         this.deepFilter = this._getValue(this._options.deepFilter, null);
         this.entryFilter = this._getValue(this._options.entryFilter, null);
         this.errorFilter = this._getValue(this._options.errorFilter, null);
-        this.pathSegmentSeparator = this._getValue(this._options.pathSegmentSeparator, path9.sep);
+        this.pathSegmentSeparator = this._getValue(this._options.pathSegmentSeparator, path8.sep);
         this.fsScandirSettings = new fsScandir.Settings({
           followSymbolicLinks: this._options.followSymbolicLinks,
           fs: this._options.fs,
@@ -1252,7 +1252,7 @@ var require_package = __commonJS({
   "package.json"(exports2, module2) {
     module2.exports = {
       name: "braintrust",
-      version: "1.0.2",
+      version: "1.0.3",
       description: "SDK for integrating Braintrust",
       repository: {
         type: "git",
@@ -1309,6 +1309,7 @@ var require_package = __commonJS({
         clean: "rm -r dist/* && rm -r dev/dist/*",
         docs: "npx typedoc --options typedoc.json src/index.ts",
         test: 'vitest run --exclude "src/wrappers/**/*.test.ts" --exclude "src/otel/**/*.test.ts" --exclude "smoke/**/*.test.ts"',
+        "test:api-compat": "vitest run src/api-compatibility.test.ts",
         "test:anthropic": "vitest run src/wrappers/anthropic.test.ts",
         "test:openai": "vitest run src/wrappers/oai.test.ts",
         "test:otel": "vitest run --dir src/otel",
@@ -1340,7 +1341,10 @@ var require_package = __commonJS({
         "@types/mustache": "^4.2.5",
         "@types/node": "^20.10.5",
         "@types/pluralize": "^0.0.30",
+        "@types/tar": "^6.1.13",
         "@types/uuid": "^9.0.7",
+        "@typescript-eslint/eslint-plugin": "^8.49.0",
+        "@typescript-eslint/parser": "^8.49.0",
         ai: "^4.3.16",
         async: "^3.2.5",
         autoevals: "^0.0.131",
@@ -1348,6 +1352,7 @@ var require_package = __commonJS({
         "npm-run-all": "^4.1.5",
         "openapi-zod-client": "^1.18.3",
         prettier: "^3.5.3",
+        tar: "^7.5.2",
         tinybench: "^4.0.1",
         "ts-jest": "^29.1.4",
         tsup: "^8.5.1",
@@ -1378,7 +1383,6 @@ var require_package = __commonJS({
         mustache: "^4.2.0",
         pluralize: "^8.0.0",
         "simple-git": "^3.21.0",
-        slugify: "^1.6.6",
         "source-map": "^0.7.4",
         "termi-link": "^1.0.1",
         uuid: "^9.0.1",
@@ -1392,7 +1396,7 @@ var require_package = __commonJS({
   }
 });
 
-// src/cli.ts
+// src/cli/index.ts
 var cli_exports = {};
 __export(cli_exports, {
   buildEvaluators: () => buildEvaluators,
@@ -1404,8 +1408,8 @@ var esbuild = __toESM(require("esbuild"));
 var dotenv2 = __toESM(require("dotenv"));
 var import_fs2 = __toESM(require("fs"));
 var import_os = __toESM(require("os"));
-var import_path6 = __toESM(require("path"));
-var import_util14 = __toESM(require("util"));
+var import_path5 = __toESM(require("path"));
+var import_util15 = __toESM(require("util"));
 var fsWalk = __toESM(require_out3());
 var import_minimatch = require("minimatch");
 var import_argparse = require("argparse");
@@ -2133,11 +2137,11 @@ function mergeDictsWithPaths({
 function mergeDictsWithPathsHelper({
   mergeInto,
   mergeFrom,
-  path: path9,
+  path: path8,
   mergePaths
 }) {
   Object.entries(mergeFrom).forEach(([k, mergeFromV]) => {
-    const fullPath = path9.concat([k]);
+    const fullPath = path8.concat([k]);
     const fullPathSerialized = JSON.stringify(fullPath);
     const mergeIntoV = recordFind(mergeInto, k);
     if (isObject(mergeIntoV) && isObject(mergeFromV) && !mergePaths.has(fullPathSerialized)) {
@@ -2168,9 +2172,9 @@ function mapAt(m, k) {
 function recordFind(m, k) {
   return m[k];
 }
-function getObjValueByPath(row, path9) {
+function getObjValueByPath(row, path8) {
   let curr = row;
-  for (const p of path9) {
+  for (const p of path8) {
     if (!isObjectOrArray(curr)) {
       return null;
     }
@@ -2449,6 +2453,93 @@ function capitalize(s, sep) {
   const items = sep ? s.split(sep) : [s];
   return items.map((s2) => s2 ? s2.charAt(0).toUpperCase() + s2.slice(1) : s2).join(sep || "");
 }
+function slugify(text, options) {
+  if (typeof text !== "string") {
+    throw new Error("slugify: string argument expected");
+  }
+  const charMap = {
+    // Currency and symbols
+    $: "dollar",
+    "%": "percent",
+    "&": "and",
+    // Latin characters
+    \u00C0: "A",
+    \u00C1: "A",
+    \u00C2: "A",
+    \u00C3: "A",
+    \u00C4: "A",
+    \u00C5: "A",
+    \u00C6: "AE",
+    \u00C7: "C",
+    \u00C8: "E",
+    \u00C9: "E",
+    \u00CA: "E",
+    \u00CB: "E",
+    \u00CC: "I",
+    \u00CD: "I",
+    \u00CE: "I",
+    \u00CF: "I",
+    \u00D1: "N",
+    \u00D2: "O",
+    \u00D3: "O",
+    \u00D4: "O",
+    \u00D5: "O",
+    \u00D6: "O",
+    \u00D8: "O",
+    \u00D9: "U",
+    \u00DA: "U",
+    \u00DB: "U",
+    \u00DC: "U",
+    \u00DD: "Y",
+    \u00E0: "a",
+    \u00E1: "a",
+    \u00E2: "a",
+    \u00E3: "a",
+    \u00E4: "a",
+    \u00E5: "a",
+    \u00E6: "ae",
+    \u00E7: "c",
+    \u00E8: "e",
+    \u00E9: "e",
+    \u00EA: "e",
+    \u00EB: "e",
+    \u00EC: "i",
+    \u00ED: "i",
+    \u00EE: "i",
+    \u00EF: "i",
+    \u00F1: "n",
+    \u00F2: "o",
+    \u00F3: "o",
+    \u00F4: "o",
+    \u00F5: "o",
+    \u00F6: "o",
+    \u00F8: "o",
+    \u00F9: "u",
+    \u00FA: "u",
+    \u00FB: "u",
+    \u00FC: "u",
+    \u00FD: "y",
+    \u00FF: "y"
+  };
+  const replacement = "-";
+  const trim = options?.trim !== false;
+  let slug = text.normalize().split("").reduce((result, ch) => {
+    const mapped = charMap[ch] || ch;
+    const appendChar = mapped === replacement ? " " : mapped;
+    return result + appendChar.replace(/[^\w\s$*_+~.()'"!\-:@]+/g, "");
+  }, "");
+  if (options?.strict) {
+    slug = slug.replace(/[^A-Za-z0-9\s]/g, "");
+  }
+  if (trim) {
+    slug = slug.trim();
+  }
+  slug = slug.replace(/\s+/g, replacement);
+  if (options?.lower) {
+    slug = slug.toLowerCase();
+  }
+  return slug;
+}
 
 // util/span_identifier_v4.ts
 var import_v34 = require("zod/v3");
@@ -4357,7 +4448,9 @@ var iso = {
   getCallerLocation: () => void 0,
   newAsyncLocalStorage: () => new DefaultAsyncLocalStorage(),
   processOn: (_0, _1) => {
-  }
+  },
+  basename: (filepath) => filepath.split(/[\\/]/).pop() || filepath,
+  writeln: (text) => console.log(text)
 };
 var isomorph_default = iso;
 
@@ -4443,10 +4536,10 @@ var DiskCache = class {
       return;
     }
     const stats = await Promise.all(
-      paths.map(async (path9) => {
-        const stat2 = await isomorph_default.stat(path9);
+      paths.map(async (path8) => {
+        const stat2 = await isomorph_default.stat(path8);
         return {
-          path: path9,
+          path: path8,
           mtime: stat2.mtime.getTime()
         };
       })
@@ -5165,9 +5258,9 @@ var HTTPConnection = class _HTTPConnection {
       this.headers["Authorization"] = `Bearer ${this.token}`;
     }
   }
-  async get(path9, params = void 0, config3) {
+  async get(path8, params = void 0, config3) {
     const { headers, ...rest } = config3 || {};
-    const url = new URL(_urljoin(this.base_url, path9));
+    const url = new URL(_urljoin(this.base_url, path8));
     url.search = new URLSearchParams(
       params ? Object.entries(params).filter(([_, v]) => v !== void 0).flatMap(
         ([k, v]) => v !== void 0 ? typeof v === "string" ? [[k, v]] : v.map((x) => [k, x]) : []
@@ -5188,13 +5281,13 @@ var HTTPConnection = class _HTTPConnection {
       })
     );
   }
-  async post(path9, params, config3) {
+  async post(path8, params, config3) {
     const { headers, ...rest } = config3 || {};
     const this_fetch = this.fetch;
     const this_base_url = this.base_url;
     const this_headers = this.headers;
     return await checkResponse(
-      await this_fetch(_urljoin(this_base_url, path9), {
+      await this_fetch(_urljoin(this_base_url, path8), {
         method: "POST",
         headers: {
           Accept: "application/json",
@@ -8444,17 +8537,11 @@ var Prompt2 = class _Prompt {
 };
 var TEST_API_KEY = "___TEST_API_KEY__THIS_IS_NOT_REAL___";
 
-// src/progress.ts
+// src/cli/reporters/progress.ts
 var import_chalk = __toESM(require("chalk"));
 var cliProgress = __toESM(require("cli-progress"));
-var MAX_NAME_LENGTH = 40;
-function fitNameToSpaces(name, length) {
-  const padded = name.padEnd(length);
-  if (padded.length <= length) {
-    return padded;
-  }
-  return padded.substring(0, length - 3) + "...";
-}
+
+// src/reporters/progress.ts
 var SimpleProgressReporter = class {
   start(name, _total) {
     console.log(`Running evaluator ${name}`);
@@ -8466,15 +8553,23 @@ var SimpleProgressReporter = class {
   setTotal(_name, _total) {
   }
 };
+
+// src/cli/reporters/progress.ts
+var MAX_NAME_LENGTH = 40;
+function fitNameToSpaces(name, length) {
+  const padded = name.padEnd(length);
+  if (padded.length <= length) {
+    return padded;
+  }
+  return padded.substring(0, length - 3) + "...";
+}
 var BarProgressReporter = class {
   multiBar;
   bars = {};
   constructor() {
     this.multiBar = new cliProgress.MultiBar(
       {
-        // clearOnComplete: true,
         format: `${import_chalk.default.blueBright("{bar}")} ${import_chalk.default.blue("{evaluator}")} {percentage}% ${import_chalk.default.gray("{value}/{total} {eta_formatted}")}`,
-        // autopadding: true,
         hideCursor: true,
         barsize: 10
       },
@@ -8498,7 +8593,7 @@ var BarProgressReporter = class {
   }
 };
 
-// src/cli.ts
+// src/cli/index.ts
 var import_chalk3 = __toESM(require("chalk"));
 var import_termi_link2 = require("termi-link");
 
@@ -9538,20 +9633,12 @@ function waterfall(tasks, callback) {
 }
 var waterfall$1 = awaitify(waterfall);
 
-// src/framework.ts
-var import_chalk2 = __toESM(require("chalk"));
-var import_termi_link = require("termi-link");
-var import_boxen = __toESM(require("boxen"));
-var import_pluralize = __toESM(require("pluralize"));
-var import_cli_table3 = __toESM(require("cli-table3"));
-
 // src/eval-parameters.ts
 var import_v310 = require("zod/v3");
 
 // src/framework2.ts
-var import_path = __toESM(require("path"));
-var import_slugify = __toESM(require("slugify"));
 var import_v39 = require("zod/v3");
+var currentFilename = typeof __filename !== "undefined" ? __filename : "unknown";
 var ProjectBuilder = class {
   create(opts) {
     return new Project2(opts);
@@ -9623,12 +9710,12 @@ var ToolBuilder = class {
     const { handler, name, slug, parameters, returns, ...rest } = opts;
     let resolvedName = name ?? handler.name;
     if (resolvedName.trim().length === 0) {
-      resolvedName = `Tool ${import_path.default.basename(__filename)} ${this.taskCounter}`;
+      resolvedName = `Tool ${isomorph_default.basename(currentFilename)} ${this.taskCounter}`;
     }
     const tool = new CodeFunction(this.project, {
       handler,
       name: resolvedName,
-      slug: slug ?? (0, import_slugify.default)(resolvedName, { lower: true, strict: true }),
+      slug: slug ?? slugify(resolvedName, { lower: true, strict: true }),
       type: "tool",
       // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/consistent-type-assertions
       parameters,
@@ -9652,9 +9739,9 @@ var ScorerBuilder = class {
       resolvedName = opts.handler.name;
     }
     if (!resolvedName || resolvedName.trim().length === 0) {
-      resolvedName = `Scorer ${import_path.default.basename(__filename)} ${this.taskCounter}`;
+      resolvedName = `Scorer ${isomorph_default.basename(currentFilename)} ${this.taskCounter}`;
     }
-    const slug = opts.slug ?? (0, import_slugify.default)(resolvedName, { lower: true, strict: true });
+    const slug = opts.slug ?? slugify(resolvedName, { lower: true, strict: true });
     if ("handler" in opts) {
       const scorer = new CodeFunction(this.project, {
         ...opts,
@@ -9825,7 +9912,7 @@ var PromptBuilder = class {
         rawTools.push(tool);
       }
     }
-    const slug = opts.slug ?? (0, import_slugify.default)(opts.name, { lower: true, strict: true });
+    const slug = opts.slug ?? slugify(opts.name, { lower: true, strict: true });
     const promptData = promptDefinitionToPromptData(opts, rawTools);
     const promptRow = {
       id: opts.id,
@@ -9950,8 +10037,11 @@ var EvalResultWithSummary = class {
     this.summary = summary;
     this.results = results;
   }
+  /**
+   * @deprecated Use `summary` instead.
+   */
   toString() {
-    return formatExperimentSummary(this.summary);
+    return JSON.stringify(this.summary);
   }
   [Symbol.for("nodejs.util.inspect.custom")]() {
     return `EvalResultWithSummary(summary="...", results=[...])`;
@@ -10030,7 +10120,7 @@ async function Eval(name, evaluator, reporterOrOpts) {
       []
     );
   }
-  const progressReporter = options.progress ?? new BarProgressReporter();
+  const progressReporter = options.progress ?? new SimpleProgressReporter();
   const shouldCollectResults = options.returnResults ?? true;
   if (typeof options.reporter === "string") {
     throw new Error(
@@ -10133,21 +10223,21 @@ function parseFilters(filters) {
     if (equalsIdx === -1) {
       throw new Error(`Invalid filter ${f}`);
     }
-    const [path9, value] = [f.slice(0, equalsIdx), f.slice(equalsIdx + 1)];
+    const [path8, value] = [f.slice(0, equalsIdx), f.slice(equalsIdx + 1)];
     let deserializedValue = deserializePlainStringAsJSON2(value).value;
     if (typeof deserializedValue !== "string") {
       deserializedValue = value;
     }
     result.push({
-      path: path9.split("."),
+      path: path8.split("."),
       pattern: new RegExp(deserializedValue)
     });
   }
   return result;
 }
 function evaluateFilter(object, filter2) {
-  const { path: path9, pattern } = filter2;
-  const key = path9.reduce(
+  const { path: path8, pattern } = filter2;
+  const key = path8.reduce(
     (acc, p) => typeof acc === "object" && acc !== null ? (
       // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
       acc[p]
@@ -10517,7 +10607,7 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
   } catch (e) {
     q.kill();
     if (e instanceof InternalAbortError) {
-      if (process.env.BRAINTRUST_VERBOSE) {
+      if (isomorph_default.getEnv("BRAINTRUST_VERBOSE")) {
         console.warn("Evaluator cancelled:", e.message);
       }
     }
@@ -10537,8 +10627,8 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
     collectResults ? collectedResults : []
   );
 }
-var error = import_chalk2.default.red;
-var warning = import_chalk2.default.yellow;
+var error = (text) => `Error: ${text}`;
+var warning = (text) => `Warning: ${text}`;
 function logError2(e, verbose) {
   if (!verbose) {
     console.error(`${e}`);
@@ -10587,11 +10677,7 @@ function reportFailures(evaluator, failingResults, { verbose, jsonl }) {
   if (failingResults.length > 0) {
     console.error(
       warning(
-        `Evaluator ${evaluator.evalName} failed with ${(0, import_pluralize.default)(
-          "error",
-          failingResults.length,
-          true
-        )}. This evaluation ("${evaluator.evalName}") will not be fully logged.`
+        `Evaluator ${evaluator.evalName} failed with ${failingResults.length} error${failingResults.length === 1 ? "" : "s"}. This evaluation ("${evaluator.evalName}") will not be fully logged.`
       )
     );
     if (jsonl) {
@@ -10623,17 +10709,92 @@ var defaultReporter = {
     if (failingResults.length > 0) {
       reportFailures(evaluator, failingResults, { verbose, jsonl });
     }
-    process.stdout.write(
-      jsonl ? JSON.stringify(summary) : formatExperimentSummary(summary)
-    );
-    process.stdout.write("\n");
+    if (jsonl) {
+      isomorph_default.writeln(JSON.stringify(summary));
+    } else {
+      isomorph_default.writeln("Experiment summary");
+      isomorph_default.writeln("==================");
+      if (summary.comparisonExperimentName) {
+        isomorph_default.writeln(
+          `${summary.comparisonExperimentName} (baseline) <- ${summary.experimentName} (comparison)`
+        );
+        isomorph_default.writeln("");
+      }
+      const hasScores = Object.keys(summary.scores).length > 0;
+      const hasMetrics = Object.keys(summary.metrics ?? {}).length > 0;
+      const hasComparison = !!summary.comparisonExperimentName;
+      if (hasScores || hasMetrics) {
+        if (hasComparison) {
+          isomorph_default.writeln(
+            "Name                Value      Change     Improvements Regressions"
+          );
+          isomorph_default.writeln(
+            "----------------------------------------------------------------"
+          );
+        }
+        for (const score of Object.values(summary.scores)) {
+          const scorePercent = (score.score * 100).toFixed(2);
+          const scoreValue = `${scorePercent}%`;
+          if (hasComparison) {
+            let diffString = "-";
+            if (!isEmpty2(score.diff)) {
+              const diffPercent = (score.diff * 100).toFixed(2);
+              const diffSign = score.diff > 0 ? "+" : "";
+              diffString = `${diffSign}${diffPercent}%`;
+            }
+            const improvements = score.improvements > 0 ? score.improvements.toString() : "-";
+            const regressions = score.regressions > 0 ? score.regressions.toString() : "-";
+            isomorph_default.writeln(
+              `${score.name.padEnd(18)} ${scoreValue.padStart(10)} ${diffString.padStart(10)} ${improvements.padStart(12)} ${regressions.padStart(11)}`
+            );
+          } else {
+            isomorph_default.writeln(`${score.name.padEnd(20)} ${scoreValue.padStart(15)}`);
+          }
+        }
+        for (const metric of Object.values(summary.metrics ?? {})) {
+          const fractionDigits = Number.isInteger(metric.metric) ? 0 : 2;
+          const formattedValue = metric.metric.toFixed(fractionDigits);
+          const metricValue = metric.unit === "$" ? `${metric.unit}${formattedValue}` : `${formattedValue}${metric.unit}`;
+          if (hasComparison) {
+            let diffString = "-";
+            if (!isEmpty2(metric.diff)) {
+              const diffPercent = (metric.diff * 100).toFixed(2);
+              const diffSign = metric.diff > 0 ? "+" : "";
+              diffString = `${diffSign}${diffPercent}%`;
+            }
+            const improvements = metric.improvements > 0 ? metric.improvements.toString() : "-";
+            const regressions = metric.regressions > 0 ? metric.regressions.toString() : "-";
+            isomorph_default.writeln(
+              `${metric.name.padEnd(18)} ${metricValue.padStart(10)} ${diffString.padStart(10)} ${improvements.padStart(12)} ${regressions.padStart(11)}`
+            );
+          } else {
+            isomorph_default.writeln(
+              `${metric.name.padEnd(20)} ${metricValue.padStart(15)}`
+            );
+          }
+        }
+      }
+      if (summary.experimentUrl) {
+        isomorph_default.writeln("");
+        isomorph_default.writeln(`View results for ${summary.experimentName}`);
+        isomorph_default.writeln(`See results at ${summary.experimentUrl}`);
+      }
+    }
+    isomorph_default.writeln("");
     return failingResults.length === 0;
   },
   async reportRun(evalReports) {
     return evalReports.every((r) => r);
   }
 };
-function formatExperimentSummary(summary) {
+
+// src/cli/reporters/eval.ts
+var import_chalk2 = __toESM(require("chalk"));
+var import_termi_link = require("termi-link");
+var import_boxen = __toESM(require("boxen"));
+var import_cli_table3 = __toESM(require("cli-table3"));
+var import_pluralize = __toESM(require("pluralize"));
+function formatExperimentSummaryFancy(summary) {
   let comparisonLine = "";
   if (summary.comparisonExperimentName) {
     comparisonLine = `${summary.comparisonExperimentName} ${import_chalk2.default.gray("(baseline)")} \u2190 ${summary.experimentName} ${import_chalk2.default.gray("(comparison)")}
@@ -10678,8 +10839,7 @@ function formatExperimentSummary(summary) {
       wordWrap: false
     });
     const scoreValues = Object.values(summary.scores);
-    for (let i = 0; i < scoreValues.length; i++) {
-      const score = scoreValues[i];
+    for (const score of scoreValues) {
       const scorePercent = (score.score * 100).toFixed(2);
       const scoreValue = import_chalk2.default.white(`${scorePercent}%`);
       let diffString = "";
@@ -10700,8 +10860,7 @@ function formatExperimentSummary(summary) {
       combinedTable.push(row);
     }
     const metricValues = Object.values(summary.metrics ?? {});
-    for (let i = 0; i < metricValues.length; i++) {
-      const metric = metricValues[i];
+    for (const metric of metricValues) {
       const fractionDigits = Number.isInteger(metric.metric) ? 0 : 2;
       const formattedValue = metric.metric.toFixed(fractionDigits);
       const metricValue = import_chalk2.default.white(
@@ -10733,18 +10892,57 @@ function formatExperimentSummary(summary) {
     { fallback: () => `See results at ${summary.experimentUrl}` }
   ) : "";
   const boxContent = [content, footer].filter(Boolean).join("\n\n");
-  return "\n" + (0, import_boxen.default)(boxContent, {
-    title: import_chalk2.default.gray("Experiment summary"),
-    titleAlignment: "left",
-    padding: 0.5,
-    borderColor: "gray",
-    borderStyle: "round"
-  });
+  try {
+    return "\n" + (0, import_boxen.default)(boxContent, {
+      title: import_chalk2.default.gray("Experiment summary"),
+      titleAlignment: "left",
+      padding: 0.5,
+      borderColor: "gray",
+      borderStyle: "round"
+    });
+  } catch (error2) {
+    return "\n" + import_chalk2.default.gray("Experiment summary") + "\n" + boxContent + "\n";
+  }
 }
+var warning2 = import_chalk2.default.yellow;
+var fancyReporter = {
+  name: "Braintrust fancy reporter",
+  async reportEval(evaluator, result, { verbose, jsonl }) {
+    const { results, summary } = result;
+    const failingResults = results.filter(
+      (r) => r.error !== void 0
+    );
+    if (failingResults.length > 0) {
+      console.error(
+        warning2(
+          `Evaluator ${evaluator.evalName} failed with ${(0, import_pluralize.default)("error", failingResults.length, true)}. This evaluation ("${evaluator.evalName}") will not be fully logged.`
+        )
+      );
+      if (jsonl) {
+        for (const result2 of failingResults) {
+          process.stdout.write(JSON.stringify(result2));
+          process.stdout.write("\n");
+        }
+      } else if (verbose) {
+        for (const result2 of failingResults) {
+          console.error(result2);
+        }
+      }
+    }
+    process.stdout.write(
+      jsonl ? JSON.stringify(summary) : formatExperimentSummaryFancy(summary)
+    );
+    process.stdout.write("\n");
+    return failingResults.length === 0;
+  },
+  async reportRun(evalReports) {
+    return evalReports.every((r) => r);
+  }
+};
 
 // src/node.ts
 var import_node_async_hooks = require("async_hooks");
-var path2 = __toESM(require("path"));
+var path = __toESM(require("path"));
 var fs = __toESM(require("fs/promises"));
 var os = __toESM(require("os"));
 var fsSync = __toESM(require("fs"));
@@ -10839,7 +11037,7 @@ async function getPastNAncestors(n = 1e3, remote = void 0) {
     return [];
   }
   const commits = await git.log({ from: ancestor, to: "HEAD", maxCount: n });
-  return commits.all.map((c) => c.hash);
+  return commits.all.slice(0, n).map((c) => c.hash);
 }
 async function attempt(fn) {
   try {
@@ -10965,7 +11163,7 @@ function getCallerLocation() {
 }
 
 // src/node.ts
-var import_util8 = require("util");
+var import_util9 = require("util");
 var zlib = __toESM(require("zlib"));
 function configureNode() {
   isomorph_default.getRepoInfo = getRepoInfo;
@@ -10976,8 +11174,10 @@ function configureNode() {
   isomorph_default.processOn = (event, handler) => {
     process.on(event, handler);
   };
-  isomorph_default.pathJoin = path2.join;
-  isomorph_default.pathDirname = path2.dirname;
+  isomorph_default.basename = path.basename;
+  isomorph_default.writeln = (text) => process.stdout.write(text + "\n");
+  isomorph_default.pathJoin = path.join;
+  isomorph_default.pathDirname = path.dirname;
   isomorph_default.mkdir = fs.mkdir;
   isomorph_default.writeFile = fs.writeFile;
   isomorph_default.readFile = fs.readFile;
@@ -10987,45 +11187,45 @@ function configureNode() {
   isomorph_default.utimes = fs.utimes;
   isomorph_default.unlink = fs.unlink;
   isomorph_default.homedir = os.homedir;
-  isomorph_default.gzip = (0, import_util8.promisify)(zlib.gzip);
-  isomorph_default.gunzip = (0, import_util8.promisify)(zlib.gunzip);
+  isomorph_default.gzip = (0, import_util9.promisify)(zlib.gzip);
+  isomorph_default.gunzip = (0, import_util9.promisify)(zlib.gunzip);
   isomorph_default.hash = (data) => crypto.createHash("sha256").update(data).digest("hex");
   _internalSetInitialState();
 }
 
-// src/cli.ts
+// src/cli/index.ts
 var import_env2 = require("@next/env");
 
-// src/functions/upload.ts
+// src/cli/functions/upload.ts
 var import_fs = __toESM(require("fs"));
-var import_path4 = __toESM(require("path"));
+var import_path3 = __toESM(require("path"));
 var import_zlib = require("zlib");
 var import_v311 = require("zod/v3");
 
-// src/functions/infer-source.ts
+// src/cli/functions/infer-source.ts
 var import_source_map = require("source-map");
 var fs2 = __toESM(require("fs/promises"));
 
-// src/jest/nodeModulesPaths.ts
-var path3 = __toESM(require("path"));
+// src/cli/jest/nodeModulesPaths.ts
+var path2 = __toESM(require("path"));
 
-// src/jest/tryRealpath.ts
+// src/cli/jest/tryRealpath.ts
 var import_graceful_fs = require("graceful-fs");
-function tryRealpath(path9) {
+function tryRealpath(path8) {
   try {
-    path9 = import_graceful_fs.realpathSync.native(path9);
+    path8 = import_graceful_fs.realpathSync.native(path8);
   } catch (error2) {
     if (error2.code !== "ENOENT" && error2.code !== "EISDIR") {
       throw error2;
     }
   }
-  return path9;
+  return path8;
 }
 
-// src/jest/nodeModulesPaths.ts
+// src/cli/jest/nodeModulesPaths.ts
 function nodeModulesPaths(basedir, options) {
   const modules = options && options.moduleDirectory ? Array.from(options.moduleDirectory) : ["node_modules"];
-  const basedirAbs = path3.resolve(basedir);
+  const basedirAbs = path2.resolve(basedir);
   let prefix = "/";
   if (/^([A-Za-z]:)/.test(basedirAbs)) {
     prefix = "";
@@ -11039,19 +11239,19 @@ function nodeModulesPaths(basedir, options) {
     physicalBasedir = basedirAbs;
   }
   const paths = [physicalBasedir];
-  let parsed = path3.parse(physicalBasedir);
+  let parsed = path2.parse(physicalBasedir);
   while (parsed.dir !== paths[paths.length - 1]) {
     paths.push(parsed.dir);
-    parsed = path3.parse(parsed.dir);
+    parsed = path2.parse(parsed.dir);
   }
   const dirs = paths.reduce((dirs2, aPath) => {
     for (const moduleDir of modules) {
-      if (path3.isAbsolute(moduleDir)) {
+      if (path2.isAbsolute(moduleDir)) {
         if (aPath === basedirAbs && moduleDir) {
           dirs2.push(moduleDir);
         }
       } else {
-        dirs2.push(path3.join(prefix, aPath, moduleDir));
+        dirs2.push(path2.join(prefix, aPath, moduleDir));
       }
     }
     return dirs2;
@@ -11062,8 +11262,8 @@ function nodeModulesPaths(basedir, options) {
   return dirs;
 }
 function findGlobalPaths() {
-  const { root } = path3.parse(process.cwd());
-  const globalPath = path3.join(root, "node_modules");
+  const { root } = path2.parse(process.cwd());
+  const globalPath = path2.join(root, "node_modules");
   const resolvePaths = require.resolve.paths("/");
   if (resolvePaths) {
     const rootIndex = resolvePaths.indexOf(globalPath);
@@ -11073,12 +11273,12 @@ function findGlobalPaths() {
 }
 var GlobalPaths = findGlobalPaths();
 
-// src/functions/load-module.ts
-var import_path2 = __toESM(require("path"));
+// src/cli/functions/load-module.ts
+var import_path = __toESM(require("path"));
 function evalWithModuleContext(inFile, evalFn) {
   const modulePaths = [...module.paths];
   try {
-    module.paths = nodeModulesPaths(import_path2.default.dirname(inFile), {});
+    module.paths = nodeModulesPaths(import_path.default.dirname(inFile), {});
     return evalFn();
   } finally {
     module.paths = modulePaths;
@@ -11098,7 +11298,7 @@ function loadModule({
     globalThis._lazy_load = true;
     globalThis.__inherited_braintrust_state = _internalGetGlobalState();
     const __filename2 = inFile;
-    const __dirname = (0, import_path2.dirname)(__filename2);
+    const __dirname = (0, import_path.dirname)(__filename2);
     new Function("require", "module", "__filename", "__dirname", moduleText)(
       require,
       module,
@@ -11109,8 +11309,8 @@ function loadModule({
   });
 }
 
-// src/functions/infer-source.ts
-var import_path3 = __toESM(require("path"));
+// src/cli/functions/infer-source.ts
+var import_path2 = __toESM(require("path"));
 async function makeSourceMapContext({
   inFile,
   outFile,
@@ -11129,7 +11329,7 @@ async function makeSourceMapContext({
     inFiles: { [inFile]: inFileContents.split("\n") },
     outFileModule: loadModule({ inFile, moduleText: outFileContents }),
     outFileLines: outFileContents.split("\n"),
-    sourceMapDir: import_path3.default.dirname(sourceMapFile),
+    sourceMapDir: import_path2.default.dirname(sourceMapFile),
     sourceMap
   };
 }
@@ -11200,7 +11400,7 @@ async function findCodeDefinition({
     return void 0;
   }
   if (!inFiles[originalPosition.source]) {
-    const originalFile = import_path3.default.join(sourceMapDir, originalPosition.source);
+    const originalFile = import_path2.default.join(sourceMapDir, originalPosition.source);
     inFiles[originalPosition.source] = (await fs2.readFile(originalFile, "utf-8")).split("\n");
   }
   const originalLines = inFiles[originalPosition.source];
@@ -11256,8 +11456,7 @@ async function getTsModule() {
   return tsModule;
 }
 
-// src/functions/upload.ts
-var import_slugify2 = __toESM(require("slugify"));
+// src/cli/functions/upload.ts
 var import_zod_to_json_schema = require("zod-to-json-schema");
 var import_pluralize2 = __toESM(require("pluralize"));
 var pathInfoSchema = import_v311.z.strictObject({
@@ -11441,7 +11640,7 @@ async function uploadBundles({
   if (isEmpty2(bundleFileName)) {
     throw new Error("No bundle file found");
   }
-  const bundleFile = import_path4.default.resolve(bundleFileName);
+  const bundleFile = import_path3.default.resolve(bundleFileName);
   const uploadPromise = (async () => {
     if (!pathInfo) {
       return true;
@@ -11532,11 +11731,11 @@ function formatNameAndSlug(pieces) {
   const nonEmptyPieces = pieces.filter((piece) => piece.trim() !== "");
   return {
     name: capitalize(nonEmptyPieces.join(" ")),
-    slug: (0, import_slugify2.default)(nonEmptyPieces.join("-"))
+    slug: slugify(nonEmptyPieces.join("-"))
   };
 }
 
-// src/cli-util/bundle.ts
+// src/cli/util/bundle.ts
 var import_env = require("@next/env");
 var dotenv = __toESM(require("dotenv"));
 async function loadCLIEnv(args) {
@@ -11603,12 +11802,11 @@ async function bundleCommand(args) {
   }
 }
 
-// src/cli-util/pull.ts
+// src/cli/util/pull.ts
 var import_v312 = require("zod/v3");
 var import_promises = __toESM(require("fs/promises"));
-var import_util11 = __toESM(require("util"));
-var import_slugify3 = __toESM(require("slugify"));
-var import_path5 = __toESM(require("path"));
+var import_util12 = __toESM(require("util"));
+var import_path4 = __toESM(require("path"));
 var import_pluralize3 = __toESM(require("pluralize"));
 async function pullCommand(args) {
   await loadCLIEnv(args);
@@ -11650,15 +11848,15 @@ async function pullCommand(args) {
   const repoRoot = await git?.revparse(["--show-toplevel"]);
   const dirtyFiles = new Set(
     (diffSummary?.files ?? []).map(
-      (f) => import_path5.default.resolve(repoRoot ?? ".", f.file)
+      (f) => import_path4.default.resolve(repoRoot ?? ".", f.file)
     )
   );
   for (const projectName of Object.keys(projectNameToFunctions)) {
-    const projectFile = import_path5.default.join(
+    const projectFile = import_path4.default.join(
       outputDir,
-      `${(0, import_slugify3.default)(projectName, { lower: true, strict: true, trim: true })}.ts`
+      `${slugify(projectName, { lower: true, strict: true, trim: true })}.ts`
     );
-    const resolvedProjectFile = import_path5.default.resolve(projectFile);
+    const resolvedProjectFile = import_path4.default.resolve(projectFile);
     const fileExists = await import_promises.default.stat(projectFile).then(
       () => true,
       () => false
@@ -11878,7 +12076,7 @@ function safeStringify(obj) {
     return JSON.stringify(obj, null, 2);
   } catch (error2) {
     try {
-      return import_util11.default.inspect(obj, {
+      return import_util12.default.inspect(obj, {
         depth: 5,
         maxStringLength: 1e3,
         breakLength: 80,
@@ -12426,7 +12624,7 @@ function makeEvalParametersSchema(parameters) {
   );
 }
 
-// src/cli-util/external-packages-plugin.ts
+// src/cli/util/external-packages-plugin.ts
 function createMarkKnownPackagesExternalPlugin(additionalPackages = []) {
   return {
     name: "make-known-packages-external",
@@ -12458,7 +12656,7 @@ function createMarkKnownPackagesExternalPlugin(additionalPackages = []) {
   };
 }
 
-// src/cli.ts
+// src/cli/index.ts
 var { version } = require_package();
 var INCLUDE_EVAL = [
   "**/*.eval.ts",
@@ -12512,7 +12710,7 @@ function resolveReporter(reporter, reporters) {
   } else if (!isEmpty2(reporter)) {
     return reporter;
   } else if (Object.keys(reporters).length === 0) {
-    return defaultReporter;
+    return fancyReporter;
   } else if (Object.keys(reporters).length === 1) {
     return reporters[Object.keys(reporters)[0]];
   } else {
@@ -12719,7 +12917,7 @@ function updateEvaluators(evaluators, buildResults, opts) {
     )) {
       if (evaluators.reporters[reporterName] && evaluators.reporters[reporterName] !== reporter) {
         console.warn(
-          warning(
+          warning2(
             `Reporter '${reporterName}' already exists. Will skip '${reporterName}' from ${result.sourceFile}.`
           )
         );
@@ -12851,7 +13049,7 @@ async function runOnce(handles, opts) {
   return allSuccess;
 }
 function checkMatch(pathInput, include_patterns, exclude_patterns) {
-  const p = import_path6.default.resolve(pathInput);
+  const p = import_path5.default.resolve(pathInput);
   if (include_patterns !== null) {
     let include = false;
     for (const pattern of include_patterns) {
@@ -12893,14 +13091,14 @@ async function collectFiles(inputPath, mode) {
     )) {
       const prefix = mode === "eval" ? ".eval" : "";
       console.warn(
-        warning(
+        warning2(
           `Reading ${inputPath} because it was specified directly. Rename it to end in ${prefix}.ts or .${prefix}.js to include it automatically when you specify a directory.`
         )
       );
     }
     files.push(inputPath);
   } else {
-    const walked = await import_util14.default.promisify(fsWalk.walk)(inputPath, {
+    const walked = await import_util15.default.promisify(fsWalk.walk)(inputPath, {
       deepFilter: (entry) => {
         return checkMatch(entry.path, null, EXCLUDE);
       },
@@ -12935,8 +13133,8 @@ var nativeNodeModulesPlugin = {
     };
     build2.onResolve({ filter: /\.node$/ }, (args) => {
       try {
-        const path9 = require.resolve(args.path, { paths: [args.resolveDir] });
-        const match = path9.match(
+        const path8 = require.resolve(args.path, { paths: [args.resolveDir] });
+        const match = path8.match(
           /node_modules[/\\]((?:@[^/\\]+[/\\])?[^/\\]+)/
         );
         if (match) {
@@ -13006,30 +13204,30 @@ async function initializeHandles({
     const newFiles = await collectFiles(inputPath, mode);
     if (newFiles.length == 0) {
       console.warn(
-        warning(
+        warning2(
           `Provided path ${inputPath} is not an eval file or a directory containing eval files, skipping...`
         )
       );
     }
     for (const file of newFiles) {
-      files[import_path6.default.resolve(file)] = true;
+      files[import_path5.default.resolve(file)] = true;
     }
   }
   if (Object.keys(files).length == 0) {
     console.warn(
-      warning("No eval files were found in any of the provided paths.")
+      warning2("No eval files were found in any of the provided paths.")
     );
     process.exit(0);
   }
-  const tmpDir = import_path6.default.join(import_os.default.tmpdir(), `btevals-${(0, import_uuid3.v4)().slice(0, 8)}`);
+  const tmpDir = import_path5.default.join(import_os.default.tmpdir(), `btevals-${(0, import_uuid3.v4)().slice(0, 8)}`);
   const initPromises = [];
   for (const file of Object.keys(files)) {
-    const baseName = `${import_path6.default.basename(
+    const baseName = `${import_path5.default.basename(
       file,
-      import_path6.default.extname(file)
+      import_path5.default.extname(file)
     )}-${(0, import_uuid3.v4)().slice(0, 8)}`;
-    const outFile = import_path6.default.join(tmpDir, `${baseName}.${OUT_EXT}`);
-    const bundleFile = import_path6.default.join(tmpDir, `${baseName}.bundle.js`);
+    const outFile = import_path5.default.join(tmpDir, `${baseName}.${OUT_EXT}`);
+    const bundleFile = import_path5.default.join(tmpDir, `${baseName}.bundle.js`);
     initPromises.push(
       initFile({
         inFile: file,