braintrust 0.0.98 → 0.0.99

package/dist/browser.js

@@ -233,10 +233,11 @@ function isEmpty(a) {
   return a === void 0 || a === null;
 }
 var LazyValue = class {
+  callable;
+  value = {
+    hasComputed: false
+  };
   constructor(callable) {
-    this.value = {
-      hasComputed: false
-    };
     this.callable = callable;
   }
   async get() {
@@ -250,8 +251,11 @@ var LazyValue = class {
 
 // src/logger.ts
 var NoopSpan = class {
+  id;
+  span_id;
+  root_span_id;
+  kind = "span";
   constructor() {
-    this.kind = "span";
     this.id = "";
     this.span_id = "";
     this.root_span_id = "";
@@ -275,15 +279,22 @@ var NoopSpan = class {
 };
 var NOOP_SPAN = new NoopSpan();
 var BraintrustState = class {
+  id;
+  currentExperiment;
+  // Note: the value of IsAsyncFlush doesn't really matter here, since we
+  // (safely) dynamically cast it whenever retrieving the logger.
+  currentLogger;
+  currentSpan;
+  appUrl = null;
+  loginToken = null;
+  orgId = null;
+  orgName = null;
+  logUrl = null;
+  loggedIn = false;
+  gitMetadataSettings;
+  _apiConn = null;
+  _logConn = null;
   constructor() {
-    this.appUrl = null;
-    this.loginToken = null;
-    this.orgId = null;
-    this.orgName = null;
-    this.logUrl = null;
-    this.loggedIn = false;
-    this._apiConn = null;
-    this._logConn = null;
     this.id = v4_default();
     this.currentExperiment = void 0;
     this.currentLogger = void 0;
@@ -330,6 +341,9 @@ function _internalSetInitialState() {
 }
 var _internalGetGlobalState = () => _state;
 var FailedHTTPResponse = class extends Error {
+  status;
+  text;
+  data;
   constructor(status, text, data = null) {
     super(`${status}: ${text}`);
     this.status = status;
@@ -349,6 +363,9 @@ async function checkResponse(resp) {
   }
 }
 var HTTPConnection = class _HTTPConnection {
+  base_url;
+  token;
+  headers;
   constructor(base_url) {
     this.base_url = base_url;
     this.token = null;
@@ -500,9 +517,13 @@ function logFeedbackImpl(bgLogger, parentIds, {
   }
 }
 var Logger = class {
+  lazyMetadata;
+  logOptions;
+  bgLogger;
+  lastStartTime;
+  // For type identification.
+  kind = "logger";
   constructor(lazyMetadata, logOptions = {}) {
-    // For type identification.
-    this.kind = "logger";
     this.lazyMetadata = lazyMetadata;
     this.logOptions = logOptions;
     const logConn = new LazyValue(
@@ -536,9 +557,19 @@ var Logger = class {
    * @param event.metadata: (Optional) a dictionary with additional data about the test example, model outputs, or just about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any JSON-serializable type, but its keys must be strings.
    * @param event.metrics: (Optional) a dictionary of metrics to log. The following keys are populated automatically: "start", "end".
    * @param event.id: (Optional) a unique identifier for the event. If you don't provide one, BrainTrust will generate one for you.
+   * @param options Additional logging options
+   * @param options.allowLogConcurrentWithActiveSpan in rare cases where you need to log at the top level separately from an active span on the logger, set this to true.
    * :returns: The `id` of the logged event.
    */
-  log(event) {
+  log(event, options) {
+    if (!options?.allowLogConcurrentWithActiveSpan) {
+      const checkCurrentSpan = currentSpan();
+      if (checkCurrentSpan instanceof SpanImpl && checkCurrentSpan.parentObject === this) {
+        throw new Error(
+          "Cannot run toplevel Logger.log method while there is an active span. To log to the span, use Span.log"
+        );
+      }
+    }
     const span = this.startSpan({ startTime: this.lastStartTime, event });
     this.lastStartTime = span.end();
     const ret = span.id;
@@ -597,6 +628,7 @@ var Logger = class {
   startSpan(args) {
     const { name, ...argsRest } = args ?? {};
     return new SpanImpl({
+      parentObject: this,
       parentIds: new LazyValue(() => this.lazyParentIds()),
       bgLogger: this.bgLogger,
       name: name ?? "root",
@@ -654,10 +686,11 @@ function now() {
   return (/* @__PURE__ */ new Date()).getTime();
 }
 var BackgroundLogger = class {
+  logConn;
+  items = [];
+  active_flush = Promise.resolve([]);
+  active_flush_resolved = true;
   constructor(logConn) {
-    this.items = [];
-    this.active_flush = Promise.resolve([]);
-    this.active_flush_resolved = true;
     this.logConn = logConn;
     isomorph_default.processOn("beforeExit", async () => {
       await this.flush();
@@ -706,12 +739,12 @@ var BackgroundLogger = class {
       }
       postPromises.push(
         (async () => {
-          const dataS = constructLogs3Data(items);
+          const dataStr = constructLogs3Data(items);
           for (let i = 0; i < NumRetries; i++) {
             const startTime = now();
             try {
               try {
-                return (await (await this.logConn.get()).post_json("logs3", dataS)).ids.map((res) => res.id);
+                return (await (await this.logConn.get()).post_json("logs3", dataStr)).ids.map((res) => res.id);
               } catch (e) {
                 const legacyDataS = constructJsonArray(
                   items.map(
@@ -730,7 +763,7 @@ var BackgroundLogger = class {
                 }
               })();
               console.warn(
-                `log request failed. Elapsed time: ${(now() - startTime) / 1e3} seconds. Payload size: ${dataS.length}. Error: ${errMsg}.${retryingText}`
+                `log request failed. Elapsed time: ${(now() - startTime) / 1e3} seconds. Payload size: ${dataStr.length}. Error: ${errMsg}.${retryingText}`
               );
             }
           }
@@ -1098,9 +1131,12 @@ async function login(options = {}) {
       }
     };
     var checkUpdatedParam = checkUpdatedParam2;
-    ;
     checkUpdatedParam2("appUrl", options.appUrl, _state.appUrl);
-    checkUpdatedParam2("apiKey", options.apiKey ? HTTPConnection.sanitize_token(options.apiKey) : void 0, _state.loginToken);
+    checkUpdatedParam2(
+      "apiKey",
+      options.apiKey ? HTTPConnection.sanitize_token(options.apiKey) : void 0,
+      _state.loginToken
+    );
     checkUpdatedParam2("orgName", options.orgName, _state.orgName);
     return;
   }
@@ -1321,8 +1357,8 @@ var ObjectFetcher = class {
     this.objectType = objectType;
     this.pinnedVersion = pinnedVersion;
     this.mutateRecord = mutateRecord;
-    this._fetchedData = void 0;
   }
+  _fetchedData = void 0;
   get id() {
     throw new Error("ObjectFetcher subclasses must have an 'id' attribute");
   }
@@ -1382,10 +1418,14 @@ var ObjectFetcher = class {
   }
 };
 var Experiment = class extends ObjectFetcher {
+  lazyMetadata;
+  dataset;
+  bgLogger;
+  lastStartTime;
+  // For type identification.
+  kind = "experiment";
   constructor(lazyMetadata, dataset) {
     super("experiment", void 0);
-    // For type identification.
-    this.kind = "experiment";
     this.lazyMetadata = lazyMetadata;
     this.dataset = dataset;
     const logConn = new LazyValue(
@@ -1426,9 +1466,19 @@ var Experiment = class extends ObjectFetcher {
    * @param event.id: (Optional) a unique identifier for the event. If you don't provide one, BrainTrust will generate one for you.
    * @param event.dataset_record_id: (Optional) the id of the dataset record that this event is associated with. This field is required if and only if the experiment is associated with a dataset.
    * @param event.inputs: (Deprecated) the same as `input` (will be removed in a future version).
+   * @param options Additional logging options
+   * @param options.allowLogConcurrentWithActiveSpan in rare cases where you need to log at the top level separately from an active span on the experiment, set this to true.
    * :returns: The `id` of the logged event.
    */
-  log(event) {
+  log(event, options) {
+    if (!options?.allowLogConcurrentWithActiveSpan) {
+      const checkCurrentSpan = currentSpan();
+      if (checkCurrentSpan instanceof SpanImpl && checkCurrentSpan.parentObject === this) {
+        throw new Error(
+          "Cannot run toplevel Experiment.log method while there is an active span. To log to the span, use Span.log"
+        );
+      }
+    }
     event = validateAndSanitizeExperimentLogFullArgs(event, !!this.dataset);
     const span = this.startSpan({ startTime: this.lastStartTime, event });
     this.lastStartTime = span.end();
@@ -1470,6 +1520,7 @@ var Experiment = class extends ObjectFetcher {
   startSpan(args) {
     const { name, ...argsRest } = args ?? {};
     return new SpanImpl({
+      parentObject: this,
       parentIds: new LazyValue(() => this.lazyParentIds()),
       bgLogger: this.bgLogger,
       name: name ?? "root",
@@ -1624,10 +1675,21 @@ var ReadonlyExperiment = class extends ObjectFetcher {
 };
 var executionCounter = 0;
 var SpanImpl = class _SpanImpl {
+  bgLogger;
+  // `internalData` contains fields that are not part of the "user-sanitized"
+  // set of fields which we want to log in just one of the span rows.
+  internalData;
+  isMerge;
+  loggedEndTime;
+  // For internal use only.
+  parentObject;
+  // These fields are logged to every span row.
+  parentIds;
+  rowIds;
+  kind = "span";
   // root_experiment should only be specified for a root span. parent_span
   // should only be specified for non-root spans.
   constructor(args) {
-    this.kind = "span";
     this.loggedEndTime = void 0;
     this.bgLogger = args.bgLogger;
     const callerLocation = isomorph_default.getCallerLocation();
@@ -1655,6 +1717,7 @@ var SpanImpl = class _SpanImpl {
       },
       created: (/* @__PURE__ */ new Date()).toISOString()
     };
+    this.parentObject = args.parentObject;
     this.parentIds = args.parentIds;
     const id = args.event?.id ?? v4_default();
     const span_id = v4_default();
@@ -1726,6 +1789,7 @@ var SpanImpl = class _SpanImpl {
   }
   startSpan(args) {
     return new _SpanImpl({
+      parentObject: this.parentObject,
       parentIds: this.parentIds,
       bgLogger: this.bgLogger,
       parentSpanInfo: {
@@ -1751,6 +1815,8 @@ var SpanImpl = class _SpanImpl {
   }
 };
 var Dataset = class extends ObjectFetcher {
+  lazyMetadata;
+  bgLogger;
   constructor(lazyMetadata, pinnedVersion, legacy) {
     const isLegacyDataset = legacy ?? DEFAULT_IS_LEGACY_DATASET;
     if (isLegacyDataset) {
@@ -2111,6 +2177,9 @@ function wrapEmbeddings(create) {
   };
 }
 var WrapperStream = class {
+  span;
+  iter;
+  startTime;
   constructor(span, startTime, iter) {
     this.span = span;
     this.iter = iter;

package/dist/cli.js

@@ -9065,7 +9065,7 @@ var require_package = __commonJS({
   "package.json"(exports2, module2) {
     module2.exports = {
       name: "braintrust",
-      version: "0.0.98",
+      version: "0.0.99",
       description: "SDK for integrating Braintrust",
       main: "./dist/index.js",
       browser: {
@@ -9108,7 +9108,7 @@ var require_package = __commonJS({
         typescript: "^5.3.3"
       },
       dependencies: {
-        "@braintrust/core": "^0.0.16",
+        "@braintrust/core": "^0.0.17",
         argparse: "^2.0.1",
         chalk: "^4.1.2",
         "cli-progress": "^3.12.0",
@@ -10664,10 +10664,11 @@ function isEmpty(a) {
   return a === void 0 || a === null;
 }
 var LazyValue = class {
+  callable;
+  value = {
+    hasComputed: false
+  };
   constructor(callable) {
-    this.value = {
-      hasComputed: false
-    };
     this.callable = callable;
   }
   async get() {
@@ -10681,8 +10682,11 @@ var LazyValue = class {
 
 // src/logger.ts
 var NoopSpan = class {
+  id;
+  span_id;
+  root_span_id;
+  kind = "span";
   constructor() {
-    this.kind = "span";
     this.id = "";
     this.span_id = "";
     this.root_span_id = "";
@@ -10706,15 +10710,22 @@ var NoopSpan = class {
 };
 var NOOP_SPAN = new NoopSpan();
 var BraintrustState = class {
+  id;
+  currentExperiment;
+  // Note: the value of IsAsyncFlush doesn't really matter here, since we
+  // (safely) dynamically cast it whenever retrieving the logger.
+  currentLogger;
+  currentSpan;
+  appUrl = null;
+  loginToken = null;
+  orgId = null;
+  orgName = null;
+  logUrl = null;
+  loggedIn = false;
+  gitMetadataSettings;
+  _apiConn = null;
+  _logConn = null;
   constructor() {
-    this.appUrl = null;
-    this.loginToken = null;
-    this.orgId = null;
-    this.orgName = null;
-    this.logUrl = null;
-    this.loggedIn = false;
-    this._apiConn = null;
-    this._logConn = null;
     this.id = v4_default();
     this.currentExperiment = void 0;
     this.currentLogger = void 0;
@@ -10761,6 +10772,9 @@ function _internalSetInitialState() {
 }
 var _internalGetGlobalState = () => _state;
 var FailedHTTPResponse = class extends Error {
+  status;
+  text;
+  data;
   constructor(status, text, data = null) {
     super(`${status}: ${text}`);
     this.status = status;
@@ -10780,6 +10794,9 @@ async function checkResponse(resp) {
   }
 }
 var HTTPConnection = class _HTTPConnection {
+  base_url;
+  token;
+  headers;
   constructor(base_url) {
     this.base_url = base_url;
     this.token = null;
@@ -10943,10 +10960,11 @@ function now() {
   return (/* @__PURE__ */ new Date()).getTime();
 }
 var BackgroundLogger = class {
+  logConn;
+  items = [];
+  active_flush = Promise.resolve([]);
+  active_flush_resolved = true;
   constructor(logConn) {
-    this.items = [];
-    this.active_flush = Promise.resolve([]);
-    this.active_flush_resolved = true;
     this.logConn = logConn;
     isomorph_default.processOn("beforeExit", async () => {
       await this.flush();
@@ -10995,12 +11013,12 @@ var BackgroundLogger = class {
       }
       postPromises.push(
         (async () => {
-          const dataS = constructLogs3Data(items);
+          const dataStr = constructLogs3Data(items);
           for (let i = 0; i < NumRetries; i++) {
             const startTime = now();
             try {
               try {
-                return (await (await this.logConn.get()).post_json("logs3", dataS)).ids.map((res) => res.id);
+                return (await (await this.logConn.get()).post_json("logs3", dataStr)).ids.map((res) => res.id);
               } catch (e) {
                 const legacyDataS = constructJsonArray(
                   items.map(
@@ -11019,7 +11037,7 @@ var BackgroundLogger = class {
                 }
               })();
               console.warn(
-                `log request failed. Elapsed time: ${(now() - startTime) / 1e3} seconds. Payload size: ${dataS.length}. Error: ${errMsg}.${retryingText}`
+                `log request failed. Elapsed time: ${(now() - startTime) / 1e3} seconds. Payload size: ${dataStr.length}. Error: ${errMsg}.${retryingText}`
               );
             }
           }
@@ -11236,9 +11254,12 @@ async function login(options = {}) {
       }
     };
     var checkUpdatedParam = checkUpdatedParam2;
-    ;
     checkUpdatedParam2("appUrl", options.appUrl, _state.appUrl);
-    checkUpdatedParam2("apiKey", options.apiKey ? HTTPConnection.sanitize_token(options.apiKey) : void 0, _state.loginToken);
+    checkUpdatedParam2(
+      "apiKey",
+      options.apiKey ? HTTPConnection.sanitize_token(options.apiKey) : void 0,
+      _state.loginToken
+    );
     checkUpdatedParam2("orgName", options.orgName, _state.orgName);
     return;
   }
@@ -11279,6 +11300,9 @@ async function login(options = {}) {
   _state.loginToken = conn.token;
   _state.loggedIn = true;
 }
+function currentSpan() {
+  return _state.currentSpan.getStore() ?? NOOP_SPAN;
+}
 function withCurrent(span, callback) {
   return _state.currentSpan.run(span, () => callback(span));
 }
@@ -11380,8 +11404,8 @@ var ObjectFetcher = class {
     this.objectType = objectType;
     this.pinnedVersion = pinnedVersion;
     this.mutateRecord = mutateRecord;
-    this._fetchedData = void 0;
   }
+  _fetchedData = void 0;
   get id() {
     throw new Error("ObjectFetcher subclasses must have an 'id' attribute");
   }
@@ -11441,10 +11465,14 @@ var ObjectFetcher = class {
   }
 };
 var Experiment = class extends ObjectFetcher {
+  lazyMetadata;
+  dataset;
+  bgLogger;
+  lastStartTime;
+  // For type identification.
+  kind = "experiment";
   constructor(lazyMetadata, dataset) {
     super("experiment", void 0);
-    // For type identification.
-    this.kind = "experiment";
     this.lazyMetadata = lazyMetadata;
     this.dataset = dataset;
     const logConn = new LazyValue(
@@ -11485,9 +11513,19 @@ var Experiment = class extends ObjectFetcher {
    * @param event.id: (Optional) a unique identifier for the event. If you don't provide one, BrainTrust will generate one for you.
    * @param event.dataset_record_id: (Optional) the id of the dataset record that this event is associated with. This field is required if and only if the experiment is associated with a dataset.
    * @param event.inputs: (Deprecated) the same as `input` (will be removed in a future version).
+   * @param options Additional logging options
+   * @param options.allowLogConcurrentWithActiveSpan in rare cases where you need to log at the top level separately from an active span on the experiment, set this to true.
    * :returns: The `id` of the logged event.
    */
-  log(event) {
+  log(event, options) {
+    if (!options?.allowLogConcurrentWithActiveSpan) {
+      const checkCurrentSpan = currentSpan();
+      if (checkCurrentSpan instanceof SpanImpl && checkCurrentSpan.parentObject === this) {
+        throw new Error(
+          "Cannot run toplevel Experiment.log method while there is an active span. To log to the span, use Span.log"
+        );
+      }
+    }
     event = validateAndSanitizeExperimentLogFullArgs(event, !!this.dataset);
     const span = this.startSpan({ startTime: this.lastStartTime, event });
     this.lastStartTime = span.end();
@@ -11529,6 +11567,7 @@ var Experiment = class extends ObjectFetcher {
   startSpan(args) {
     const { name, ...argsRest } = args ?? {};
     return new SpanImpl({
+      parentObject: this,
       parentIds: new LazyValue(() => this.lazyParentIds()),
       bgLogger: this.bgLogger,
       name: name ?? "root",
@@ -11683,10 +11722,21 @@ var ReadonlyExperiment = class extends ObjectFetcher {
 };
 var executionCounter = 0;
 var SpanImpl = class _SpanImpl {
+  bgLogger;
+  // `internalData` contains fields that are not part of the "user-sanitized"
+  // set of fields which we want to log in just one of the span rows.
+  internalData;
+  isMerge;
+  loggedEndTime;
+  // For internal use only.
+  parentObject;
+  // These fields are logged to every span row.
+  parentIds;
+  rowIds;
+  kind = "span";
   // root_experiment should only be specified for a root span. parent_span
   // should only be specified for non-root spans.
   constructor(args) {
-    this.kind = "span";
     this.loggedEndTime = void 0;
     this.bgLogger = args.bgLogger;
     const callerLocation = isomorph_default.getCallerLocation();
@@ -11714,6 +11764,7 @@ var SpanImpl = class _SpanImpl {
       },
       created: (/* @__PURE__ */ new Date()).toISOString()
     };
+    this.parentObject = args.parentObject;
     this.parentIds = args.parentIds;
     const id = args.event?.id ?? v4_default();
     const span_id = v4_default();
@@ -11785,6 +11836,7 @@ var SpanImpl = class _SpanImpl {
   }
   startSpan(args) {
     return new _SpanImpl({
+      parentObject: this.parentObject,
       parentIds: this.parentIds,
       bgLogger: this.bgLogger,
       parentSpanInfo: {
@@ -11830,8 +11882,9 @@ var SimpleProgressReporter = class {
   }
 };
 var BarProgressReporter = class {
+  multiBar;
+  bars = {};
   constructor() {
-    this.bars = {};
     this.multiBar = new cliProgress.MultiBar(
       {
         clearOnComplete: false,
@@ -11973,6 +12026,9 @@ function evaluateFilter(object, filter2) {
   }
   return pattern.test(serializeJSONWithPlainString(key));
 }
+function scorerName(scorer, scorer_idx) {
+  return scorer.name || `scorer_${scorer_idx}`;
+}
 async function runEvaluator(experiment, evaluator, progressReporter, filters) {
   if (typeof evaluator.data === "string") {
     throw new Error("Unimplemented: string data paths");
@@ -12016,11 +12072,13 @@ async function runEvaluator(experiment, evaluator, progressReporter, filters) {
   );
   progressReporter.start(evaluator.evalName, data.length);
   const evals = data.map(async (datum) => {
-    let metadata = { ..."metadata" in datum ? datum.metadata : {} };
-    let output = void 0;
-    let error2 = void 0;
-    let scores = {};
     const callback = async (rootSpan) => {
+      let metadata = {
+        ..."metadata" in datum ? datum.metadata : {}
+      };
+      let output = void 0;
+      let error2 = void 0;
+      let scores = {};
       try {
         const meta = (o) => metadata = { ...metadata, ...o };
         await rootSpan.traced(
@@ -12037,42 +12095,55 @@ async function runEvaluator(experiment, evaluator, progressReporter, filters) {
         );
         rootSpan.log({ output });
         const scoringArgs = { ...datum, metadata, output };
+        const scorerNames = evaluator.scores.map(scorerName);
         const scoreResults = await Promise.all(
           evaluator.scores.map(async (score, score_idx) => {
-            return rootSpan.traced(
-              async (span) => {
-                const scoreResult = score(scoringArgs);
-                const result = scoreResult instanceof Promise ? await scoreResult : scoreResult;
-                const {
-                  metadata: resultMetadata,
-                  name: _,
-                  ...resultRest
-                } = result;
-                span.log({
-                  output: resultRest,
-                  metadata: resultMetadata
-                });
-                return result;
-              },
-              {
-                name: score.name || `scorer_${score_idx}`,
-                spanAttributes: {
-                  type: SpanTypeAttribute.SCORE
+            try {
+              const result = await rootSpan.traced(
+                async (span) => {
+                  const scoreResult = score(scoringArgs);
+                  const result2 = scoreResult instanceof Promise ? await scoreResult : scoreResult;
+                  const {
+                    metadata: resultMetadata,
+                    name: _,
+                    ...resultRest
+                  } = result2;
+                  span.log({
+                    output: resultRest,
+                    metadata: resultMetadata
+                  });
+                  return result2;
                 },
-                event: { input: scoringArgs }
-              }
-            );
+                {
+                  name: scorerNames[score_idx],
+                  spanAttributes: {
+                    type: SpanTypeAttribute.SCORE
+                  },
+                  event: { input: scoringArgs }
+                }
+              );
+              return { kind: "score", value: result };
+            } catch (e) {
+              return { kind: "error", value: e };
+            }
           })
         );
+        const passingScorersAndResults = [];
+        const failingScorersAndResults = [];
+        scoreResults.forEach((result, i) => {
+          const name = scorerNames[i];
+          if (result.kind === "score") {
+            passingScorersAndResults.push({ name, score: result.value });
+          } else {
+            failingScorersAndResults.push({ name, error: result.value });
+          }
+        });
         const scoreMetadata = {};
-        for (const scoreResult of scoreResults) {
+        for (const { score: scoreResult } of passingScorersAndResults) {
           scores[scoreResult.name] = scoreResult.score;
           const metadata2 = {
             ...scoreResult.metadata
           };
-          if (scoreResult.error !== void 0) {
-            metadata2.error = scoreResult.error;
-          }
           if (Object.keys(metadata2).length > 0) {
             scoreMetadata[scoreResult.name] = metadata2;
           }
@@ -12081,6 +12152,21 @@ async function runEvaluator(experiment, evaluator, progressReporter, filters) {
           meta({ scores: scoreMetadata });
         }
         rootSpan.log({ scores, metadata });
+        if (failingScorersAndResults.length) {
+          const scorerErrors = Object.fromEntries(
+            failingScorersAndResults.map(({ name, error: error3 }) => [
+              name,
+              error3 instanceof Error ? error3.stack : `${error3}`
+            ])
+          );
+          metadata["scorer_errors"] = scorerErrors;
+          const names = Object.keys(scorerErrors).join(", ");
+          const errors = failingScorersAndResults.map((item) => item.error);
+          throw new AggregateError(
+            errors,
+            `Found exceptions for the following scorers: ${names}`
+          );
+        }
       } catch (e) {
         error2 = e;
       } finally {
@@ -12159,7 +12245,8 @@ function reportEvaluatorResult(evaluatorName, evaluatorResult, {
     if (!verbose && !jsonl) {
       console.error(warning("Add --verbose to see full stack traces."));
     }
-  } else if (summary) {
+  }
+  if (summary) {
     console.log(jsonl ? JSON.stringify(summary) : summary);
   } else {
     const scoresByName = {};

package/dist/framework.d.ts

@@ -91,7 +91,7 @@ export declare function parseFilters(filters: string[]): Filter[];
 export declare function runEvaluator(experiment: Experiment | null, evaluator: EvaluatorDef<any, any, any | void, any | void>, progressReporter: ProgressReporter, filters: Filter[]): Promise<{
     results: {
         output: any;
-        metadata: object;
+        metadata: Record<string, unknown>;
         scores: Record<string, number | null>;
         error: unknown;
     }[];