braintrust 0.0.113 → 0.0.114

package/dist/cli.js

@@ -4995,9 +4995,9 @@ var require_pluralize = __commonJS({
   }
 });
 
-// ../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.3.1_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/lib/zod-is-type.js
+// ../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.4.0_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/lib/zod-is-type.js
 var require_zod_is_type = __commonJS({
-  "../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.3.1_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/lib/zod-is-type.js"(exports2) {
+  "../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.4.0_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/lib/zod-is-type.js"(exports2) {
     "use strict";
     Object.defineProperty(exports2, "__esModule", { value: true });
     exports2.isAnyZodType = exports2.isZodType = void 0;
@@ -5013,9 +5013,9 @@ var require_zod_is_type = __commonJS({
   }
 });
 
-// ../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.3.1_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/zod-extensions.js
+// ../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.4.0_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/zod-extensions.js
 var require_zod_extensions = __commonJS({
-  "../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.3.1_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/zod-extensions.js"(exports2) {
+  "../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.4.0_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/zod-extensions.js"(exports2) {
     "use strict";
     var __rest = exports2 && exports2.__rest || function(s, e) {
       var t = {};
@@ -5103,9 +5103,9 @@ var require_zod_extensions = __commonJS({
   }
 });
 
-// ../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.3.1_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/lib/object-set.js
+// ../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.4.0_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/lib/object-set.js
 var require_object_set = __commonJS({
-  "../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.3.1_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/lib/object-set.js"(exports2) {
+  "../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.4.0_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/lib/object-set.js"(exports2) {
     "use strict";
     Object.defineProperty(exports2, "__esModule", { value: true });
     exports2.ObjectSet = exports2.isEqual = void 0;
@@ -5201,12 +5201,12 @@ var require_object_set = __commonJS({
   }
 });
 
-// ../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.3.1_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/lib/lodash.js
+// ../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.4.0_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/lib/lodash.js
 var require_lodash = __commonJS({
-  "../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.3.1_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/lib/lodash.js"(exports2) {
+  "../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.4.0_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/lib/lodash.js"(exports2) {
     "use strict";
     Object.defineProperty(exports2, "__esModule", { value: true });
-    exports2.uniq = exports2.objectEquals = exports2.compact = exports2.omitBy = exports2.omit = exports2.mapValues = exports2.isNil = exports2.isUndefined = void 0;
+    exports2.isString = exports2.uniq = exports2.objectEquals = exports2.compact = exports2.omitBy = exports2.omit = exports2.mapValues = exports2.isNil = exports2.isUndefined = void 0;
     var object_set_1 = require_object_set();
     function isUndefined(value) {
       return value === void 0;
@@ -5255,12 +5255,16 @@ var require_lodash = __commonJS({
       return [...set.values()];
     }
     exports2.uniq = uniq;
+    function isString(val) {
+      return typeof val === "string";
+    }
+    exports2.isString = isString;
   }
 });
 
-// ../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.3.1_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/openapi-metadata.js
+// ../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.4.0_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/openapi-metadata.js
 var require_openapi_metadata = __commonJS({
-  "../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.3.1_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/openapi-metadata.js"(exports2) {
+  "../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.4.0_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/openapi-metadata.js"(exports2) {
     "use strict";
     Object.defineProperty(exports2, "__esModule", { value: true });
     exports2.getOpenApiMetadata = void 0;
@@ -5273,9 +5277,9 @@ var require_openapi_metadata = __commonJS({
   }
 });
 
-// ../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.3.1_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/openapi-registry.js
+// ../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.4.0_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/openapi-registry.js
 var require_openapi_registry = __commonJS({
-  "../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.3.1_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/openapi-registry.js"(exports2) {
+  "../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.4.0_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/openapi-registry.js"(exports2) {
     "use strict";
     Object.defineProperty(exports2, "__esModule", { value: true });
     exports2.OpenAPIRegistry = void 0;
@@ -5357,9 +5361,9 @@ var require_openapi_registry = __commonJS({
   }
 });
 
-// ../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.3.1_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/errors.js
+// ../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.4.0_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/errors.js
 var require_errors = __commonJS({
-  "../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.3.1_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/errors.js"(exports2) {
+  "../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.4.0_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/errors.js"(exports2) {
     "use strict";
     Object.defineProperty(exports2, "__esModule", { value: true });
     exports2.UnknownZodTypeError = exports2.MissingParameterDataError = exports2.ConflictError = exports2.ZodToOpenAPIError = void 0;
@@ -5393,9 +5397,9 @@ var require_errors = __commonJS({
   }
 });
 
-// ../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.3.1_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/lib/enum-info.js
+// ../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.4.0_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/lib/enum-info.js
 var require_enum_info = __commonJS({
-  "../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.3.1_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/lib/enum-info.js"(exports2) {
+  "../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.4.0_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/lib/enum-info.js"(exports2) {
     "use strict";
     Object.defineProperty(exports2, "__esModule", { value: true });
     exports2.enumInfo = void 0;
@@ -5410,9 +5414,9 @@ var require_enum_info = __commonJS({
   }
 });
 
-// ../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.3.1_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/openapi-generator.js
+// ../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.4.0_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/openapi-generator.js
 var require_openapi_generator = __commonJS({
-  "../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.3.1_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/openapi-generator.js"(exports2) {
+  "../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.4.0_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/openapi-generator.js"(exports2) {
     "use strict";
     var __rest = exports2 && exports2.__rest || function(s, e) {
       var t = {};
@@ -5753,7 +5757,7 @@ var require_openapi_generator = __commonJS({
       }
       getBodyContent(content) {
         return (0, lodash_1.mapValues)(content, (config) => {
-          if (!(0, zod_is_type_1.isAnyZodType)(config.schema)) {
+          if (!config || !(0, zod_is_type_1.isAnyZodType)(config.schema)) {
             return config;
           }
           const { schema: configSchema } = config, rest = __rest(config, ["schema"]);
@@ -5941,7 +5945,14 @@ var require_openapi_generator = __commonJS({
         }
         if ((0, zod_is_type_1.isZodType)(zodSchema, "ZodRecord")) {
           const propertiesType = zodSchema._def.valueType;
-          return Object.assign(Object.assign({}, this.mapNullableType("object", isNullable)), { additionalProperties: this.generateSchemaWithRef(propertiesType), default: defaultValue });
+          const keyType = zodSchema._def.keyType;
+          const propertiesSchema = this.generateSchemaWithRef(propertiesType);
+          if ((0, zod_is_type_1.isZodType)(keyType, "ZodEnum") || (0, zod_is_type_1.isZodType)(keyType, "ZodNativeEnum")) {
+            const keys = Object.values(keyType.enum).filter(lodash_1.isString);
+            const properties = keys.reduce((acc, curr) => Object.assign(Object.assign({}, acc), { [curr]: propertiesSchema }), {});
+            return Object.assign(Object.assign({}, this.mapNullableType("object", isNullable)), { properties, default: defaultValue });
+          }
+          return Object.assign(Object.assign({}, this.mapNullableType("object", isNullable)), { additionalProperties: propertiesSchema, default: defaultValue });
         }
         if ((0, zod_is_type_1.isZodType)(zodSchema, "ZodUnknown") || (0, zod_is_type_1.isZodType)(zodSchema, "ZodAny")) {
           return this.mapNullableType(void 0, isNullable);
@@ -6107,9 +6118,9 @@ var require_openapi_generator = __commonJS({
   }
 });
 
-// ../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.3.1_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/v3.0/specifics.js
+// ../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.4.0_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/v3.0/specifics.js
 var require_specifics = __commonJS({
-  "../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.3.1_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/v3.0/specifics.js"(exports2) {
+  "../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.4.0_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/v3.0/specifics.js"(exports2) {
     "use strict";
     Object.defineProperty(exports2, "__esModule", { value: true });
     exports2.OpenApiGeneratorV30Specifics = void 0;
@@ -6143,9 +6154,9 @@ var require_specifics = __commonJS({
   }
 });
 
-// ../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.3.1_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/v3.0/openapi-generator.js
+// ../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.4.0_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/v3.0/openapi-generator.js
 var require_openapi_generator2 = __commonJS({
-  "../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.3.1_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/v3.0/openapi-generator.js"(exports2) {
+  "../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.4.0_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/v3.0/openapi-generator.js"(exports2) {
     "use strict";
     Object.defineProperty(exports2, "__esModule", { value: true });
     exports2.OpenApiGeneratorV3 = void 0;
@@ -6168,9 +6179,9 @@ var require_openapi_generator2 = __commonJS({
   }
 });
 
-// ../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.3.1_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/v3.1/specifics.js
+// ../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.4.0_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/v3.1/specifics.js
 var require_specifics2 = __commonJS({
-  "../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.3.1_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/v3.1/specifics.js"(exports2) {
+  "../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.4.0_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/v3.1/specifics.js"(exports2) {
     "use strict";
     Object.defineProperty(exports2, "__esModule", { value: true });
     exports2.OpenApiGeneratorV31Specifics = void 0;
@@ -6214,9 +6225,9 @@ var require_specifics2 = __commonJS({
   }
 });
 
-// ../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.3.1_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/v3.1/openapi-generator.js
+// ../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.4.0_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/v3.1/openapi-generator.js
 var require_openapi_generator3 = __commonJS({
-  "../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.3.1_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/v3.1/openapi-generator.js"(exports2) {
+  "../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.4.0_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/v3.1/openapi-generator.js"(exports2) {
     "use strict";
     Object.defineProperty(exports2, "__esModule", { value: true });
     exports2.OpenApiGeneratorV31 = void 0;
@@ -6250,9 +6261,9 @@ var require_openapi_generator3 = __commonJS({
   }
 });
 
-// ../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.3.1_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/index.js
+// ../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.4.0_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/index.js
 var require_dist = __commonJS({
-  "../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.3.1_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/index.js"(exports2) {
+  "../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.4.0_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/index.js"(exports2) {
     "use strict";
     var __createBinding = exports2 && exports2.__createBinding || (Object.create ? function(o, m, k, k2) {
       if (k2 === void 0)
@@ -10366,7 +10377,7 @@ var require_package = __commonJS({
   "package.json"(exports2, module2) {
     module2.exports = {
       name: "braintrust",
-      version: "0.0.113",
+      version: "0.0.114",
       description: "SDK for integrating Braintrust",
       main: "./dist/index.js",
       browser: {
@@ -11811,7 +11822,7 @@ var v4_default = v4;
 // src/cli.ts
 var import_pluralize2 = __toESM(require_pluralize());
 
-// ../core/js/dist/index.mjs
+// ../../node_modules/.pnpm/@braintrust+core@0.0.27/node_modules/@braintrust/core/dist/index.mjs
 var TRANSACTION_ID_FIELD = "_xact_id";
 var IS_MERGE_FIELD = "_is_merge";
 var MERGE_PATHS_FIELD = "_merge_paths";
@@ -12085,7 +12096,7 @@ var MOD = BigInt(1) << BigInt(64);
 var COPRIME = BigInt("205891132094649");
 var COPRIME_INVERSE = BigInt("1522336535492693385");
 
-// ../core/js/typespecs/dist/index.mjs
+// ../../node_modules/.pnpm/@braintrust+core@0.0.27/node_modules/@braintrust/core/typespecs/dist/index.mjs
 var import_zod_to_openapi = __toESM(require_dist(), 1);
 
 // ../../node_modules/.pnpm/zod@3.22.4/node_modules/zod/lib/index.mjs
@@ -15821,7 +15832,7 @@ var z = /* @__PURE__ */ Object.freeze({
   ZodError
 });
 
-// ../core/js/typespecs/dist/index.mjs
+// ../../node_modules/.pnpm/@braintrust+core@0.0.27/node_modules/@braintrust/core/typespecs/dist/index.mjs
 var import_zod_to_openapi2 = __toESM(require_dist(), 1);
 var import_zod_to_openapi3 = __toESM(require_dist(), 1);
 var literalSchema = z.union([
@@ -15866,12 +15877,12 @@ var modeToTypes = {
 };
 var customTypes = modeToTypes[mode];
 var chatCompletionSystemMessageParamSchema = z.object({
-  content: z.string(),
+  content: z.string().default(""),
   role: z.literal("system"),
   name: z.string().optional()
 });
 var chatCompletionContentPartTextSchema = z.object({
-  text: z.string(),
+  text: z.string().default(""),
   type: z.literal("text")
 });
 var functionCallSchema = z.object({
@@ -15887,12 +15898,12 @@ var imageURLSchema = z.object({
   detail: z.union([z.literal("auto"), z.literal("low"), z.literal("high")]).optional()
 });
 var chatCompletionToolMessageParamSchema = z.object({
-  content: z.string(),
+  content: z.string().default(""),
   role: z.literal("tool"),
   tool_call_id: z.string()
 });
 var chatCompletionFunctionMessageParamSchema = z.object({
-  content: z.string().nullable(),
+  content: z.string().default(""),
   name: z.string(),
   role: z.literal("function")
 });
@@ -15911,13 +15922,16 @@ var chatCompletionContentPartSchema = z.union([
 ]);
 var chatCompletionAssistantMessageParamSchema = z.object({
   role: z.literal("assistant"),
-  content: z.string().optional().nullable(),
+  content: z.string().nullish(),
   function_call: functionCallSchema.optional(),
   name: z.string().optional(),
   tool_calls: z.array(chatCompletionMessageToolCallSchema).optional()
 });
 var chatCompletionUserMessageParamSchema = z.object({
-  content: z.union([z.string(), z.array(chatCompletionContentPartSchema)]),
+  content: z.union([
+    z.string().default(""),
+    z.array(chatCompletionContentPartSchema)
+  ]),
   role: z.literal("user"),
   name: z.string().optional()
 });
@@ -17281,6 +17295,7 @@ var BraintrustState = class {
   currentLogger;
   currentSpan;
   appUrl = null;
+  appPublicUrl = null;
   loginToken = null;
   orgId = null;
   orgName = null;
@@ -17891,8 +17906,10 @@ async function login(options = {}) {
     apiKey = isomorph_default.getEnv("BRAINTRUST_API_KEY"),
     orgName = isomorph_default.getEnv("BRAINTRUST_ORG_NAME")
   } = options || {};
+  const appPublicUrl = isomorph_default.getEnv("BRAINTRUST_APP_PUBLIC_URL") || appUrl;
   _state.resetLoginInfo();
   _state.appUrl = appUrl;
+  _state.appPublicUrl = appPublicUrl;
   let conn = null;
   if (apiKey !== void 0) {
     const resp = await checkResponse(
@@ -18202,12 +18219,13 @@ var Experiment = class extends ObjectFetcher {
    * See `traced` for full details.
    */
   startSpan(args) {
-    const { name, ...argsRest } = args ?? {};
+    const { name, type, ...argsRest } = args ?? {};
     return new SpanImpl({
       parentObject: this,
       parentIds: new LazyValue(() => this.lazyParentIds()),
       bgLogger: this.bgLogger,
       name: name ?? "root",
+      type: type ?? SpanTypeAttribute.EVAL,
       ...argsRest
     });
   }
@@ -18243,7 +18261,7 @@ var Experiment = class extends ObjectFetcher {
     let { summarizeScores = true, comparisonExperimentId = void 0 } = options || {};
     await this.bgLogger.flush();
     const state = await this.getState();
-    const projectUrl = `${state.appUrl}/app/${encodeURIComponent(
+    const projectUrl = `${state.appPublicUrl}/app/${encodeURIComponent(
       state.orgName
     )}/p/${encodeURIComponent((await this.project).name)}`;
     const experimentUrl = `${projectUrl}/${encodeURIComponent(
@@ -18279,7 +18297,7 @@ var Experiment = class extends ObjectFetcher {
       projectUrl,
       experimentUrl,
       comparisonExperimentName,
-      scores,
+      scores: scores ?? {},
       metrics
     };
   }
@@ -18397,6 +18415,7 @@ var SpanImpl = class _SpanImpl {
       },
       context: { ...callerLocation },
       span_attributes: {
+        type: args.type,
         ...args.spanAttributes,
         name,
         exec_counter: executionCounter++
@@ -18640,7 +18659,10 @@ function initExperiment(projectName, options = {}) {
     setCurrent: false
   });
 }
-globalThis._evals = {};
+globalThis._evals = {
+  evaluators: {},
+  reporters: {}
+};
 function serializeJSONWithPlainString(v) {
   if (typeof v === "string") {
     return v;
@@ -18759,27 +18781,44 @@ async function runEvaluator(experiment, evaluator, progressReporter, filters) {
         const scoreResults = await Promise.all(
           evaluator.scores.map(async (score, score_idx) => {
             try {
-              const result = await rootSpan.traced(
+              const results2 = await rootSpan.traced(
                 async (span) => {
                   const scoreResult = score(scoringArgs);
                   const scoreValue = scoreResult instanceof Promise ? await scoreResult : scoreResult;
                   if (scoreValue === null) {
                     return null;
                   }
-                  const result2 = typeof scoreValue === "object" ? scoreValue : { name: scorerNames[score_idx], score: scoreValue };
-                  const {
-                    metadata: resultMetadata,
-                    name,
-                    ...resultRest
-                  } = result2;
+                  const scoreValues = Array.isArray(scoreValue) ? scoreValue : [scoreValue];
+                  const results3 = scoreValues.map(
+                    (scoreValue2, idx) => typeof scoreValue2 === "object" ? scoreValue2 : {
+                      name: scoreValues.length > 1 ? `${scorerNames[score_idx]}_${idx}` : scorerNames[score_idx],
+                      score: scoreValue2
+                    }
+                  );
+                  const getOtherFields = (s) => {
+                    const { metadata: metadata2, name, ...rest } = s;
+                    return rest;
+                  };
+                  const resultMetadata = results3.length === 1 ? results3[0].metadata : results3.reduce(
+                    (prev, s) => mergeDicts(prev, {
+                      [s.name]: s.metadata
+                    }),
+                    {}
+                  );
+                  const resultOutput = results3.length === 1 ? getOtherFields(results3[0]) : results3.reduce(
+                    (prev, s) => mergeDicts(prev, { [s.name]: getOtherFields(s) }),
+                    {}
+                  );
+                  const scores2 = results3.reduce(
+                    (prev, s) => mergeDicts(prev, { [s.name]: s.score }),
+                    {}
+                  );
                   span.log({
-                    output: resultRest,
+                    output: resultOutput,
                     metadata: resultMetadata,
-                    scores: {
-                      [name]: resultRest.score
-                    }
+                    scores: scores2
                   });
-                  return result2;
+                  return results3;
                 },
                 {
                   name: scorerNames[score_idx],
@@ -18789,7 +18828,7 @@ async function runEvaluator(experiment, evaluator, progressReporter, filters) {
                   event: { input: scoringArgs }
                 }
               );
-              return { kind: "score", value: result };
+              return { kind: "score", value: results2 };
             } catch (e) {
               return { kind: "error", value: e };
             }
@@ -18797,12 +18836,18 @@ async function runEvaluator(experiment, evaluator, progressReporter, filters) {
         );
         const passingScorersAndResults = [];
         const failingScorersAndResults = [];
-        scoreResults.forEach((result, i) => {
+        scoreResults.forEach((results2, i) => {
           const name = scorerNames[i];
-          if (result.kind === "score") {
-            passingScorersAndResults.push({ name, score: result.value });
+          if (results2.kind === "score") {
+            (results2.value || []).forEach((result) => {
+              passingScorersAndResults.push({
+                name: result.name,
+                score: result
+              });
+              scores[result.name] = result.score;
+            });
           } else {
-            failingScorersAndResults.push({ name, error: result.value });
+            failingScorersAndResults.push({ name, error: results2.value });
           }
         });
         if (failingScorersAndResults.length) {
@@ -18827,7 +18872,10 @@ async function runEvaluator(experiment, evaluator, progressReporter, filters) {
         progressReporter.increment(evaluator.evalName);
       }
       return {
+        input: datum.input,
+        ..."expected" in datum ? { expected: datum.expected } : {},
         output,
+        tags: datum.tags,
         metadata,
         scores,
         error: error2
@@ -18850,10 +18898,10 @@ async function runEvaluator(experiment, evaluator, progressReporter, filters) {
     }
   });
   const results = await Promise.all(evals);
-  const summary = experiment ? await experiment.summarize() : null;
+  const summary = experiment ? await experiment.summarize() : buildLocalSummary(evaluator, results);
   return {
-    results,
-    summary
+    summary,
+    results
   };
 }
 var error = import_chalk.default.bold.red;
@@ -18865,28 +18913,46 @@ function logError(e, verbose) {
     console.error(e);
   }
 }
-function reportEvaluatorResult(evaluatorName, evaluatorResult, {
-  verbose,
-  jsonl
-}) {
-  const { results, summary } = evaluatorResult;
-  const failingResults = results.filter(
-    (r) => r.error !== void 0
-  );
+function buildLocalSummary(evaluator, results) {
+  const scoresByName = {};
+  for (const result of results) {
+    for (const [name, score] of Object.entries(result.scores)) {
+      const { total, count } = scoresByName[name] || { total: 0, count: 0 };
+      if (score === null) {
+        continue;
+      }
+      scoresByName[name] = { total: total + score, count: count + 1 };
+    }
+  }
+  return {
+    projectName: evaluator.projectName,
+    experimentName: evaluator.evalName,
+    scores: Object.fromEntries(
+      Object.entries(scoresByName).map(([name, { total, count }]) => [
+        name,
+        {
+          name,
+          score: total / count
+        }
+      ])
+    )
+  };
+}
+function reportFailures(evaluator, failingResults, { verbose, jsonl }) {
   if (failingResults.length > 0) {
     console.error(
       warning(
-        `Evaluator ${evaluatorName} failed with ${(0, import_pluralize.default)(
+        `Evaluator ${evaluator.evalName} failed with ${(0, import_pluralize.default)(
           "error",
           failingResults.length,
           true
-        )}. This evaluation ("${evaluatorName}") will not be fully logged.`
+        )}. This evaluation ("${evaluator.evalName}") will not be fully logged.`
       )
     );
     if (jsonl) {
       console.log(
         JSON.stringify({
-          evaluatorName,
+          evaluatorName: evaluator.evalName,
           errors: failingResults.map(
             (r) => `${r.error instanceof Error ? r.error.stack : r.error}`
           )
@@ -18901,33 +18967,24 @@ function reportEvaluatorResult(evaluatorName, evaluatorResult, {
       console.error(warning("Add --verbose to see full stack traces."));
     }
   }
-  if (summary) {
-    console.log(jsonl ? JSON.stringify(summary) : summary);
-  } else {
-    const scoresByName = {};
-    for (const result of results) {
-      for (const [name, score] of Object.entries(result.scores)) {
-        const { total, count } = scoresByName[name] || { total: 0, count: 0 };
-        if (score === null) {
-          continue;
-        }
-        scoresByName[name] = { total: total + score, count: count + 1 };
-      }
+}
+var defaultReporter = {
+  name: "Braintrust default reporter",
+  async reportEval(evaluator, result, { verbose, jsonl }) {
+    const { results, summary } = result;
+    const failingResults = results.filter(
+      (r) => r.error !== void 0
+    );
+    if (failingResults.length > 0) {
+      reportFailures(evaluator, failingResults, { verbose, jsonl });
     }
-    const summary2 = {
-      scores: Object.fromEntries(
-        Object.entries(scoresByName).map(([name, { total, count }]) => [
-          name,
-          {
-            name,
-            score: total / count
-          }
-        ])
-      )
-    };
-    console.log(jsonl ? JSON.stringify(summary2) : summary2);
+    console.log(jsonl ? JSON.stringify(summary) : summary);
+    return failingResults.length === 0;
+  },
+  async reportRun(evalReports) {
+    return evalReports.every((r) => r);
   }
-}
+};
 
 // src/node.ts
 var import_node_async_hooks = require("node:async_hooks");
@@ -23109,7 +23166,10 @@ function evaluateBuildResults(inFile, buildResult) {
   }
   const moduleText = buildResult.outputFiles[0].text;
   return evalWithModuleContext(inFile, () => {
-    globalThis._evals = {};
+    globalThis._evals = {
+      evaluators: {},
+      reporters: {}
+    };
     globalThis._lazy_load = true;
     globalThis.__inherited_braintrust_state = _internalGetGlobalState();
     const __filename = inFile;
@@ -23133,7 +23193,39 @@ async function initLogger(projectName, experimentName, metadata) {
   );
   return logger;
 }
+function resolveReporter(reporter, reporters) {
+  if (typeof reporter === "string") {
+    if (!reporters[reporter]) {
+      throw new Error(`Reporter ${reporter} not found`);
+    }
+    return reporters[reporter];
+  } else if (!isEmpty(reporter)) {
+    return reporter;
+  } else if (Object.keys(reporters).length === 0) {
+    return defaultReporter;
+  } else if (Object.keys(reporters).length === 1) {
+    return reporters[Object.keys(reporters)[0]];
+  } else {
+    const reporterNames = Object.keys(reporters).join(", ");
+    throw new Error(
+      `Multiple reporters found (${reporterNames}). Please specify a reporter explicitly.`
+    );
+  }
+}
+function addReport(evalReports, reporter, report) {
+  if (!evalReports[reporter.name]) {
+    evalReports[reporter.name] = {
+      reporter,
+      results: []
+    };
+  }
+  evalReports[reporter.name].results.push(report);
+}
 function buildWatchPluginForEvaluator(inFile, opts) {
+  const evaluators = {
+    evaluators: {},
+    reporters: {}
+  };
   const plugin = {
     name: "run-evalutator-on-end",
     setup(build) {
@@ -23152,7 +23244,23 @@ function buildWatchPluginForEvaluator(inFile, opts) {
         if (!evalResult) {
           return;
         }
-        for (const evaluator of Object.values(evalResult)) {
+        for (const [evalName, evaluator] of Object.entries(
+          evalResult.evaluators
+        )) {
+          evaluators.evaluators[evalName] = {
+            sourceFile: inFile,
+            evaluator: evaluator.evaluator,
+            reporter: evaluator.reporter
+          };
+        }
+        for (const [reporterName, reporter] of Object.entries(
+          evalResult.reporters
+        )) {
+          evaluators.reporters[reporterName] = reporter;
+        }
+        const evalReports = {};
+        for (const evaluatorDef of Object.values(evalResult.evaluators)) {
+          const { evaluator, reporter } = evaluatorDef;
           const logger = opts.noSendLogs ? null : await initLogger(
             evaluator.projectName,
             evaluator.experimentName,
@@ -23164,10 +23272,28 @@ function buildWatchPluginForEvaluator(inFile, opts) {
             opts.progressReporter,
             opts.filters
           );
-          reportEvaluatorResult(evaluator.evalName, evaluatorResult, {
-            verbose: true,
-            jsonl: opts.jsonl
-          });
+          const resolvedReporter = resolveReporter(
+            reporter,
+            evaluators.reporters
+            // Let these accumulate across all files.
+          );
+          const report = resolvedReporter.reportEval(
+            evaluator,
+            evaluatorResult,
+            {
+              verbose: opts.verbose,
+              jsonl: opts.jsonl
+            }
+          );
+          addReport(evalReports, resolvedReporter, report);
+        }
+        for (const [reporterName, { reporter, results }] of Object.entries(
+          evalReports
+        )) {
+          const success = await reporter.reportRun(await Promise.all(results));
+          if (!success) {
+            console.error(error(`Reporter ${reporterName} failed.`));
+          }
         }
       });
     }
@@ -23190,7 +23316,10 @@ async function initFile(inFile, outFile, opts, args) {
             sourceFile: inFile
           };
         }
-        const evaluator = evaluateBuildResults(inFile, result) || {};
+        const evaluator = evaluateBuildResults(inFile, result) || {
+          evaluators: {},
+          reporters: {}
+        };
         return { type: "success", result, evaluator, sourceFile: inFile };
       } catch (e) {
         return { type: "failure", error: e, sourceFile: inFile };
@@ -23219,20 +23348,36 @@ function updateEvaluators(evaluators, buildResults, opts) {
       }
       continue;
     }
-    for (const [evalName, evaluator] of Object.entries(result.evaluator)) {
-      if (evaluators[evalName] && (evaluators[evalName].sourceFile !== result.sourceFile || evaluators[evalName].evaluator !== evaluator)) {
+    for (const [evalName, evaluator] of Object.entries(
+      result.evaluator.evaluators
+    )) {
+      if (evaluators.evaluators[evalName] && (evaluators.evaluators[evalName].sourceFile !== result.sourceFile || evaluators.evaluators[evalName] !== evaluator)) {
         console.warn(
           warning(
-            `Evaluator ${evalName} already exists (in ${evaluators[evalName].sourceFile} and ${result.sourceFile}). Will skip ${evalName} in ${result.sourceFile}.`
+            `Evaluator ${evalName} already exists (in ${evaluators.evaluators[evalName].sourceFile} and ${result.sourceFile}). Will skip ${evalName} in ${result.sourceFile}.`
           )
         );
         continue;
       }
-      evaluators[evalName] = {
+      evaluators.evaluators[evalName] = {
         sourceFile: result.sourceFile,
-        evaluator
+        evaluator: evaluator.evaluator,
+        reporter: evaluator.reporter
       };
     }
+    for (const [reporterName, reporter] of Object.entries(
+      result.evaluator.reporters
+    )) {
+      if (evaluators.reporters[reporterName] && evaluators.reporters[reporterName] !== reporter) {
+        console.warn(
+          warning(
+            `Reporter ${reporterName} already exists. Will skip ${reporterName}.`
+          )
+        );
+        continue;
+      }
+      evaluators.reporters[reporterName] = reporter;
+    }
   }
 }
 async function runAndWatch(handles, opts) {
@@ -23257,40 +23402,63 @@ async function runOnce(handles, opts) {
     (handle) => handle.rebuild()
   );
   const buildResults = await Promise.all(buildPromises);
-  const evaluators = {};
+  const evaluators = {
+    evaluators: {},
+    reporters: {}
+  };
   updateEvaluators(evaluators, buildResults, opts);
-  const resultPromises = Object.values(evaluators).map(async (evaluator) => {
-    const logger = opts.noSendLogs ? null : await initLogger(
-      evaluator.evaluator.projectName,
-      evaluator.evaluator.experimentName,
-      evaluator.evaluator.metadata
-    );
-    try {
-      return await runEvaluator(
-        logger,
-        evaluator.evaluator,
-        opts.progressReporter,
-        opts.filters
+  const resultPromises = Object.values(evaluators.evaluators).map(
+    async (evaluator) => {
+      const logger = opts.noSendLogs ? null : await initLogger(
+        evaluator.evaluator.projectName,
+        evaluator.evaluator.experimentName,
+        evaluator.evaluator.metadata
       );
-    } finally {
-      if (logger) {
-        await logger.flush();
+      try {
+        return await runEvaluator(
+          logger,
+          evaluator.evaluator,
+          opts.progressReporter,
+          opts.filters
+        );
+      } finally {
+        if (logger) {
+          await logger.flush();
+        }
       }
     }
-  });
+  );
   console.error(`Processing ${resultPromises.length} evaluators...`);
   const allEvalsResults = await Promise.all(resultPromises);
   opts.progressReporter.stop();
   console.error("");
-  for (const [evaluator, idx] of Object.keys(evaluators).map((k, i) => [
-    k,
-    i
-  ])) {
-    reportEvaluatorResult(evaluator, allEvalsResults[idx], {
-      verbose: opts.verbose,
-      jsonl: opts.jsonl
-    });
+  const evalReports = {};
+  for (const [evaluatorName, idx] of Object.keys(evaluators.evaluators).map(
+    (k, i) => [k, i]
+  )) {
+    const evaluator = evaluators.evaluators[evaluatorName];
+    const resolvedReporter = resolveReporter(
+      evaluator.reporter,
+      evaluators.reporters
+    );
+    const report = resolvedReporter.reportEval(
+      evaluator.evaluator,
+      allEvalsResults[idx],
+      {
+        verbose: opts.verbose,
+        jsonl: opts.jsonl
+      }
+    );
+    addReport(evalReports, resolvedReporter, report);
+  }
+  let allSuccess = true;
+  for (const [reporterName, { reporter, results }] of Object.entries(
+    evalReports
+  )) {
+    const success = await reporter.reportRun(await Promise.all(results));
+    allSuccess = allSuccess && success;
   }
+  return allSuccess;
 }
 function checkMatch(pathInput, include_patterns, exclude_patterns) {
   const p = import_path.default.resolve(pathInput);
@@ -23328,9 +23496,14 @@ async function collectFiles(inputPath) {
   }
   let files = [];
   if (!pathStat.isDirectory()) {
-    if (checkMatch(inputPath, INCLUDE, EXCLUDE)) {
-      files.push(inputPath);
+    if (!checkMatch(inputPath, INCLUDE, EXCLUDE)) {
+      console.warn(
+        warning(
+          `Reading ${inputPath} because it was specified directly. Rename it to end in .eval.ts or .eval.js to include it automatically when you specify a directory.`
+        )
+      );
     }
+    files.push(inputPath);
   } else {
     const walked = await import_util3.default.promisify(fsWalk.walk)(inputPath, {
       deepFilter: (entry) => {
@@ -23427,6 +23600,7 @@ async function run(args) {
     filters: args.filter ? parseFilters(args.filter) : []
   };
   const handles = await initializeHandles(args, evaluatorOpts);
+  let success = true;
   try {
     if (!evaluatorOpts.noSendLogs) {
       await login({
@@ -23438,13 +23612,16 @@ async function run(args) {
     if (args.watch) {
       await runAndWatch(handles, evaluatorOpts);
     } else {
-      runOnce(handles, evaluatorOpts);
+      success = await runOnce(handles, evaluatorOpts);
     }
   } finally {
     for (const handle of Object.values(handles)) {
       await handle.destroy();
     }
   }
+  if (!success) {
+    process.exit(1);
+  }
 }
 async function main() {
   const [, ...args] = process.argv;