braintrust 0.0.113 → 0.0.114

package/dist/index.js

@@ -872,9 +872,9 @@ var require_dist2 = __commonJS({
   }
 });
 
-// ../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.3.1_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/lib/zod-is-type.js
+// ../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.4.0_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/lib/zod-is-type.js
 var require_zod_is_type = __commonJS({
-  "../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.3.1_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/lib/zod-is-type.js"(exports) {
+  "../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.4.0_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/lib/zod-is-type.js"(exports) {
     "use strict";
     Object.defineProperty(exports, "__esModule", { value: true });
     exports.isAnyZodType = exports.isZodType = void 0;
@@ -890,9 +890,9 @@ var require_zod_is_type = __commonJS({
   }
 });
 
-// ../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.3.1_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/zod-extensions.js
+// ../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.4.0_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/zod-extensions.js
 var require_zod_extensions = __commonJS({
-  "../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.3.1_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/zod-extensions.js"(exports) {
+  "../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.4.0_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/zod-extensions.js"(exports) {
     "use strict";
     var __rest = exports && exports.__rest || function(s, e) {
       var t = {};
@@ -980,9 +980,9 @@ var require_zod_extensions = __commonJS({
   }
 });
 
-// ../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.3.1_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/lib/object-set.js
+// ../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.4.0_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/lib/object-set.js
 var require_object_set = __commonJS({
-  "../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.3.1_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/lib/object-set.js"(exports) {
+  "../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.4.0_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/lib/object-set.js"(exports) {
     "use strict";
     Object.defineProperty(exports, "__esModule", { value: true });
     exports.ObjectSet = exports.isEqual = void 0;
@@ -1078,12 +1078,12 @@ var require_object_set = __commonJS({
   }
 });
 
-// ../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.3.1_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/lib/lodash.js
+// ../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.4.0_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/lib/lodash.js
 var require_lodash = __commonJS({
-  "../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.3.1_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/lib/lodash.js"(exports) {
+  "../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.4.0_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/lib/lodash.js"(exports) {
     "use strict";
     Object.defineProperty(exports, "__esModule", { value: true });
-    exports.uniq = exports.objectEquals = exports.compact = exports.omitBy = exports.omit = exports.mapValues = exports.isNil = exports.isUndefined = void 0;
+    exports.isString = exports.uniq = exports.objectEquals = exports.compact = exports.omitBy = exports.omit = exports.mapValues = exports.isNil = exports.isUndefined = void 0;
     var object_set_1 = require_object_set();
     function isUndefined(value) {
       return value === void 0;
@@ -1132,12 +1132,16 @@ var require_lodash = __commonJS({
       return [...set.values()];
     }
     exports.uniq = uniq;
+    function isString(val) {
+      return typeof val === "string";
+    }
+    exports.isString = isString;
   }
 });
 
-// ../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.3.1_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/openapi-metadata.js
+// ../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.4.0_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/openapi-metadata.js
 var require_openapi_metadata = __commonJS({
-  "../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.3.1_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/openapi-metadata.js"(exports) {
+  "../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.4.0_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/openapi-metadata.js"(exports) {
     "use strict";
     Object.defineProperty(exports, "__esModule", { value: true });
     exports.getOpenApiMetadata = void 0;
@@ -1150,9 +1154,9 @@ var require_openapi_metadata = __commonJS({
   }
 });
 
-// ../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.3.1_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/openapi-registry.js
+// ../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.4.0_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/openapi-registry.js
 var require_openapi_registry = __commonJS({
-  "../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.3.1_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/openapi-registry.js"(exports) {
+  "../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.4.0_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/openapi-registry.js"(exports) {
     "use strict";
     Object.defineProperty(exports, "__esModule", { value: true });
     exports.OpenAPIRegistry = void 0;
@@ -1234,9 +1238,9 @@ var require_openapi_registry = __commonJS({
   }
 });
 
-// ../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.3.1_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/errors.js
+// ../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.4.0_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/errors.js
 var require_errors = __commonJS({
-  "../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.3.1_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/errors.js"(exports) {
+  "../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.4.0_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/errors.js"(exports) {
     "use strict";
     Object.defineProperty(exports, "__esModule", { value: true });
     exports.UnknownZodTypeError = exports.MissingParameterDataError = exports.ConflictError = exports.ZodToOpenAPIError = void 0;
@@ -1270,9 +1274,9 @@ var require_errors = __commonJS({
   }
 });
 
-// ../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.3.1_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/lib/enum-info.js
+// ../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.4.0_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/lib/enum-info.js
 var require_enum_info = __commonJS({
-  "../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.3.1_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/lib/enum-info.js"(exports) {
+  "../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.4.0_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/lib/enum-info.js"(exports) {
     "use strict";
     Object.defineProperty(exports, "__esModule", { value: true });
     exports.enumInfo = void 0;
@@ -1287,9 +1291,9 @@ var require_enum_info = __commonJS({
   }
 });
 
-// ../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.3.1_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/openapi-generator.js
+// ../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.4.0_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/openapi-generator.js
 var require_openapi_generator = __commonJS({
-  "../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.3.1_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/openapi-generator.js"(exports) {
+  "../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.4.0_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/openapi-generator.js"(exports) {
     "use strict";
     var __rest = exports && exports.__rest || function(s, e) {
       var t = {};
@@ -1630,7 +1634,7 @@ var require_openapi_generator = __commonJS({
       }
       getBodyContent(content) {
         return (0, lodash_1.mapValues)(content, (config) => {
-          if (!(0, zod_is_type_1.isAnyZodType)(config.schema)) {
+          if (!config || !(0, zod_is_type_1.isAnyZodType)(config.schema)) {
             return config;
           }
           const { schema: configSchema } = config, rest = __rest(config, ["schema"]);
@@ -1818,7 +1822,14 @@ var require_openapi_generator = __commonJS({
         }
         if ((0, zod_is_type_1.isZodType)(zodSchema, "ZodRecord")) {
           const propertiesType = zodSchema._def.valueType;
-          return Object.assign(Object.assign({}, this.mapNullableType("object", isNullable)), { additionalProperties: this.generateSchemaWithRef(propertiesType), default: defaultValue });
+          const keyType = zodSchema._def.keyType;
+          const propertiesSchema = this.generateSchemaWithRef(propertiesType);
+          if ((0, zod_is_type_1.isZodType)(keyType, "ZodEnum") || (0, zod_is_type_1.isZodType)(keyType, "ZodNativeEnum")) {
+            const keys = Object.values(keyType.enum).filter(lodash_1.isString);
+            const properties = keys.reduce((acc, curr) => Object.assign(Object.assign({}, acc), { [curr]: propertiesSchema }), {});
+            return Object.assign(Object.assign({}, this.mapNullableType("object", isNullable)), { properties, default: defaultValue });
+          }
+          return Object.assign(Object.assign({}, this.mapNullableType("object", isNullable)), { additionalProperties: propertiesSchema, default: defaultValue });
         }
         if ((0, zod_is_type_1.isZodType)(zodSchema, "ZodUnknown") || (0, zod_is_type_1.isZodType)(zodSchema, "ZodAny")) {
           return this.mapNullableType(void 0, isNullable);
@@ -1984,9 +1995,9 @@ var require_openapi_generator = __commonJS({
   }
 });
 
-// ../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.3.1_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/v3.0/specifics.js
+// ../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.4.0_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/v3.0/specifics.js
 var require_specifics = __commonJS({
-  "../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.3.1_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/v3.0/specifics.js"(exports) {
+  "../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.4.0_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/v3.0/specifics.js"(exports) {
     "use strict";
     Object.defineProperty(exports, "__esModule", { value: true });
     exports.OpenApiGeneratorV30Specifics = void 0;
@@ -2020,9 +2031,9 @@ var require_specifics = __commonJS({
   }
 });
 
-// ../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.3.1_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/v3.0/openapi-generator.js
+// ../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.4.0_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/v3.0/openapi-generator.js
 var require_openapi_generator2 = __commonJS({
-  "../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.3.1_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/v3.0/openapi-generator.js"(exports) {
+  "../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.4.0_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/v3.0/openapi-generator.js"(exports) {
     "use strict";
     Object.defineProperty(exports, "__esModule", { value: true });
     exports.OpenApiGeneratorV3 = void 0;
@@ -2045,9 +2056,9 @@ var require_openapi_generator2 = __commonJS({
   }
 });
 
-// ../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.3.1_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/v3.1/specifics.js
+// ../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.4.0_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/v3.1/specifics.js
 var require_specifics2 = __commonJS({
-  "../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.3.1_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/v3.1/specifics.js"(exports) {
+  "../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.4.0_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/v3.1/specifics.js"(exports) {
     "use strict";
     Object.defineProperty(exports, "__esModule", { value: true });
     exports.OpenApiGeneratorV31Specifics = void 0;
@@ -2091,9 +2102,9 @@ var require_specifics2 = __commonJS({
   }
 });
 
-// ../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.3.1_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/v3.1/openapi-generator.js
+// ../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.4.0_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/v3.1/openapi-generator.js
 var require_openapi_generator3 = __commonJS({
-  "../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.3.1_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/v3.1/openapi-generator.js"(exports) {
+  "../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.4.0_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/v3.1/openapi-generator.js"(exports) {
     "use strict";
     Object.defineProperty(exports, "__esModule", { value: true });
     exports.OpenApiGeneratorV31 = void 0;
@@ -2127,9 +2138,9 @@ var require_openapi_generator3 = __commonJS({
   }
 });
 
-// ../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.3.1_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/index.js
+// ../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.4.0_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/index.js
 var require_dist3 = __commonJS({
-  "../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.3.1_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/index.js"(exports) {
+  "../../node_modules/.pnpm/@asteasolutions+zod-to-openapi@6.4.0_zod@3.22.4/node_modules/@asteasolutions/zod-to-openapi/dist/index.js"(exports) {
     "use strict";
     var __createBinding = exports && exports.__createBinding || (Object.create ? function(o, m, k, k2) {
       if (k2 === void 0)
@@ -4956,9 +4967,11 @@ __export(src_exports, {
   NoopSpan: () => NoopSpan,
   Prompt: () => Prompt,
   ReadonlyExperiment: () => ReadonlyExperiment,
+  Reporter: () => Reporter,
   SpanImpl: () => SpanImpl,
   _internalGetGlobalState: () => _internalGetGlobalState,
   _internalSetInitialState: () => _internalSetInitialState,
+  buildLocalSummary: () => buildLocalSummary,
   currentExperiment: () => currentExperiment,
   currentLogger: () => currentLogger,
   currentSpan: () => currentSpan,
@@ -4970,6 +4983,7 @@ __export(src_exports, {
   loadPrompt: () => loadPrompt,
   log: () => log,
   login: () => login,
+  reportFailures: () => reportFailures,
   startSpan: () => startSpan,
   summarize: () => summarize,
   traced: () => traced,
@@ -9193,7 +9207,7 @@ function v4(options, buf, offset) {
 }
 var v4_default = v4;
 
-// ../core/js/dist/index.mjs
+// ../../node_modules/.pnpm/@braintrust+core@0.0.27/node_modules/@braintrust/core/dist/index.mjs
 var TRANSACTION_ID_FIELD = "_xact_id";
 var IS_MERGE_FIELD = "_is_merge";
 var MERGE_PATHS_FIELD = "_merge_paths";
@@ -9498,7 +9512,7 @@ var MOD = BigInt(1) << BigInt(64);
 var COPRIME = BigInt("205891132094649");
 var COPRIME_INVERSE = BigInt("1522336535492693385");
 
-// ../core/js/typespecs/dist/index.mjs
+// ../../node_modules/.pnpm/@braintrust+core@0.0.27/node_modules/@braintrust/core/typespecs/dist/index.mjs
 var import_zod_to_openapi = __toESM(require_dist3(), 1);
 
 // ../../node_modules/.pnpm/zod@3.22.4/node_modules/zod/lib/index.mjs
@@ -13234,7 +13248,7 @@ var z = /* @__PURE__ */ Object.freeze({
   ZodError
 });
 
-// ../core/js/typespecs/dist/index.mjs
+// ../../node_modules/.pnpm/@braintrust+core@0.0.27/node_modules/@braintrust/core/typespecs/dist/index.mjs
 var import_zod_to_openapi2 = __toESM(require_dist3(), 1);
 var import_zod_to_openapi3 = __toESM(require_dist3(), 1);
 var literalSchema = z.union([
@@ -13279,12 +13293,12 @@ var modeToTypes = {
 };
 var customTypes = modeToTypes[mode];
 var chatCompletionSystemMessageParamSchema = z.object({
-  content: z.string(),
+  content: z.string().default(""),
   role: z.literal("system"),
   name: z.string().optional()
 });
 var chatCompletionContentPartTextSchema = z.object({
-  text: z.string(),
+  text: z.string().default(""),
   type: z.literal("text")
 });
 var functionCallSchema = z.object({
@@ -13300,12 +13314,12 @@ var imageURLSchema = z.object({
   detail: z.union([z.literal("auto"), z.literal("low"), z.literal("high")]).optional()
 });
 var chatCompletionToolMessageParamSchema = z.object({
-  content: z.string(),
+  content: z.string().default(""),
   role: z.literal("tool"),
   tool_call_id: z.string()
 });
 var chatCompletionFunctionMessageParamSchema = z.object({
-  content: z.string().nullable(),
+  content: z.string().default(""),
   name: z.string(),
   role: z.literal("function")
 });
@@ -13324,13 +13338,16 @@ var chatCompletionContentPartSchema = z.union([
 ]);
 var chatCompletionAssistantMessageParamSchema = z.object({
   role: z.literal("assistant"),
-  content: z.string().optional().nullable(),
+  content: z.string().nullish(),
   function_call: functionCallSchema.optional(),
   name: z.string().optional(),
   tool_calls: z.array(chatCompletionMessageToolCallSchema).optional()
 });
 var chatCompletionUserMessageParamSchema = z.object({
-  content: z.union([z.string(), z.array(chatCompletionContentPartSchema)]),
+  content: z.union([
+    z.string().default(""),
+    z.array(chatCompletionContentPartSchema)
+  ]),
   role: z.literal("user"),
   name: z.string().optional()
 });
@@ -14672,6 +14689,7 @@ var BraintrustState = class {
   currentLogger;
   currentSpan;
   appUrl = null;
+  appPublicUrl = null;
   loginToken = null;
   orgId = null;
   orgName = null;
@@ -15015,12 +15033,13 @@ var Logger = class {
    * See `traced` for full details.
    */
   startSpan(args) {
-    const { name, ...argsRest } = args ?? {};
+    const { name, type, ...argsRest } = args ?? {};
     return new SpanImpl({
       parentObject: this,
       parentIds: new LazyValue(() => this.lazyParentIds()),
       bgLogger: this.bgLogger,
       name: name ?? "root",
+      type: type ?? SpanTypeAttribute.TASK,
       ...argsRest
     });
   }
@@ -15631,8 +15650,10 @@ async function login(options = {}) {
     apiKey = isomorph_default.getEnv("BRAINTRUST_API_KEY"),
     orgName = isomorph_default.getEnv("BRAINTRUST_ORG_NAME")
   } = options || {};
+  const appPublicUrl = isomorph_default.getEnv("BRAINTRUST_APP_PUBLIC_URL") || appUrl;
   _state.resetLoginInfo();
   _state.appUrl = appUrl;
+  _state.appPublicUrl = appPublicUrl;
   let conn = null;
   if (apiKey !== void 0) {
     const resp = await checkResponse(
@@ -16018,12 +16039,13 @@ var Experiment = class extends ObjectFetcher {
    * See `traced` for full details.
    */
   startSpan(args) {
-    const { name, ...argsRest } = args ?? {};
+    const { name, type, ...argsRest } = args ?? {};
     return new SpanImpl({
       parentObject: this,
       parentIds: new LazyValue(() => this.lazyParentIds()),
       bgLogger: this.bgLogger,
       name: name ?? "root",
+      type: type ?? SpanTypeAttribute.EVAL,
       ...argsRest
     });
   }
@@ -16059,7 +16081,7 @@ var Experiment = class extends ObjectFetcher {
     let { summarizeScores = true, comparisonExperimentId = void 0 } = options || {};
     await this.bgLogger.flush();
     const state = await this.getState();
-    const projectUrl = `${state.appUrl}/app/${encodeURIComponent(
+    const projectUrl = `${state.appPublicUrl}/app/${encodeURIComponent(
       state.orgName
     )}/p/${encodeURIComponent((await this.project).name)}`;
     const experimentUrl = `${projectUrl}/${encodeURIComponent(
@@ -16095,7 +16117,7 @@ var Experiment = class extends ObjectFetcher {
       projectUrl,
       experimentUrl,
       comparisonExperimentName,
-      scores,
+      scores: scores ?? {},
       metrics
     };
   }
@@ -16213,6 +16235,7 @@ var SpanImpl = class _SpanImpl {
       },
       context: { ...callerLocation },
       span_attributes: {
+        type: args.type,
         ...args.spanAttributes,
         name,
         exec_counter: executionCounter++
@@ -16447,7 +16470,7 @@ var Dataset = class extends ObjectFetcher {
     let { summarizeData = true } = options || {};
     await this.bgLogger.flush();
     const state = await this.getState();
-    const projectUrl = `${state.appUrl}/app/${encodeURIComponent(
+    const projectUrl = `${state.appPublicUrl}/app/${encodeURIComponent(
       state.orgName
     )}/p/${encodeURIComponent((await this.project).name)}`;
     const datasetUrl = `${projectUrl}/d/${encodeURIComponent(await this.name)}`;
@@ -16662,25 +16685,37 @@ function initExperiment2(projectName, options = {}) {
     setCurrent: false
   });
 }
-globalThis._evals = {};
-async function Eval(name, evaluator) {
+globalThis._evals = {
+  evaluators: {},
+  reporters: {}
+};
+async function Eval(name, evaluator, reporter) {
   let evalName = makeEvalName(name, evaluator.experimentName);
-  if (_evals[evalName]) {
+  if (_evals.evaluators[evalName]) {
     evalName = `${evalName}_${Object.keys(_evals).length}`;
   }
   if (globalThis._lazy_load) {
-    _evals[evalName] = { evalName, projectName: name, ...evaluator };
+    _evals.evaluators[evalName] = {
+      evaluator: { evalName, projectName: name, ...evaluator },
+      reporter
+    };
     return {
-      projectName: "_lazy_load",
-      experimentName: "_lazy_load",
-      projectUrl: "",
-      experimentUrl: "",
-      comparisonExperimentName: "",
-      scores: {},
-      metrics: {}
+      summary: {
+        scores: {},
+        metrics: {},
+        projectName: "",
+        experimentName: ""
+      },
+      results: []
     };
   }
   const progressReporter = new BarProgressReporter();
+  if (typeof reporter === "string") {
+    throw new Error(
+      "Must specify a reporter object, not a name. Can only specify reporter names when running 'braintrust eval'"
+    );
+  }
+  const resolvedReporter = reporter || defaultReporter;
   try {
     const experiment = initExperiment2(name, {
       experiment: evaluator.experimentName,
@@ -16688,18 +16723,17 @@ async function Eval(name, evaluator) {
       isPublic: evaluator.isPublic
     });
     try {
-      const ret = await runEvaluator(
-        experiment,
-        {
-          evalName,
-          projectName: name,
-          ...evaluator
-        },
-        progressReporter,
-        []
-      );
-      reportEvaluatorResult(name, ret, { verbose: true, jsonl: false });
-      return ret.summary;
+      const evalDef = {
+        evalName,
+        projectName: name,
+        ...evaluator
+      };
+      const ret = await runEvaluator(experiment, evalDef, progressReporter, []);
+      resolvedReporter.reportEval(evalDef, ret, {
+        verbose: true,
+        jsonl: false
+      });
+      return ret;
     } finally {
       experiment.flush();
     }
@@ -16707,6 +16741,16 @@ async function Eval(name, evaluator) {
     progressReporter.stop();
   }
 }
+function Reporter(name, reporter) {
+  const ret = { name, ...reporter };
+  if (_evals.reporters[name]) {
+    throw new Error(`Reporter ${name} already exists`);
+  }
+  if (globalThis._lazy_load) {
+    _evals.reporters[name] = ret;
+  }
+  return ret;
+}
 function serializeJSONWithPlainString(v) {
   if (typeof v === "string") {
     return v;
@@ -16799,27 +16843,44 @@ async function runEvaluator(experiment, evaluator, progressReporter, filters) {
         const scoreResults = await Promise.all(
           evaluator.scores.map(async (score, score_idx) => {
             try {
-              const result = await rootSpan.traced(
+              const results2 = await rootSpan.traced(
                 async (span) => {
                   const scoreResult = score(scoringArgs);
                   const scoreValue = scoreResult instanceof Promise ? await scoreResult : scoreResult;
                   if (scoreValue === null) {
                     return null;
                   }
-                  const result2 = typeof scoreValue === "object" ? scoreValue : { name: scorerNames[score_idx], score: scoreValue };
-                  const {
-                    metadata: resultMetadata,
-                    name,
-                    ...resultRest
-                  } = result2;
+                  const scoreValues = Array.isArray(scoreValue) ? scoreValue : [scoreValue];
+                  const results3 = scoreValues.map(
+                    (scoreValue2, idx) => typeof scoreValue2 === "object" ? scoreValue2 : {
+                      name: scoreValues.length > 1 ? `${scorerNames[score_idx]}_${idx}` : scorerNames[score_idx],
+                      score: scoreValue2
+                    }
+                  );
+                  const getOtherFields = (s) => {
+                    const { metadata: metadata2, name, ...rest } = s;
+                    return rest;
+                  };
+                  const resultMetadata = results3.length === 1 ? results3[0].metadata : results3.reduce(
+                    (prev, s) => mergeDicts(prev, {
+                      [s.name]: s.metadata
+                    }),
+                    {}
+                  );
+                  const resultOutput = results3.length === 1 ? getOtherFields(results3[0]) : results3.reduce(
+                    (prev, s) => mergeDicts(prev, { [s.name]: getOtherFields(s) }),
+                    {}
+                  );
+                  const scores2 = results3.reduce(
+                    (prev, s) => mergeDicts(prev, { [s.name]: s.score }),
+                    {}
+                  );
                   span.log({
-                    output: resultRest,
+                    output: resultOutput,
                     metadata: resultMetadata,
-                    scores: {
-                      [name]: resultRest.score
-                    }
+                    scores: scores2
                   });
-                  return result2;
+                  return results3;
                 },
                 {
                   name: scorerNames[score_idx],
@@ -16829,7 +16890,7 @@ async function runEvaluator(experiment, evaluator, progressReporter, filters) {
                   event: { input: scoringArgs }
                 }
               );
-              return { kind: "score", value: result };
+              return { kind: "score", value: results2 };
             } catch (e) {
               return { kind: "error", value: e };
             }
@@ -16837,12 +16898,18 @@ async function runEvaluator(experiment, evaluator, progressReporter, filters) {
         );
         const passingScorersAndResults = [];
         const failingScorersAndResults = [];
-        scoreResults.forEach((result, i) => {
+        scoreResults.forEach((results2, i) => {
           const name = scorerNames[i];
-          if (result.kind === "score") {
-            passingScorersAndResults.push({ name, score: result.value });
+          if (results2.kind === "score") {
+            (results2.value || []).forEach((result) => {
+              passingScorersAndResults.push({
+                name: result.name,
+                score: result
+              });
+              scores[result.name] = result.score;
+            });
           } else {
-            failingScorersAndResults.push({ name, error: result.value });
+            failingScorersAndResults.push({ name, error: results2.value });
           }
         });
         if (failingScorersAndResults.length) {
@@ -16867,7 +16934,10 @@ async function runEvaluator(experiment, evaluator, progressReporter, filters) {
         progressReporter.increment(evaluator.evalName);
       }
       return {
+        input: datum.input,
+        ..."expected" in datum ? { expected: datum.expected } : {},
         output,
+        tags: datum.tags,
         metadata,
         scores,
         error: error2
@@ -16890,10 +16960,10 @@ async function runEvaluator(experiment, evaluator, progressReporter, filters) {
     }
   });
   const results = await Promise.all(evals);
-  const summary = experiment ? await experiment.summarize() : null;
+  const summary = experiment ? await experiment.summarize() : buildLocalSummary(evaluator, results);
   return {
-    results,
-    summary
+    summary,
+    results
   };
 }
 var error = import_chalk.default.bold.red;
@@ -16905,28 +16975,46 @@ function logError(e, verbose) {
     console.error(e);
   }
 }
-function reportEvaluatorResult(evaluatorName, evaluatorResult, {
-  verbose,
-  jsonl
-}) {
-  const { results, summary } = evaluatorResult;
-  const failingResults = results.filter(
-    (r) => r.error !== void 0
-  );
+function buildLocalSummary(evaluator, results) {
+  const scoresByName = {};
+  for (const result of results) {
+    for (const [name, score] of Object.entries(result.scores)) {
+      const { total, count } = scoresByName[name] || { total: 0, count: 0 };
+      if (score === null) {
+        continue;
+      }
+      scoresByName[name] = { total: total + score, count: count + 1 };
+    }
+  }
+  return {
+    projectName: evaluator.projectName,
+    experimentName: evaluator.evalName,
+    scores: Object.fromEntries(
+      Object.entries(scoresByName).map(([name, { total, count }]) => [
+        name,
+        {
+          name,
+          score: total / count
+        }
+      ])
+    )
+  };
+}
+function reportFailures(evaluator, failingResults, { verbose, jsonl }) {
   if (failingResults.length > 0) {
     console.error(
       warning(
-        `Evaluator ${evaluatorName} failed with ${(0, import_pluralize.default)(
+        `Evaluator ${evaluator.evalName} failed with ${(0, import_pluralize.default)(
           "error",
           failingResults.length,
           true
-        )}. This evaluation ("${evaluatorName}") will not be fully logged.`
+        )}. This evaluation ("${evaluator.evalName}") will not be fully logged.`
       )
     );
     if (jsonl) {
       console.log(
         JSON.stringify({
-          evaluatorName,
+          evaluatorName: evaluator.evalName,
           errors: failingResults.map(
             (r) => `${r.error instanceof Error ? r.error.stack : r.error}`
           )
@@ -16941,33 +17029,24 @@ function reportEvaluatorResult(evaluatorName, evaluatorResult, {
       console.error(warning("Add --verbose to see full stack traces."));
     }
   }
-  if (summary) {
-    console.log(jsonl ? JSON.stringify(summary) : summary);
-  } else {
-    const scoresByName = {};
-    for (const result of results) {
-      for (const [name, score] of Object.entries(result.scores)) {
-        const { total, count } = scoresByName[name] || { total: 0, count: 0 };
-        if (score === null) {
-          continue;
-        }
-        scoresByName[name] = { total: total + score, count: count + 1 };
-      }
+}
+var defaultReporter = {
+  name: "Braintrust default reporter",
+  async reportEval(evaluator, result, { verbose, jsonl }) {
+    const { results, summary } = result;
+    const failingResults = results.filter(
+      (r) => r.error !== void 0
+    );
+    if (failingResults.length > 0) {
+      reportFailures(evaluator, failingResults, { verbose, jsonl });
     }
-    const summary2 = {
-      scores: Object.fromEntries(
-        Object.entries(scoresByName).map(([name, { total, count }]) => [
-          name,
-          {
-            name,
-            score: total / count
-          }
-        ])
-      )
-    };
-    console.log(jsonl ? JSON.stringify(summary2) : summary2);
+    console.log(jsonl ? JSON.stringify(summary) : summary);
+    return failingResults.length === 0;
+  },
+  async reportRun(evalReports) {
+    return evalReports.every((r) => r);
   }
-}
+};
 
 // src/oai.ts
 function wrapOpenAI(openai) {
@@ -17232,9 +17311,11 @@ configureNode();
   NoopSpan,
   Prompt,
   ReadonlyExperiment,
+  Reporter,
   SpanImpl,
   _internalGetGlobalState,
   _internalSetInitialState,
+  buildLocalSummary,
   currentExperiment,
   currentLogger,
   currentSpan,
@@ -17246,6 +17327,7 @@ configureNode();
   loadPrompt,
   log,
   login,
+  reportFailures,
   startSpan,
   summarize,
   traced,