braintrust 3.0.0 → 3.1.0

package/dev/dist/index.d.mts

@@ -13759,6 +13759,10 @@ interface Evaluator<Input, Output, Expected, Metadata extends BaseMetadata = Def
      * Optional additional metadata for the experiment.
      */
     metadata?: Record<string, unknown>;
+    /**
+     * Optional tags for the experiment.
+     */
+    tags?: string[];
     /**
      * Whether the experiment should be public. Defaults to false.
      */

package/dev/dist/index.d.ts

@@ -13759,6 +13759,10 @@ interface Evaluator<Input, Output, Expected, Metadata extends BaseMetadata = Def
      * Optional additional metadata for the experiment.
      */
     metadata?: Record<string, unknown>;
+    /**
+     * Optional tags for the experiment.
+     */
+    tags?: string[];
     /**
      * Whether the experiment should be public. Defaults to false.
      */

package/dev/dist/index.js

@@ -6288,6 +6288,7 @@ function init(projectOrOptions, optionalOptions) {
     forceLogin,
     fetch: fetch2,
     metadata,
+    tags,
     gitMetadataSettings,
     projectId,
     baseExperimentId,
@@ -6400,6 +6401,10 @@ function init(projectOrOptions, optionalOptions) {
       if (metadata) {
         args["metadata"] = metadata;
       }
+      if (tags) {
+        validateTags(tags);
+        args["tags"] = tags;
+      }
       let response = null;
       while (true) {
         try {
@@ -6798,6 +6803,7 @@ function validateTags(tags) {
     if (seen.has(tag)) {
       throw new Error(`duplicate tag: ${tag}`);
     }
+    seen.add(tag);
   }
 }
 function validateAndSanitizeExperimentLogPartialArgs(event) {
@@ -10089,6 +10095,7 @@ async function Eval(name, evaluator, reporterOrOpts) {
       experiment: evaluator.experimentName,
       description: evaluator.description,
       metadata: evaluator.metadata,
+      tags: evaluator.tags,
       isPublic: evaluator.isPublic,
       update: evaluator.update,
       baseExperiment: _nullishCoalesce(evaluator.baseExperimentName, () => ( defaultBaseExperiment)),

package/dev/dist/index.mjs

@@ -6288,6 +6288,7 @@ function init(projectOrOptions, optionalOptions) {
     forceLogin,
     fetch: fetch2,
     metadata,
+    tags,
     gitMetadataSettings,
     projectId,
     baseExperimentId,
@@ -6400,6 +6401,10 @@ function init(projectOrOptions, optionalOptions) {
       if (metadata) {
         args["metadata"] = metadata;
       }
+      if (tags) {
+        validateTags(tags);
+        args["tags"] = tags;
+      }
       let response = null;
       while (true) {
         try {
@@ -6798,6 +6803,7 @@ function validateTags(tags) {
     if (seen.has(tag)) {
       throw new Error(`duplicate tag: ${tag}`);
     }
+    seen.add(tag);
   }
 }
 function validateAndSanitizeExperimentLogPartialArgs(event) {
@@ -10089,6 +10095,7 @@ async function Eval(name, evaluator, reporterOrOpts) {
       experiment: evaluator.experimentName,
       description: evaluator.description,
       metadata: evaluator.metadata,
+      tags: evaluator.tags,
       isPublic: evaluator.isPublic,
       update: evaluator.update,
       baseExperiment: evaluator.baseExperimentName ?? defaultBaseExperiment,

package/dist/browser.d.mts

@@ -19917,6 +19917,7 @@ type InitOptions<IsOpen extends boolean> = FullLoginOptions & {
     baseExperiment?: string;
     isPublic?: boolean;
     metadata?: Record<string, unknown>;
+    tags?: string[];
     gitMetadataSettings?: GitMetadataSettingsType;
     projectId?: string;
     baseExperimentId?: string;
@@ -19949,6 +19950,7 @@ type InitializedExperiment<IsOpen extends boolean | undefined> = IsOpen extends
  * @param options.projectId The id of the project to create the experiment in. This takes precedence over `project` if specified.
  * @param options.baseExperimentId An optional experiment id to use as a base. If specified, the new experiment will be summarized and compared to this. This takes precedence over `baseExperiment` if specified.
  * @param options.repoInfo (Optional) Explicitly specify the git metadata for this experiment. This takes precedence over `gitMetadataSettings` if specified.
+ * @param options.tags (Optional) A list of tags to attach to the experiment.
  * @returns The newly created Experiment.
  */
 declare function init<IsOpen extends boolean = false>(options: Readonly<FullInitOptions<IsOpen>>): InitializedExperiment<IsOpen>;
@@ -20340,6 +20342,7 @@ declare function setFetch(fetch: typeof globalThis.fetch): void;
  */
 declare function withCurrent<R>(span: Span, callback: (span: Span) => R, state?: BraintrustState | undefined): R;
 declare function withParent<R>(parent: string, callback: () => R, state?: BraintrustState | undefined): R;
+declare function validateTags(tags: readonly string[]): void;
 /**
  * Creates a deep copy of the given event. Replaces references to user objects
  * with placeholder strings to ensure serializability, except for
@@ -20896,6 +20899,7 @@ declare const _exportsForTestingOnly: {
     isGeneratorFunction: typeof isGeneratorFunction;
     isAsyncGeneratorFunction: typeof isAsyncGeneratorFunction;
     resetIdGenStateForTests: typeof resetIdGenStateForTests;
+    validateTags: typeof validateTags;
     isomorph: Common;
 };
 
@@ -40026,6 +40030,10 @@ interface Evaluator<Input, Output, Expected, Metadata extends BaseMetadata = Def
      * Optional additional metadata for the experiment.
      */
     metadata?: Record<string, unknown>;
+    /**
+     * Optional tags for the experiment.
+     */
+    tags?: string[];
     /**
      * Whether the experiment should be public. Defaults to false.
      */

package/dist/browser.d.ts

@@ -19917,6 +19917,7 @@ type InitOptions<IsOpen extends boolean> = FullLoginOptions & {
     baseExperiment?: string;
     isPublic?: boolean;
     metadata?: Record<string, unknown>;
+    tags?: string[];
     gitMetadataSettings?: GitMetadataSettingsType;
     projectId?: string;
     baseExperimentId?: string;
@@ -19949,6 +19950,7 @@ type InitializedExperiment<IsOpen extends boolean | undefined> = IsOpen extends
  * @param options.projectId The id of the project to create the experiment in. This takes precedence over `project` if specified.
  * @param options.baseExperimentId An optional experiment id to use as a base. If specified, the new experiment will be summarized and compared to this. This takes precedence over `baseExperiment` if specified.
  * @param options.repoInfo (Optional) Explicitly specify the git metadata for this experiment. This takes precedence over `gitMetadataSettings` if specified.
+ * @param options.tags (Optional) A list of tags to attach to the experiment.
  * @returns The newly created Experiment.
  */
 declare function init<IsOpen extends boolean = false>(options: Readonly<FullInitOptions<IsOpen>>): InitializedExperiment<IsOpen>;
@@ -20340,6 +20342,7 @@ declare function setFetch(fetch: typeof globalThis.fetch): void;
  */
 declare function withCurrent<R>(span: Span, callback: (span: Span) => R, state?: BraintrustState | undefined): R;
 declare function withParent<R>(parent: string, callback: () => R, state?: BraintrustState | undefined): R;
+declare function validateTags(tags: readonly string[]): void;
 /**
  * Creates a deep copy of the given event. Replaces references to user objects
  * with placeholder strings to ensure serializability, except for
@@ -20896,6 +20899,7 @@ declare const _exportsForTestingOnly: {
     isGeneratorFunction: typeof isGeneratorFunction;
     isAsyncGeneratorFunction: typeof isAsyncGeneratorFunction;
     resetIdGenStateForTests: typeof resetIdGenStateForTests;
+    validateTags: typeof validateTags;
     isomorph: Common;
 };
 
@@ -40026,6 +40030,10 @@ interface Evaluator<Input, Output, Expected, Metadata extends BaseMetadata = Def
      * Optional additional metadata for the experiment.
      */
     metadata?: Record<string, unknown>;
+    /**
+     * Optional tags for the experiment.
+     */
+    tags?: string[];
     /**
      * Whether the experiment should be public. Defaults to false.
      */

package/dist/browser.js

@@ -6429,6 +6429,7 @@ function init(projectOrOptions, optionalOptions) {
     forceLogin,
     fetch: fetch2,
     metadata,
+    tags,
     gitMetadataSettings,
     projectId,
     baseExperimentId,
@@ -6541,6 +6542,10 @@ function init(projectOrOptions, optionalOptions) {
       if (metadata) {
         args["metadata"] = metadata;
       }
+      if (tags) {
+        validateTags(tags);
+        args["tags"] = tags;
+      }
       let response = null;
       while (true) {
         try {
@@ -7487,6 +7492,7 @@ function validateTags(tags) {
     if (seen.has(tag)) {
       throw new Error(`duplicate tag: ${tag}`);
     }
+    seen.add(tag);
   }
 }
 function validateAndSanitizeExperimentLogPartialArgs(event) {
@@ -9211,6 +9217,7 @@ var _exportsForTestingOnly = {
   isGeneratorFunction,
   isAsyncGeneratorFunction,
   resetIdGenStateForTests,
+  validateTags,
   isomorph: isomorph_default
   // Expose isomorph for build type detection
 };
@@ -10559,9 +10566,13 @@ var makeGenerateTextWrapper = (name, options, generateText, aiSDK) => {
     const processedInput = await processInputAttachments2(params);
     return traced(
       async (span) => {
+        const { wrappedModel, getMetrics } = wrapModelAndGetMetrics(
+          params.model,
+          aiSDK
+        );
         const result = await generateText({
           ...params,
-          model: wrapModel(params.model, aiSDK),
+          model: wrappedModel,
           tools: wrapTools(params.tools)
         });
         const gatewayInfo = extractGatewayRoutingInfo(result);
@@ -10574,7 +10585,7 @@ var makeGenerateTextWrapper = (name, options, generateText, aiSDK) => {
         }
         span.log({
           output: await processOutput(result, options.denyOutputPaths),
-          metrics: extractTokenMetrics(result),
+          metrics: getMetrics(result),
           ...Object.keys(resolvedMetadata).length > 0 ? { metadata: resolvedMetadata } : {}
         });
         return result;
@@ -10795,6 +10806,14 @@ var wrapModel = (model, ai) => {
 var wrapGenerateText = (generateText, options = {}, aiSDK) => {
   return makeGenerateTextWrapper("generateText", options, generateText, aiSDK);
 };
+var wrapModelAndGetMetrics = (model, aiSDK) => {
+  const wrappedModel = wrapModel(model, aiSDK);
+  const modelIsWrapped = wrappedModel?._braintrustWrapped === true;
+  return {
+    wrappedModel,
+    getMetrics: (result) => modelIsWrapped ? void 0 : extractTokenMetrics(result)
+  };
+};
 var wrapGenerateObject = (generateObject, options = {}, aiSDK) => {
   return async function generateObjectWrapper(allParams) {
     const { span_info, ...params } = allParams;
@@ -10807,9 +10826,13 @@ var wrapGenerateObject = (generateObject, options = {}, aiSDK) => {
     const processedInput = await processInputAttachments2(params);
     return traced(
       async (span) => {
+        const { wrappedModel, getMetrics } = wrapModelAndGetMetrics(
+          params.model,
+          aiSDK
+        );
         const result = await generateObject({
           ...params,
-          model: wrapModel(params.model, aiSDK),
+          model: wrappedModel,
           tools: wrapTools(params.tools)
         });
         const output = await processOutput(result, options.denyOutputPaths);
@@ -10823,7 +10846,7 @@ var wrapGenerateObject = (generateObject, options = {}, aiSDK) => {
         }
         span.log({
           output,
-          metrics: extractTokenMetrics(result),
+          metrics: getMetrics(result),
           ...Object.keys(resolvedMetadata).length > 0 ? { metadata: resolvedMetadata } : {}
         });
         return result;
@@ -10888,11 +10911,15 @@ var makeStreamTextWrapper = (name, options, streamText, aiSDK) => {
     try {
       const startTime = Date.now();
       let receivedFirst = false;
+      const { wrappedModel, getMetrics } = wrapModelAndGetMetrics(
+        params.model,
+        aiSDK
+      );
       const result = withCurrent(
         span,
         () => streamText({
           ...params,
-          model: wrapModel(params.model, aiSDK),
+          model: wrappedModel,
           tools: wrapTools(params.tools),
           onChunk: (chunk) => {
             if (!receivedFirst) {
@@ -10917,7 +10944,7 @@ var makeStreamTextWrapper = (name, options, streamText, aiSDK) => {
             }
             span.log({
               output: await processOutput(event, options.denyOutputPaths),
-              metrics: extractTokenMetrics(event),
+              metrics: getMetrics(event),
               ...Object.keys(resolvedMetadata).length > 0 ? { metadata: resolvedMetadata } : {}
             });
             span.end();
@@ -11012,11 +11039,15 @@ var wrapStreamObject = (streamObject, options = {}, aiSDK) => {
     try {
       const startTime = Date.now();
       let receivedFirst = false;
+      const { wrappedModel, getMetrics } = wrapModelAndGetMetrics(
+        params.model,
+        aiSDK
+      );
       const result = withCurrent(
         span,
         () => streamObject({
           ...params,
-          model: wrapModel(params.model, aiSDK),
+          model: wrappedModel,
           tools: wrapTools(params.tools),
           onChunk: (chunk) => {
             if (!receivedFirst) {
@@ -11041,7 +11072,7 @@ var wrapStreamObject = (streamObject, options = {}, aiSDK) => {
             }
             span.log({
               output: await processOutput(event, options.denyOutputPaths),
-              metrics: extractTokenMetrics(event),
+              metrics: getMetrics(event),
               ...Object.keys(resolvedMetadata).length > 0 ? { metadata: resolvedMetadata } : {}
             });
             span.end();
@@ -15635,6 +15666,7 @@ async function Eval(name, evaluator, reporterOrOpts) {
       experiment: evaluator.experimentName,
       description: evaluator.description,
       metadata: evaluator.metadata,
+      tags: evaluator.tags,
       isPublic: evaluator.isPublic,
       update: evaluator.update,
       baseExperiment: evaluator.baseExperimentName ?? defaultBaseExperiment,

package/dist/browser.mjs

@@ -6273,6 +6273,7 @@ function init(projectOrOptions, optionalOptions) {
     forceLogin,
     fetch: fetch2,
     metadata,
+    tags,
     gitMetadataSettings,
     projectId,
     baseExperimentId,
@@ -6385,6 +6386,10 @@ function init(projectOrOptions, optionalOptions) {
       if (metadata) {
         args["metadata"] = metadata;
       }
+      if (tags) {
+        validateTags(tags);
+        args["tags"] = tags;
+      }
       let response = null;
       while (true) {
         try {
@@ -7331,6 +7336,7 @@ function validateTags(tags) {
     if (seen.has(tag)) {
       throw new Error(`duplicate tag: ${tag}`);
     }
+    seen.add(tag);
   }
 }
 function validateAndSanitizeExperimentLogPartialArgs(event) {
@@ -9055,6 +9061,7 @@ var _exportsForTestingOnly = {
   isGeneratorFunction,
   isAsyncGeneratorFunction,
   resetIdGenStateForTests,
+  validateTags,
   isomorph: isomorph_default
   // Expose isomorph for build type detection
 };
@@ -10403,9 +10410,13 @@ var makeGenerateTextWrapper = (name, options, generateText, aiSDK) => {
     const processedInput = await processInputAttachments2(params);
     return traced(
       async (span) => {
+        const { wrappedModel, getMetrics } = wrapModelAndGetMetrics(
+          params.model,
+          aiSDK
+        );
         const result = await generateText({
           ...params,
-          model: wrapModel(params.model, aiSDK),
+          model: wrappedModel,
           tools: wrapTools(params.tools)
         });
         const gatewayInfo = extractGatewayRoutingInfo(result);
@@ -10418,7 +10429,7 @@ var makeGenerateTextWrapper = (name, options, generateText, aiSDK) => {
         }
         span.log({
           output: await processOutput(result, options.denyOutputPaths),
-          metrics: extractTokenMetrics(result),
+          metrics: getMetrics(result),
           ...Object.keys(resolvedMetadata).length > 0 ? { metadata: resolvedMetadata } : {}
         });
         return result;
@@ -10639,6 +10650,14 @@ var wrapModel = (model, ai) => {
 var wrapGenerateText = (generateText, options = {}, aiSDK) => {
   return makeGenerateTextWrapper("generateText", options, generateText, aiSDK);
 };
+var wrapModelAndGetMetrics = (model, aiSDK) => {
+  const wrappedModel = wrapModel(model, aiSDK);
+  const modelIsWrapped = wrappedModel?._braintrustWrapped === true;
+  return {
+    wrappedModel,
+    getMetrics: (result) => modelIsWrapped ? void 0 : extractTokenMetrics(result)
+  };
+};
 var wrapGenerateObject = (generateObject, options = {}, aiSDK) => {
   return async function generateObjectWrapper(allParams) {
     const { span_info, ...params } = allParams;
@@ -10651,9 +10670,13 @@ var wrapGenerateObject = (generateObject, options = {}, aiSDK) => {
     const processedInput = await processInputAttachments2(params);
     return traced(
       async (span) => {
+        const { wrappedModel, getMetrics } = wrapModelAndGetMetrics(
+          params.model,
+          aiSDK
+        );
         const result = await generateObject({
           ...params,
-          model: wrapModel(params.model, aiSDK),
+          model: wrappedModel,
           tools: wrapTools(params.tools)
         });
         const output = await processOutput(result, options.denyOutputPaths);
@@ -10667,7 +10690,7 @@ var wrapGenerateObject = (generateObject, options = {}, aiSDK) => {
         }
         span.log({
           output,
-          metrics: extractTokenMetrics(result),
+          metrics: getMetrics(result),
           ...Object.keys(resolvedMetadata).length > 0 ? { metadata: resolvedMetadata } : {}
         });
         return result;
@@ -10732,11 +10755,15 @@ var makeStreamTextWrapper = (name, options, streamText, aiSDK) => {
     try {
       const startTime = Date.now();
       let receivedFirst = false;
+      const { wrappedModel, getMetrics } = wrapModelAndGetMetrics(
+        params.model,
+        aiSDK
+      );
       const result = withCurrent(
         span,
         () => streamText({
           ...params,
-          model: wrapModel(params.model, aiSDK),
+          model: wrappedModel,
           tools: wrapTools(params.tools),
           onChunk: (chunk) => {
             if (!receivedFirst) {
@@ -10761,7 +10788,7 @@ var makeStreamTextWrapper = (name, options, streamText, aiSDK) => {
             }
             span.log({
               output: await processOutput(event, options.denyOutputPaths),
-              metrics: extractTokenMetrics(event),
+              metrics: getMetrics(event),
               ...Object.keys(resolvedMetadata).length > 0 ? { metadata: resolvedMetadata } : {}
             });
             span.end();
@@ -10856,11 +10883,15 @@ var wrapStreamObject = (streamObject, options = {}, aiSDK) => {
     try {
       const startTime = Date.now();
       let receivedFirst = false;
+      const { wrappedModel, getMetrics } = wrapModelAndGetMetrics(
+        params.model,
+        aiSDK
+      );
       const result = withCurrent(
         span,
         () => streamObject({
           ...params,
-          model: wrapModel(params.model, aiSDK),
+          model: wrappedModel,
           tools: wrapTools(params.tools),
           onChunk: (chunk) => {
             if (!receivedFirst) {
@@ -10885,7 +10916,7 @@ var wrapStreamObject = (streamObject, options = {}, aiSDK) => {
             }
             span.log({
               output: await processOutput(event, options.denyOutputPaths),
-              metrics: extractTokenMetrics(event),
+              metrics: getMetrics(event),
               ...Object.keys(resolvedMetadata).length > 0 ? { metadata: resolvedMetadata } : {}
             });
             span.end();
@@ -15479,6 +15510,7 @@ async function Eval(name, evaluator, reporterOrOpts) {
       experiment: evaluator.experimentName,
       description: evaluator.description,
       metadata: evaluator.metadata,
+      tags: evaluator.tags,
       isPublic: evaluator.isPublic,
       update: evaluator.update,
       baseExperiment: evaluator.baseExperimentName ?? defaultBaseExperiment,

package/dist/cli.js

@@ -1275,7 +1275,7 @@ var require_package = __commonJS({
   "package.json"(exports2, module2) {
     module2.exports = {
       name: "braintrust",
-      version: "3.0.0",
+      version: "3.1.0",
       description: "SDK for integrating Braintrust",
       repository: {
         type: "git",
@@ -7543,6 +7543,7 @@ function init(projectOrOptions, optionalOptions) {
     forceLogin,
     fetch: fetch2,
     metadata,
+    tags,
     gitMetadataSettings,
     projectId,
     baseExperimentId,
@@ -7655,6 +7656,10 @@ function init(projectOrOptions, optionalOptions) {
       if (metadata) {
         args["metadata"] = metadata;
       }
+      if (tags) {
+        validateTags(tags);
+        args["tags"] = tags;
+      }
       let response = null;
       while (true) {
         try {
@@ -8053,6 +8058,7 @@ function validateTags(tags) {
     if (seen.has(tag)) {
       throw new Error(`duplicate tag: ${tag}`);
     }
+    seen.add(tag);
   }
 }
 function validateAndSanitizeExperimentLogPartialArgs(event) {
@@ -11351,6 +11357,7 @@ async function Eval(name, evaluator, reporterOrOpts) {
       experiment: evaluator.experimentName,
       description: evaluator.description,
       metadata: evaluator.metadata,
+      tags: evaluator.tags,
       isPublic: evaluator.isPublic,
       update: evaluator.update,
       baseExperiment: evaluator.baseExperimentName ?? defaultBaseExperiment,
@@ -14462,6 +14469,7 @@ async function initExperiment2(evaluator, evaluatorData) {
     experiment: evaluator.experimentName,
     description: evaluator.description,
     metadata: evaluator.metadata,
+    tags: evaluator.tags,
     isPublic: evaluator.isPublic,
     update: evaluator.update,
     baseExperiment: evaluator.baseExperimentName ?? defaultBaseExperiment,