braintrust 0.0.175 → 0.0.177

package/dist/browser.js

@@ -3423,6 +3423,21 @@ function renderMessage(render, message) {
             return _exhaustiveCheck;
         }
       })
+    } : {},
+    ..."tool_calls" in message ? {
+      tool_calls: isEmpty(message.tool_calls) ? void 0 : message.tool_calls.map((t) => {
+        return {
+          type: t.type,
+          id: render(t.id),
+          function: {
+            name: render(t.function.name),
+            arguments: render(t.function.arguments)
+          }
+        };
+      })
+    } : {},
+    ..."tool_call_id" in message ? {
+      tool_call_id: render(message.tool_call_id)
     } : {}
   };
 }

package/dist/browser.mjs

@@ -3373,6 +3373,21 @@ function renderMessage(render, message) {
             return _exhaustiveCheck;
         }
       })
+    } : {},
+    ..."tool_calls" in message ? {
+      tool_calls: isEmpty(message.tool_calls) ? void 0 : message.tool_calls.map((t) => {
+        return {
+          type: t.type,
+          id: render(t.id),
+          function: {
+            name: render(t.function.name),
+            arguments: render(t.function.arguments)
+          }
+        };
+      })
+    } : {},
+    ..."tool_call_id" in message ? {
+      tool_call_id: render(message.tool_call_id)
     } : {}
   };
 }

package/dist/cli.js

@@ -1236,7 +1236,7 @@ var require_package = __commonJS({
   "package.json"(exports2, module2) {
     module2.exports = {
       name: "braintrust",
-      version: "0.0.175",
+      version: "0.0.177",
       description: "SDK for integrating Braintrust",
       repository: {
         type: "git",
@@ -1311,7 +1311,7 @@ var require_package = __commonJS({
       },
       dependencies: {
         "@ai-sdk/provider": "^1.0.1",
-        "@braintrust/core": "0.0.69",
+        "@braintrust/core": "0.0.71",
         "@next/env": "^14.2.3",
         "@vercel/functions": "^1.0.2",
         ai: "^3.2.16",
@@ -4392,6 +4392,21 @@ function renderMessage(render, message) {
             return _exhaustiveCheck;
         }
       })
+    } : {},
+    ..."tool_calls" in message ? {
+      tool_calls: isEmpty(message.tool_calls) ? void 0 : message.tool_calls.map((t) => {
+        return {
+          type: t.type,
+          id: render(t.id),
+          function: {
+            name: render(t.function.name),
+            arguments: render(t.function.arguments)
+          }
+        };
+      })
+    } : {},
+    ..."tool_call_id" in message ? {
+      tool_call_id: render(message.tool_call_id)
     } : {}
   };
 }
@@ -4574,9 +4589,7 @@ var BarProgressReporter = class {
 };
 
 // src/framework.ts
-var import_chalk = __toESM(require("chalk"));
 var import_core2 = require("@braintrust/core");
-var import_pluralize = __toESM(require("pluralize"));
 
 // ../../node_modules/.pnpm/async@3.2.5/node_modules/async/dist/async.mjs
 function initialParams(fn) {
@@ -5682,6 +5695,8 @@ function waterfall(tasks, callback) {
 var waterfall$1 = awaitify(waterfall);
 
 // src/framework.ts
+var import_chalk = __toESM(require("chalk"));
+var import_pluralize = __toESM(require("pluralize"));
 var EvalResultWithSummary = class {
   constructor(summary, results) {
     this.summary = summary;
@@ -5858,7 +5873,11 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
           const meta = (o) => metadata = { ...metadata, ...o };
           await rootSpan.traced(
             async (span) => {
-              const outputResult = evaluator.task(datum.input, { meta, span });
+              const outputResult = evaluator.task(datum.input, {
+                meta,
+                metadata,
+                span
+              });
               if (outputResult instanceof Promise) {
                 output = await outputResult;
               } else {
@@ -6720,6 +6739,7 @@ async function uploadHandleBundles({
           function_data: {
             type: "prompt"
           },
+          function_type: prompt.functionType,
           prompt_data,
           if_exists: prompt.ifExists
         });
@@ -7045,8 +7065,8 @@ async function bundleCommand(args) {
 }
 
 // src/cli-util/pull.ts
-var import_typespecs4 = require("@braintrust/core/typespecs");
-var import_zod5 = require("zod");
+var import_typespecs5 = require("@braintrust/core/typespecs");
+var import_zod4 = require("zod");
 var import_promises = __toESM(require("fs/promises"));
 var import_util4 = __toESM(require("util"));
 var import_slugify3 = __toESM(require("slugify"));
@@ -7056,7 +7076,7 @@ var import_core4 = require("@braintrust/core");
 // src/framework2.ts
 var import_path4 = __toESM(require("path"));
 var import_slugify2 = __toESM(require("slugify"));
-var import_zod4 = require("zod");
+var import_typespecs4 = require("@braintrust/core/typespecs");
 var ProjectBuilder = class {
   create(opts) {
     return new Project(opts);
@@ -7068,12 +7088,14 @@ var Project = class {
   id;
   tools;
   prompts;
+  scorers;
   constructor(args) {
     _initializeSpanContext();
     this.name = "name" in args ? args.name : void 0;
     this.id = "id" in args ? args.id : void 0;
     this.tools = new ToolBuilder(this);
     this.prompts = new PromptBuilder(this);
+    this.scorers = new ScorerBuilder(this);
   }
 };
 var ToolBuilder = class {
@@ -7107,6 +7129,70 @@ var ToolBuilder = class {
     return tool;
   }
 };
+var ScorerBuilder = class {
+  constructor(project) {
+    this.project = project;
+  }
+  taskCounter = 0;
+  create(opts) {
+    this.taskCounter++;
+    let resolvedName = opts.name;
+    if (!resolvedName && "handler" in opts) {
+      resolvedName = opts.handler.name;
+    }
+    if (!resolvedName || resolvedName.trim().length === 0) {
+      resolvedName = `Scorer ${import_path4.default.basename(__filename)} ${this.taskCounter}`;
+    }
+    const slug = opts.slug ?? (0, import_slugify2.default)(resolvedName, { lower: true, strict: true });
+    if ("handler" in opts) {
+      const scorer = new CodeFunction(this.project, {
+        ...opts,
+        name: resolvedName,
+        slug,
+        type: "scorer"
+      });
+      if (globalThis._lazy_load) {
+        globalThis._evals.functions.push(
+          scorer
+        );
+      }
+    } else {
+      const promptBlock = "messages" in opts ? {
+        type: "chat",
+        messages: opts.messages
+      } : {
+        type: "completion",
+        content: opts.prompt
+      };
+      const promptData = {
+        prompt: promptBlock,
+        options: {
+          model: opts.model,
+          params: opts.params
+        },
+        parser: {
+          type: "llm_classifier",
+          use_cot: opts.useCot,
+          choice_scores: opts.choiceScores
+        }
+      };
+      const codePrompt = new CodePrompt(
+        this.project,
+        promptData,
+        [],
+        {
+          ...opts,
+          name: resolvedName,
+          slug
+        },
+        "scorer"
+      );
+      if (globalThis._lazy_load) {
+        globalThis._evals.prompts.push(codePrompt);
+      }
+    }
+  }
+};
 var CodeFunction = class {
   constructor(project, opts) {
     this.project = project;
@@ -7146,8 +7232,9 @@ var CodePrompt = class {
   ifExists;
   description;
   id;
+  functionType;
   toolFunctions;
-  constructor(project, prompt, toolFunctions, opts) {
+  constructor(project, prompt, toolFunctions, opts, functionType) {
     this.project = project;
     this.name = opts.name;
     this.slug = opts.slug;
@@ -7156,17 +7243,9 @@ var CodePrompt = class {
     this.ifExists = opts.ifExists;
     this.description = opts.description;
     this.id = opts.id;
+    this.functionType = functionType;
   }
 };
-var toolFunctionDefinitionSchema = import_zod4.z.object({
-  type: import_zod4.z.literal("function"),
-  function: import_zod4.z.object({
-    name: import_zod4.z.string(),
-    description: import_zod4.z.string().optional(),
-    parameters: import_zod4.z.record(import_zod4.z.unknown()).optional(),
-    strict: import_zod4.z.boolean().optional()
-  })
-});
 var PromptBuilder = class {
   constructor(project) {
     this.project = project;
@@ -7235,11 +7314,11 @@ async function pullCommand(args) {
     ...args.id ? { ids: [args.id] } : {},
     ...args.version ? { version: (0, import_core4.loadPrettyXact)(args.version) } : {}
   });
-  const functionObjects = import_zod5.z.object({ objects: import_zod5.z.array(import_zod5.z.unknown()) }).parse(functions);
+  const functionObjects = import_zod4.z.object({ objects: import_zod4.z.array(import_zod4.z.unknown()) }).parse(functions);
   const projectNameToFunctions = {};
   const projectNameIdMap = new ProjectNameIdMap();
   for (const rawFunc of functionObjects.objects) {
-    const parsedFunc = import_typespecs4.functionSchema.safeParse(rawFunc);
+    const parsedFunc = import_typespecs5.functionSchema.safeParse(rawFunc);
     if (!parsedFunc.success) {
       const id = typeof rawFunc === "object" && rawFunc && "id" in rawFunc ? ` ${rawFunc.id}` : "";
       console.warn(
@@ -7388,7 +7467,7 @@ function makeFunctionDefinition({
   const objectType = "prompt";
   const prompt = func.prompt_data.prompt;
   const promptContents = prompt.type === "completion" ? `prompt: ${doubleQuote(prompt.content)}` : `messages: ${import_util4.default.inspect(prompt.messages, { depth: null }).trimStart()}`;
-  const rawToolsParsed = prompt.type === "chat" && prompt.tools && prompt.tools.length > 0 ? import_zod5.z.array(toolFunctionDefinitionSchema).safeParse(JSON.parse(prompt.tools)) : void 0;
+  const rawToolsParsed = prompt.type === "chat" && prompt.tools && prompt.tools.length > 0 ? import_zod4.z.array(import_typespecs4.toolFunctionDefinitionSchema).safeParse(JSON.parse(prompt.tools)) : void 0;
   if (rawToolsParsed && !rawToolsParsed.success) {
     console.warn(
       warning(

package/dist/index.d.mts

@@ -1,6 +1,7 @@
 import { LogFeedbackFullArgs, ExperimentEvent, BackgroundLogEvent, ExperimentLogFullArgs, ExperimentLogPartialArgs, IdField, SpanType, SpanComponentsV3, DEFAULT_IS_LEGACY_DATASET, TRANSACTION_ID_FIELD, TransactionId, SpanObjectTypeV3, DatasetRecord, Score, CommentEvent, InputField, LogCommentFullArgs, OtherExperimentLogFields, ParentExperimentIds, ParentProjectLogIds } from '@braintrust/core';
 export { CommentEvent, DatasetRecord, ExperimentLogFullArgs, ExperimentLogPartialArgs, IdField, InputField, LogCommentFullArgs, LogFeedbackFullArgs, OtherExperimentLogFields, ParentExperimentIds, ParentProjectLogIds } from '@braintrust/core';
-import { GitMetadataSettings, AttachmentReference, AttachmentStatus, RepoInfo, PromptData, OpenAIMessage, Tools, AnyModelParam, Message, Prompt as Prompt$1, PromptSessionEvent, StreamingMode, FunctionType, IfExists, SavedFunctionId, ModelParams } from '@braintrust/core/typespecs';
+import { GitMetadataSettings, AttachmentReference, AttachmentStatus, RepoInfo, PromptData, OpenAIMessage, Tools, AnyModelParam, Message, Prompt as Prompt$1, PromptSessionEvent, StreamingMode, FunctionType, IfExists, SavedFunctionId, ModelParams, ToolFunctionDefinition, toolFunctionDefinitionSchema } from '@braintrust/core/typespecs';
+export { ToolFunctionDefinition, toolFunctionDefinitionSchema } from '@braintrust/core/typespecs';
 import { z } from 'zod';
 
 interface IsoAsyncLocalStorage<T> {
@@ -1572,13 +1573,20 @@ declare class Project {
     readonly id?: string;
     tools: ToolBuilder;
     prompts: PromptBuilder;
+    scorers: ScorerBuilder;
     constructor(args: CreateProjectOpts);
 }
 declare class ToolBuilder {
     private readonly project;
     private taskCounter;
     constructor(project: Project);
-    create<Input, Output, Fn extends GenericFunction<Input, Output>>(opts: ToolOpts<Input, Output, Fn>): CodeFunction<Input, Output, Fn>;
+    create<Input, Output, Fn extends GenericFunction<Input, Output>>(opts: CodeOpts<Input, Output, Fn>): CodeFunction<Input, Output, Fn>;
+}
+declare class ScorerBuilder {
+    private readonly project;
+    private taskCounter;
+    constructor(project: Project);
+    create<Output, Input, Params, Returns, Fn extends GenericFunction<Exact<Params, ScorerArgs<Output, Input>>, Returns>>(opts: ScorerOpts<Output, Input, Params, Returns, Fn>): void;
 }
 type Schema<Input, Output> = Partial<{
     parameters: z.ZodSchema<Input>;
@@ -1590,9 +1598,21 @@ interface BaseFnOpts {
     description: string;
     ifExists: IfExists;
 }
-type ToolOpts<Params, Returns, Fn extends GenericFunction<Params, Returns>> = Partial<BaseFnOpts> & {
+type CodeOpts<Params, Returns, Fn extends GenericFunction<Params, Returns>> = Partial<BaseFnOpts> & {
     handler: Fn;
 } & Schema<Params, Returns>;
+type ScorerPromptOpts = Partial<BaseFnOpts> & PromptOpts<false, false, false, false> & {
+    useCot: boolean;
+    choiceScores: Record<string, number>;
+};
+type ScorerArgs<Output, Input> = {
+    output: Output;
+    expected?: Output;
+    input?: Input;
+    metadata?: Record<string, unknown>;
+};
+type Exact<T, Shape> = T extends Shape ? Exclude<keyof T, keyof Shape> extends never ? T : never : never;
+type ScorerOpts<Output, Input, Params, Returns, Fn extends GenericFunction<Exact<Params, ScorerArgs<Output, Input>>, Returns>> = CodeOpts<Exact<Params, ScorerArgs<Output, Input>>, Returns, Fn> | ScorerPromptOpts;
 declare class CodeFunction<Input, Output, Fn extends GenericFunction<Input, Output>> {
     readonly project: Project;
     readonly handler: Fn;
@@ -1603,7 +1623,7 @@ declare class CodeFunction<Input, Output, Fn extends GenericFunction<Input, Outp
     readonly parameters?: z.ZodSchema<Input>;
     readonly returns?: z.ZodSchema<Output>;
     readonly ifExists?: IfExists;
-    constructor(project: Project, opts: Omit<ToolOpts<Input, Output, Fn>, "name" | "slug"> & {
+    constructor(project: Project, opts: Omit<CodeOpts<Input, Output, Fn>, "name" | "slug"> & {
         name: string;
         slug: string;
         type: FunctionType;
@@ -1619,66 +1639,35 @@ declare class CodePrompt {
     readonly ifExists?: IfExists;
     readonly description?: string;
     readonly id?: string;
+    readonly functionType?: FunctionType;
     readonly toolFunctions: (SavedFunctionId | GenericCodeFunction)[];
-    constructor(project: Project, prompt: PromptData, toolFunctions: (SavedFunctionId | GenericCodeFunction)[], opts: Omit<PromptOpts<false, false>, "name" | "slug"> & {
+    constructor(project: Project, prompt: PromptData, toolFunctions: (SavedFunctionId | GenericCodeFunction)[], opts: Omit<PromptOpts<false, false, false, false>, "name" | "slug"> & {
         name: string;
         slug: string;
-    });
+    }, functionType?: FunctionType);
 }
-declare const toolFunctionDefinitionSchema: z.ZodObject<{
-    type: z.ZodLiteral<"function">;
-    function: z.ZodObject<{
-        name: z.ZodString;
-        description: z.ZodOptional<z.ZodString>;
-        parameters: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
-        strict: z.ZodOptional<z.ZodBoolean>;
-    }, "strip", z.ZodTypeAny, {
-        name: string;
-        description?: string | undefined;
-        parameters?: Record<string, unknown> | undefined;
-        strict?: boolean | undefined;
-    }, {
-        name: string;
-        description?: string | undefined;
-        parameters?: Record<string, unknown> | undefined;
-        strict?: boolean | undefined;
-    }>;
-}, "strip", z.ZodTypeAny, {
-    function: {
-        name: string;
-        description?: string | undefined;
-        parameters?: Record<string, unknown> | undefined;
-        strict?: boolean | undefined;
-    };
-    type: "function";
-}, {
-    function: {
-        name: string;
-        description?: string | undefined;
-        parameters?: Record<string, unknown> | undefined;
-        strict?: boolean | undefined;
-    };
-    type: "function";
-}>;
-type ToolFunctionDefinition = z.infer<typeof toolFunctionDefinitionSchema>;
 interface PromptId {
     id: string;
 }
 interface PromptVersion {
     version: TransactionId;
 }
+interface PromptTools {
+    tools: (GenericCodeFunction | SavedFunctionId | ToolFunctionDefinition)[];
+}
+interface PromptNoTrace {
+    noTrace: boolean;
+}
 type PromptContents = {
     prompt: string;
 } | {
     messages: Message[];
 };
-type PromptOpts<HasId extends boolean, HasVersion extends boolean> = (Partial<Omit<BaseFnOpts, "name">> & {
+type PromptOpts<HasId extends boolean, HasVersion extends boolean, HasTools extends boolean = true, HasNoTrace extends boolean = true> = (Partial<Omit<BaseFnOpts, "name">> & {
     name: string;
-}) & (HasId extends true ? PromptId : Partial<PromptId>) & (HasVersion extends true ? PromptVersion : Partial<PromptVersion>) & PromptContents & {
+}) & (HasId extends true ? PromptId : Partial<PromptId>) & (HasVersion extends true ? PromptVersion : Partial<PromptVersion>) & (HasTools extends true ? Partial<PromptTools> : {}) & (HasNoTrace extends true ? Partial<PromptNoTrace> : {}) & PromptContents & {
     model: string;
     params?: ModelParams;
-    tools?: (GenericCodeFunction | SavedFunctionId | ToolFunctionDefinition)[];
-    noTrace?: boolean;
 };
 declare class PromptBuilder {
     private readonly project;
@@ -1707,7 +1696,14 @@ declare function BaseExperiment<Input = unknown, Expected = unknown, Metadata ex
 type EvalData<Input, Expected, Metadata extends BaseMetadata = DefaultMetadataType> = EvalCase<Input, Expected, Metadata>[] | (() => EvalCase<Input, Expected, Metadata>[]) | Promise<EvalCase<Input, Expected, Metadata>[]> | (() => Promise<EvalCase<Input, Expected, Metadata>[]>) | AsyncGenerator<EvalCase<Input, Expected, Metadata>> | AsyncIterable<EvalCase<Input, Expected, Metadata>> | BaseExperiment<Input, Expected, Metadata> | (() => BaseExperiment<Input, Expected, Metadata>);
 type EvalTask<Input, Output> = ((input: Input, hooks: EvalHooks) => Promise<Output>) | ((input: Input, hooks: EvalHooks) => Output);
 interface EvalHooks {
+    /**
+     * @deprecated Use `metadata` instead.
+     */
     meta: (info: Record<string, unknown>) => void;
+    /**
+     * The metadata object for the current evaluation. You can mutate this object to add or remove metadata.
+     */
+    metadata: Record<string, unknown>;
     span: Span;
 }
 type EvalScorerArgs<Input, Output, Expected, Metadata extends BaseMetadata = DefaultMetadataType> = EvalCase<Input, Expected, Metadata> & {
@@ -1923,6 +1919,7 @@ type braintrust_BraintrustStreamChunk = BraintrustStreamChunk;
 type braintrust_ChatPrompt = ChatPrompt;
 type braintrust_CodeFunction<Input, Output, Fn extends GenericFunction<Input, Output>> = CodeFunction<Input, Output, Fn>;
 declare const braintrust_CodeFunction: typeof CodeFunction;
+type braintrust_CodeOpts<Params, Returns, Fn extends GenericFunction<Params, Returns>> = CodeOpts<Params, Returns, Fn>;
 type braintrust_CodePrompt = CodePrompt;
 declare const braintrust_CodePrompt: typeof CodePrompt;
 declare const braintrust_CommentEvent: typeof CommentEvent;
@@ -1986,7 +1983,7 @@ type braintrust_Prompt<HasId extends boolean = true, HasVersion extends boolean
 declare const braintrust_Prompt: typeof Prompt;
 type braintrust_PromptBuilder = PromptBuilder;
 declare const braintrust_PromptBuilder: typeof PromptBuilder;
-type braintrust_PromptOpts<HasId extends boolean, HasVersion extends boolean> = PromptOpts<HasId, HasVersion>;
+type braintrust_PromptOpts<HasId extends boolean, HasVersion extends boolean, HasTools extends boolean = true, HasNoTrace extends boolean = true> = PromptOpts<HasId, HasVersion, HasTools, HasNoTrace>;
 type braintrust_PromptRowWithId<HasId extends boolean = true, HasVersion extends boolean = true> = PromptRowWithId<HasId, HasVersion>;
 type braintrust_ReadonlyAttachment = ReadonlyAttachment;
 declare const braintrust_ReadonlyAttachment: typeof ReadonlyAttachment;
@@ -1995,6 +1992,9 @@ declare const braintrust_ReadonlyExperiment: typeof ReadonlyExperiment;
 declare const braintrust_Reporter: typeof Reporter;
 type braintrust_ReporterBody<EvalReport> = ReporterBody<EvalReport>;
 type braintrust_ScoreSummary = ScoreSummary;
+type braintrust_ScorerBuilder = ScorerBuilder;
+declare const braintrust_ScorerBuilder: typeof ScorerBuilder;
+type braintrust_ScorerOpts<Output, Input, Params, Returns, Fn extends GenericFunction<Exact<Params, ScorerArgs<Output, Input>>, Returns>> = ScorerOpts<Output, Input, Params, Returns, Fn>;
 type braintrust_SerializedBraintrustState = SerializedBraintrustState;
 type braintrust_SetCurrentArg = SetCurrentArg;
 type braintrust_Span = Span;
@@ -2004,8 +2004,7 @@ declare const braintrust_SpanImpl: typeof SpanImpl;
 type braintrust_StartSpanArgs = StartSpanArgs;
 type braintrust_ToolBuilder = ToolBuilder;
 declare const braintrust_ToolBuilder: typeof ToolBuilder;
-type braintrust_ToolFunctionDefinition = ToolFunctionDefinition;
-type braintrust_ToolOpts<Params, Returns, Fn extends GenericFunction<Params, Returns>> = ToolOpts<Params, Returns, Fn>;
+declare const braintrust_ToolFunctionDefinition: typeof ToolFunctionDefinition;
 type braintrust_WithTransactionId<R> = WithTransactionId<R>;
 declare const braintrust_X_CACHED_HEADER: typeof X_CACHED_HEADER;
 declare const braintrust__exportsForTestingOnly: typeof _exportsForTestingOnly;
@@ -2053,7 +2052,7 @@ declare const braintrust_wrapOpenAI: typeof wrapOpenAI;
 declare const braintrust_wrapOpenAIv4: typeof wrapOpenAIv4;
 declare const braintrust_wrapTraced: typeof wrapTraced;
 declare namespace braintrust {
-  export { type braintrust_AnyDataset as AnyDataset, braintrust_Attachment as Attachment, type braintrust_AttachmentParams as AttachmentParams, type braintrust_BackgroundLoggerOpts as BackgroundLoggerOpts, braintrust_BaseExperiment as BaseExperiment, type braintrust_BaseMetadata as BaseMetadata, braintrust_BraintrustState as BraintrustState, braintrust_BraintrustStream as BraintrustStream, type braintrust_BraintrustStreamChunk as BraintrustStreamChunk, type braintrust_ChatPrompt as ChatPrompt, braintrust_CodeFunction as CodeFunction, braintrust_CodePrompt as CodePrompt, braintrust_CommentEvent as CommentEvent, type braintrust_CompiledPrompt as CompiledPrompt, type braintrust_CompiledPromptParams as CompiledPromptParams, type braintrust_CompletionPrompt as CompletionPrompt, type braintrust_CreateProjectOpts as CreateProjectOpts, type braintrust_DataSummary as DataSummary, braintrust_Dataset as Dataset, braintrust_DatasetRecord as DatasetRecord, type braintrust_DatasetSummary as DatasetSummary, type braintrust_DefaultMetadataType as DefaultMetadataType, type braintrust_DefaultPromptArgs as DefaultPromptArgs, type braintrust_EndSpanArgs as EndSpanArgs, braintrust_Eval as Eval, type braintrust_EvalCase as EvalCase, type braintrust_EvalResult as EvalResult, type braintrust_EvalScorer as EvalScorer, type braintrust_EvalScorerArgs as EvalScorerArgs, type braintrust_EvalTask as EvalTask, type braintrust_Evaluator as Evaluator, type braintrust_EvaluatorDef as EvaluatorDef, type braintrust_EvaluatorFile as EvaluatorFile, braintrust_Experiment as Experiment, braintrust_ExperimentLogFullArgs as ExperimentLogFullArgs, braintrust_ExperimentLogPartialArgs as ExperimentLogPartialArgs, type braintrust_ExperimentSummary as ExperimentSummary, type braintrust_Exportable as Exportable, braintrust_FailedHTTPResponse as FailedHTTPResponse, type braintrust_FullInitOptions as FullInitOptions, type braintrust_FullLoginOptions as FullLoginOptions, braintrust_IdField as IdField, type braintrust_InitOptions as InitOptions, braintrust_InputField as InputField, type braintrust_InvokeFunctionArgs as InvokeFunctionArgs, type braintrust_InvokeReturn as InvokeReturn, braintrust_LEGACY_CACHED_HEADER as LEGACY_CACHED_HEADER, braintrust_LazyValue as LazyValue, braintrust_LogCommentFullArgs as LogCommentFullArgs, braintrust_LogFeedbackFullArgs as LogFeedbackFullArgs, type braintrust_LogOptions as LogOptions, braintrust_Logger as Logger, type braintrust_LoginOptions as LoginOptions, type braintrust_MetricSummary as MetricSummary, braintrust_NOOP_SPAN as NOOP_SPAN, braintrust_NoopSpan as NoopSpan, type braintrust_ObjectMetadata as ObjectMetadata, braintrust_OtherExperimentLogFields as OtherExperimentLogFields, braintrust_ParentExperimentIds as ParentExperimentIds, braintrust_ParentProjectLogIds as ParentProjectLogIds, braintrust_Project as Project, type braintrust_PromiseUnless as PromiseUnless, braintrust_Prompt as Prompt, braintrust_PromptBuilder as PromptBuilder, type braintrust_PromptOpts as PromptOpts, type braintrust_PromptRowWithId as PromptRowWithId, braintrust_ReadonlyAttachment as ReadonlyAttachment, braintrust_ReadonlyExperiment as ReadonlyExperiment, braintrust_Reporter as Reporter, type braintrust_ReporterBody as ReporterBody, type braintrust_ScoreSummary as ScoreSummary, type braintrust_SerializedBraintrustState as SerializedBraintrustState, type braintrust_SetCurrentArg as SetCurrentArg, type braintrust_Span as Span, type braintrust_SpanContext as SpanContext, braintrust_SpanImpl as SpanImpl, type braintrust_StartSpanArgs as StartSpanArgs, braintrust_ToolBuilder as ToolBuilder, type braintrust_ToolFunctionDefinition as ToolFunctionDefinition, type braintrust_ToolOpts as ToolOpts, type braintrust_WithTransactionId as WithTransactionId, braintrust_X_CACHED_HEADER as X_CACHED_HEADER, braintrust__exportsForTestingOnly as _exportsForTestingOnly, braintrust__internalGetGlobalState as _internalGetGlobalState, braintrust__internalSetInitialState as _internalSetInitialState, braintrust_braintrustStreamChunkSchema as braintrustStreamChunkSchema, braintrust_buildLocalSummary as buildLocalSummary, braintrust_createFinalValuePassThroughStream as createFinalValuePassThroughStream, braintrust_currentExperiment as currentExperiment, braintrust_currentLogger as currentLogger, braintrust_currentSpan as currentSpan, braintrust_devNullWritableStream as devNullWritableStream, braintrust_flush as flush, braintrust_getSpanParentObject as getSpanParentObject, braintrust_init as init, braintrust_initDataset as initDataset, braintrust_initExperiment as initExperiment, braintrust_initLogger as initLogger, braintrust_invoke as invoke, braintrust_loadPrompt as loadPrompt, braintrust_log as log, braintrust_logError as logError, braintrust_login as login, braintrust_loginToState as loginToState, braintrust_newId as newId, braintrust_parseCachedHeader as parseCachedHeader, braintrust_permalink as permalink, braintrust_projects as projects, braintrust_renderMessage as renderMessage, braintrust_reportFailures as reportFailures, braintrust_setFetch as setFetch, braintrust_spanComponentsToObjectId as spanComponentsToObjectId, braintrust_startSpan as startSpan, braintrust_summarize as summarize, braintrust_toolFunctionDefinitionSchema as toolFunctionDefinitionSchema, braintrust_traceable as traceable, braintrust_traced as traced, braintrust_updateSpan as updateSpan, braintrust_withCurrent as withCurrent, braintrust_withDataset as withDataset, braintrust_withExperiment as withExperiment, braintrust_withLogger as withLogger, braintrust_wrapAISDKModel as wrapAISDKModel, braintrust_wrapOpenAI as wrapOpenAI, braintrust_wrapOpenAIv4 as wrapOpenAIv4, braintrust_wrapTraced as wrapTraced };
+  export { type braintrust_AnyDataset as AnyDataset, braintrust_Attachment as Attachment, type braintrust_AttachmentParams as AttachmentParams, type braintrust_BackgroundLoggerOpts as BackgroundLoggerOpts, braintrust_BaseExperiment as BaseExperiment, type braintrust_BaseMetadata as BaseMetadata, braintrust_BraintrustState as BraintrustState, braintrust_BraintrustStream as BraintrustStream, type braintrust_BraintrustStreamChunk as BraintrustStreamChunk, type braintrust_ChatPrompt as ChatPrompt, braintrust_CodeFunction as CodeFunction, type braintrust_CodeOpts as CodeOpts, braintrust_CodePrompt as CodePrompt, braintrust_CommentEvent as CommentEvent, type braintrust_CompiledPrompt as CompiledPrompt, type braintrust_CompiledPromptParams as CompiledPromptParams, type braintrust_CompletionPrompt as CompletionPrompt, type braintrust_CreateProjectOpts as CreateProjectOpts, type braintrust_DataSummary as DataSummary, braintrust_Dataset as Dataset, braintrust_DatasetRecord as DatasetRecord, type braintrust_DatasetSummary as DatasetSummary, type braintrust_DefaultMetadataType as DefaultMetadataType, type braintrust_DefaultPromptArgs as DefaultPromptArgs, type braintrust_EndSpanArgs as EndSpanArgs, braintrust_Eval as Eval, type braintrust_EvalCase as EvalCase, type braintrust_EvalResult as EvalResult, type braintrust_EvalScorer as EvalScorer, type braintrust_EvalScorerArgs as EvalScorerArgs, type braintrust_EvalTask as EvalTask, type braintrust_Evaluator as Evaluator, type braintrust_EvaluatorDef as EvaluatorDef, type braintrust_EvaluatorFile as EvaluatorFile, braintrust_Experiment as Experiment, braintrust_ExperimentLogFullArgs as ExperimentLogFullArgs, braintrust_ExperimentLogPartialArgs as ExperimentLogPartialArgs, type braintrust_ExperimentSummary as ExperimentSummary, type braintrust_Exportable as Exportable, braintrust_FailedHTTPResponse as FailedHTTPResponse, type braintrust_FullInitOptions as FullInitOptions, type braintrust_FullLoginOptions as FullLoginOptions, braintrust_IdField as IdField, type braintrust_InitOptions as InitOptions, braintrust_InputField as InputField, type braintrust_InvokeFunctionArgs as InvokeFunctionArgs, type braintrust_InvokeReturn as InvokeReturn, braintrust_LEGACY_CACHED_HEADER as LEGACY_CACHED_HEADER, braintrust_LazyValue as LazyValue, braintrust_LogCommentFullArgs as LogCommentFullArgs, braintrust_LogFeedbackFullArgs as LogFeedbackFullArgs, type braintrust_LogOptions as LogOptions, braintrust_Logger as Logger, type braintrust_LoginOptions as LoginOptions, type braintrust_MetricSummary as MetricSummary, braintrust_NOOP_SPAN as NOOP_SPAN, braintrust_NoopSpan as NoopSpan, type braintrust_ObjectMetadata as ObjectMetadata, braintrust_OtherExperimentLogFields as OtherExperimentLogFields, braintrust_ParentExperimentIds as ParentExperimentIds, braintrust_ParentProjectLogIds as ParentProjectLogIds, braintrust_Project as Project, type braintrust_PromiseUnless as PromiseUnless, braintrust_Prompt as Prompt, braintrust_PromptBuilder as PromptBuilder, type braintrust_PromptOpts as PromptOpts, type braintrust_PromptRowWithId as PromptRowWithId, braintrust_ReadonlyAttachment as ReadonlyAttachment, braintrust_ReadonlyExperiment as ReadonlyExperiment, braintrust_Reporter as Reporter, type braintrust_ReporterBody as ReporterBody, type braintrust_ScoreSummary as ScoreSummary, braintrust_ScorerBuilder as ScorerBuilder, type braintrust_ScorerOpts as ScorerOpts, type braintrust_SerializedBraintrustState as SerializedBraintrustState, type braintrust_SetCurrentArg as SetCurrentArg, type braintrust_Span as Span, type braintrust_SpanContext as SpanContext, braintrust_SpanImpl as SpanImpl, type braintrust_StartSpanArgs as StartSpanArgs, braintrust_ToolBuilder as ToolBuilder, braintrust_ToolFunctionDefinition as ToolFunctionDefinition, type braintrust_WithTransactionId as WithTransactionId, braintrust_X_CACHED_HEADER as X_CACHED_HEADER, braintrust__exportsForTestingOnly as _exportsForTestingOnly, braintrust__internalGetGlobalState as _internalGetGlobalState, braintrust__internalSetInitialState as _internalSetInitialState, braintrust_braintrustStreamChunkSchema as braintrustStreamChunkSchema, braintrust_buildLocalSummary as buildLocalSummary, braintrust_createFinalValuePassThroughStream as createFinalValuePassThroughStream, braintrust_currentExperiment as currentExperiment, braintrust_currentLogger as currentLogger, braintrust_currentSpan as currentSpan, braintrust_devNullWritableStream as devNullWritableStream, braintrust_flush as flush, braintrust_getSpanParentObject as getSpanParentObject, braintrust_init as init, braintrust_initDataset as initDataset, braintrust_initExperiment as initExperiment, braintrust_initLogger as initLogger, braintrust_invoke as invoke, braintrust_loadPrompt as loadPrompt, braintrust_log as log, braintrust_logError as logError, braintrust_login as login, braintrust_loginToState as loginToState, braintrust_newId as newId, braintrust_parseCachedHeader as parseCachedHeader, braintrust_permalink as permalink, braintrust_projects as projects, braintrust_renderMessage as renderMessage, braintrust_reportFailures as reportFailures, braintrust_setFetch as setFetch, braintrust_spanComponentsToObjectId as spanComponentsToObjectId, braintrust_startSpan as startSpan, braintrust_summarize as summarize, braintrust_toolFunctionDefinitionSchema as toolFunctionDefinitionSchema, braintrust_traceable as traceable, braintrust_traced as traced, braintrust_updateSpan as updateSpan, braintrust_withCurrent as withCurrent, braintrust_withDataset as withDataset, braintrust_withExperiment as withExperiment, braintrust_withLogger as withLogger, braintrust_wrapAISDKModel as wrapAISDKModel, braintrust_wrapOpenAI as wrapOpenAI, braintrust_wrapOpenAIv4 as wrapOpenAIv4, braintrust_wrapTraced as wrapTraced };
 }
 
 /**
@@ -2109,4 +2108,4 @@ declare namespace braintrust {
  * @module braintrust
  */
 
-export { type AnyDataset, Attachment, type AttachmentParams, type BackgroundLoggerOpts, BaseExperiment, type BaseMetadata, BraintrustState, BraintrustStream, type BraintrustStreamChunk, type ChatPrompt, CodeFunction, CodePrompt, type CompiledPrompt, type CompiledPromptParams, type CompletionPrompt, type CreateProjectOpts, type DataSummary, Dataset, type DatasetSummary, type DefaultMetadataType, type DefaultPromptArgs, type EndSpanArgs, Eval, type EvalCase, type EvalResult, type EvalScorer, type EvalScorerArgs, type EvalTask, type Evaluator, type EvaluatorDef, type EvaluatorFile, Experiment, type ExperimentSummary, type Exportable, FailedHTTPResponse, type FullInitOptions, type FullLoginOptions, type InitOptions, type InvokeFunctionArgs, type InvokeReturn, LEGACY_CACHED_HEADER, LazyValue, type LogOptions, Logger, type LoginOptions, type MetricSummary, NOOP_SPAN, NoopSpan, type ObjectMetadata, Project, type PromiseUnless, Prompt, PromptBuilder, type PromptOpts, type PromptRowWithId, ReadonlyAttachment, ReadonlyExperiment, Reporter, type ReporterBody, type ScoreSummary, type SerializedBraintrustState, type SetCurrentArg, type Span, type SpanContext, SpanImpl, type StartSpanArgs, ToolBuilder, type ToolFunctionDefinition, type ToolOpts, type WithTransactionId, X_CACHED_HEADER, _exportsForTestingOnly, _internalGetGlobalState, _internalSetInitialState, braintrustStreamChunkSchema, buildLocalSummary, createFinalValuePassThroughStream, currentExperiment, currentLogger, currentSpan, braintrust as default, devNullWritableStream, flush, getSpanParentObject, init, initDataset, initExperiment, initLogger, invoke, loadPrompt, log, logError, login, loginToState, newId, parseCachedHeader, permalink, projects, renderMessage, reportFailures, setFetch, spanComponentsToObjectId, startSpan, summarize, toolFunctionDefinitionSchema, traceable, traced, updateSpan, withCurrent, withDataset, withExperiment, withLogger, wrapAISDKModel, wrapOpenAI, wrapOpenAIv4, wrapTraced };
+export { type AnyDataset, Attachment, type AttachmentParams, type BackgroundLoggerOpts, BaseExperiment, type BaseMetadata, BraintrustState, BraintrustStream, type BraintrustStreamChunk, type ChatPrompt, CodeFunction, type CodeOpts, CodePrompt, type CompiledPrompt, type CompiledPromptParams, type CompletionPrompt, type CreateProjectOpts, type DataSummary, Dataset, type DatasetSummary, type DefaultMetadataType, type DefaultPromptArgs, type EndSpanArgs, Eval, type EvalCase, type EvalResult, type EvalScorer, type EvalScorerArgs, type EvalTask, type Evaluator, type EvaluatorDef, type EvaluatorFile, Experiment, type ExperimentSummary, type Exportable, FailedHTTPResponse, type FullInitOptions, type FullLoginOptions, type InitOptions, type InvokeFunctionArgs, type InvokeReturn, LEGACY_CACHED_HEADER, LazyValue, type LogOptions, Logger, type LoginOptions, type MetricSummary, NOOP_SPAN, NoopSpan, type ObjectMetadata, Project, type PromiseUnless, Prompt, PromptBuilder, type PromptOpts, type PromptRowWithId, ReadonlyAttachment, ReadonlyExperiment, Reporter, type ReporterBody, type ScoreSummary, ScorerBuilder, type ScorerOpts, type SerializedBraintrustState, type SetCurrentArg, type Span, type SpanContext, SpanImpl, type StartSpanArgs, ToolBuilder, type WithTransactionId, X_CACHED_HEADER, _exportsForTestingOnly, _internalGetGlobalState, _internalSetInitialState, braintrustStreamChunkSchema, buildLocalSummary, createFinalValuePassThroughStream, currentExperiment, currentLogger, currentSpan, braintrust as default, devNullWritableStream, flush, getSpanParentObject, init, initDataset, initExperiment, initLogger, invoke, loadPrompt, log, logError, login, loginToState, newId, parseCachedHeader, permalink, projects, renderMessage, reportFailures, setFetch, spanComponentsToObjectId, startSpan, summarize, traceable, traced, updateSpan, withCurrent, withDataset, withExperiment, withLogger, wrapAISDKModel, wrapOpenAI, wrapOpenAIv4, wrapTraced };

package/dist/index.d.ts

@@ -1,6 +1,7 @@
 import { LogFeedbackFullArgs, ExperimentEvent, BackgroundLogEvent, ExperimentLogFullArgs, ExperimentLogPartialArgs, IdField, SpanType, SpanComponentsV3, DEFAULT_IS_LEGACY_DATASET, TRANSACTION_ID_FIELD, TransactionId, SpanObjectTypeV3, DatasetRecord, Score, CommentEvent, InputField, LogCommentFullArgs, OtherExperimentLogFields, ParentExperimentIds, ParentProjectLogIds } from '@braintrust/core';
 export { CommentEvent, DatasetRecord, ExperimentLogFullArgs, ExperimentLogPartialArgs, IdField, InputField, LogCommentFullArgs, LogFeedbackFullArgs, OtherExperimentLogFields, ParentExperimentIds, ParentProjectLogIds } from '@braintrust/core';
-import { GitMetadataSettings, AttachmentReference, AttachmentStatus, RepoInfo, PromptData, OpenAIMessage, Tools, AnyModelParam, Message, Prompt as Prompt$1, PromptSessionEvent, StreamingMode, FunctionType, IfExists, SavedFunctionId, ModelParams } from '@braintrust/core/typespecs';
+import { GitMetadataSettings, AttachmentReference, AttachmentStatus, RepoInfo, PromptData, OpenAIMessage, Tools, AnyModelParam, Message, Prompt as Prompt$1, PromptSessionEvent, StreamingMode, FunctionType, IfExists, SavedFunctionId, ModelParams, ToolFunctionDefinition, toolFunctionDefinitionSchema } from '@braintrust/core/typespecs';
+export { ToolFunctionDefinition, toolFunctionDefinitionSchema } from '@braintrust/core/typespecs';
 import { z } from 'zod';
 
 interface IsoAsyncLocalStorage<T> {
@@ -1572,13 +1573,20 @@ declare class Project {
     readonly id?: string;
     tools: ToolBuilder;
     prompts: PromptBuilder;
+    scorers: ScorerBuilder;
     constructor(args: CreateProjectOpts);
 }
 declare class ToolBuilder {
     private readonly project;
     private taskCounter;
     constructor(project: Project);
-    create<Input, Output, Fn extends GenericFunction<Input, Output>>(opts: ToolOpts<Input, Output, Fn>): CodeFunction<Input, Output, Fn>;
+    create<Input, Output, Fn extends GenericFunction<Input, Output>>(opts: CodeOpts<Input, Output, Fn>): CodeFunction<Input, Output, Fn>;
+}
+declare class ScorerBuilder {
+    private readonly project;
+    private taskCounter;
+    constructor(project: Project);
+    create<Output, Input, Params, Returns, Fn extends GenericFunction<Exact<Params, ScorerArgs<Output, Input>>, Returns>>(opts: ScorerOpts<Output, Input, Params, Returns, Fn>): void;
 }
 type Schema<Input, Output> = Partial<{
     parameters: z.ZodSchema<Input>;
@@ -1590,9 +1598,21 @@ interface BaseFnOpts {
     description: string;
     ifExists: IfExists;
 }
-type ToolOpts<Params, Returns, Fn extends GenericFunction<Params, Returns>> = Partial<BaseFnOpts> & {
+type CodeOpts<Params, Returns, Fn extends GenericFunction<Params, Returns>> = Partial<BaseFnOpts> & {
     handler: Fn;
 } & Schema<Params, Returns>;
+type ScorerPromptOpts = Partial<BaseFnOpts> & PromptOpts<false, false, false, false> & {
+    useCot: boolean;
+    choiceScores: Record<string, number>;
+};
+type ScorerArgs<Output, Input> = {
+    output: Output;
+    expected?: Output;
+    input?: Input;
+    metadata?: Record<string, unknown>;
+};
+type Exact<T, Shape> = T extends Shape ? Exclude<keyof T, keyof Shape> extends never ? T : never : never;
+type ScorerOpts<Output, Input, Params, Returns, Fn extends GenericFunction<Exact<Params, ScorerArgs<Output, Input>>, Returns>> = CodeOpts<Exact<Params, ScorerArgs<Output, Input>>, Returns, Fn> | ScorerPromptOpts;
 declare class CodeFunction<Input, Output, Fn extends GenericFunction<Input, Output>> {
     readonly project: Project;
     readonly handler: Fn;
@@ -1603,7 +1623,7 @@ declare class CodeFunction<Input, Output, Fn extends GenericFunction<Input, Outp
     readonly parameters?: z.ZodSchema<Input>;
     readonly returns?: z.ZodSchema<Output>;
     readonly ifExists?: IfExists;
-    constructor(project: Project, opts: Omit<ToolOpts<Input, Output, Fn>, "name" | "slug"> & {
+    constructor(project: Project, opts: Omit<CodeOpts<Input, Output, Fn>, "name" | "slug"> & {
         name: string;
         slug: string;
         type: FunctionType;
@@ -1619,66 +1639,35 @@ declare class CodePrompt {
     readonly ifExists?: IfExists;
     readonly description?: string;
     readonly id?: string;
+    readonly functionType?: FunctionType;
     readonly toolFunctions: (SavedFunctionId | GenericCodeFunction)[];
-    constructor(project: Project, prompt: PromptData, toolFunctions: (SavedFunctionId | GenericCodeFunction)[], opts: Omit<PromptOpts<false, false>, "name" | "slug"> & {
+    constructor(project: Project, prompt: PromptData, toolFunctions: (SavedFunctionId | GenericCodeFunction)[], opts: Omit<PromptOpts<false, false, false, false>, "name" | "slug"> & {
         name: string;
         slug: string;
-    });
+    }, functionType?: FunctionType);
 }
-declare const toolFunctionDefinitionSchema: z.ZodObject<{
-    type: z.ZodLiteral<"function">;
-    function: z.ZodObject<{
-        name: z.ZodString;
-        description: z.ZodOptional<z.ZodString>;
-        parameters: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
-        strict: z.ZodOptional<z.ZodBoolean>;
-    }, "strip", z.ZodTypeAny, {
-        name: string;
-        description?: string | undefined;
-        parameters?: Record<string, unknown> | undefined;
-        strict?: boolean | undefined;
-    }, {
-        name: string;
-        description?: string | undefined;
-        parameters?: Record<string, unknown> | undefined;
-        strict?: boolean | undefined;
-    }>;
-}, "strip", z.ZodTypeAny, {
-    function: {
-        name: string;
-        description?: string | undefined;
-        parameters?: Record<string, unknown> | undefined;
-        strict?: boolean | undefined;
-    };
-    type: "function";
-}, {
-    function: {
-        name: string;
-        description?: string | undefined;
-        parameters?: Record<string, unknown> | undefined;
-        strict?: boolean | undefined;
-    };
-    type: "function";
-}>;
-type ToolFunctionDefinition = z.infer<typeof toolFunctionDefinitionSchema>;
 interface PromptId {
     id: string;
 }
 interface PromptVersion {
     version: TransactionId;
 }
+interface PromptTools {
+    tools: (GenericCodeFunction | SavedFunctionId | ToolFunctionDefinition)[];
+}
+interface PromptNoTrace {
+    noTrace: boolean;
+}
 type PromptContents = {
     prompt: string;
 } | {
     messages: Message[];
 };
-type PromptOpts<HasId extends boolean, HasVersion extends boolean> = (Partial<Omit<BaseFnOpts, "name">> & {
+type PromptOpts<HasId extends boolean, HasVersion extends boolean, HasTools extends boolean = true, HasNoTrace extends boolean = true> = (Partial<Omit<BaseFnOpts, "name">> & {
     name: string;
-}) & (HasId extends true ? PromptId : Partial<PromptId>) & (HasVersion extends true ? PromptVersion : Partial<PromptVersion>) & PromptContents & {
+}) & (HasId extends true ? PromptId : Partial<PromptId>) & (HasVersion extends true ? PromptVersion : Partial<PromptVersion>) & (HasTools extends true ? Partial<PromptTools> : {}) & (HasNoTrace extends true ? Partial<PromptNoTrace> : {}) & PromptContents & {
     model: string;
     params?: ModelParams;
-    tools?: (GenericCodeFunction | SavedFunctionId | ToolFunctionDefinition)[];
-    noTrace?: boolean;
 };
 declare class PromptBuilder {
     private readonly project;
@@ -1707,7 +1696,14 @@ declare function BaseExperiment<Input = unknown, Expected = unknown, Metadata ex
 type EvalData<Input, Expected, Metadata extends BaseMetadata = DefaultMetadataType> = EvalCase<Input, Expected, Metadata>[] | (() => EvalCase<Input, Expected, Metadata>[]) | Promise<EvalCase<Input, Expected, Metadata>[]> | (() => Promise<EvalCase<Input, Expected, Metadata>[]>) | AsyncGenerator<EvalCase<Input, Expected, Metadata>> | AsyncIterable<EvalCase<Input, Expected, Metadata>> | BaseExperiment<Input, Expected, Metadata> | (() => BaseExperiment<Input, Expected, Metadata>);
 type EvalTask<Input, Output> = ((input: Input, hooks: EvalHooks) => Promise<Output>) | ((input: Input, hooks: EvalHooks) => Output);
 interface EvalHooks {
+    /**
+     * @deprecated Use `metadata` instead.
+     */
     meta: (info: Record<string, unknown>) => void;
+    /**
+     * The metadata object for the current evaluation. You can mutate this object to add or remove metadata.
+     */
+    metadata: Record<string, unknown>;
     span: Span;
 }
 type EvalScorerArgs<Input, Output, Expected, Metadata extends BaseMetadata = DefaultMetadataType> = EvalCase<Input, Expected, Metadata> & {
@@ -1923,6 +1919,7 @@ type braintrust_BraintrustStreamChunk = BraintrustStreamChunk;
 type braintrust_ChatPrompt = ChatPrompt;
 type braintrust_CodeFunction<Input, Output, Fn extends GenericFunction<Input, Output>> = CodeFunction<Input, Output, Fn>;
 declare const braintrust_CodeFunction: typeof CodeFunction;
+type braintrust_CodeOpts<Params, Returns, Fn extends GenericFunction<Params, Returns>> = CodeOpts<Params, Returns, Fn>;
 type braintrust_CodePrompt = CodePrompt;
 declare const braintrust_CodePrompt: typeof CodePrompt;
 declare const braintrust_CommentEvent: typeof CommentEvent;
@@ -1986,7 +1983,7 @@ type braintrust_Prompt<HasId extends boolean = true, HasVersion extends boolean
 declare const braintrust_Prompt: typeof Prompt;
 type braintrust_PromptBuilder = PromptBuilder;
 declare const braintrust_PromptBuilder: typeof PromptBuilder;
-type braintrust_PromptOpts<HasId extends boolean, HasVersion extends boolean> = PromptOpts<HasId, HasVersion>;
+type braintrust_PromptOpts<HasId extends boolean, HasVersion extends boolean, HasTools extends boolean = true, HasNoTrace extends boolean = true> = PromptOpts<HasId, HasVersion, HasTools, HasNoTrace>;
 type braintrust_PromptRowWithId<HasId extends boolean = true, HasVersion extends boolean = true> = PromptRowWithId<HasId, HasVersion>;
 type braintrust_ReadonlyAttachment = ReadonlyAttachment;
 declare const braintrust_ReadonlyAttachment: typeof ReadonlyAttachment;
@@ -1995,6 +1992,9 @@ declare const braintrust_ReadonlyExperiment: typeof ReadonlyExperiment;
 declare const braintrust_Reporter: typeof Reporter;
 type braintrust_ReporterBody<EvalReport> = ReporterBody<EvalReport>;
 type braintrust_ScoreSummary = ScoreSummary;
+type braintrust_ScorerBuilder = ScorerBuilder;
+declare const braintrust_ScorerBuilder: typeof ScorerBuilder;
+type braintrust_ScorerOpts<Output, Input, Params, Returns, Fn extends GenericFunction<Exact<Params, ScorerArgs<Output, Input>>, Returns>> = ScorerOpts<Output, Input, Params, Returns, Fn>;
 type braintrust_SerializedBraintrustState = SerializedBraintrustState;
 type braintrust_SetCurrentArg = SetCurrentArg;
 type braintrust_Span = Span;
@@ -2004,8 +2004,7 @@ declare const braintrust_SpanImpl: typeof SpanImpl;
 type braintrust_StartSpanArgs = StartSpanArgs;
 type braintrust_ToolBuilder = ToolBuilder;
 declare const braintrust_ToolBuilder: typeof ToolBuilder;
-type braintrust_ToolFunctionDefinition = ToolFunctionDefinition;
-type braintrust_ToolOpts<Params, Returns, Fn extends GenericFunction<Params, Returns>> = ToolOpts<Params, Returns, Fn>;
+declare const braintrust_ToolFunctionDefinition: typeof ToolFunctionDefinition;
 type braintrust_WithTransactionId<R> = WithTransactionId<R>;
 declare const braintrust_X_CACHED_HEADER: typeof X_CACHED_HEADER;
 declare const braintrust__exportsForTestingOnly: typeof _exportsForTestingOnly;
@@ -2053,7 +2052,7 @@ declare const braintrust_wrapOpenAI: typeof wrapOpenAI;
 declare const braintrust_wrapOpenAIv4: typeof wrapOpenAIv4;
 declare const braintrust_wrapTraced: typeof wrapTraced;
 declare namespace braintrust {
-  export { type braintrust_AnyDataset as AnyDataset, braintrust_Attachment as Attachment, type braintrust_AttachmentParams as AttachmentParams, type braintrust_BackgroundLoggerOpts as BackgroundLoggerOpts, braintrust_BaseExperiment as BaseExperiment, type braintrust_BaseMetadata as BaseMetadata, braintrust_BraintrustState as BraintrustState, braintrust_BraintrustStream as BraintrustStream, type braintrust_BraintrustStreamChunk as BraintrustStreamChunk, type braintrust_ChatPrompt as ChatPrompt, braintrust_CodeFunction as CodeFunction, braintrust_CodePrompt as CodePrompt, braintrust_CommentEvent as CommentEvent, type braintrust_CompiledPrompt as CompiledPrompt, type braintrust_CompiledPromptParams as CompiledPromptParams, type braintrust_CompletionPrompt as CompletionPrompt, type braintrust_CreateProjectOpts as CreateProjectOpts, type braintrust_DataSummary as DataSummary, braintrust_Dataset as Dataset, braintrust_DatasetRecord as DatasetRecord, type braintrust_DatasetSummary as DatasetSummary, type braintrust_DefaultMetadataType as DefaultMetadataType, type braintrust_DefaultPromptArgs as DefaultPromptArgs, type braintrust_EndSpanArgs as EndSpanArgs, braintrust_Eval as Eval, type braintrust_EvalCase as EvalCase, type braintrust_EvalResult as EvalResult, type braintrust_EvalScorer as EvalScorer, type braintrust_EvalScorerArgs as EvalScorerArgs, type braintrust_EvalTask as EvalTask, type braintrust_Evaluator as Evaluator, type braintrust_EvaluatorDef as EvaluatorDef, type braintrust_EvaluatorFile as EvaluatorFile, braintrust_Experiment as Experiment, braintrust_ExperimentLogFullArgs as ExperimentLogFullArgs, braintrust_ExperimentLogPartialArgs as ExperimentLogPartialArgs, type braintrust_ExperimentSummary as ExperimentSummary, type braintrust_Exportable as Exportable, braintrust_FailedHTTPResponse as FailedHTTPResponse, type braintrust_FullInitOptions as FullInitOptions, type braintrust_FullLoginOptions as FullLoginOptions, braintrust_IdField as IdField, type braintrust_InitOptions as InitOptions, braintrust_InputField as InputField, type braintrust_InvokeFunctionArgs as InvokeFunctionArgs, type braintrust_InvokeReturn as InvokeReturn, braintrust_LEGACY_CACHED_HEADER as LEGACY_CACHED_HEADER, braintrust_LazyValue as LazyValue, braintrust_LogCommentFullArgs as LogCommentFullArgs, braintrust_LogFeedbackFullArgs as LogFeedbackFullArgs, type braintrust_LogOptions as LogOptions, braintrust_Logger as Logger, type braintrust_LoginOptions as LoginOptions, type braintrust_MetricSummary as MetricSummary, braintrust_NOOP_SPAN as NOOP_SPAN, braintrust_NoopSpan as NoopSpan, type braintrust_ObjectMetadata as ObjectMetadata, braintrust_OtherExperimentLogFields as OtherExperimentLogFields, braintrust_ParentExperimentIds as ParentExperimentIds, braintrust_ParentProjectLogIds as ParentProjectLogIds, braintrust_Project as Project, type braintrust_PromiseUnless as PromiseUnless, braintrust_Prompt as Prompt, braintrust_PromptBuilder as PromptBuilder, type braintrust_PromptOpts as PromptOpts, type braintrust_PromptRowWithId as PromptRowWithId, braintrust_ReadonlyAttachment as ReadonlyAttachment, braintrust_ReadonlyExperiment as ReadonlyExperiment, braintrust_Reporter as Reporter, type braintrust_ReporterBody as ReporterBody, type braintrust_ScoreSummary as ScoreSummary, type braintrust_SerializedBraintrustState as SerializedBraintrustState, type braintrust_SetCurrentArg as SetCurrentArg, type braintrust_Span as Span, type braintrust_SpanContext as SpanContext, braintrust_SpanImpl as SpanImpl, type braintrust_StartSpanArgs as StartSpanArgs, braintrust_ToolBuilder as ToolBuilder, type braintrust_ToolFunctionDefinition as ToolFunctionDefinition, type braintrust_ToolOpts as ToolOpts, type braintrust_WithTransactionId as WithTransactionId, braintrust_X_CACHED_HEADER as X_CACHED_HEADER, braintrust__exportsForTestingOnly as _exportsForTestingOnly, braintrust__internalGetGlobalState as _internalGetGlobalState, braintrust__internalSetInitialState as _internalSetInitialState, braintrust_braintrustStreamChunkSchema as braintrustStreamChunkSchema, braintrust_buildLocalSummary as buildLocalSummary, braintrust_createFinalValuePassThroughStream as createFinalValuePassThroughStream, braintrust_currentExperiment as currentExperiment, braintrust_currentLogger as currentLogger, braintrust_currentSpan as currentSpan, braintrust_devNullWritableStream as devNullWritableStream, braintrust_flush as flush, braintrust_getSpanParentObject as getSpanParentObject, braintrust_init as init, braintrust_initDataset as initDataset, braintrust_initExperiment as initExperiment, braintrust_initLogger as initLogger, braintrust_invoke as invoke, braintrust_loadPrompt as loadPrompt, braintrust_log as log, braintrust_logError as logError, braintrust_login as login, braintrust_loginToState as loginToState, braintrust_newId as newId, braintrust_parseCachedHeader as parseCachedHeader, braintrust_permalink as permalink, braintrust_projects as projects, braintrust_renderMessage as renderMessage, braintrust_reportFailures as reportFailures, braintrust_setFetch as setFetch, braintrust_spanComponentsToObjectId as spanComponentsToObjectId, braintrust_startSpan as startSpan, braintrust_summarize as summarize, braintrust_toolFunctionDefinitionSchema as toolFunctionDefinitionSchema, braintrust_traceable as traceable, braintrust_traced as traced, braintrust_updateSpan as updateSpan, braintrust_withCurrent as withCurrent, braintrust_withDataset as withDataset, braintrust_withExperiment as withExperiment, braintrust_withLogger as withLogger, braintrust_wrapAISDKModel as wrapAISDKModel, braintrust_wrapOpenAI as wrapOpenAI, braintrust_wrapOpenAIv4 as wrapOpenAIv4, braintrust_wrapTraced as wrapTraced };
+  export { type braintrust_AnyDataset as AnyDataset, braintrust_Attachment as Attachment, type braintrust_AttachmentParams as AttachmentParams, type braintrust_BackgroundLoggerOpts as BackgroundLoggerOpts, braintrust_BaseExperiment as BaseExperiment, type braintrust_BaseMetadata as BaseMetadata, braintrust_BraintrustState as BraintrustState, braintrust_BraintrustStream as BraintrustStream, type braintrust_BraintrustStreamChunk as BraintrustStreamChunk, type braintrust_ChatPrompt as ChatPrompt, braintrust_CodeFunction as CodeFunction, type braintrust_CodeOpts as CodeOpts, braintrust_CodePrompt as CodePrompt, braintrust_CommentEvent as CommentEvent, type braintrust_CompiledPrompt as CompiledPrompt, type braintrust_CompiledPromptParams as CompiledPromptParams, type braintrust_CompletionPrompt as CompletionPrompt, type braintrust_CreateProjectOpts as CreateProjectOpts, type braintrust_DataSummary as DataSummary, braintrust_Dataset as Dataset, braintrust_DatasetRecord as DatasetRecord, type braintrust_DatasetSummary as DatasetSummary, type braintrust_DefaultMetadataType as DefaultMetadataType, type braintrust_DefaultPromptArgs as DefaultPromptArgs, type braintrust_EndSpanArgs as EndSpanArgs, braintrust_Eval as Eval, type braintrust_EvalCase as EvalCase, type braintrust_EvalResult as EvalResult, type braintrust_EvalScorer as EvalScorer, type braintrust_EvalScorerArgs as EvalScorerArgs, type braintrust_EvalTask as EvalTask, type braintrust_Evaluator as Evaluator, type braintrust_EvaluatorDef as EvaluatorDef, type braintrust_EvaluatorFile as EvaluatorFile, braintrust_Experiment as Experiment, braintrust_ExperimentLogFullArgs as ExperimentLogFullArgs, braintrust_ExperimentLogPartialArgs as ExperimentLogPartialArgs, type braintrust_ExperimentSummary as ExperimentSummary, type braintrust_Exportable as Exportable, braintrust_FailedHTTPResponse as FailedHTTPResponse, type braintrust_FullInitOptions as FullInitOptions, type braintrust_FullLoginOptions as FullLoginOptions, braintrust_IdField as IdField, type braintrust_InitOptions as InitOptions, braintrust_InputField as InputField, type braintrust_InvokeFunctionArgs as InvokeFunctionArgs, type braintrust_InvokeReturn as InvokeReturn, braintrust_LEGACY_CACHED_HEADER as LEGACY_CACHED_HEADER, braintrust_LazyValue as LazyValue, braintrust_LogCommentFullArgs as LogCommentFullArgs, braintrust_LogFeedbackFullArgs as LogFeedbackFullArgs, type braintrust_LogOptions as LogOptions, braintrust_Logger as Logger, type braintrust_LoginOptions as LoginOptions, type braintrust_MetricSummary as MetricSummary, braintrust_NOOP_SPAN as NOOP_SPAN, braintrust_NoopSpan as NoopSpan, type braintrust_ObjectMetadata as ObjectMetadata, braintrust_OtherExperimentLogFields as OtherExperimentLogFields, braintrust_ParentExperimentIds as ParentExperimentIds, braintrust_ParentProjectLogIds as ParentProjectLogIds, braintrust_Project as Project, type braintrust_PromiseUnless as PromiseUnless, braintrust_Prompt as Prompt, braintrust_PromptBuilder as PromptBuilder, type braintrust_PromptOpts as PromptOpts, type braintrust_PromptRowWithId as PromptRowWithId, braintrust_ReadonlyAttachment as ReadonlyAttachment, braintrust_ReadonlyExperiment as ReadonlyExperiment, braintrust_Reporter as Reporter, type braintrust_ReporterBody as ReporterBody, type braintrust_ScoreSummary as ScoreSummary, braintrust_ScorerBuilder as ScorerBuilder, type braintrust_ScorerOpts as ScorerOpts, type braintrust_SerializedBraintrustState as SerializedBraintrustState, type braintrust_SetCurrentArg as SetCurrentArg, type braintrust_Span as Span, type braintrust_SpanContext as SpanContext, braintrust_SpanImpl as SpanImpl, type braintrust_StartSpanArgs as StartSpanArgs, braintrust_ToolBuilder as ToolBuilder, braintrust_ToolFunctionDefinition as ToolFunctionDefinition, type braintrust_WithTransactionId as WithTransactionId, braintrust_X_CACHED_HEADER as X_CACHED_HEADER, braintrust__exportsForTestingOnly as _exportsForTestingOnly, braintrust__internalGetGlobalState as _internalGetGlobalState, braintrust__internalSetInitialState as _internalSetInitialState, braintrust_braintrustStreamChunkSchema as braintrustStreamChunkSchema, braintrust_buildLocalSummary as buildLocalSummary, braintrust_createFinalValuePassThroughStream as createFinalValuePassThroughStream, braintrust_currentExperiment as currentExperiment, braintrust_currentLogger as currentLogger, braintrust_currentSpan as currentSpan, braintrust_devNullWritableStream as devNullWritableStream, braintrust_flush as flush, braintrust_getSpanParentObject as getSpanParentObject, braintrust_init as init, braintrust_initDataset as initDataset, braintrust_initExperiment as initExperiment, braintrust_initLogger as initLogger, braintrust_invoke as invoke, braintrust_loadPrompt as loadPrompt, braintrust_log as log, braintrust_logError as logError, braintrust_login as login, braintrust_loginToState as loginToState, braintrust_newId as newId, braintrust_parseCachedHeader as parseCachedHeader, braintrust_permalink as permalink, braintrust_projects as projects, braintrust_renderMessage as renderMessage, braintrust_reportFailures as reportFailures, braintrust_setFetch as setFetch, braintrust_spanComponentsToObjectId as spanComponentsToObjectId, braintrust_startSpan as startSpan, braintrust_summarize as summarize, braintrust_toolFunctionDefinitionSchema as toolFunctionDefinitionSchema, braintrust_traceable as traceable, braintrust_traced as traced, braintrust_updateSpan as updateSpan, braintrust_withCurrent as withCurrent, braintrust_withDataset as withDataset, braintrust_withExperiment as withExperiment, braintrust_withLogger as withLogger, braintrust_wrapAISDKModel as wrapAISDKModel, braintrust_wrapOpenAI as wrapOpenAI, braintrust_wrapOpenAIv4 as wrapOpenAIv4, braintrust_wrapTraced as wrapTraced };
 }
 
 /**
@@ -2109,4 +2108,4 @@ declare namespace braintrust {
  * @module braintrust
  */
 
-export { type AnyDataset, Attachment, type AttachmentParams, type BackgroundLoggerOpts, BaseExperiment, type BaseMetadata, BraintrustState, BraintrustStream, type BraintrustStreamChunk, type ChatPrompt, CodeFunction, CodePrompt, type CompiledPrompt, type CompiledPromptParams, type CompletionPrompt, type CreateProjectOpts, type DataSummary, Dataset, type DatasetSummary, type DefaultMetadataType, type DefaultPromptArgs, type EndSpanArgs, Eval, type EvalCase, type EvalResult, type EvalScorer, type EvalScorerArgs, type EvalTask, type Evaluator, type EvaluatorDef, type EvaluatorFile, Experiment, type ExperimentSummary, type Exportable, FailedHTTPResponse, type FullInitOptions, type FullLoginOptions, type InitOptions, type InvokeFunctionArgs, type InvokeReturn, LEGACY_CACHED_HEADER, LazyValue, type LogOptions, Logger, type LoginOptions, type MetricSummary, NOOP_SPAN, NoopSpan, type ObjectMetadata, Project, type PromiseUnless, Prompt, PromptBuilder, type PromptOpts, type PromptRowWithId, ReadonlyAttachment, ReadonlyExperiment, Reporter, type ReporterBody, type ScoreSummary, type SerializedBraintrustState, type SetCurrentArg, type Span, type SpanContext, SpanImpl, type StartSpanArgs, ToolBuilder, type ToolFunctionDefinition, type ToolOpts, type WithTransactionId, X_CACHED_HEADER, _exportsForTestingOnly, _internalGetGlobalState, _internalSetInitialState, braintrustStreamChunkSchema, buildLocalSummary, createFinalValuePassThroughStream, currentExperiment, currentLogger, currentSpan, braintrust as default, devNullWritableStream, flush, getSpanParentObject, init, initDataset, initExperiment, initLogger, invoke, loadPrompt, log, logError, login, loginToState, newId, parseCachedHeader, permalink, projects, renderMessage, reportFailures, setFetch, spanComponentsToObjectId, startSpan, summarize, toolFunctionDefinitionSchema, traceable, traced, updateSpan, withCurrent, withDataset, withExperiment, withLogger, wrapAISDKModel, wrapOpenAI, wrapOpenAIv4, wrapTraced };
+export { type AnyDataset, Attachment, type AttachmentParams, type BackgroundLoggerOpts, BaseExperiment, type BaseMetadata, BraintrustState, BraintrustStream, type BraintrustStreamChunk, type ChatPrompt, CodeFunction, type CodeOpts, CodePrompt, type CompiledPrompt, type CompiledPromptParams, type CompletionPrompt, type CreateProjectOpts, type DataSummary, Dataset, type DatasetSummary, type DefaultMetadataType, type DefaultPromptArgs, type EndSpanArgs, Eval, type EvalCase, type EvalResult, type EvalScorer, type EvalScorerArgs, type EvalTask, type Evaluator, type EvaluatorDef, type EvaluatorFile, Experiment, type ExperimentSummary, type Exportable, FailedHTTPResponse, type FullInitOptions, type FullLoginOptions, type InitOptions, type InvokeFunctionArgs, type InvokeReturn, LEGACY_CACHED_HEADER, LazyValue, type LogOptions, Logger, type LoginOptions, type MetricSummary, NOOP_SPAN, NoopSpan, type ObjectMetadata, Project, type PromiseUnless, Prompt, PromptBuilder, type PromptOpts, type PromptRowWithId, ReadonlyAttachment, ReadonlyExperiment, Reporter, type ReporterBody, type ScoreSummary, ScorerBuilder, type ScorerOpts, type SerializedBraintrustState, type SetCurrentArg, type Span, type SpanContext, SpanImpl, type StartSpanArgs, ToolBuilder, type WithTransactionId, X_CACHED_HEADER, _exportsForTestingOnly, _internalGetGlobalState, _internalSetInitialState, braintrustStreamChunkSchema, buildLocalSummary, createFinalValuePassThroughStream, currentExperiment, currentLogger, currentSpan, braintrust as default, devNullWritableStream, flush, getSpanParentObject, init, initDataset, initExperiment, initLogger, invoke, loadPrompt, log, logError, login, loginToState, newId, parseCachedHeader, permalink, projects, renderMessage, reportFailures, setFetch, spanComponentsToObjectId, startSpan, summarize, traceable, traced, updateSpan, withCurrent, withDataset, withExperiment, withLogger, wrapAISDKModel, wrapOpenAI, wrapOpenAIv4, wrapTraced };

package/dist/index.js

@@ -51,6 +51,7 @@ __export(src_exports, {
   ReadonlyAttachment: () => ReadonlyAttachment,
   ReadonlyExperiment: () => ReadonlyExperiment,
   Reporter: () => Reporter,
+  ScorerBuilder: () => ScorerBuilder,
   SpanImpl: () => SpanImpl,
   ToolBuilder: () => ToolBuilder,
   X_CACHED_HEADER: () => X_CACHED_HEADER,
@@ -87,7 +88,7 @@ __export(src_exports, {
   spanComponentsToObjectId: () => spanComponentsToObjectId,
   startSpan: () => startSpan,
   summarize: () => summarize,
-  toolFunctionDefinitionSchema: () => toolFunctionDefinitionSchema,
+  toolFunctionDefinitionSchema: () => import_typespecs4.toolFunctionDefinitionSchema,
   traceable: () => traceable,
   traced: () => traced,
   updateSpan: () => updateSpan,
@@ -3656,6 +3657,21 @@ function renderMessage(render, message) {
             return _exhaustiveCheck;
         }
       })
+    } : {},
+    ..."tool_calls" in message ? {
+      tool_calls: isEmpty(message.tool_calls) ? void 0 : message.tool_calls.map((t) => {
+        return {
+          type: t.type,
+          id: render(t.id),
+          function: {
+            name: render(t.function.name),
+            arguments: render(t.function.arguments)
+          }
+        };
+      })
+    } : {},
+    ..."tool_call_id" in message ? {
+      tool_call_id: render(message.tool_call_id)
     } : {}
   };
 }
@@ -3834,6 +3850,7 @@ __export(exports_node_exports, {
   ReadonlyAttachment: () => ReadonlyAttachment,
   ReadonlyExperiment: () => ReadonlyExperiment,
   Reporter: () => Reporter,
+  ScorerBuilder: () => ScorerBuilder,
   SpanImpl: () => SpanImpl,
   ToolBuilder: () => ToolBuilder,
   X_CACHED_HEADER: () => X_CACHED_HEADER,
@@ -3869,7 +3886,7 @@ __export(exports_node_exports, {
   spanComponentsToObjectId: () => spanComponentsToObjectId,
   startSpan: () => startSpan,
   summarize: () => summarize,
-  toolFunctionDefinitionSchema: () => toolFunctionDefinitionSchema,
+  toolFunctionDefinitionSchema: () => import_typespecs4.toolFunctionDefinitionSchema,
   traceable: () => traceable,
   traced: () => traced,
   updateSpan: () => updateSpan,
@@ -3948,49 +3965,8 @@ async function invoke(args) {
 }
 
 // src/framework.ts
-var import_chalk = __toESM(require("chalk"));
 var import_core2 = require("@braintrust/core");
 
-// src/progress.ts
-var cliProgress = __toESM(require("cli-progress"));
-var MAX_NAME_LENGTH = 40;
-function fitNameToSpaces(name, length) {
-  const padded = name.padEnd(length);
-  if (padded.length <= length) {
-    return padded;
-  }
-  return padded.substring(0, length - 3) + "...";
-}
-var BarProgressReporter = class {
-  multiBar;
-  bars = {};
-  constructor() {
-    this.multiBar = new cliProgress.MultiBar(
-      {
-        clearOnComplete: false,
-        format: " {bar} | {evaluator} | {percentage}% | {value}/{total} datapoints",
-        autopadding: true
-      },
-      cliProgress.Presets.shades_grey
-    );
-  }
-  start(name, total) {
-    const bar = this.multiBar.create(total, 0);
-    this.bars[name] = bar;
-  }
-  stop() {
-    this.multiBar.stop();
-  }
-  increment(name) {
-    this.bars[name].increment({
-      evaluator: fitNameToSpaces(name, MAX_NAME_LENGTH)
-    });
-  }
-};
-
-// src/framework.ts
-var import_pluralize = __toESM(require("pluralize"));
-
 // ../../node_modules/.pnpm/async@3.2.5/node_modules/async/dist/async.mjs
 function initialParams(fn) {
   return function(...args) {
@@ -5094,6 +5070,47 @@ function waterfall(tasks, callback) {
 }
 var waterfall$1 = awaitify(waterfall);
 
+// src/framework.ts
+var import_chalk = __toESM(require("chalk"));
+var import_pluralize = __toESM(require("pluralize"));
+
+// src/progress.ts
+var cliProgress = __toESM(require("cli-progress"));
+var MAX_NAME_LENGTH = 40;
+function fitNameToSpaces(name, length) {
+  const padded = name.padEnd(length);
+  if (padded.length <= length) {
+    return padded;
+  }
+  return padded.substring(0, length - 3) + "...";
+}
+var BarProgressReporter = class {
+  multiBar;
+  bars = {};
+  constructor() {
+    this.multiBar = new cliProgress.MultiBar(
+      {
+        clearOnComplete: false,
+        format: " {bar} | {evaluator} | {percentage}% | {value}/{total} datapoints",
+        autopadding: true
+      },
+      cliProgress.Presets.shades_grey
+    );
+  }
+  start(name, total) {
+    const bar = this.multiBar.create(total, 0);
+    this.bars[name] = bar;
+  }
+  stop() {
+    this.multiBar.stop();
+  }
+  increment(name) {
+    this.bars[name].increment({
+      evaluator: fitNameToSpaces(name, MAX_NAME_LENGTH)
+    });
+  }
+};
+
 // src/framework.ts
 function BaseExperiment(options = {}) {
   return { _type: "BaseExperiment", ...options };
@@ -5339,7 +5356,11 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
           const meta = (o) => metadata = { ...metadata, ...o };
           await rootSpan.traced(
             async (span) => {
-              const outputResult = evaluator.task(datum.input, { meta, span });
+              const outputResult = evaluator.task(datum.input, {
+                meta,
+                metadata,
+                span
+              });
               if (outputResult instanceof Promise) {
                 output = await outputResult;
               } else {
@@ -5622,7 +5643,7 @@ function formatMetricSummary(summary, longestMetricName) {
 // src/framework2.ts
 var import_path = __toESM(require("path"));
 var import_slugify = __toESM(require("slugify"));
-var import_zod3 = require("zod");
+var import_typespecs4 = require("@braintrust/core/typespecs");
 var ProjectBuilder = class {
   create(opts) {
     return new Project(opts);
@@ -5634,12 +5655,14 @@ var Project = class {
   id;
   tools;
   prompts;
+  scorers;
   constructor(args) {
     _initializeSpanContext();
     this.name = "name" in args ? args.name : void 0;
     this.id = "id" in args ? args.id : void 0;
     this.tools = new ToolBuilder(this);
     this.prompts = new PromptBuilder(this);
+    this.scorers = new ScorerBuilder(this);
   }
 };
 var ToolBuilder = class {
@@ -5673,6 +5696,70 @@ var ToolBuilder = class {
     return tool;
   }
 };
+var ScorerBuilder = class {
+  constructor(project) {
+    this.project = project;
+  }
+  taskCounter = 0;
+  create(opts) {
+    this.taskCounter++;
+    let resolvedName = opts.name;
+    if (!resolvedName && "handler" in opts) {
+      resolvedName = opts.handler.name;
+    }
+    if (!resolvedName || resolvedName.trim().length === 0) {
+      resolvedName = `Scorer ${import_path.default.basename(__filename)} ${this.taskCounter}`;
+    }
+    const slug = opts.slug ?? (0, import_slugify.default)(resolvedName, { lower: true, strict: true });
+    if ("handler" in opts) {
+      const scorer = new CodeFunction(this.project, {
+        ...opts,
+        name: resolvedName,
+        slug,
+        type: "scorer"
+      });
+      if (globalThis._lazy_load) {
+        globalThis._evals.functions.push(
+          scorer
+        );
+      }
+    } else {
+      const promptBlock = "messages" in opts ? {
+        type: "chat",
+        messages: opts.messages
+      } : {
+        type: "completion",
+        content: opts.prompt
+      };
+      const promptData = {
+        prompt: promptBlock,
+        options: {
+          model: opts.model,
+          params: opts.params
+        },
+        parser: {
+          type: "llm_classifier",
+          use_cot: opts.useCot,
+          choice_scores: opts.choiceScores
+        }
+      };
+      const codePrompt = new CodePrompt(
+        this.project,
+        promptData,
+        [],
+        {
+          ...opts,
+          name: resolvedName,
+          slug
+        },
+        "scorer"
+      );
+      if (globalThis._lazy_load) {
+        globalThis._evals.prompts.push(codePrompt);
+      }
+    }
+  }
+};
 var CodeFunction = class {
   constructor(project, opts) {
     this.project = project;
@@ -5712,8 +5799,9 @@ var CodePrompt = class {
   ifExists;
   description;
   id;
+  functionType;
   toolFunctions;
-  constructor(project, prompt, toolFunctions, opts) {
+  constructor(project, prompt, toolFunctions, opts, functionType) {
     this.project = project;
     this.name = opts.name;
     this.slug = opts.slug;
@@ -5722,17 +5810,9 @@ var CodePrompt = class {
     this.ifExists = opts.ifExists;
     this.description = opts.description;
     this.id = opts.id;
+    this.functionType = functionType;
   }
 };
-var toolFunctionDefinitionSchema = import_zod3.z.object({
-  type: import_zod3.z.literal("function"),
-  function: import_zod3.z.object({
-    name: import_zod3.z.string(),
-    description: import_zod3.z.string().optional(),
-    parameters: import_zod3.z.record(import_zod3.z.unknown()).optional(),
-    strict: import_zod3.z.boolean().optional()
-  })
-});
 var PromptBuilder = class {
   constructor(project) {
     this.project = project;
@@ -6456,6 +6536,7 @@ var src_default = exports_node_exports;
   ReadonlyAttachment,
   ReadonlyExperiment,
   Reporter,
+  ScorerBuilder,
   SpanImpl,
   ToolBuilder,
   X_CACHED_HEADER,

package/dist/index.mjs

@@ -3592,6 +3592,21 @@ function renderMessage(render, message) {
             return _exhaustiveCheck;
         }
       })
+    } : {},
+    ..."tool_calls" in message ? {
+      tool_calls: isEmpty(message.tool_calls) ? void 0 : message.tool_calls.map((t) => {
+        return {
+          type: t.type,
+          id: render(t.id),
+          function: {
+            name: render(t.function.name),
+            arguments: render(t.function.arguments)
+          }
+        };
+      })
+    } : {},
+    ..."tool_call_id" in message ? {
+      tool_call_id: render(message.tool_call_id)
     } : {}
   };
 }
@@ -3770,6 +3785,7 @@ __export(exports_node_exports, {
   ReadonlyAttachment: () => ReadonlyAttachment,
   ReadonlyExperiment: () => ReadonlyExperiment,
   Reporter: () => Reporter,
+  ScorerBuilder: () => ScorerBuilder,
   SpanImpl: () => SpanImpl,
   ToolBuilder: () => ToolBuilder,
   X_CACHED_HEADER: () => X_CACHED_HEADER,
@@ -3886,49 +3902,8 @@ async function invoke(args) {
 }
 
 // src/framework.ts
-import chalk from "chalk";
 import { SpanTypeAttribute as SpanTypeAttribute2, mergeDicts as mergeDicts2 } from "@braintrust/core";
 
-// src/progress.ts
-import * as cliProgress from "cli-progress";
-var MAX_NAME_LENGTH = 40;
-function fitNameToSpaces(name, length) {
-  const padded = name.padEnd(length);
-  if (padded.length <= length) {
-    return padded;
-  }
-  return padded.substring(0, length - 3) + "...";
-}
-var BarProgressReporter = class {
-  multiBar;
-  bars = {};
-  constructor() {
-    this.multiBar = new cliProgress.MultiBar(
-      {
-        clearOnComplete: false,
-        format: " {bar} | {evaluator} | {percentage}% | {value}/{total} datapoints",
-        autopadding: true
-      },
-      cliProgress.Presets.shades_grey
-    );
-  }
-  start(name, total) {
-    const bar = this.multiBar.create(total, 0);
-    this.bars[name] = bar;
-  }
-  stop() {
-    this.multiBar.stop();
-  }
-  increment(name) {
-    this.bars[name].increment({
-      evaluator: fitNameToSpaces(name, MAX_NAME_LENGTH)
-    });
-  }
-};
-
-// src/framework.ts
-import pluralize from "pluralize";
-
 // ../../node_modules/.pnpm/async@3.2.5/node_modules/async/dist/async.mjs
 function initialParams(fn) {
   return function(...args) {
@@ -5032,6 +5007,47 @@ function waterfall(tasks, callback) {
 }
 var waterfall$1 = awaitify(waterfall);
 
+// src/framework.ts
+import chalk from "chalk";
+import pluralize from "pluralize";
+
+// src/progress.ts
+import * as cliProgress from "cli-progress";
+var MAX_NAME_LENGTH = 40;
+function fitNameToSpaces(name, length) {
+  const padded = name.padEnd(length);
+  if (padded.length <= length) {
+    return padded;
+  }
+  return padded.substring(0, length - 3) + "...";
+}
+var BarProgressReporter = class {
+  multiBar;
+  bars = {};
+  constructor() {
+    this.multiBar = new cliProgress.MultiBar(
+      {
+        clearOnComplete: false,
+        format: " {bar} | {evaluator} | {percentage}% | {value}/{total} datapoints",
+        autopadding: true
+      },
+      cliProgress.Presets.shades_grey
+    );
+  }
+  start(name, total) {
+    const bar = this.multiBar.create(total, 0);
+    this.bars[name] = bar;
+  }
+  stop() {
+    this.multiBar.stop();
+  }
+  increment(name) {
+    this.bars[name].increment({
+      evaluator: fitNameToSpaces(name, MAX_NAME_LENGTH)
+    });
+  }
+};
+
 // src/framework.ts
 function BaseExperiment(options = {}) {
   return { _type: "BaseExperiment", ...options };
@@ -5277,7 +5293,11 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
           const meta = (o) => metadata = { ...metadata, ...o };
           await rootSpan.traced(
             async (span) => {
-              const outputResult = evaluator.task(datum.input, { meta, span });
+              const outputResult = evaluator.task(datum.input, {
+                meta,
+                metadata,
+                span
+              });
               if (outputResult instanceof Promise) {
                 output = await outputResult;
               } else {
@@ -5560,7 +5580,9 @@ function formatMetricSummary(summary, longestMetricName) {
 // src/framework2.ts
 import path2 from "path";
 import slugifyLib from "slugify";
-import { z as z3 } from "zod";
+import {
+  toolFunctionDefinitionSchema
+} from "@braintrust/core/typespecs";
 var ProjectBuilder = class {
   create(opts) {
     return new Project(opts);
@@ -5572,12 +5594,14 @@ var Project = class {
   id;
   tools;
   prompts;
+  scorers;
   constructor(args) {
     _initializeSpanContext();
     this.name = "name" in args ? args.name : void 0;
     this.id = "id" in args ? args.id : void 0;
     this.tools = new ToolBuilder(this);
     this.prompts = new PromptBuilder(this);
+    this.scorers = new ScorerBuilder(this);
   }
 };
 var ToolBuilder = class {
@@ -5611,6 +5635,70 @@ var ToolBuilder = class {
     return tool;
   }
 };
+var ScorerBuilder = class {
+  constructor(project) {
+    this.project = project;
+  }
+  taskCounter = 0;
+  create(opts) {
+    this.taskCounter++;
+    let resolvedName = opts.name;
+    if (!resolvedName && "handler" in opts) {
+      resolvedName = opts.handler.name;
+    }
+    if (!resolvedName || resolvedName.trim().length === 0) {
+      resolvedName = `Scorer ${path2.basename(__filename)} ${this.taskCounter}`;
+    }
+    const slug = opts.slug ?? slugifyLib(resolvedName, { lower: true, strict: true });
+    if ("handler" in opts) {
+      const scorer = new CodeFunction(this.project, {
+        ...opts,
+        name: resolvedName,
+        slug,
+        type: "scorer"
+      });
+      if (globalThis._lazy_load) {
+        globalThis._evals.functions.push(
+          scorer
+        );
+      }
+    } else {
+      const promptBlock = "messages" in opts ? {
+        type: "chat",
+        messages: opts.messages
+      } : {
+        type: "completion",
+        content: opts.prompt
+      };
+      const promptData = {
+        prompt: promptBlock,
+        options: {
+          model: opts.model,
+          params: opts.params
+        },
+        parser: {
+          type: "llm_classifier",
+          use_cot: opts.useCot,
+          choice_scores: opts.choiceScores
+        }
+      };
+      const codePrompt = new CodePrompt(
+        this.project,
+        promptData,
+        [],
+        {
+          ...opts,
+          name: resolvedName,
+          slug
+        },
+        "scorer"
+      );
+      if (globalThis._lazy_load) {
+        globalThis._evals.prompts.push(codePrompt);
+      }
+    }
+  }
+};
 var CodeFunction = class {
   constructor(project, opts) {
     this.project = project;
@@ -5650,8 +5738,9 @@ var CodePrompt = class {
   ifExists;
   description;
   id;
+  functionType;
   toolFunctions;
-  constructor(project, prompt, toolFunctions, opts) {
+  constructor(project, prompt, toolFunctions, opts, functionType) {
     this.project = project;
     this.name = opts.name;
     this.slug = opts.slug;
@@ -5660,17 +5749,9 @@ var CodePrompt = class {
     this.ifExists = opts.ifExists;
     this.description = opts.description;
     this.id = opts.id;
+    this.functionType = functionType;
   }
 };
-var toolFunctionDefinitionSchema = z3.object({
-  type: z3.literal("function"),
-  function: z3.object({
-    name: z3.string(),
-    description: z3.string().optional(),
-    parameters: z3.record(z3.unknown()).optional(),
-    strict: z3.boolean().optional()
-  })
-});
 var PromptBuilder = class {
   constructor(project) {
     this.project = project;
@@ -6393,6 +6474,7 @@ export {
   ReadonlyAttachment,
   ReadonlyExperiment,
   Reporter,
+  ScorerBuilder,
   SpanImpl,
   ToolBuilder,
   X_CACHED_HEADER,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "braintrust",
-  "version": "0.0.175",
+  "version": "0.0.177",
   "description": "SDK for integrating Braintrust",
   "repository": {
     "type": "git",
@@ -75,7 +75,7 @@
   },
   "dependencies": {
     "@ai-sdk/provider": "^1.0.1",
-    "@braintrust/core": "0.0.69",
+    "@braintrust/core": "0.0.71",
     "@next/env": "^14.2.3",
     "@vercel/functions": "^1.0.2",
     "ai": "^3.2.16",