braintrust 0.0.144 → 0.0.146

package/dist/cli.js

@@ -1232,7 +1232,7 @@ var require_package = __commonJS({
   "package.json"(exports2, module2) {
     module2.exports = {
       name: "braintrust",
-      version: "0.0.144",
+      version: "0.0.146",
       description: "SDK for integrating Braintrust",
       repository: {
         type: "git",
@@ -1302,7 +1302,7 @@ var require_package = __commonJS({
       },
       dependencies: {
         "@ai-sdk/provider": "^0.0.11",
-        "@braintrust/core": "0.0.45",
+        "@braintrust/core": "0.0.47",
         "@next/env": "^14.2.3",
         "@vercel/functions": "^1.0.2",
         ai: "^3.2.16",
@@ -4872,6 +4872,17 @@ function initExperiment(state, options = {}) {
     setCurrent: false
   });
 }
+function callEvaluatorData(data) {
+  let dataResult = typeof data === "function" ? data() : data;
+  let baseExperiment = void 0;
+  if ("_type" in dataResult && dataResult._type === "BaseExperiment") {
+    baseExperiment = dataResult.name;
+  }
+  return {
+    data: dataResult,
+    baseExperiment
+  };
+}
 globalThis._evals = {
   evaluators: {},
   reporters: {}
@@ -5705,10 +5716,11 @@ function evaluateBuildResults(inFile, buildResult) {
     return { ...globalThis._evals };
   });
 }
-async function initLogger(projectName, experimentName, metadata) {
+async function initLogger(projectName, experimentName, metadata, baseExperiment) {
   const logger = init(projectName, {
     experiment: experimentName,
-    metadata
+    metadata,
+    baseExperiment
   });
   const info = await logger.summarize({ summarizeScores: false });
   console.error(
@@ -5952,15 +5964,22 @@ async function runOnce(handles, opts) {
   }
   const experimentIdToEvaluator = {};
   const resultPromises = evaluators.evaluators.map(async (evaluator) => {
+    const { data, baseExperiment } = callEvaluatorData(
+      evaluator.evaluator.data
+    );
     const logger = opts.noSendLogs ? null : await initLogger(
       evaluator.evaluator.projectName,
       evaluator.evaluator.experimentName,
-      evaluator.evaluator.metadata
+      evaluator.evaluator.metadata,
+      baseExperiment
     );
     try {
       return await runEvaluator(
         logger,
-        evaluator.evaluator,
+        {
+          ...evaluator.evaluator,
+          data
+        },
         opts.progressReporter,
         opts.filters
       );

package/dist/index.d.mts

@@ -1200,7 +1200,7 @@ type BaseExperiment<Input, Expected, Metadata extends BaseMetadata = DefaultMeta
 declare function BaseExperiment<Input = unknown, Expected = unknown, Metadata extends BaseMetadata = DefaultMetadataType>(options?: {
     name?: string;
 }): BaseExperiment<Input, Expected, Metadata>;
-type EvalData<Input, Expected, Metadata extends BaseMetadata = DefaultMetadataType> = EvalCase<Input, Expected, Metadata>[] | (() => EvalCase<Input, Expected, Metadata>[]) | (() => Promise<EvalCase<Input, Expected, Metadata>[]>) | AsyncGenerator<EvalCase<Input, Expected, Metadata>> | AsyncIterable<EvalCase<Input, Expected, Metadata>> | BaseExperiment<Input, Expected, Metadata> | (() => BaseExperiment<Input, Expected, Metadata>);
+type EvalData<Input, Expected, Metadata extends BaseMetadata = DefaultMetadataType> = EvalCase<Input, Expected, Metadata>[] | (() => EvalCase<Input, Expected, Metadata>[]) | Promise<EvalCase<Input, Expected, Metadata>[]> | (() => Promise<EvalCase<Input, Expected, Metadata>[]>) | AsyncGenerator<EvalCase<Input, Expected, Metadata>> | AsyncIterable<EvalCase<Input, Expected, Metadata>> | BaseExperiment<Input, Expected, Metadata> | (() => BaseExperiment<Input, Expected, Metadata>);
 type EvalTask<Input, Output> = ((input: Input, hooks: EvalHooks) => Promise<Output>) | ((input: Input, hooks: EvalHooks) => Output);
 interface EvalHooks {
     meta: (info: Record<string, unknown>) => void;

package/dist/index.d.ts

@@ -1200,7 +1200,7 @@ type BaseExperiment<Input, Expected, Metadata extends BaseMetadata = DefaultMeta
 declare function BaseExperiment<Input = unknown, Expected = unknown, Metadata extends BaseMetadata = DefaultMetadataType>(options?: {
     name?: string;
 }): BaseExperiment<Input, Expected, Metadata>;
-type EvalData<Input, Expected, Metadata extends BaseMetadata = DefaultMetadataType> = EvalCase<Input, Expected, Metadata>[] | (() => EvalCase<Input, Expected, Metadata>[]) | (() => Promise<EvalCase<Input, Expected, Metadata>[]>) | AsyncGenerator<EvalCase<Input, Expected, Metadata>> | AsyncIterable<EvalCase<Input, Expected, Metadata>> | BaseExperiment<Input, Expected, Metadata> | (() => BaseExperiment<Input, Expected, Metadata>);
+type EvalData<Input, Expected, Metadata extends BaseMetadata = DefaultMetadataType> = EvalCase<Input, Expected, Metadata>[] | (() => EvalCase<Input, Expected, Metadata>[]) | Promise<EvalCase<Input, Expected, Metadata>[]> | (() => Promise<EvalCase<Input, Expected, Metadata>[]>) | AsyncGenerator<EvalCase<Input, Expected, Metadata>> | AsyncIterable<EvalCase<Input, Expected, Metadata>> | BaseExperiment<Input, Expected, Metadata> | (() => BaseExperiment<Input, Expected, Metadata>);
 type EvalTask<Input, Output> = ((input: Input, hooks: EvalHooks) => Promise<Output>) | ((input: Input, hooks: EvalHooks) => Output);
 interface EvalHooks {
     meta: (info: Record<string, unknown>) => void;

package/dist/index.js

@@ -4311,6 +4311,17 @@ function initExperiment2(state, options = {}) {
     setCurrent: false
   });
 }
+function callEvaluatorData(data) {
+  let dataResult = typeof data === "function" ? data() : data;
+  let baseExperiment = void 0;
+  if ("_type" in dataResult && dataResult._type === "BaseExperiment") {
+    baseExperiment = dataResult.name;
+  }
+  return {
+    data: dataResult,
+    baseExperiment
+  };
+}
 globalThis._evals = {
   evaluators: {},
   reporters: {}
@@ -4345,12 +4356,14 @@ async function Eval(name, evaluator, reporterOrOpts) {
   }
   const resolvedReporter = options.reporter || defaultReporter;
   try {
+    const { data, baseExperiment } = callEvaluatorData(evaluator.data);
     const experiment = initExperiment2(evaluator.state, {
       ...evaluator.projectId ? { projectId: evaluator.projectId } : { project: name },
       experiment: evaluator.experimentName,
       metadata: evaluator.metadata,
       isPublic: evaluator.isPublic,
-      update: evaluator.update
+      update: evaluator.update,
+      baseExperiment
     });
     if (options.onStart) {
       experiment.summarize({ summarizeScores: false }).then(options.onStart);
@@ -4359,7 +4372,8 @@ async function Eval(name, evaluator, reporterOrOpts) {
       const evalDef = {
         evalName,
         projectName: name,
-        ...evaluator
+        ...evaluator,
+        data
       };
       const ret = await runEvaluator(experiment, evalDef, progressReporter, []);
       progressReporter.stop();

package/dist/index.mjs

@@ -4254,6 +4254,17 @@ function initExperiment2(state, options = {}) {
     setCurrent: false
   });
 }
+function callEvaluatorData(data) {
+  let dataResult = typeof data === "function" ? data() : data;
+  let baseExperiment = void 0;
+  if ("_type" in dataResult && dataResult._type === "BaseExperiment") {
+    baseExperiment = dataResult.name;
+  }
+  return {
+    data: dataResult,
+    baseExperiment
+  };
+}
 globalThis._evals = {
   evaluators: {},
   reporters: {}
@@ -4288,12 +4299,14 @@ async function Eval(name, evaluator, reporterOrOpts) {
   }
   const resolvedReporter = options.reporter || defaultReporter;
   try {
+    const { data, baseExperiment } = callEvaluatorData(evaluator.data);
     const experiment = initExperiment2(evaluator.state, {
       ...evaluator.projectId ? { projectId: evaluator.projectId } : { project: name },
       experiment: evaluator.experimentName,
       metadata: evaluator.metadata,
       isPublic: evaluator.isPublic,
-      update: evaluator.update
+      update: evaluator.update,
+      baseExperiment
     });
     if (options.onStart) {
       experiment.summarize({ summarizeScores: false }).then(options.onStart);
@@ -4302,7 +4315,8 @@ async function Eval(name, evaluator, reporterOrOpts) {
       const evalDef = {
         evalName,
         projectName: name,
-        ...evaluator
+        ...evaluator,
+        data
       };
       const ret = await runEvaluator(experiment, evalDef, progressReporter, []);
       progressReporter.stop();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "braintrust",
-  "version": "0.0.144",
+  "version": "0.0.146",
   "description": "SDK for integrating Braintrust",
   "repository": {
     "type": "git",
@@ -70,7 +70,7 @@
   },
   "dependencies": {
     "@ai-sdk/provider": "^0.0.11",
-    "@braintrust/core": "0.0.45",
+    "@braintrust/core": "0.0.47",
     "@next/env": "^14.2.3",
     "@vercel/functions": "^1.0.2",
     "ai": "^3.2.16",