braintrust 0.1.0 → 0.2.0

package/dev/dist/index.d.mts

@@ -1531,6 +1531,52 @@ declare const promptDefinitionSchema: z.ZodIntersection<z.ZodUnion<[z.ZodObject<
         name: string;
         content: string | null;
         role: "function";
+    }>, z.ZodObject<{
+        content: z.ZodUnion<[z.ZodDefault<z.ZodString>, z.ZodArray<z.ZodObject<{
+            text: z.ZodDefault<z.ZodString>;
+            type: z.ZodLiteral<"text">;
+            cache_control: z.ZodOptional<z.ZodObject<{
+                type: z.ZodEnum<["ephemeral"]>;
+            }, "strip", z.ZodTypeAny, {
+                type: "ephemeral";
+            }, {
+                type: "ephemeral";
+            }>>;
+        }, "strip", z.ZodTypeAny, {
+            type: "text";
+            text: string;
+            cache_control?: {
+                type: "ephemeral";
+            } | undefined;
+        }, {
+            type: "text";
+            text?: string | undefined;
+            cache_control?: {
+                type: "ephemeral";
+            } | undefined;
+        }>, "many">]>;
+        role: z.ZodLiteral<"developer">;
+        name: z.ZodOptional<z.ZodString>;
+    }, "strip", z.ZodTypeAny, {
+        content: string | {
+            type: "text";
+            text: string;
+            cache_control?: {
+                type: "ephemeral";
+            } | undefined;
+        }[];
+        role: "developer";
+        name?: string | undefined;
+    }, {
+        role: "developer";
+        name?: string | undefined;
+        content?: string | {
+            type: "text";
+            text?: string | undefined;
+            cache_control?: {
+                type: "ephemeral";
+            } | undefined;
+        }[] | undefined;
     }>]>, z.ZodObject<{
         role: z.ZodEnum<["model"]>;
         content: z.ZodOptional<z.ZodNullable<z.ZodString>>;
@@ -1608,6 +1654,16 @@ declare const promptDefinitionSchema: z.ZodIntersection<z.ZodUnion<[z.ZodObject<
         name: string;
         content: string | null;
         role: "function";
+    } | {
+        content: string | {
+            type: "text";
+            text: string;
+            cache_control?: {
+                type: "ephemeral";
+            } | undefined;
+        }[];
+        role: "developer";
+        name?: string | undefined;
     } | {
         role: "model";
         content?: string | null | undefined;
@@ -1679,6 +1735,16 @@ declare const promptDefinitionSchema: z.ZodIntersection<z.ZodUnion<[z.ZodObject<
         name: string;
         content: string | null;
         role: "function";
+    } | {
+        role: "developer";
+        name?: string | undefined;
+        content?: string | {
+            type: "text";
+            text?: string | undefined;
+            cache_control?: {
+                type: "ephemeral";
+            } | undefined;
+        }[] | undefined;
     } | {
         role: "model";
         content?: string | null | undefined;
@@ -2612,6 +2678,52 @@ declare const evalParametersSchema: z.ZodRecord<z.ZodString, z.ZodUnion<[z.ZodOb
             name: string;
             content: string | null;
             role: "function";
+        }>, z.ZodObject<{
+            content: z.ZodUnion<[z.ZodDefault<z.ZodString>, z.ZodArray<z.ZodObject<{
+                text: z.ZodDefault<z.ZodString>;
+                type: z.ZodLiteral<"text">;
+                cache_control: z.ZodOptional<z.ZodObject<{
+                    type: z.ZodEnum<["ephemeral"]>;
+                }, "strip", z.ZodTypeAny, {
+                    type: "ephemeral";
+                }, {
+                    type: "ephemeral";
+                }>>;
+            }, "strip", z.ZodTypeAny, {
+                type: "text";
+                text: string;
+                cache_control?: {
+                    type: "ephemeral";
+                } | undefined;
+            }, {
+                type: "text";
+                text?: string | undefined;
+                cache_control?: {
+                    type: "ephemeral";
+                } | undefined;
+            }>, "many">]>;
+            role: z.ZodLiteral<"developer">;
+            name: z.ZodOptional<z.ZodString>;
+        }, "strip", z.ZodTypeAny, {
+            content: string | {
+                type: "text";
+                text: string;
+                cache_control?: {
+                    type: "ephemeral";
+                } | undefined;
+            }[];
+            role: "developer";
+            name?: string | undefined;
+        }, {
+            role: "developer";
+            name?: string | undefined;
+            content?: string | {
+                type: "text";
+                text?: string | undefined;
+                cache_control?: {
+                    type: "ephemeral";
+                } | undefined;
+            }[] | undefined;
         }>]>, z.ZodObject<{
             role: z.ZodEnum<["model"]>;
             content: z.ZodOptional<z.ZodNullable<z.ZodString>>;
@@ -2689,6 +2801,16 @@ declare const evalParametersSchema: z.ZodRecord<z.ZodString, z.ZodUnion<[z.ZodOb
             name: string;
             content: string | null;
             role: "function";
+        } | {
+            content: string | {
+                type: "text";
+                text: string;
+                cache_control?: {
+                    type: "ephemeral";
+                } | undefined;
+            }[];
+            role: "developer";
+            name?: string | undefined;
         } | {
             role: "model";
             content?: string | null | undefined;
@@ -2760,6 +2882,16 @@ declare const evalParametersSchema: z.ZodRecord<z.ZodString, z.ZodUnion<[z.ZodOb
             name: string;
             content: string | null;
             role: "function";
+        } | {
+            role: "developer";
+            name?: string | undefined;
+            content?: string | {
+                type: "text";
+                text?: string | undefined;
+                cache_control?: {
+                    type: "ephemeral";
+                } | undefined;
+            }[] | undefined;
         } | {
             role: "model";
             content?: string | null | undefined;
@@ -3447,6 +3579,16 @@ declare const evalParametersSchema: z.ZodRecord<z.ZodString, z.ZodUnion<[z.ZodOb
             name: string;
             content: string | null;
             role: "function";
+        } | {
+            content: string | {
+                type: "text";
+                text: string;
+                cache_control?: {
+                    type: "ephemeral";
+                } | undefined;
+            }[];
+            role: "developer";
+            name?: string | undefined;
         } | {
             role: "model";
             content?: string | null | undefined;
@@ -3646,6 +3788,16 @@ declare const evalParametersSchema: z.ZodRecord<z.ZodString, z.ZodUnion<[z.ZodOb
             name: string;
             content: string | null;
             role: "function";
+        } | {
+            role: "developer";
+            name?: string | undefined;
+            content?: string | {
+                type: "text";
+                text?: string | undefined;
+                cache_control?: {
+                    type: "ephemeral";
+                } | undefined;
+            }[] | undefined;
         } | {
             role: "model";
             content?: string | null | undefined;
@@ -3830,6 +3982,10 @@ interface EvalHooks<Expected, Metadata extends BaseMetadata, Parameters extends
      * Report progress that will show up in the playground.
      */
     reportProgress: (progress: TaskProgressEvent) => void;
+    /**
+     * The index of the current trial (0-based). This is useful when trialCount > 1.
+     */
+    trialIndex: number;
 }
 type EvalScorerArgs<Input, Output, Expected, Metadata extends BaseMetadata = DefaultMetadataType> = EvalCase<Input, Expected, Metadata> & {
     output: Output;

package/dev/dist/index.d.ts

@@ -1531,6 +1531,52 @@ declare const promptDefinitionSchema: z.ZodIntersection<z.ZodUnion<[z.ZodObject<
         name: string;
         content: string | null;
         role: "function";
+    }>, z.ZodObject<{
+        content: z.ZodUnion<[z.ZodDefault<z.ZodString>, z.ZodArray<z.ZodObject<{
+            text: z.ZodDefault<z.ZodString>;
+            type: z.ZodLiteral<"text">;
+            cache_control: z.ZodOptional<z.ZodObject<{
+                type: z.ZodEnum<["ephemeral"]>;
+            }, "strip", z.ZodTypeAny, {
+                type: "ephemeral";
+            }, {
+                type: "ephemeral";
+            }>>;
+        }, "strip", z.ZodTypeAny, {
+            type: "text";
+            text: string;
+            cache_control?: {
+                type: "ephemeral";
+            } | undefined;
+        }, {
+            type: "text";
+            text?: string | undefined;
+            cache_control?: {
+                type: "ephemeral";
+            } | undefined;
+        }>, "many">]>;
+        role: z.ZodLiteral<"developer">;
+        name: z.ZodOptional<z.ZodString>;
+    }, "strip", z.ZodTypeAny, {
+        content: string | {
+            type: "text";
+            text: string;
+            cache_control?: {
+                type: "ephemeral";
+            } | undefined;
+        }[];
+        role: "developer";
+        name?: string | undefined;
+    }, {
+        role: "developer";
+        name?: string | undefined;
+        content?: string | {
+            type: "text";
+            text?: string | undefined;
+            cache_control?: {
+                type: "ephemeral";
+            } | undefined;
+        }[] | undefined;
     }>]>, z.ZodObject<{
         role: z.ZodEnum<["model"]>;
         content: z.ZodOptional<z.ZodNullable<z.ZodString>>;
@@ -1608,6 +1654,16 @@ declare const promptDefinitionSchema: z.ZodIntersection<z.ZodUnion<[z.ZodObject<
         name: string;
         content: string | null;
         role: "function";
+    } | {
+        content: string | {
+            type: "text";
+            text: string;
+            cache_control?: {
+                type: "ephemeral";
+            } | undefined;
+        }[];
+        role: "developer";
+        name?: string | undefined;
     } | {
         role: "model";
         content?: string | null | undefined;
@@ -1679,6 +1735,16 @@ declare const promptDefinitionSchema: z.ZodIntersection<z.ZodUnion<[z.ZodObject<
         name: string;
         content: string | null;
         role: "function";
+    } | {
+        role: "developer";
+        name?: string | undefined;
+        content?: string | {
+            type: "text";
+            text?: string | undefined;
+            cache_control?: {
+                type: "ephemeral";
+            } | undefined;
+        }[] | undefined;
     } | {
         role: "model";
         content?: string | null | undefined;
@@ -2612,6 +2678,52 @@ declare const evalParametersSchema: z.ZodRecord<z.ZodString, z.ZodUnion<[z.ZodOb
             name: string;
             content: string | null;
             role: "function";
+        }>, z.ZodObject<{
+            content: z.ZodUnion<[z.ZodDefault<z.ZodString>, z.ZodArray<z.ZodObject<{
+                text: z.ZodDefault<z.ZodString>;
+                type: z.ZodLiteral<"text">;
+                cache_control: z.ZodOptional<z.ZodObject<{
+                    type: z.ZodEnum<["ephemeral"]>;
+                }, "strip", z.ZodTypeAny, {
+                    type: "ephemeral";
+                }, {
+                    type: "ephemeral";
+                }>>;
+            }, "strip", z.ZodTypeAny, {
+                type: "text";
+                text: string;
+                cache_control?: {
+                    type: "ephemeral";
+                } | undefined;
+            }, {
+                type: "text";
+                text?: string | undefined;
+                cache_control?: {
+                    type: "ephemeral";
+                } | undefined;
+            }>, "many">]>;
+            role: z.ZodLiteral<"developer">;
+            name: z.ZodOptional<z.ZodString>;
+        }, "strip", z.ZodTypeAny, {
+            content: string | {
+                type: "text";
+                text: string;
+                cache_control?: {
+                    type: "ephemeral";
+                } | undefined;
+            }[];
+            role: "developer";
+            name?: string | undefined;
+        }, {
+            role: "developer";
+            name?: string | undefined;
+            content?: string | {
+                type: "text";
+                text?: string | undefined;
+                cache_control?: {
+                    type: "ephemeral";
+                } | undefined;
+            }[] | undefined;
         }>]>, z.ZodObject<{
             role: z.ZodEnum<["model"]>;
             content: z.ZodOptional<z.ZodNullable<z.ZodString>>;
@@ -2689,6 +2801,16 @@ declare const evalParametersSchema: z.ZodRecord<z.ZodString, z.ZodUnion<[z.ZodOb
             name: string;
             content: string | null;
             role: "function";
+        } | {
+            content: string | {
+                type: "text";
+                text: string;
+                cache_control?: {
+                    type: "ephemeral";
+                } | undefined;
+            }[];
+            role: "developer";
+            name?: string | undefined;
         } | {
             role: "model";
             content?: string | null | undefined;
@@ -2760,6 +2882,16 @@ declare const evalParametersSchema: z.ZodRecord<z.ZodString, z.ZodUnion<[z.ZodOb
             name: string;
             content: string | null;
             role: "function";
+        } | {
+            role: "developer";
+            name?: string | undefined;
+            content?: string | {
+                type: "text";
+                text?: string | undefined;
+                cache_control?: {
+                    type: "ephemeral";
+                } | undefined;
+            }[] | undefined;
         } | {
             role: "model";
             content?: string | null | undefined;
@@ -3447,6 +3579,16 @@ declare const evalParametersSchema: z.ZodRecord<z.ZodString, z.ZodUnion<[z.ZodOb
             name: string;
             content: string | null;
             role: "function";
+        } | {
+            content: string | {
+                type: "text";
+                text: string;
+                cache_control?: {
+                    type: "ephemeral";
+                } | undefined;
+            }[];
+            role: "developer";
+            name?: string | undefined;
         } | {
             role: "model";
             content?: string | null | undefined;
@@ -3646,6 +3788,16 @@ declare const evalParametersSchema: z.ZodRecord<z.ZodString, z.ZodUnion<[z.ZodOb
             name: string;
             content: string | null;
             role: "function";
+        } | {
+            role: "developer";
+            name?: string | undefined;
+            content?: string | {
+                type: "text";
+                text?: string | undefined;
+                cache_control?: {
+                    type: "ephemeral";
+                } | undefined;
+            }[] | undefined;
         } | {
             role: "model";
             content?: string | null | undefined;
@@ -3830,6 +3982,10 @@ interface EvalHooks<Expected, Metadata extends BaseMetadata, Parameters extends
      * Report progress that will show up in the playground.
      */
     reportProgress: (progress: TaskProgressEvent) => void;
+    /**
+     * The index of the current trial (0-based). This is useful when trialCount > 1.
+     */
+    trialIndex: number;
 }
 type EvalScorerArgs<Input, Output, Expected, Metadata extends BaseMetadata = DefaultMetadataType> = EvalCase<Input, Expected, Metadata> & {
     output: Output;

package/dev/dist/index.js

@@ -100,7 +100,7 @@ async function getBaseBranchAncestor(remote = void 0) {
     return void 0;
   }
 }
-async function getPastNAncestors(n = 10, remote = void 0) {
+async function getPastNAncestors(n = 1e3, remote = void 0) {
   const git = await currentRepo();
   if (git === null) {
     return [];
@@ -117,7 +117,7 @@ async function getPastNAncestors(n = 10, remote = void 0) {
   if (!ancestor) {
     return [];
   }
-  const commits = await git.log({ from: ancestor, to: "HEAD" });
+  const commits = await git.log({ from: ancestor, to: "HEAD", maxCount: n });
   return commits.all.map((c) => c.hash);
 }
 async function attempt(fn) {
@@ -6151,14 +6151,20 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
   } else {
     data = dataResult;
   }
-  data = data.filter((d) => filters.every((f) => evaluateFilter(d, f))).flatMap(
-    (datum) => [...Array(_nullishCoalesce(evaluator.trialCount, () => ( 1))).keys()].map(() => datum)
+  const dataWithTrials = data.filter((d) => filters.every((f) => evaluateFilter(d, f))).flatMap(
+    (datum) => [...Array(_nullishCoalesce(evaluator.trialCount, () => ( 1))).keys()].map((trialIndex) => ({
+      datum,
+      trialIndex
+    }))
   );
-  progressReporter.start(evaluator.evalName, data.length);
+  progressReporter.start(evaluator.evalName, dataWithTrials.length);
   const results = [];
   const q = queue(
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    async (datum) => {
+    async ({
+      datum,
+      trialIndex
+    }) => {
       const eventDataset = experiment ? experiment.dataset : Dataset.isDataset(evaluator.data) ? evaluator.data : void 0;
       const baseEvent = {
         name: "eval",
@@ -6207,7 +6213,8 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
                     name: evaluator.evalName,
                     object_type: "task"
                   })]);
-                }
+                },
+                trialIndex
               });
               if (outputResult instanceof Promise) {
                 output = await outputResult;
@@ -6357,9 +6364,9 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
         return await experiment.traced(callback, baseEvent);
       }
     },
-    Math.max(_nullishCoalesce(evaluator.maxConcurrency, () => ( data.length)), 1)
+    Math.max(_nullishCoalesce(evaluator.maxConcurrency, () => ( dataWithTrials.length)), 1)
   );
-  q.push(data);
+  q.push(dataWithTrials);
   const cancel = async () => {
     await new Promise((_, reject2) => {
       if (evaluator.timeout) {

package/dev/dist/index.mjs

@@ -100,7 +100,7 @@ async function getBaseBranchAncestor(remote = void 0) {
     return void 0;
   }
 }
-async function getPastNAncestors(n = 10, remote = void 0) {
+async function getPastNAncestors(n = 1e3, remote = void 0) {
   const git = await currentRepo();
   if (git === null) {
     return [];
@@ -117,7 +117,7 @@ async function getPastNAncestors(n = 10, remote = void 0) {
   if (!ancestor) {
     return [];
   }
-  const commits = await git.log({ from: ancestor, to: "HEAD" });
+  const commits = await git.log({ from: ancestor, to: "HEAD", maxCount: n });
   return commits.all.map((c) => c.hash);
 }
 async function attempt(fn) {
@@ -6151,14 +6151,20 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
   } else {
     data = dataResult;
   }
-  data = data.filter((d) => filters.every((f) => evaluateFilter(d, f))).flatMap(
-    (datum) => [...Array(evaluator.trialCount ?? 1).keys()].map(() => datum)
+  const dataWithTrials = data.filter((d) => filters.every((f) => evaluateFilter(d, f))).flatMap(
+    (datum) => [...Array(evaluator.trialCount ?? 1).keys()].map((trialIndex) => ({
+      datum,
+      trialIndex
+    }))
   );
-  progressReporter.start(evaluator.evalName, data.length);
+  progressReporter.start(evaluator.evalName, dataWithTrials.length);
   const results = [];
   const q = queue(
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    async (datum) => {
+    async ({
+      datum,
+      trialIndex
+    }) => {
       const eventDataset = experiment ? experiment.dataset : Dataset.isDataset(evaluator.data) ? evaluator.data : void 0;
       const baseEvent = {
         name: "eval",
@@ -6207,7 +6213,8 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
                     name: evaluator.evalName,
                     object_type: "task"
                   });
-                }
+                },
+                trialIndex
               });
               if (outputResult instanceof Promise) {
                 output = await outputResult;
@@ -6357,9 +6364,9 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
         return await experiment.traced(callback, baseEvent);
       }
     },
-    Math.max(evaluator.maxConcurrency ?? data.length, 1)
+    Math.max(evaluator.maxConcurrency ?? dataWithTrials.length, 1)
   );
-  q.push(data);
+  q.push(dataWithTrials);
   const cancel = async () => {
     await new Promise((_, reject2) => {
       if (evaluator.timeout) {