braintrust 0.1.1 → 0.2.0

package/dev/dist/index.d.mts

@@ -191,8 +191,6 @@ declare class LazyValue<T> {
     get hasSucceeded(): boolean;
 }
 
-/// <reference lib="dom" />
-
 type SetCurrentArg = {
     setCurrent?: boolean;
 };
@@ -384,10 +382,10 @@ declare const loginSchema: z.ZodObject<{
         fields: z.ZodOptional<z.ZodArray<z.ZodEnum<["dirty", "tag", "commit", "branch", "author_name", "author_email", "commit_message", "commit_time", "git_diff"]>, "many">>;
     }, "strict", z.ZodTypeAny, {
         collect: "some" | "none" | "all";
-        fields?: ("branch" | "dirty" | "tag" | "commit" | "author_name" | "author_email" | "commit_message" | "commit_time" | "git_diff")[] | undefined;
+        fields?: ("dirty" | "tag" | "commit" | "branch" | "author_name" | "author_email" | "commit_message" | "commit_time" | "git_diff")[] | undefined;
     }, {
         collect: "some" | "none" | "all";
-        fields?: ("branch" | "dirty" | "tag" | "commit" | "author_name" | "author_email" | "commit_message" | "commit_time" | "git_diff")[] | undefined;
+        fields?: ("dirty" | "tag" | "commit" | "branch" | "author_name" | "author_email" | "commit_message" | "commit_time" | "git_diff")[] | undefined;
     }>>>;
 }, "strict", z.ZodTypeAny, {
     appUrl: string;
@@ -399,7 +397,7 @@ declare const loginSchema: z.ZodObject<{
     orgId?: string | null | undefined;
     gitMetadataSettings?: {
         collect: "some" | "none" | "all";
-        fields?: ("branch" | "dirty" | "tag" | "commit" | "author_name" | "author_email" | "commit_message" | "commit_time" | "git_diff")[] | undefined;
+        fields?: ("dirty" | "tag" | "commit" | "branch" | "author_name" | "author_email" | "commit_message" | "commit_time" | "git_diff")[] | undefined;
     } | null | undefined;
 }, {
     appUrl: string;
@@ -411,7 +409,7 @@ declare const loginSchema: z.ZodObject<{
     orgId?: string | null | undefined;
     gitMetadataSettings?: {
         collect: "some" | "none" | "all";
-        fields?: ("branch" | "dirty" | "tag" | "commit" | "author_name" | "author_email" | "commit_message" | "commit_time" | "git_diff")[] | undefined;
+        fields?: ("dirty" | "tag" | "commit" | "branch" | "author_name" | "author_email" | "commit_message" | "commit_time" | "git_diff")[] | undefined;
     } | null | undefined;
 }>;
 type SerializedBraintrustState = z.infer<typeof loginSchema>;
@@ -1270,23 +1268,23 @@ declare const promptDefinitionSchema: z.ZodIntersection<z.ZodUnion<[z.ZodObject<
                 detail: z.ZodOptional<z.ZodUnion<[z.ZodLiteral<"auto">, z.ZodLiteral<"low">, z.ZodLiteral<"high">]>>;
             }, "strip", z.ZodTypeAny, {
                 url: string;
-                detail?: "auto" | "high" | "low" | undefined;
+                detail?: "auto" | "low" | "high" | undefined;
             }, {
                 url?: unknown;
-                detail?: "auto" | "high" | "low" | undefined;
+                detail?: "auto" | "low" | "high" | undefined;
             }>;
             type: z.ZodLiteral<"image_url">;
         }, "strip", z.ZodTypeAny, {
             type: "image_url";
             image_url: {
                 url: string;
-                detail?: "auto" | "high" | "low" | undefined;
+                detail?: "auto" | "low" | "high" | undefined;
             };
         }, {
             type: "image_url";
             image_url: {
                 url?: unknown;
-                detail?: "auto" | "high" | "low" | undefined;
+                detail?: "auto" | "low" | "high" | undefined;
             };
         }>]>, "many">]>;
         role: z.ZodLiteral<"user">;
@@ -1302,7 +1300,7 @@ declare const promptDefinitionSchema: z.ZodIntersection<z.ZodUnion<[z.ZodObject<
             type: "image_url";
             image_url: {
                 url: string;
-                detail?: "auto" | "high" | "low" | undefined;
+                detail?: "auto" | "low" | "high" | undefined;
             };
         })[];
         role: "user";
@@ -1320,7 +1318,7 @@ declare const promptDefinitionSchema: z.ZodIntersection<z.ZodUnion<[z.ZodObject<
             type: "image_url";
             image_url: {
                 url?: unknown;
-                detail?: "auto" | "high" | "low" | undefined;
+                detail?: "auto" | "low" | "high" | undefined;
             };
         })[] | undefined;
     }>, z.ZodObject<{
@@ -1611,7 +1609,7 @@ declare const promptDefinitionSchema: z.ZodIntersection<z.ZodUnion<[z.ZodObject<
             type: "image_url";
             image_url: {
                 url: string;
-                detail?: "auto" | "high" | "low" | undefined;
+                detail?: "auto" | "low" | "high" | undefined;
             };
         })[];
         role: "user";
@@ -1694,7 +1692,7 @@ declare const promptDefinitionSchema: z.ZodIntersection<z.ZodUnion<[z.ZodObject<
             type: "image_url";
             image_url: {
                 url?: unknown;
-                detail?: "auto" | "high" | "low" | undefined;
+                detail?: "auto" | "low" | "high" | undefined;
             };
         })[] | undefined;
     } | {
@@ -2417,23 +2415,23 @@ declare const evalParametersSchema: z.ZodRecord<z.ZodString, z.ZodUnion<[z.ZodOb
                     detail: z.ZodOptional<z.ZodUnion<[z.ZodLiteral<"auto">, z.ZodLiteral<"low">, z.ZodLiteral<"high">]>>;
                 }, "strip", z.ZodTypeAny, {
                     url: string;
-                    detail?: "auto" | "high" | "low" | undefined;
+                    detail?: "auto" | "low" | "high" | undefined;
                 }, {
                     url?: unknown;
-                    detail?: "auto" | "high" | "low" | undefined;
+                    detail?: "auto" | "low" | "high" | undefined;
                 }>;
                 type: z.ZodLiteral<"image_url">;
             }, "strip", z.ZodTypeAny, {
                 type: "image_url";
                 image_url: {
                     url: string;
-                    detail?: "auto" | "high" | "low" | undefined;
+                    detail?: "auto" | "low" | "high" | undefined;
                 };
             }, {
                 type: "image_url";
                 image_url: {
                     url?: unknown;
-                    detail?: "auto" | "high" | "low" | undefined;
+                    detail?: "auto" | "low" | "high" | undefined;
                 };
             }>]>, "many">]>;
             role: z.ZodLiteral<"user">;
@@ -2449,7 +2447,7 @@ declare const evalParametersSchema: z.ZodRecord<z.ZodString, z.ZodUnion<[z.ZodOb
                 type: "image_url";
                 image_url: {
                     url: string;
-                    detail?: "auto" | "high" | "low" | undefined;
+                    detail?: "auto" | "low" | "high" | undefined;
                 };
             })[];
             role: "user";
@@ -2467,7 +2465,7 @@ declare const evalParametersSchema: z.ZodRecord<z.ZodString, z.ZodUnion<[z.ZodOb
                 type: "image_url";
                 image_url: {
                     url?: unknown;
-                    detail?: "auto" | "high" | "low" | undefined;
+                    detail?: "auto" | "low" | "high" | undefined;
                 };
             })[] | undefined;
         }>, z.ZodObject<{
@@ -2758,7 +2756,7 @@ declare const evalParametersSchema: z.ZodRecord<z.ZodString, z.ZodUnion<[z.ZodOb
                 type: "image_url";
                 image_url: {
                     url: string;
-                    detail?: "auto" | "high" | "low" | undefined;
+                    detail?: "auto" | "low" | "high" | undefined;
                 };
             })[];
             role: "user";
@@ -2841,7 +2839,7 @@ declare const evalParametersSchema: z.ZodRecord<z.ZodString, z.ZodUnion<[z.ZodOb
                 type: "image_url";
                 image_url: {
                     url?: unknown;
-                    detail?: "auto" | "high" | "low" | undefined;
+                    detail?: "auto" | "low" | "high" | undefined;
                 };
             })[] | undefined;
         } | {
@@ -3536,7 +3534,7 @@ declare const evalParametersSchema: z.ZodRecord<z.ZodString, z.ZodUnion<[z.ZodOb
                 type: "image_url";
                 image_url: {
                     url: string;
-                    detail?: "auto" | "high" | "low" | undefined;
+                    detail?: "auto" | "low" | "high" | undefined;
                 };
             })[];
             role: "user";
@@ -3747,7 +3745,7 @@ declare const evalParametersSchema: z.ZodRecord<z.ZodString, z.ZodUnion<[z.ZodOb
                 type: "image_url";
                 image_url: {
                     url?: unknown;
-                    detail?: "auto" | "high" | "low" | undefined;
+                    detail?: "auto" | "low" | "high" | undefined;
                 };
             })[] | undefined;
         } | {
@@ -3984,6 +3982,10 @@ interface EvalHooks<Expected, Metadata extends BaseMetadata, Parameters extends
      * Report progress that will show up in the playground.
      */
     reportProgress: (progress: TaskProgressEvent) => void;
+    /**
+     * The index of the current trial (0-based). This is useful when trialCount > 1.
+     */
+    trialIndex: number;
 }
 type EvalScorerArgs<Input, Output, Expected, Metadata extends BaseMetadata = DefaultMetadataType> = EvalCase<Input, Expected, Metadata> & {
     output: Output;

package/dev/dist/index.d.ts

@@ -191,8 +191,6 @@ declare class LazyValue<T> {
     get hasSucceeded(): boolean;
 }
 
-/// <reference lib="dom" />
-
 type SetCurrentArg = {
     setCurrent?: boolean;
 };
@@ -384,10 +382,10 @@ declare const loginSchema: z.ZodObject<{
         fields: z.ZodOptional<z.ZodArray<z.ZodEnum<["dirty", "tag", "commit", "branch", "author_name", "author_email", "commit_message", "commit_time", "git_diff"]>, "many">>;
     }, "strict", z.ZodTypeAny, {
         collect: "some" | "none" | "all";
-        fields?: ("branch" | "dirty" | "tag" | "commit" | "author_name" | "author_email" | "commit_message" | "commit_time" | "git_diff")[] | undefined;
+        fields?: ("dirty" | "tag" | "commit" | "branch" | "author_name" | "author_email" | "commit_message" | "commit_time" | "git_diff")[] | undefined;
     }, {
         collect: "some" | "none" | "all";
-        fields?: ("branch" | "dirty" | "tag" | "commit" | "author_name" | "author_email" | "commit_message" | "commit_time" | "git_diff")[] | undefined;
+        fields?: ("dirty" | "tag" | "commit" | "branch" | "author_name" | "author_email" | "commit_message" | "commit_time" | "git_diff")[] | undefined;
     }>>>;
 }, "strict", z.ZodTypeAny, {
     appUrl: string;
@@ -399,7 +397,7 @@ declare const loginSchema: z.ZodObject<{
     orgId?: string | null | undefined;
     gitMetadataSettings?: {
         collect: "some" | "none" | "all";
-        fields?: ("branch" | "dirty" | "tag" | "commit" | "author_name" | "author_email" | "commit_message" | "commit_time" | "git_diff")[] | undefined;
+        fields?: ("dirty" | "tag" | "commit" | "branch" | "author_name" | "author_email" | "commit_message" | "commit_time" | "git_diff")[] | undefined;
     } | null | undefined;
 }, {
     appUrl: string;
@@ -411,7 +409,7 @@ declare const loginSchema: z.ZodObject<{
     orgId?: string | null | undefined;
     gitMetadataSettings?: {
         collect: "some" | "none" | "all";
-        fields?: ("branch" | "dirty" | "tag" | "commit" | "author_name" | "author_email" | "commit_message" | "commit_time" | "git_diff")[] | undefined;
+        fields?: ("dirty" | "tag" | "commit" | "branch" | "author_name" | "author_email" | "commit_message" | "commit_time" | "git_diff")[] | undefined;
     } | null | undefined;
 }>;
 type SerializedBraintrustState = z.infer<typeof loginSchema>;
@@ -1270,23 +1268,23 @@ declare const promptDefinitionSchema: z.ZodIntersection<z.ZodUnion<[z.ZodObject<
                 detail: z.ZodOptional<z.ZodUnion<[z.ZodLiteral<"auto">, z.ZodLiteral<"low">, z.ZodLiteral<"high">]>>;
             }, "strip", z.ZodTypeAny, {
                 url: string;
-                detail?: "auto" | "high" | "low" | undefined;
+                detail?: "auto" | "low" | "high" | undefined;
             }, {
                 url?: unknown;
-                detail?: "auto" | "high" | "low" | undefined;
+                detail?: "auto" | "low" | "high" | undefined;
             }>;
             type: z.ZodLiteral<"image_url">;
         }, "strip", z.ZodTypeAny, {
             type: "image_url";
             image_url: {
                 url: string;
-                detail?: "auto" | "high" | "low" | undefined;
+                detail?: "auto" | "low" | "high" | undefined;
             };
         }, {
             type: "image_url";
             image_url: {
                 url?: unknown;
-                detail?: "auto" | "high" | "low" | undefined;
+                detail?: "auto" | "low" | "high" | undefined;
             };
         }>]>, "many">]>;
         role: z.ZodLiteral<"user">;
@@ -1302,7 +1300,7 @@ declare const promptDefinitionSchema: z.ZodIntersection<z.ZodUnion<[z.ZodObject<
             type: "image_url";
             image_url: {
                 url: string;
-                detail?: "auto" | "high" | "low" | undefined;
+                detail?: "auto" | "low" | "high" | undefined;
             };
         })[];
         role: "user";
@@ -1320,7 +1318,7 @@ declare const promptDefinitionSchema: z.ZodIntersection<z.ZodUnion<[z.ZodObject<
             type: "image_url";
             image_url: {
                 url?: unknown;
-                detail?: "auto" | "high" | "low" | undefined;
+                detail?: "auto" | "low" | "high" | undefined;
             };
         })[] | undefined;
     }>, z.ZodObject<{
@@ -1611,7 +1609,7 @@ declare const promptDefinitionSchema: z.ZodIntersection<z.ZodUnion<[z.ZodObject<
             type: "image_url";
             image_url: {
                 url: string;
-                detail?: "auto" | "high" | "low" | undefined;
+                detail?: "auto" | "low" | "high" | undefined;
             };
         })[];
         role: "user";
@@ -1694,7 +1692,7 @@ declare const promptDefinitionSchema: z.ZodIntersection<z.ZodUnion<[z.ZodObject<
             type: "image_url";
             image_url: {
                 url?: unknown;
-                detail?: "auto" | "high" | "low" | undefined;
+                detail?: "auto" | "low" | "high" | undefined;
             };
         })[] | undefined;
     } | {
@@ -2417,23 +2415,23 @@ declare const evalParametersSchema: z.ZodRecord<z.ZodString, z.ZodUnion<[z.ZodOb
                     detail: z.ZodOptional<z.ZodUnion<[z.ZodLiteral<"auto">, z.ZodLiteral<"low">, z.ZodLiteral<"high">]>>;
                 }, "strip", z.ZodTypeAny, {
                     url: string;
-                    detail?: "auto" | "high" | "low" | undefined;
+                    detail?: "auto" | "low" | "high" | undefined;
                 }, {
                     url?: unknown;
-                    detail?: "auto" | "high" | "low" | undefined;
+                    detail?: "auto" | "low" | "high" | undefined;
                 }>;
                 type: z.ZodLiteral<"image_url">;
             }, "strip", z.ZodTypeAny, {
                 type: "image_url";
                 image_url: {
                     url: string;
-                    detail?: "auto" | "high" | "low" | undefined;
+                    detail?: "auto" | "low" | "high" | undefined;
                 };
             }, {
                 type: "image_url";
                 image_url: {
                     url?: unknown;
-                    detail?: "auto" | "high" | "low" | undefined;
+                    detail?: "auto" | "low" | "high" | undefined;
                 };
             }>]>, "many">]>;
             role: z.ZodLiteral<"user">;
@@ -2449,7 +2447,7 @@ declare const evalParametersSchema: z.ZodRecord<z.ZodString, z.ZodUnion<[z.ZodOb
                 type: "image_url";
                 image_url: {
                     url: string;
-                    detail?: "auto" | "high" | "low" | undefined;
+                    detail?: "auto" | "low" | "high" | undefined;
                 };
             })[];
             role: "user";
@@ -2467,7 +2465,7 @@ declare const evalParametersSchema: z.ZodRecord<z.ZodString, z.ZodUnion<[z.ZodOb
                 type: "image_url";
                 image_url: {
                     url?: unknown;
-                    detail?: "auto" | "high" | "low" | undefined;
+                    detail?: "auto" | "low" | "high" | undefined;
                 };
             })[] | undefined;
         }>, z.ZodObject<{
@@ -2758,7 +2756,7 @@ declare const evalParametersSchema: z.ZodRecord<z.ZodString, z.ZodUnion<[z.ZodOb
                 type: "image_url";
                 image_url: {
                     url: string;
-                    detail?: "auto" | "high" | "low" | undefined;
+                    detail?: "auto" | "low" | "high" | undefined;
                 };
             })[];
             role: "user";
@@ -2841,7 +2839,7 @@ declare const evalParametersSchema: z.ZodRecord<z.ZodString, z.ZodUnion<[z.ZodOb
                 type: "image_url";
                 image_url: {
                     url?: unknown;
-                    detail?: "auto" | "high" | "low" | undefined;
+                    detail?: "auto" | "low" | "high" | undefined;
                 };
             })[] | undefined;
         } | {
@@ -3536,7 +3534,7 @@ declare const evalParametersSchema: z.ZodRecord<z.ZodString, z.ZodUnion<[z.ZodOb
                 type: "image_url";
                 image_url: {
                     url: string;
-                    detail?: "auto" | "high" | "low" | undefined;
+                    detail?: "auto" | "low" | "high" | undefined;
                 };
             })[];
             role: "user";
@@ -3747,7 +3745,7 @@ declare const evalParametersSchema: z.ZodRecord<z.ZodString, z.ZodUnion<[z.ZodOb
                 type: "image_url";
                 image_url: {
                     url?: unknown;
-                    detail?: "auto" | "high" | "low" | undefined;
+                    detail?: "auto" | "low" | "high" | undefined;
                 };
             })[] | undefined;
         } | {
@@ -3984,6 +3982,10 @@ interface EvalHooks<Expected, Metadata extends BaseMetadata, Parameters extends
      * Report progress that will show up in the playground.
      */
     reportProgress: (progress: TaskProgressEvent) => void;
+    /**
+     * The index of the current trial (0-based). This is useful when trialCount > 1.
+     */
+    trialIndex: number;
 }
 type EvalScorerArgs<Input, Output, Expected, Metadata extends BaseMetadata = DefaultMetadataType> = EvalCase<Input, Expected, Metadata> & {
     output: Output;

package/dev/dist/index.js

@@ -100,7 +100,7 @@ async function getBaseBranchAncestor(remote = void 0) {
     return void 0;
   }
 }
-async function getPastNAncestors(n = 10, remote = void 0) {
+async function getPastNAncestors(n = 1e3, remote = void 0) {
   const git = await currentRepo();
   if (git === null) {
     return [];
@@ -117,7 +117,7 @@ async function getPastNAncestors(n = 10, remote = void 0) {
   if (!ancestor) {
     return [];
   }
-  const commits = await git.log({ from: ancestor, to: "HEAD" });
+  const commits = await git.log({ from: ancestor, to: "HEAD", maxCount: n });
   return commits.all.map((c) => c.hash);
 }
 async function attempt(fn) {
@@ -4441,7 +4441,7 @@ var _cors = require('cors'); var _cors2 = _interopRequireDefault(_cors);
 // src/framework.ts
 
 
-// ../node_modules/.pnpm/async@3.2.5/node_modules/async/dist/async.mjs
+// ../../node_modules/.pnpm/async@3.2.5/node_modules/async/dist/async.mjs
 function initialParams(fn) {
   return function(...args) {
     var callback = args.pop();
@@ -6151,14 +6151,20 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
   } else {
     data = dataResult;
   }
-  data = data.filter((d) => filters.every((f) => evaluateFilter(d, f))).flatMap(
-    (datum) => [...Array(_nullishCoalesce(evaluator.trialCount, () => ( 1))).keys()].map(() => datum)
+  const dataWithTrials = data.filter((d) => filters.every((f) => evaluateFilter(d, f))).flatMap(
+    (datum) => [...Array(_nullishCoalesce(evaluator.trialCount, () => ( 1))).keys()].map((trialIndex) => ({
+      datum,
+      trialIndex
+    }))
   );
-  progressReporter.start(evaluator.evalName, data.length);
+  progressReporter.start(evaluator.evalName, dataWithTrials.length);
   const results = [];
   const q = queue(
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    async (datum) => {
+    async ({
+      datum,
+      trialIndex
+    }) => {
       const eventDataset = experiment ? experiment.dataset : Dataset.isDataset(evaluator.data) ? evaluator.data : void 0;
       const baseEvent = {
         name: "eval",
@@ -6207,7 +6213,8 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
                     name: evaluator.evalName,
                     object_type: "task"
                   })]);
-                }
+                },
+                trialIndex
               });
               if (outputResult instanceof Promise) {
                 output = await outputResult;
@@ -6357,9 +6364,9 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
         return await experiment.traced(callback, baseEvent);
       }
     },
-    Math.max(_nullishCoalesce(evaluator.maxConcurrency, () => ( data.length)), 1)
+    Math.max(_nullishCoalesce(evaluator.maxConcurrency, () => ( dataWithTrials.length)), 1)
   );
-  q.push(data);
+  q.push(dataWithTrials);
   const cancel = async () => {
     await new Promise((_, reject2) => {
       if (evaluator.timeout) {

package/dev/dist/index.mjs

@@ -100,7 +100,7 @@ async function getBaseBranchAncestor(remote = void 0) {
     return void 0;
   }
 }
-async function getPastNAncestors(n = 10, remote = void 0) {
+async function getPastNAncestors(n = 1e3, remote = void 0) {
   const git = await currentRepo();
   if (git === null) {
     return [];
@@ -117,7 +117,7 @@ async function getPastNAncestors(n = 10, remote = void 0) {
   if (!ancestor) {
     return [];
   }
-  const commits = await git.log({ from: ancestor, to: "HEAD" });
+  const commits = await git.log({ from: ancestor, to: "HEAD", maxCount: n });
   return commits.all.map((c) => c.hash);
 }
 async function attempt(fn) {
@@ -4441,7 +4441,7 @@ import cors from "cors";
 // src/framework.ts
 import { SpanTypeAttribute as SpanTypeAttribute2, mergeDicts as mergeDicts2 } from "@braintrust/core";
 
-// ../node_modules/.pnpm/async@3.2.5/node_modules/async/dist/async.mjs
+// ../../node_modules/.pnpm/async@3.2.5/node_modules/async/dist/async.mjs
 function initialParams(fn) {
   return function(...args) {
     var callback = args.pop();
@@ -6151,14 +6151,20 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
   } else {
     data = dataResult;
   }
-  data = data.filter((d) => filters.every((f) => evaluateFilter(d, f))).flatMap(
-    (datum) => [...Array(evaluator.trialCount ?? 1).keys()].map(() => datum)
+  const dataWithTrials = data.filter((d) => filters.every((f) => evaluateFilter(d, f))).flatMap(
+    (datum) => [...Array(evaluator.trialCount ?? 1).keys()].map((trialIndex) => ({
+      datum,
+      trialIndex
+    }))
   );
-  progressReporter.start(evaluator.evalName, data.length);
+  progressReporter.start(evaluator.evalName, dataWithTrials.length);
   const results = [];
   const q = queue(
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    async (datum) => {
+    async ({
+      datum,
+      trialIndex
+    }) => {
       const eventDataset = experiment ? experiment.dataset : Dataset.isDataset(evaluator.data) ? evaluator.data : void 0;
       const baseEvent = {
         name: "eval",
@@ -6207,7 +6213,8 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
                     name: evaluator.evalName,
                     object_type: "task"
                   });
-                }
+                },
+                trialIndex
               });
               if (outputResult instanceof Promise) {
                 output = await outputResult;
@@ -6357,9 +6364,9 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
         return await experiment.traced(callback, baseEvent);
       }
     },
-    Math.max(evaluator.maxConcurrency ?? data.length, 1)
+    Math.max(evaluator.maxConcurrency ?? dataWithTrials.length, 1)
   );
-  q.push(data);
+  q.push(dataWithTrials);
   const cancel = async () => {
     await new Promise((_, reject2) => {
       if (evaluator.timeout) {