braintrust 3.27.0 → 3.28.0

package/dist/index.mjs

@@ -55,9 +55,11 @@ import {
   X_CACHED_HEADER,
   _exportsForTestingOnly,
   _internalGetGlobalState,
+  _internalResumeSpan,
   _internalSetInitialState,
   _internalStartSpanWithInitialMerge,
   addAzureBlobHeaders,
+  base64ToUint8Array,
   braintrustAISDKTelemetry,
   braintrustFlueInstrumentation,
   braintrustFlueObserver,
@@ -105,7 +107,6 @@ import {
   loginToState,
   logs3OverflowUploadSchema,
   makeScorerPropagatedEvent,
-  mergeDicts,
   newId,
   parseCachedHeader,
   parseTemplateFormat,
@@ -127,8 +128,10 @@ import {
   summarize,
   summarizeAndFlush,
   templateRegistry,
+  toNumericMetrics,
   traceable,
   traced,
+  uint8ArrayToBase64,
   updateSpan,
   uploadLogs3OverflowPayload,
   utf8ByteLength,
@@ -141,7 +144,7 @@ import {
   wrapMastraAgent,
   wrapTraced,
   zodToJsonSchema
-} from "./chunk-YKD22IMR.mjs";
+} from "./chunk-UPFNQCGB.mjs";
 import {
   INSTRUMENTATION_NAMES,
   __export,
@@ -172,8 +175,9 @@ import {
   openRouterChannels,
   piCodingAgentChannels,
   strandsAgentSDKChannels,
+  voyageAIChannels,
   withSpanInstrumentationName
-} from "./chunk-CZM5JIQL.mjs";
+} from "./chunk-ZHUHZWFY.mjs";
 
 // src/exports.ts
 var exports_exports = {};
@@ -186,6 +190,8 @@ __export(exports_exports, {
   BRAINTRUST_PARENT_KEY: () => BRAINTRUST_PARENT_KEY,
   BaseAttachment: () => BaseAttachment,
   BaseExperiment: () => BaseExperiment,
+  BatchScorer: () => BatchScorer,
+  BatchTask: () => BatchTask,
   BraintrustLangChainCallbackHandler: () => BraintrustLangChainCallbackHandler,
   BraintrustMiddleware: () => BraintrustMiddleware,
   BraintrustObservabilityExporter: () => BraintrustObservabilityExporter,
@@ -199,6 +205,8 @@ __export(exports_exports, {
   DEFAULT_MAX_REQUEST_SIZE: () => DEFAULT_MAX_REQUEST_SIZE,
   Dataset: () => Dataset,
   DatasetPipeline: () => DatasetPipeline,
+  DurableEvalMemoryStore: () => DurableEvalMemoryStore,
+  DurableEvalRedisStore: () => DurableEvalRedisStore,
   ERR_PERMALINK: () => ERR_PERMALINK,
   Eval: () => Eval,
   EvalResultWithSummary: () => EvalResultWithSummary,
@@ -256,6 +264,7 @@ __export(exports_exports, {
   currentSpan: () => currentSpan,
   deepCopyEvent: () => deepCopyEvent,
   defaultErrorScoreHandler: () => defaultErrorScoreHandler,
+  defineDurableEval: () => defineDurableEval,
   deserializePlainStringAsJSON: () => deserializePlainStringAsJSON,
   devNullWritableStream: () => devNullWritableStream,
   evaluatorDefinitionSchema: () => evaluatorDefinitionSchema,
@@ -351,7 +360,8 @@ __export(exports_exports, {
   wrapPiCodingAgentSDK: () => wrapPiCodingAgentSDK,
   wrapStrandsAgentSDK: () => wrapStrandsAgentSDK,
   wrapTraced: () => wrapTraced,
-  wrapVitest: () => wrapVitest
+  wrapVitest: () => wrapVitest,
+  wrapVoyageAI: () => wrapVoyageAI
 });
 
 // src/sandbox.ts
@@ -1612,7 +1622,7 @@ function normalizeUsageMetrics(usage, provider, providerMetadata) {
         cacheCreationTokens
       );
       Object.assign(metrics, cacheTokens);
-      Object.assign(metrics, finalizeAnthropicTokens(metrics));
+      return toNumericMetrics(finalizeAnthropicTokens(metrics));
     }
   }
   return metrics;
@@ -5725,6 +5735,84 @@ function wrapRerank3(rerank) {
   });
 }
 
+// src/wrappers/voyageai.ts
+function wrapVoyageAI(client) {
+  if (!isSupportedVoyageAIClient(client)) {
+    console.warn("Unsupported Voyage AI library. Not wrapping.");
+    return client;
+  }
+  return voyageAIProxy(client);
+}
+var voyageAIProxyCache = /* @__PURE__ */ new WeakMap();
+function isObject2(value) {
+  return typeof value === "object" && value !== null;
+}
+function hasFunction5(value, name) {
+  return typeof Reflect.get(value, name) === "function";
+}
+function isSupportedVoyageAIClient(value) {
+  if (!isObject2(value)) {
+    return false;
+  }
+  return hasFunction5(value, "embed") || hasFunction5(value, "multimodalEmbed") || hasFunction5(value, "rerank") || hasFunction5(value, "contextualizedEmbed");
+}
+function voyageAIProxy(client) {
+  const cached = voyageAIProxyCache.get(client);
+  if (cached) {
+    return cached;
+  }
+  const proxy = new Proxy(client, {
+    get(target, prop, receiver) {
+      switch (prop) {
+        case "embed":
+          if (typeof target.embed !== "function") {
+            return target.embed;
+          }
+          return (request, options) => voyageAIChannels.embed.invoke(
+            target.embed,
+            target,
+            [request, options],
+            {}
+          );
+        case "multimodalEmbed":
+          if (typeof target.multimodalEmbed !== "function") {
+            return target.multimodalEmbed;
+          }
+          return (request, options) => voyageAIChannels.multimodalEmbed.invoke(
+            target.multimodalEmbed,
+            target,
+            [request, options],
+            {}
+          );
+        case "rerank":
+          if (typeof target.rerank !== "function") {
+            return target.rerank;
+          }
+          return (request, options) => voyageAIChannels.rerank.invoke(
+            target.rerank,
+            target,
+            [request, options],
+            {}
+          );
+        case "contextualizedEmbed":
+          if (typeof target.contextualizedEmbed !== "function") {
+            return target.contextualizedEmbed;
+          }
+          return (request, options) => voyageAIChannels.contextualizedEmbed.invoke(
+            target.contextualizedEmbed,
+            target,
+            [request, options],
+            {}
+          );
+        default:
+          return Reflect.get(target, prop, receiver);
+      }
+    }
+  });
+  voyageAIProxyCache.set(client, proxy);
+  return proxy;
+}
+
 // src/wrappers/groq.ts
 function wrapGroq(groq) {
   if (isSupportedGroqClient(groq)) {
@@ -5736,14 +5824,14 @@ function wrapGroq(groq) {
 function isRecord4(value) {
   return typeof value === "object" && value !== null;
 }
-function hasFunction5(value, methodName) {
+function hasFunction6(value, methodName) {
   return isRecord4(value) && methodName in value && typeof value[methodName] === "function";
 }
 function hasChat2(value) {
-  return isRecord4(value) && isRecord4(value.completions) && hasFunction5(value.completions, "create");
+  return isRecord4(value) && isRecord4(value.completions) && hasFunction6(value.completions, "create");
 }
 function hasEmbeddings2(value) {
-  return hasFunction5(value, "create");
+  return hasFunction6(value, "create");
 }
 function isSupportedGroqClient(value) {
   return isRecord4(value) && (value.chat !== void 0 && hasChat2(value.chat) || value.embeddings !== void 0 && hasEmbeddings2(value.embeddings));
@@ -6285,7 +6373,7 @@ async function flushExperimentWithSync(context, config) {
 // src/wrappers/shared/scorers.ts
 async function runScorers(args) {
   const { scorers, output, expected, input, metadata, span } = args;
-  const scorerArgs = {
+  const scorerArgs2 = {
     output,
     expected,
     input,
@@ -6294,7 +6382,7 @@ async function runScorers(args) {
   await Promise.all(
     scorers.map(async (scorer) => {
       try {
-        const result = await scorer(scorerArgs);
+        const result = await scorer(scorerArgs2);
         const scores = normalizeScores(result);
         if (scores.length > 0) {
           const accScores = {};
@@ -8653,6 +8741,26 @@ async function getExperimentParametersRef(parameters) {
     version: resolvedParameters.version
   };
 }
+async function _internalInitEvaluatorExperiment(projectName, evaluator, data, options = {}) {
+  if (options.disabled) return null;
+  const { baseExperiment } = callEvaluatorData(data);
+  const parameters = await getExperimentParametersRef(evaluator.parameters);
+  return initExperiment2(evaluator.state, {
+    ...evaluator.projectId ? { projectId: evaluator.projectId } : { project: projectName },
+    experiment: options.experimentName ?? evaluator.experimentName,
+    description: evaluator.description,
+    metadata: evaluator.metadata,
+    tags: evaluator.tags,
+    isPublic: evaluator.isPublic,
+    update: options.update ?? evaluator.update,
+    baseExperiment: evaluator.baseExperimentName ?? baseExperiment,
+    baseExperimentId: evaluator.baseExperimentId,
+    gitMetadataSettings: evaluator.gitMetadataSettings,
+    repoInfo: evaluator.repoInfo,
+    dataset: Dataset.isDataset(data) ? data : void 0,
+    parameters
+  });
+}
 function callEvaluatorData(data) {
   const dataResult = typeof data === "function" ? data() : data;
   let baseExperiment = void 0;
@@ -8670,6 +8778,48 @@ function isAsyncIterable2(value) {
 function isIterable(value) {
   return typeof value === "object" && value !== null && Symbol.iterator in value && typeof value[Symbol.iterator] === "function";
 }
+async function _internalResolveEvaluatorData(evaluator, experiment) {
+  if (typeof evaluator.data === "string") {
+    throw new Error("Unimplemented: string data paths");
+  }
+  let dataResult = typeof evaluator.data === "function" ? evaluator.data() : evaluator.data;
+  if ("_type" in dataResult) {
+    if (dataResult._type !== "BaseExperiment") {
+      throw new Error("Invalid _type");
+    }
+    if (!experiment) {
+      throw new Error(
+        "Cannot use BaseExperiment() without connecting to Braintrust (you most likely set --no-send-logs)"
+      );
+    }
+    let name = dataResult.name;
+    if (isEmpty(name)) {
+      const baseExperiment = await experiment.fetchBaseExperiment();
+      if (!baseExperiment) {
+        throw new Error("BaseExperiment() failed to fetch base experiment");
+      }
+      name = baseExperiment.name;
+    }
+    dataResult = initExperiment2(evaluator.state, {
+      ...evaluator.projectId ? { projectId: evaluator.projectId } : { project: evaluator.projectName },
+      experiment: name,
+      open: true
+    }).asDataset();
+  }
+  const resolvedDataResult = dataResult instanceof Promise ? await dataResult : dataResult;
+  if (isAsyncIterable2(resolvedDataResult)) {
+    return resolvedDataResult;
+  }
+  if (Array.isArray(resolvedDataResult) || isIterable(resolvedDataResult)) {
+    const iterable = resolvedDataResult;
+    return (async function* () {
+      for (const datum of iterable) yield datum;
+    })();
+  }
+  throw new Error(
+    "Evaluator data must be an array, iterable, or async iterable"
+  );
+}
 globalThis._evals = {
   functions: [],
   prompts: [],
@@ -8716,25 +8866,13 @@ async function Eval(name, evaluator, reporterOrOpts) {
   }
   const resolvedReporter = options.reporter || defaultReporter;
   try {
-    const { data, baseExperiment: defaultBaseExperiment } = callEvaluatorData(
-      evaluator.data
+    const { data } = callEvaluatorData(evaluator.data);
+    const experiment = await _internalInitEvaluatorExperiment(
+      name,
+      evaluator,
+      data,
+      { disabled: Boolean(options.parent || options.noSendLogs) }
     );
-    const parameters = await getExperimentParametersRef(evaluator.parameters);
-    const experiment = options.parent || options.noSendLogs ? null : initExperiment2(evaluator.state, {
-      ...evaluator.projectId ? { projectId: evaluator.projectId } : { project: name },
-      experiment: evaluator.experimentName,
-      description: evaluator.description,
-      metadata: evaluator.metadata,
-      tags: evaluator.tags,
-      isPublic: evaluator.isPublic,
-      update: evaluator.update,
-      baseExperiment: evaluator.baseExperimentName ?? defaultBaseExperiment,
-      baseExperimentId: evaluator.baseExperimentId,
-      gitMetadataSettings: evaluator.gitMetadataSettings,
-      repoInfo: evaluator.repoInfo,
-      dataset: Dataset.isDataset(data) ? data : void 0,
-      parameters
-    });
     if (experiment && typeof process !== "undefined" && globalThis.BRAINTRUST_CONTEXT_MANAGER !== void 0) {
       await experiment._waitForId();
     }
@@ -8832,19 +8970,74 @@ function scorerName(scorer, scorer_idx) {
 function classifierName(classifier, classifier_idx) {
   return classifier.name || `classifier_${classifier_idx}`;
 }
+async function _internalRunEvaluatorTask(task, datum, trialIndex, parameters, span, reportProgress = () => void 0) {
+  const metadata = {
+    ..."metadata" in datum ? datum.metadata : {}
+  };
+  const hooks = {
+    meta(value) {
+      Object.assign(metadata, value);
+    },
+    metadata,
+    expected: "expected" in datum ? datum.expected : void 0,
+    span,
+    parameters,
+    reportProgress,
+    trialIndex,
+    tags: [...datum.tags ?? []]
+  };
+  const output = await task(datum.input, hooks);
+  span.log({ output });
+  return {
+    output,
+    metadata: hooks.metadata,
+    tags: hooks.tags ?? []
+  };
+}
 function buildSpanMetadata(results) {
-  return results.length === 1 ? results[0].metadata : results.reduce(
-    (prev, s) => mergeDicts(prev, { [s.name]: s.metadata }),
-    {}
+  return results.length === 1 ? results[0].metadata : Object.fromEntries(
+    results.map((result) => [result.name, result.metadata])
   );
 }
 function buildSpanScores(results) {
-  const scoresRecord = results.reduce(
-    (prev, s) => mergeDicts(prev, { [s.name]: s.score }),
-    {}
+  const scoresRecord = Object.fromEntries(
+    results.map((result) => [result.name, result.score])
   );
   return { resultMetadata: buildSpanMetadata(results), scoresRecord };
 }
+function _internalPrepareEvaluatorScore(scoreValue, name) {
+  if (scoreValue === null) return { results: null };
+  if (Array.isArray(scoreValue)) {
+    for (const score of scoreValue) {
+      if (!(typeof score === "object" && !isEmpty(score))) {
+        throw new Error(
+          `When returning an array of scores, each score must be a non-empty object. Got: ${JSON.stringify(score)}`
+        );
+      }
+    }
+  }
+  let results;
+  if (Array.isArray(scoreValue)) {
+    results = scoreValue;
+  } else if (typeof scoreValue === "object" && !isEmpty(scoreValue)) {
+    results = [scoreValue];
+  } else {
+    results = [{ name, score: scoreValue }];
+  }
+  const { resultMetadata, scoresRecord } = buildSpanScores(results);
+  const fields = (score) => {
+    const { metadata: _metadata, name: _name, ...rest } = score;
+    return rest;
+  };
+  return {
+    results,
+    output: results.length === 1 ? fields(results[0]) : Object.fromEntries(
+      results.map((score) => [score.name ?? name, fields(score)])
+    ),
+    metadata: resultMetadata,
+    scores: scoresRecord
+  };
+}
 async function runInScorerSpan(rootSpan, spanName, spanType, propagatedEvent, eventInput, fn) {
   try {
     const value = await rootSpan.traced(fn, {
@@ -8889,6 +9082,27 @@ function toClassificationItem(c) {
     ...c.metadata !== void 0 ? { metadata: c.metadata } : {}
   };
 }
+function _internalPrepareEvaluatorClassification(value, name) {
+  if (value === null) return { results: null };
+  const results = (Array.isArray(value) ? value : [value]).map(
+    (result) => validateClassificationResult(result, name)
+  );
+  const classifications = /* @__PURE__ */ Object.create(null);
+  for (const result of results) {
+    (classifications[result.name] ??= []).push(toClassificationItem(result));
+  }
+  return {
+    results,
+    output: results.length === 1 ? toClassificationItem(results[0]) : Object.fromEntries(
+      results.map((result) => [
+        result.name,
+        toClassificationItem(result)
+      ])
+    ),
+    metadata: buildSpanMetadata(results),
+    classifications
+  };
+}
 function logScoringFailures(kind, failures, metadata, rootSpan, state) {
   if (!failures.length) return [];
   const errorMap = Object.fromEntries(
@@ -8936,54 +9150,14 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
     (evaluator.state ?? _internalGetGlobalState())?.spanCache?.start();
   }
   try {
-    if (typeof evaluator.data === "string") {
-      throw new Error("Unimplemented: string data paths");
-    }
-    let dataResult = typeof evaluator.data === "function" ? evaluator.data() : evaluator.data;
     parameters = await validateParameters(
       parameters ?? {},
       evaluator.parameters
     );
-    if ("_type" in dataResult) {
-      if (dataResult._type !== "BaseExperiment") {
-        throw new Error("Invalid _type");
-      }
-      if (!experiment) {
-        throw new Error(
-          "Cannot use BaseExperiment() without connecting to Braintrust (you most likely set --no-send-logs)"
-        );
-      }
-      let name = dataResult.name;
-      if (isEmpty(name)) {
-        const baseExperiment = await experiment.fetchBaseExperiment();
-        if (!baseExperiment) {
-          throw new Error("BaseExperiment() failed to fetch base experiment");
-        }
-        name = baseExperiment.name;
-      }
-      dataResult = initExperiment2(evaluator.state, {
-        ...evaluator.projectId ? { projectId: evaluator.projectId } : { project: evaluator.projectName },
-        experiment: name,
-        open: true
-      }).asDataset();
-    }
-    const resolvedDataResult = dataResult instanceof Promise ? await dataResult : dataResult;
-    const dataIterable = (() => {
-      if (isAsyncIterable2(resolvedDataResult)) {
-        return resolvedDataResult;
-      }
-      if (Array.isArray(resolvedDataResult) || isIterable(resolvedDataResult)) {
-        const iterable = resolvedDataResult;
-        return (async function* () {
-          for (const datum of iterable) {
-            yield datum;
-          }
-        })();
-      }
-      throw new Error(
-        "Evaluator data must be an array, iterable, or async iterable"
-      );
-    })();
+    const dataIterable = await _internalResolveEvaluatorData(
+      evaluator,
+      experiment
+    );
     progressReporter.start(evaluator.evalName, 0);
     const experimentIdPromise = experiment ? (async () => {
       try {
@@ -9052,57 +9226,45 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
             ensureSpansFlushed,
             state
           }) : void 0;
-          let metadata = {
-            ..."metadata" in datum ? datum.metadata : {}
-          };
+          let metadata = {};
           const expected = "expected" in datum ? datum.expected : void 0;
           let output = void 0;
           let error = void 0;
-          let tags = [...datum.tags ?? []];
-          const scores = {};
-          const classifications = {};
+          let tags = [];
+          const scores = /* @__PURE__ */ Object.create(null);
+          const classifications = /* @__PURE__ */ Object.create(null);
           const scorerNames = (evaluator.scores ?? []).map(scorerName);
           const classifierNames = (evaluator.classifiers ?? []).map(
             classifierName
           );
           let unhandledScores = scorerNames;
           try {
-            const meta = (o) => metadata = { ...metadata, ...o };
-            await rootSpan.traced(
-              async (span) => {
-                const hooksForTask = {
-                  meta,
-                  metadata,
-                  expected,
-                  span,
-                  parameters: parameters ?? {},
-                  reportProgress: (event) => {
-                    stream?.({
-                      ...event,
-                      id: rootSpan.id,
-                      origin: baseEvent.event?.origin,
-                      name: evaluator.evalName,
-                      object_type: "task"
-                    });
-                  },
-                  trialIndex,
-                  tags
-                };
-                const outputResult = evaluator.task(datum.input, hooksForTask);
-                if (outputResult instanceof Promise) {
-                  output = await outputResult;
-                } else {
-                  output = outputResult;
+            const taskResult = await rootSpan.traced(
+              (span) => _internalRunEvaluatorTask(
+                evaluator.task,
+                datum,
+                trialIndex,
+                parameters ?? {},
+                span,
+                (event) => {
+                  stream?.({
+                    ...event,
+                    id: rootSpan.id,
+                    origin: baseEvent.event?.origin,
+                    name: evaluator.evalName,
+                    object_type: "task"
+                  });
                 }
-                tags = hooksForTask.tags ?? [];
-                span.log({ output });
-              },
+              ),
               {
                 name: "task",
                 spanAttributes: { type: "task" /* TASK */ },
                 event: { input: datum.input }
               }
             );
+            output = taskResult.output;
+            metadata = taskResult.metadata;
+            tags = taskResult.tags;
             if (tags.length) {
               rootSpan.log({ output, metadata, expected, tags });
             } else {
@@ -9112,20 +9274,18 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
               await rootSpan.flush();
             }
             const scoringArgs = {
+              id: datum.id,
               input: datum.input,
               expected: "expected" in datum ? datum.expected : void 0,
               metadata,
               output,
+              tags,
               trace
             };
             const { trace: _trace, ...scoringArgsForLogging } = scoringArgs;
             const propagatedEvent = makeScorerPropagatedEvent(
               await rootSpan.export()
             );
-            const getOtherFields = (s) => {
-              const { metadata: _metadata, name: _name, ...rest } = s;
-              return rest;
-            };
             const [scoreResults, classificationResults] = await Promise.all([
               Promise.all(
                 (evaluator.scores ?? []).map(
@@ -9139,35 +9299,17 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
                       const scoreValue = await Promise.resolve(
                         score(scoringArgs)
                       );
-                      if (scoreValue === null) return null;
-                      if (Array.isArray(scoreValue)) {
-                        for (const s of scoreValue) {
-                          if (!(typeof s === "object" && !isEmpty(s))) {
-                            throw new Error(
-                              `When returning an array of scores, each score must be a non-empty object. Got: ${JSON.stringify(s)}`
-                            );
-                          }
-                        }
-                      }
-                      const results = Array.isArray(scoreValue) ? scoreValue : typeof scoreValue === "object" && !isEmpty(scoreValue) ? [scoreValue] : [
-                        {
-                          name: scorerNames[score_idx],
-                          score: scoreValue
-                        }
-                      ];
-                      const { resultMetadata, scoresRecord } = buildSpanScores(results);
-                      const resultOutput = results.length === 1 ? getOtherFields(results[0]) : results.reduce(
-                        (prev, s) => mergeDicts(prev, {
-                          [s.name]: getOtherFields(s)
-                        }),
-                        {}
+                      const prepared = _internalPrepareEvaluatorScore(
+                        scoreValue,
+                        scorerNames[score_idx]
                       );
+                      if (prepared.results === null) return null;
                       span.log({
-                        output: resultOutput,
-                        metadata: resultMetadata,
-                        scores: scoresRecord
+                        output: prepared.output,
+                        metadata: prepared.metadata,
+                        scores: prepared.scores
                       });
-                      return results;
+                      return prepared.results;
                     }
                   )
                 )
@@ -9184,24 +9326,16 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
                       const classifierValue = await Promise.resolve(
                         classifier(scoringArgs)
                       );
-                      if (classifierValue === null) return null;
-                      const rawResults = (Array.isArray(classifierValue) ? classifierValue : [classifierValue]).map(
-                        (result) => validateClassificationResult(
-                          result,
-                          classifierNames[idx]
-                        )
-                      );
-                      const resultOutput = rawResults.length === 1 ? toClassificationItem(rawResults[0]) : rawResults.reduce(
-                        (prev, r) => mergeDicts(prev, {
-                          [r.name]: toClassificationItem(r)
-                        }),
-                        {}
+                      const prepared = _internalPrepareEvaluatorClassification(
+                        classifierValue,
+                        classifierNames[idx]
                       );
+                      if (prepared.results === null) return null;
                       span.log({
-                        output: resultOutput,
-                        metadata: buildSpanMetadata(rawResults)
+                        output: prepared.output,
+                        metadata: prepared.metadata
                       });
-                      return rawResults;
+                      return prepared.results;
                     }
                   )
                 )
@@ -9430,7 +9564,7 @@ function accumulateScores(accumulator, scores) {
   }
 }
 function ensureScoreAccumulator(results) {
-  const accumulator = {};
+  const accumulator = /* @__PURE__ */ Object.create(null);
   for (const result of results) {
     accumulateScores(accumulator, result.scores);
   }
@@ -9573,6 +9707,1156 @@ var defaultReporter = {
   }
 };
 
+// src/durable-eval.ts
+var encoder = new TextEncoder();
+var decoder = new TextDecoder();
+var BATCH_TASK_KIND = "braintrust.durable.batch-task";
+var BATCH_SCORER_KIND = "braintrust.durable.batch-scorer";
+var DEFAULT_BATCH_SIZE = 1e3;
+var DurableEvalMemoryStore = class {
+  values = /* @__PURE__ */ new Map();
+  async read(key) {
+    return this.values.get(key)?.slice();
+  }
+  async write(key, value) {
+    this.values.set(key, value.slice());
+  }
+  async getOrSet(key, value) {
+    const existing = this.values.get(key);
+    if (existing) return { value: existing.slice(), created: false };
+    this.values.set(key, value.slice());
+    return { value: value.slice(), created: true };
+  }
+};
+var DurableEvalRedisStore = class {
+  client;
+  keyPrefix;
+  ttlMs;
+  constructor(options) {
+    this.client = options.client;
+    this.keyPrefix = options.keyPrefix ?? "braintrust-eval:";
+    this.ttlMs = options.ttlMs ?? 1e3 * 60 * 60 * 24 * 7;
+    if (!Number.isInteger(this.ttlMs) || this.ttlMs < 1) {
+      throw new Error("DurableEvalRedisStore ttlMs must be a positive integer");
+    }
+  }
+  async read(key) {
+    const value = await this.client.get(`${this.keyPrefix}${key}`);
+    if (value == null) return void 0;
+    if (typeof value !== "string") {
+      throw new Error("DurableEvalRedisStore expected GET to return a string");
+    }
+    return base64ToUint8Array(value);
+  }
+  async write(key, value) {
+    const client = this.client;
+    const set = client.set;
+    const redisKey = `${this.keyPrefix}${key}`;
+    const encoded = uint8ArrayToBase64(value);
+    if (typeof client.defineCommand === "function") {
+      await set.call(client, redisKey, encoded, "PX", this.ttlMs);
+    } else if (typeof client.sendCommand === "function") {
+      await set.call(client, redisKey, encoded, { PX: this.ttlMs });
+    } else if (typeof client.createScript === "function") {
+      await set.call(client, redisKey, encoded, { px: this.ttlMs });
+    } else {
+      throw new Error(
+        "DurableEvalRedisStore requires a node-redis, ioredis, or @upstash/redis client"
+      );
+    }
+  }
+  async getOrSet(key, value) {
+    const redisKey = `${this.keyPrefix}${key}`;
+    const encoded = uint8ArrayToBase64(value);
+    const client = this.client;
+    const set = client.set;
+    let setOptions;
+    if (typeof client.defineCommand === "function") {
+      setOptions = ["PX", this.ttlMs, "NX", "GET"];
+    } else if (typeof client.sendCommand === "function") {
+      setOptions = [{ PX: this.ttlMs, NX: true, GET: true }];
+    } else if (typeof client.createScript === "function") {
+      setOptions = [{ px: this.ttlMs, nx: true, get: true }];
+    } else {
+      throw new Error(
+        "DurableEvalRedisStore getOrSet requires a node-redis, ioredis, or @upstash/redis client"
+      );
+    }
+    const existing = await set.call(client, redisKey, encoded, ...setOptions);
+    if (existing === null) return { value: value.slice(), created: true };
+    if (typeof existing !== "string") {
+      throw new Error(
+        "DurableEvalRedisStore expected atomic SET to return a string or null"
+      );
+    }
+    return { value: base64ToUint8Array(existing), created: false };
+  }
+};
+var BatchTask = class {
+  constructor(processor) {
+    this.processor = processor;
+  }
+  processor;
+  kind = BATCH_TASK_KIND;
+};
+var BatchScorer = class {
+  constructor(processor) {
+    this.processor = processor;
+    this.name = processor.name;
+  }
+  processor;
+  kind = BATCH_SCORER_KIND;
+  name;
+};
+function defineDurableEval(projectName, evaluator) {
+  const definition = {
+    projectName,
+    evalName: evaluator.experimentName ?? projectName,
+    evaluator
+  };
+  return {
+    start: (options = {}) => startDurableEval(definition, options),
+    status: (options) => getDurableEvalStatus(definition, options),
+    poll: (options) => pollDurableEval(definition, options),
+    processBatchResult: (result) => processDurableBatchResult(definition, result)
+  };
+}
+async function startDurableEval(definition, options) {
+  const store = definition.evaluator.store;
+  const runId = newId();
+  const key = runKey(definition.projectName, definition.evalName, runId);
+  const { data } = callEvaluatorData(definition.evaluator.data);
+  const parameters = await validateParameters(
+    options.parameters ?? {},
+    definition.evaluator.parameters
+  );
+  const experimentName = definition.evaluator.experimentName ?? `${definition.evalName}-${runId}`;
+  const experiment = await _internalInitEvaluatorExperiment(
+    definition.projectName,
+    { ...definition.evaluator, data },
+    data,
+    {
+      disabled: options.noSendLogs ?? false,
+      experimentName,
+      update: true
+    }
+  );
+  if (!isBatchTask(definition.evaluator.task) && !(definition.evaluator.scores ?? []).some(isBatchScorer)) {
+    const result = await runEvaluator(
+      experiment,
+      {
+        ...definition.evaluator,
+        projectName: definition.projectName,
+        evalName: definition.evalName,
+        data
+      },
+      {
+        start: () => void 0,
+        stop: () => void 0,
+        increment: () => void 0
+      },
+      [],
+      void 0,
+      parameters,
+      true,
+      true
+    );
+    const state2 = {
+      runId,
+      experimentName,
+      noSendLogs: options.noSendLogs ?? false,
+      parameters: assertJsonValue(parameters, "eval parameters"),
+      status: "completed",
+      summary: result.summary,
+      cases: [],
+      batches: []
+    };
+    await experiment?.flush();
+    await writeRunRecord(store, key, state2);
+    return currentStatus(definition, state2);
+  }
+  const state = {
+    runId,
+    experimentName,
+    noSendLogs: options.noSendLogs ?? false,
+    parameters: assertJsonValue(parameters, "eval parameters"),
+    status: "running",
+    cases: await materializeCases(definition, data, experiment),
+    batches: []
+  };
+  await writeCaseBaseRecords(store, key, state.cases);
+  await writeRunRecord(store, key, state);
+  return advanceDurableEval(definition, state, store, key, experiment);
+}
+async function getDurableEvalStatus(definition, options) {
+  const store = definition.evaluator.store;
+  const state = await readRunState(
+    definition,
+    store,
+    runKey(definition.projectName, definition.evalName, options.runId)
+  );
+  if (!state) throw new Error(`Durable eval run ${options.runId} is missing`);
+  return currentStatus(definition, state);
+}
+async function processDurableBatchResult(definition, result) {
+  if (!result.batchId && !result.externalId) {
+    throw new Error("Batch results require batchId or externalId");
+  }
+  const store = definition.evaluator.store;
+  const key = runKey(definition.projectName, definition.evalName, result.runId);
+  const state = await readRunState(definition, store, key);
+  if (!state) throw new Error(`Durable eval run ${result.runId} is missing`);
+  const byBatch = result.batchId ? state.batches.find((candidate) => candidate.id === result.batchId) : void 0;
+  const byExternal = result.externalId ? state.batches.find(
+    (candidate) => candidate.externalId === result.externalId
+  ) : void 0;
+  if (byBatch && byExternal && byBatch.id !== byExternal.id) {
+    throw new Error("batchId and externalId identify different batches");
+  }
+  const batch = byBatch ?? byExternal;
+  if (!batch) throw new Error("No submitted batch matches this result");
+  if (batch.status !== "complete") {
+    const records = await collectBatch(definition, state, batch);
+    batch.status = "complete";
+    await writeCaseRecords(store, key, records);
+    await writeBatchRecords(store, key, [batch]);
+  }
+  return advanceDurableEval(
+    definition,
+    await readRunState(definition, store, key),
+    store,
+    key
+  );
+}
+async function pollDurableEval(definition, options) {
+  const store = definition.evaluator.store;
+  const key = runKey(
+    definition.projectName,
+    definition.evalName,
+    options.runId
+  );
+  const state = await readRunState(definition, store, key);
+  if (!state) throw new Error(`Durable eval run ${options.runId} is missing`);
+  const batches = state.batches.filter((batch) => {
+    if (batch.status === "complete") return false;
+    return processorForStage(definition, batch.kind, batch.scorerName).completion.mode === "poll";
+  });
+  const results = await Promise.all(
+    batches.map(async (batch) => ({
+      batch,
+      result: await processorForStage(definition, batch.kind, batch.scorerName).completion.poll(batch.submissionData, {
+        runId: state.runId,
+        batchId: batch.id
+      })
+    }))
+  );
+  const changedCases = /* @__PURE__ */ new Map();
+  const changedBatches = [];
+  for (const { batch, result } of results) {
+    if (result.status === "failed") throw asError(result.error);
+    if (result.status !== "complete") continue;
+    for (const record of await collectBatch(definition, state, batch)) {
+      changedCases.set(record.id, record);
+    }
+    batch.status = "complete";
+    changedBatches.push(batch);
+  }
+  if (changedBatches.length > 0) {
+    await writeCaseRecords(store, key, [...changedCases.values()]);
+    await writeBatchRecords(store, key, changedBatches);
+  }
+  const currentState = changedBatches.length > 0 ? await readRunState(definition, store, key) : state;
+  return advanceDurableEval(definition, currentState, store, key);
+}
+async function openDurableExperiment(definition, state) {
+  const data = [];
+  return await _internalInitEvaluatorExperiment(
+    definition.projectName,
+    { ...definition.evaluator, data },
+    data,
+    {
+      disabled: state.noSendLogs,
+      experimentName: state.experimentName,
+      update: true
+    }
+  );
+}
+async function advanceDurableEval(definition, state, store, key, existingExperiment) {
+  if (state.status === "completed") return currentStatus(definition, state);
+  const experiment = existingExperiment === void 0 ? await openDurableExperiment(definition, state) : existingExperiment;
+  await runTaskStage(definition, state, store, key, experiment);
+  await logCompletedTasks(definition, state, store, key, experiment);
+  state = await readRunState(definition, store, key) ?? state;
+  if (state.cases.some((record) => !record.taskComplete || !record.taskLogged)) {
+    return currentStatus(definition, state);
+  }
+  await runScoreStages(definition, state, store, key, experiment);
+  state = await readRunState(definition, store, key) ?? state;
+  const scorerNames = resolveScorers(definition.evaluator.scores ?? []).map(
+    ({ name }) => name
+  );
+  const classifierNames = (definition.evaluator.classifiers ?? []).map(
+    classifierName
+  );
+  if (state.cases.some(
+    (record) => scorerNames.some(
+      (name) => !Object.hasOwn(record.scores, name) || !Object.hasOwn(record.loggedScores, name)
+    ) || classifierNames.some(
+      (name) => !Object.hasOwn(record.loggedClassifications, name)
+    )
+  )) {
+    return currentStatus(definition, state);
+  }
+  if (!await claimAction(store, key, "finish")) {
+    const latest = await readRunState(definition, store, key);
+    return currentStatus(definition, latest ?? state);
+  }
+  state.summary = await finishExperiment(definition, state, experiment);
+  state.status = "completed";
+  await writeRunRecord(store, key, state);
+  return currentStatus(definition, state);
+}
+function currentStatus(definition, state) {
+  if (state.status === "completed") {
+    if (!state.summary) {
+      throw new Error(`Durable eval run ${state.runId} has no saved summary`);
+    }
+    return {
+      status: "completed",
+      runId: state.runId,
+      pending: { poll: 0, webhook: 0 },
+      summary: state.summary
+    };
+  }
+  const pending = { poll: 0, webhook: 0 };
+  for (const batch of state.batches) {
+    if (batch.status === "complete") continue;
+    pending[processorForStage(definition, batch.kind, batch.scorerName).completion.mode]++;
+  }
+  return { status: "waiting", runId: state.runId, pending };
+}
+async function startCaseRoot(definition, state, record, experiment) {
+  if (!experiment) return NOOP_SPAN;
+  const datum = record.datum;
+  return _internalStartSpanWithInitialMerge({
+    ...definition.evaluator.state ? { state: definition.evaluator.state } : {},
+    parent: await experiment.export(),
+    name: "eval",
+    spanId: deterministicId(`${state.runId}:${record.id}:span`),
+    spanAttributes: { type: "eval" /* EVAL */ },
+    event: {
+      id: deterministicId(`${state.runId}:${record.id}:row`),
+      input: datum.input,
+      expected: "expected" in datum ? datum.expected : void 0,
+      tags: datum.tags,
+      origin: datum.origin
+    }
+  });
+}
+async function logTaskResult(definition, state, record, experiment, task) {
+  const datum = record.datum;
+  const root = await startCaseRoot(definition, state, record, experiment);
+  try {
+    if (task) {
+      const result = await root.traced(
+        (span) => _internalRunEvaluatorTask(
+          task,
+          datum,
+          record.trialIndex,
+          state.parameters,
+          span
+        ),
+        {
+          name: "task",
+          spanId: deterministicId(`${state.runId}:${record.id}:task`),
+          spanAttributes: { type: "task" /* TASK */ },
+          event: { input: datum.input }
+        }
+      );
+      record.output = assertJsonValue(
+        result.output,
+        `task output for ${record.caseId}`
+      );
+      record.metadata = assertJsonValue(result.metadata, "task metadata");
+      record.tags = result.tags;
+      record.taskComplete = true;
+    } else {
+      await root.traced((span) => span.log({ output: record.output }), {
+        name: "task",
+        spanId: deterministicId(`${state.runId}:${record.id}:task`),
+        spanAttributes: { type: "task" /* TASK */ },
+        event: { input: datum.input }
+      });
+    }
+    root.log({
+      output: record.output,
+      expected: "expected" in datum ? datum.expected : void 0,
+      metadata: {
+        ...record.metadata,
+        durable_eval: {
+          run_id: state.runId,
+          case_id: record.caseId,
+          trial_index: record.trialIndex
+        }
+      },
+      tags: record.tags
+    });
+    record.rootSpan = await root.export();
+    record.taskLogged = true;
+  } catch (error) {
+    logError(root, error);
+    throw error;
+  } finally {
+    root.end();
+  }
+}
+async function logCompletedTasks(definition, state, store, key, experiment) {
+  const changed = [];
+  for (const record of state.cases) {
+    if (!record.taskComplete || record.taskLogged) continue;
+    if (!await claimAction(store, key, "task-log", record.id)) continue;
+    await logTaskResult(definition, state, record, experiment);
+    changed.push(record);
+  }
+  if (changed.length > 0) {
+    await experiment?.flush();
+    await writeCaseRecords(store, key, changed);
+  }
+}
+async function runTaskStage(definition, state, store, key, experiment) {
+  if (isBatchTask(definition.evaluator.task)) {
+    await ensureBatches(definition, state, store, key, "task");
+    return;
+  }
+  const task = definition.evaluator.task;
+  const changed = [];
+  for (const record of state.cases) {
+    if (record.taskComplete) continue;
+    if (!await claimAction(store, key, "task", record.id)) continue;
+    await logTaskResult(definition, state, record, experiment, task);
+    changed.push(record);
+  }
+  if (changed.length > 0) {
+    await experiment?.flush();
+    await writeCaseRecords(store, key, changed);
+  }
+}
+async function runScoreStages(definition, state, store, key, experiment) {
+  const scorers = resolveScorers(definition.evaluator.scores ?? []);
+  const changed = /* @__PURE__ */ new Map();
+  const persistChangedCases = async () => {
+    if (changed.size === 0) return;
+    await experiment?.flush();
+    await writeCaseRecords(store, key, [...changed.values()]);
+    changed.clear();
+  };
+  for (const { name, scorer } of scorers) {
+    if (isBatchScorer(scorer)) {
+      for (const record of state.cases) {
+        if (Object.hasOwn(record.scores, name) && !Object.hasOwn(record.loggedScores, name)) {
+          if (!await claimAction(store, key, "score-log", record.id, name)) {
+            continue;
+          }
+          await evaluateAndLogScore(
+            definition,
+            state,
+            record,
+            name,
+            experiment
+          );
+          changed.set(record.id, record);
+        }
+      }
+      await persistChangedCases();
+      await ensureBatches(definition, state, store, key, "score", name);
+      continue;
+    }
+    for (const record of state.cases) {
+      if (Object.hasOwn(record.loggedScores, name)) continue;
+      if (!await claimAction(store, key, "score", record.id, name)) continue;
+      await evaluateAndLogScore(
+        definition,
+        state,
+        record,
+        name,
+        experiment,
+        scorer
+      );
+      changed.set(record.id, record);
+    }
+  }
+  for (const [index, classifier] of (definition.evaluator.classifiers ?? []).entries()) {
+    const name = classifierName(classifier, index);
+    for (const record of state.cases) {
+      if (Object.hasOwn(record.loggedClassifications, name)) continue;
+      if (!await claimAction(store, key, "classification", record.id, name)) {
+        continue;
+      }
+      await evaluateAndLogClassification(
+        definition,
+        state,
+        record,
+        name,
+        classifier,
+        experiment
+      );
+      changed.set(record.id, record);
+    }
+  }
+  await persistChangedCases();
+}
+function scorerArgs(record) {
+  const datum = record.datum;
+  return {
+    ...datum,
+    metadata: record.metadata,
+    output: record.output
+  };
+}
+function resumeCaseRoot(definition, record, experiment) {
+  if (!experiment) return NOOP_SPAN;
+  if (!record.rootSpan) {
+    throw new Error(`Durable eval case ${record.caseId} has no root span`);
+  }
+  return _internalResumeSpan({
+    exported: record.rootSpan,
+    state: definition.evaluator.state
+  });
+}
+async function evaluateAndLogScore(definition, state, record, name, experiment, scorer) {
+  const root = resumeCaseRoot(definition, record, experiment);
+  try {
+    const rootExport = await root.export();
+    const prepared = await root.traced(
+      async (span) => {
+        const value = scorer ? await scorer(scorerArgs(record)) : record.scores[name];
+        if (scorer) {
+          record.scores[name] = assertJsonValue(value, `scorer ${name} output`);
+        }
+        const result = _internalPrepareEvaluatorScore(value, name);
+        if (result.results !== null) {
+          span.log({
+            output: result.output,
+            metadata: result.metadata,
+            scores: result.scores
+          });
+        }
+        return result;
+      },
+      {
+        name,
+        spanId: deterministicId(`${state.runId}:${record.id}:score:${name}`),
+        spanAttributes: {
+          type: "score" /* SCORE */,
+          purpose: "scorer"
+        },
+        propagatedEvent: makeScorerPropagatedEvent(rootExport || void 0),
+        event: { input: scorerArgs(record) }
+      }
+    );
+    if (prepared.scores) root.log({ scores: prepared.scores });
+    record.loggedScores[name] = true;
+  } catch (error) {
+    logError(root, error);
+    throw error;
+  } finally {
+    root.end();
+  }
+}
+async function evaluateAndLogClassification(definition, state, record, name, classifier, experiment) {
+  const root = resumeCaseRoot(definition, record, experiment);
+  try {
+    const rootExport = await root.export();
+    const prepared = await root.traced(
+      async (span) => {
+        const value = await classifier(scorerArgs(record));
+        record.classifications[name] = assertJsonValue(
+          value,
+          `classifier ${name} output`
+        );
+        const result = _internalPrepareEvaluatorClassification(value, name);
+        if (result.results !== null) {
+          span.log({ output: result.output, metadata: result.metadata });
+        }
+        return result;
+      },
+      {
+        name,
+        spanId: deterministicId(
+          `${state.runId}:${record.id}:classification:${name}`
+        ),
+        spanAttributes: {
+          type: "classifier" /* CLASSIFIER */,
+          purpose: "scorer"
+        },
+        propagatedEvent: makeScorerPropagatedEvent(rootExport || void 0),
+        event: { input: scorerArgs(record) }
+      }
+    );
+    if (prepared.classifications) {
+      root.log({ classifications: prepared.classifications });
+    }
+    record.loggedClassifications[name] = true;
+  } catch (error) {
+    logError(root, error);
+    throw error;
+  } finally {
+    root.end();
+  }
+}
+async function ensureBatches(definition, state, store, key, kind, scorerName2) {
+  const processor = processorForStage(definition, kind, scorerName2);
+  const plans = plannedBatches(
+    definition,
+    state.runId,
+    state.cases.map(({ id }) => id)
+  );
+  const casesById = new Map(state.cases.map((record) => [record.id, record]));
+  for (const plan of plans) {
+    if (plan.kind !== kind || plan.scorerName !== scorerName2) continue;
+    if (state.batches.some(({ id }) => id === plan.id)) continue;
+    const records = plan.itemIds.map((id) => casesById.get(id));
+    const ready = records.every(
+      (record) => kind === "task" ? !record.taskComplete : !Object.hasOwn(record.scores, scorerName2)
+    );
+    if (!ready) continue;
+    const batchId = plan.id;
+    const claim = await store.getOrSet(
+      claimRecordKey(key, "batch", batchId),
+      encoder.encode(batchId)
+    );
+    if (!claim.created) continue;
+    const context = { runId: state.runId, batchId };
+    const items = records.map(
+      (record) => kind === "task" ? taskBatchItem(record, state.parameters) : scorerBatchItem(record)
+    );
+    const submissionData = assertJsonValue(
+      await processor.submit(items, context),
+      `submission data for batch ${batchId}`
+    );
+    const externalId = processor.completion.mode === "webhook" ? processor.completion.getExternalId(submissionData, context) : void 0;
+    if (externalId !== void 0 && !externalId.trim()) {
+      throw new Error(`Batch ${batchId} produced an empty externalId`);
+    }
+    const batch = {
+      id: batchId,
+      kind,
+      scorerName: scorerName2,
+      itemIds: records.map((record) => record.id),
+      submissionData,
+      externalId,
+      status: "submitted"
+    };
+    state.batches.push(batch);
+    await writeBatchRecords(store, key, [batch]);
+  }
+}
+async function collectBatch(definition, state, batch) {
+  const processor = processorForStage(definition, batch.kind, batch.scorerName);
+  const context = { runId: state.runId, batchId: batch.id };
+  const results = await processor.collect(batch.submissionData, context);
+  if (!Array.isArray(results)) {
+    throw new Error(`collect for batch ${batch.id} must return an array`);
+  }
+  const expectedIds = new Set(batch.itemIds);
+  const seen = /* @__PURE__ */ new Set();
+  const records = [];
+  for (const result of results) {
+    const id = resultItemId(result);
+    if (!expectedIds.has(id)) {
+      throw new Error(`Batch ${batch.id} returned unknown item ${id}`);
+    }
+    if (seen.has(id)) {
+      throw new Error(`Batch ${batch.id} returned item ${id} more than once`);
+    }
+    seen.add(id);
+    const record = state.cases.find((candidate) => candidate.id === id);
+    records.push(record);
+    if (batch.kind === "task") {
+      record.output = assertJsonValue(
+        result.output,
+        `task output for item ${id}`
+      );
+      if ("metadata" in result && result.metadata !== void 0) {
+        record.metadata = assertJsonValue(
+          {
+            ...record.metadata,
+            ...result.metadata
+          },
+          `metadata for ${id}`
+        );
+      }
+      if ("tags" in result && result.tags !== void 0)
+        record.tags = result.tags;
+      record.taskComplete = true;
+    } else {
+      record.scores[batch.scorerName] = assertJsonValue(
+        result.score,
+        `score output for item ${id}`
+      );
+    }
+  }
+  const missing = batch.itemIds.filter((id) => !seen.has(id));
+  if (missing.length > 0) {
+    throw new Error(
+      `Batch ${batch.id} did not return results for: ${missing.join(", ")}`
+    );
+  }
+  return records;
+}
+function processorForStage(definition, kind, scorerName2) {
+  if (kind === "task") {
+    if (!isBatchTask(definition.evaluator.task)) {
+      throw new Error("Definition no longer contains the batch task");
+    }
+    return definition.evaluator.task.processor;
+  }
+  const scorer = resolveScorers(definition.evaluator.scores ?? []).find(
+    ({ name }) => name === scorerName2
+  )?.scorer;
+  if (!isBatchScorer(scorer)) {
+    throw new Error(`Definition no longer contains scorer ${scorerName2}`);
+  }
+  return scorer.processor;
+}
+async function materializeCases(definition, data, experiment) {
+  const evaluator = definition.evaluator;
+  const iterable = await _internalResolveEvaluatorData(
+    {
+      data,
+      projectName: definition.projectName,
+      projectId: evaluator.projectId,
+      state: evaluator.state
+    },
+    experiment
+  );
+  const records = [];
+  const seen = /* @__PURE__ */ new Set();
+  for await (const datum of iterable) {
+    const caseId = datum.id ?? datum.upsert_id ?? (evaluator.caseId ? await evaluator.caseId(datum) : void 0);
+    if (!caseId) {
+      throw new Error(
+        "Every durable eval case requires id, upsert_id, or caseId"
+      );
+    }
+    if (seen.has(caseId))
+      throw new Error(`Duplicate durable eval case id: ${caseId}`);
+    seen.add(caseId);
+    const trialCount = datum.trialCount ?? evaluator.trialCount ?? 1;
+    if (!Number.isInteger(trialCount) || trialCount < 1) {
+      throw new Error(`Invalid trialCount for durable eval case ${caseId}`);
+    }
+    for (let trialIndex = 0; trialIndex < trialCount; trialIndex++) {
+      records.push({
+        id: `${caseId}:trial:${trialIndex}`,
+        caseId,
+        trialIndex,
+        datum: assertJsonValue(datum, `case ${caseId}`),
+        metadata: assertJsonValue(
+          "metadata" in datum ? datum.metadata : {},
+          `metadata for ${caseId}`
+        ),
+        tags: datum.tags,
+        taskComplete: false,
+        taskLogged: false,
+        scores: /* @__PURE__ */ Object.create(null),
+        loggedScores: /* @__PURE__ */ Object.create(null),
+        classifications: /* @__PURE__ */ Object.create(null),
+        loggedClassifications: /* @__PURE__ */ Object.create(null)
+      });
+    }
+  }
+  return records;
+}
+function taskBatchItem(record, parameters) {
+  const datum = record.datum;
+  return {
+    id: record.id,
+    input: datum.input,
+    expected: "expected" in datum ? datum.expected : void 0,
+    metadata: record.metadata,
+    tags: record.tags,
+    parameters,
+    trialIndex: record.trialIndex
+  };
+}
+function scorerBatchItem(record) {
+  const datum = record.datum;
+  return {
+    id: record.id,
+    input: datum.input,
+    output: record.output,
+    expected: "expected" in datum ? datum.expected : void 0,
+    metadata: record.metadata,
+    tags: record.tags,
+    trialIndex: record.trialIndex
+  };
+}
+async function finishExperiment(definition, state, experiment) {
+  const scorerNames = resolveScorers(definition.evaluator.scores ?? []).map(
+    ({ name }) => name
+  );
+  const results = state.cases.map((record) => {
+    const datum = record.datum;
+    const scores = Object.fromEntries(
+      scorerNames.flatMap(
+        (name) => Object.entries(
+          _internalPrepareEvaluatorScore(
+            record.scores[name],
+            name
+          ).scores ?? {}
+        )
+      )
+    );
+    const classifications = Object.fromEntries(
+      Object.entries(record.classifications).flatMap(
+        ([name, value]) => Object.entries(
+          _internalPrepareEvaluatorClassification(value, name).classifications ?? {}
+        )
+      )
+    );
+    return {
+      ...datum,
+      output: record.output,
+      metadata: record.metadata,
+      tags: record.tags,
+      scores,
+      error: void 0,
+      ...Object.keys(classifications).length > 0 ? { classifications } : {}
+    };
+  });
+  if (!experiment) {
+    return buildLocalSummary(
+      {
+        ...definition.evaluator,
+        projectName: definition.projectName,
+        evalName: state.experimentName
+      },
+      results
+    );
+  }
+  await experiment.flush();
+  let comparisonExperimentId = definition.evaluator.baseExperimentId;
+  if (!comparisonExperimentId) {
+    try {
+      comparisonExperimentId = await experiment._getBaseExperimentId();
+    } catch {
+      comparisonExperimentId = void 0;
+    }
+  }
+  return await experiment.summarize({
+    summarizeScores: definition.evaluator.summarizeScores,
+    ...comparisonExperimentId ? { comparisonExperimentId } : {}
+  });
+}
+function resolveScorers(scorers) {
+  return scorers.map((scorer, index) => ({
+    name: isBatchScorer(scorer) ? scorer.name : scorer.name || `scorer_${index}`,
+    scorer
+  }));
+}
+function runKey(projectName, evalName, runId) {
+  return `durable-eval/v1/runs/${contentVersion(encoder.encode(`${projectName}\0${evalName}\0${runId}`))}`;
+}
+function encodedKeyPart(value) {
+  return uint8ArrayToBase64(encoder.encode(value)).replaceAll("+", "-").replaceAll("/", "_").replace(/=+$/, "");
+}
+function caseRecordKey(key, caseId, kind, ...names) {
+  const suffix = names.map(encodedKeyPart).join("/");
+  return `${key}/cases/${encodedKeyPart(caseId)}/${kind}${suffix ? `/${suffix}` : ""}`;
+}
+function batchRecordKey(key, batchId) {
+  return `${key}/batches/${encodedKeyPart(batchId)}`;
+}
+function claimRecordKey(key, kind, ...parts) {
+  const identity = stableStringify([kind, parts]);
+  return `${key}/claims/${contentVersion(encoder.encode(identity))}`;
+}
+async function claimAction(store, key, kind, ...parts) {
+  return (await store.getOrSet(
+    claimRecordKey(key, kind, ...parts),
+    encoder.encode("claimed")
+  )).created;
+}
+function plannedBatches(definition, runId, caseIds) {
+  const stages = [];
+  if (isBatchTask(definition.evaluator.task)) stages.push({ kind: "task" });
+  for (const { name, scorer } of resolveScorers(
+    definition.evaluator.scores ?? []
+  )) {
+    if (isBatchScorer(scorer)) {
+      stages.push({ kind: "score", scorerName: name });
+    }
+  }
+  const plans = [];
+  for (const { kind, scorerName: scorerName2 } of stages) {
+    const batchSize = processorForStage(definition, kind, scorerName2).batchSize ?? DEFAULT_BATCH_SIZE;
+    if (!Number.isInteger(batchSize) || batchSize < 1) {
+      throw new Error(
+        `Invalid batchSize for ${scorerName2 ?? "task"}: ${batchSize}`
+      );
+    }
+    for (let offset = 0; offset < caseIds.length; offset += batchSize) {
+      const itemIds = caseIds.slice(offset, offset + batchSize);
+      plans.push({
+        id: deterministicId(
+          stableStringify([runId, kind, scorerName2, itemIds])
+        ),
+        kind,
+        scorerName: scorerName2,
+        itemIds
+      });
+    }
+  }
+  return plans;
+}
+async function readCaseRecord(definition, store, key, id) {
+  const scorers = resolveScorers(definition.evaluator.scores ?? []);
+  const classifiers = (definition.evaluator.classifiers ?? []).map(
+    classifierName
+  );
+  const [
+    base,
+    task,
+    taskLog,
+    scoreValues,
+    scoreLogValues,
+    classificationValues,
+    classificationLogValues
+  ] = await Promise.all([
+    readJson(store, caseRecordKey(key, id, "base")),
+    readJson(store, caseRecordKey(key, id, "task")),
+    readJson(store, caseRecordKey(key, id, "task-log")),
+    Promise.all(
+      scorers.map(async ({ name }) => ({
+        name,
+        value: await readJson(
+          store,
+          caseRecordKey(key, id, "score", name)
+        )
+      }))
+    ),
+    Promise.all(
+      scorers.map(async ({ name }) => ({
+        name,
+        value: await readJson(
+          store,
+          caseRecordKey(key, id, "score-log", name)
+        )
+      }))
+    ),
+    Promise.all(
+      classifiers.map(async (name) => ({
+        name,
+        value: await readJson(
+          store,
+          caseRecordKey(key, id, "classification", name)
+        )
+      }))
+    ),
+    Promise.all(
+      classifiers.map(async (name) => ({
+        name,
+        value: await readJson(
+          store,
+          caseRecordKey(key, id, "classification-log", name)
+        )
+      }))
+    )
+  ]);
+  if (!base) throw new Error(`Durable eval case ${id} is missing`);
+  const scores = /* @__PURE__ */ Object.create(null);
+  for (const { name, value } of scoreValues) {
+    if (value !== void 0) scores[name] = value;
+  }
+  const loggedScores = /* @__PURE__ */ Object.create(null);
+  for (const { name, value } of scoreLogValues) {
+    if (value) loggedScores[name] = true;
+  }
+  const classifications = /* @__PURE__ */ Object.create(null);
+  for (const { name, value } of classificationValues) {
+    if (value !== void 0) classifications[name] = value;
+  }
+  const loggedClassifications = /* @__PURE__ */ Object.create(null);
+  for (const { name, value } of classificationLogValues) {
+    if (value) loggedClassifications[name] = true;
+  }
+  return {
+    ...base,
+    metadata: task?.metadata ?? base.metadata,
+    tags: task ? task.tags : base.tags,
+    taskComplete: task !== void 0,
+    taskLogged: taskLog !== void 0,
+    output: task?.output,
+    rootSpan: taskLog?.rootSpan,
+    scores,
+    loggedScores,
+    classifications,
+    loggedClassifications
+  };
+}
+async function readRunState(definition, store, key) {
+  const record = await readJson(store, key);
+  if (!record) return void 0;
+  const plans = plannedBatches(definition, record.runId, record.caseIds);
+  const [cases, batchRecords] = await Promise.all([
+    Promise.all(
+      record.caseIds.map((id) => readCaseRecord(definition, store, key, id))
+    ),
+    Promise.all(
+      plans.map(async ({ id }) => {
+        return readJson(store, batchRecordKey(key, id));
+      })
+    )
+  ]);
+  const batches = batchRecords.filter(
+    (value) => value !== void 0
+  );
+  const { caseIds: _caseIds, ...state } = record;
+  return { ...state, cases, batches };
+}
+async function writeRunRecord(store, key, state) {
+  const { cases, batches: _batches, ...record } = state;
+  await writeJson(store, key, {
+    ...record,
+    caseIds: cases.map(({ id }) => id)
+  });
+}
+async function writeCaseBaseRecords(store, key, records) {
+  await Promise.all(
+    records.map(
+      ({ id, caseId, trialIndex, datum, metadata, tags }) => writeJson(store, caseRecordKey(key, id, "base"), {
+        id,
+        caseId,
+        trialIndex,
+        datum,
+        metadata,
+        tags
+      })
+    )
+  );
+}
+async function writeCaseRecords(store, key, records) {
+  const writes = [];
+  for (const record of records) {
+    if (record.taskComplete) {
+      writes.push(
+        writeJson(store, caseRecordKey(key, record.id, "task"), {
+          output: record.output,
+          metadata: record.metadata,
+          tags: record.tags,
+          taskComplete: true
+        })
+      );
+    }
+    if (record.taskLogged) {
+      writes.push(
+        writeJson(store, caseRecordKey(key, record.id, "task-log"), {
+          rootSpan: record.rootSpan,
+          taskLogged: true
+        })
+      );
+    }
+    for (const [name, value] of Object.entries(record.scores)) {
+      writes.push(
+        writeJson(store, caseRecordKey(key, record.id, "score", name), value)
+      );
+    }
+    for (const name of Object.keys(record.loggedScores)) {
+      writes.push(
+        writeJson(
+          store,
+          caseRecordKey(key, record.id, "score-log", name),
+          true
+        )
+      );
+    }
+    for (const [name, value] of Object.entries(record.classifications)) {
+      writes.push(
+        writeJson(
+          store,
+          caseRecordKey(key, record.id, "classification", name),
+          value
+        )
+      );
+    }
+    for (const name of Object.keys(record.loggedClassifications)) {
+      writes.push(
+        writeJson(
+          store,
+          caseRecordKey(key, record.id, "classification-log", name),
+          true
+        )
+      );
+    }
+  }
+  await Promise.all(writes);
+}
+async function writeBatchRecords(store, key, records) {
+  await Promise.all(
+    records.map(
+      (record) => writeJson(store, batchRecordKey(key, record.id), record)
+    )
+  );
+}
+function deterministicId(value) {
+  const hex = contentVersion(encoder.encode(value)).padEnd(32, "0").slice(0, 32);
+  return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-5${hex.slice(13, 16)}-a${hex.slice(17, 20)}-${hex.slice(20, 32)}`;
+}
+function contentVersion(value) {
+  if (isomorph_default.hash) return isomorph_default.hash(decoder.decode(value));
+  let hash = 2166136261;
+  for (const byte of value) {
+    hash ^= byte;
+    hash = Math.imul(hash, 16777619);
+  }
+  return (hash >>> 0).toString(16).padStart(8, "0");
+}
+async function readJson(store, key) {
+  const value = await store.read(key);
+  return value ? JSON.parse(decoder.decode(value)) : void 0;
+}
+async function writeJson(store, key, value) {
+  await store.write(key, encoder.encode(stableStringify(value)));
+}
+function stableStringify(value) {
+  return JSON.stringify(value, (_key, nested) => {
+    if (nested && typeof nested === "object" && !Array.isArray(nested)) {
+      return Object.fromEntries(
+        Object.entries(nested).sort(
+          ([left], [right]) => left.localeCompare(right)
+        )
+      );
+    }
+    return nested;
+  });
+}
+function assertJsonValue(value, label) {
+  try {
+    const serialized = JSON.stringify(value);
+    if (serialized === void 0)
+      throw new Error("value serializes to undefined");
+    return JSON.parse(serialized);
+  } catch (error) {
+    throw new Error(`${label} must be JSON serializable`, { cause: error });
+  }
+}
+function resultItemId(value) {
+  if (typeof value !== "object" || value === null || !("id" in value) || typeof value.id !== "string") {
+    throw new Error("Batch results must contain a string id");
+  }
+  return value.id;
+}
+function isBatchTask(value) {
+  return typeof value === "object" && value !== null && "kind" in value && value.kind === BATCH_TASK_KIND;
+}
+function isBatchScorer(value) {
+  return typeof value === "object" && value !== null && "kind" in value && value.kind === BATCH_SCORER_KIND;
+}
+function asError(error) {
+  return error instanceof Error ? error : new Error(String(error));
+}
+
 // src/agent-assertions.ts
 function agentAssertionScorer(callback, options = {}) {
   return async (args) => {
@@ -10505,6 +11789,8 @@ export {
   BRAINTRUST_PARENT_KEY,
   BaseAttachment,
   BaseExperiment,
+  BatchScorer,
+  BatchTask,
   BraintrustLangChainCallbackHandler,
   BraintrustMiddleware,
   BraintrustObservabilityExporter,
@@ -10518,6 +11804,8 @@ export {
   DEFAULT_MAX_REQUEST_SIZE,
   Dataset,
   DatasetPipeline,
+  DurableEvalMemoryStore,
+  DurableEvalRedisStore,
   ERR_PERMALINK,
   Eval,
   EvalResultWithSummary,
@@ -10576,6 +11864,7 @@ export {
   deepCopyEvent,
   exports_exports as default,
   defaultErrorScoreHandler,
+  defineDurableEval,
   deserializePlainStringAsJSON,
   devNullWritableStream,
   evaluatorDefinitionSchema,
@@ -10671,5 +11960,6 @@ export {
   wrapPiCodingAgentSDK,
   wrapStrandsAgentSDK,
   wrapTraced,
-  wrapVitest
+  wrapVitest,
+  wrapVoyageAI
 };