braintrust 3.31.0 → 3.33.0

package/dist/index.mjs

@@ -54,6 +54,7 @@ import {
   UUIDGenerator,
   X_CACHED_HEADER,
   _exportsForTestingOnly,
+  _internalExportParentSynchronously,
   _internalGetGlobalState,
   _internalResumeSpan,
   _internalSetInitialState,
@@ -117,6 +118,7 @@ import {
   renderMessage,
   renderPromptParams,
   renderTemplateContent,
+  resolveUseLegacyUuidIds,
   runWithAutoInstrumentationSuppressed,
   setFetch,
   setMaskingFunction,
@@ -143,10 +145,9 @@ import {
   wrapMastraAgent,
   wrapTraced,
   zodToJsonSchema
-} from "./chunk-AJT4FR25.mjs";
+} from "./chunk-MZLBAFVJ.mjs";
 import {
   INSTRUMENTATION_NAMES,
-  __export,
   aiSDKChannels,
   anthropicChannels,
   bedrockRuntimeChannels,
@@ -156,16 +157,19 @@ import {
   cohereChannels,
   cursorSDKChannels,
   debugLogger,
+  elevenLabsChannels,
   genkitChannels,
   gitHubCopilotChannels,
   googleADKChannels,
   googleGenAIChannels,
+  googleGenerativeAIChannels,
   groqChannels,
   harnessAgentChannels,
   huggingFaceChannels,
   huggingFaceTransformersChannels,
   isSupportedHuggingFaceTransformersTask,
   isomorph_default,
+  langGraphSDKChannels,
   langSmithChannels,
   mistralChannels,
   ollamaChannels,
@@ -177,7 +181,10 @@ import {
   strandsAgentSDKChannels,
   voyageAIChannels,
   withSpanInstrumentationName
-} from "./chunk-CE3BLB2L.mjs";
+} from "./chunk-TZVZN2JJ.mjs";
+import {
+  __export
+} from "./chunk-7P6ASYW6.mjs";
 
 // src/exports.ts
 var exports_exports = {};
@@ -190,8 +197,6 @@ __export(exports_exports, {
   BRAINTRUST_PARENT_KEY: () => BRAINTRUST_PARENT_KEY,
   BaseAttachment: () => BaseAttachment,
   BaseExperiment: () => BaseExperiment,
-  BatchScorer: () => BatchScorer,
-  BatchTask: () => BatchTask,
   BraintrustLangChainCallbackHandler: () => BraintrustLangChainCallbackHandler,
   BraintrustMiddleware: () => BraintrustMiddleware,
   BraintrustObservabilityExporter: () => BraintrustObservabilityExporter,
@@ -205,8 +210,6 @@ __export(exports_exports, {
   DEFAULT_MAX_REQUEST_SIZE: () => DEFAULT_MAX_REQUEST_SIZE,
   Dataset: () => Dataset,
   DatasetPipeline: () => DatasetPipeline,
-  DurableEvalMemoryStore: () => DurableEvalMemoryStore,
-  DurableEvalRedisStore: () => DurableEvalRedisStore,
   ERR_PERMALINK: () => ERR_PERMALINK,
   Eval: () => Eval,
   EvalResultWithSummary: () => EvalResultWithSummary,
@@ -241,6 +244,10 @@ __export(exports_exports, {
   TestBackgroundLogger: () => TestBackgroundLogger,
   ToolBuilder: () => ToolBuilder,
   UUIDGenerator: () => UUIDGenerator,
+  WorkflowEvalMemoryStore: () => WorkflowEvalMemoryStore,
+  WorkflowEvalRedisStore: () => WorkflowEvalRedisStore,
+  WorkflowScorer: () => WorkflowScorer,
+  WorkflowTask: () => WorkflowTask,
   X_CACHED_HEADER: () => X_CACHED_HEADER,
   _exportsForTestingOnly: () => _exportsForTestingOnly,
   _internalGetGlobalState: () => _internalGetGlobalState,
@@ -265,12 +272,13 @@ __export(exports_exports, {
   currentSpan: () => currentSpan,
   deepCopyEvent: () => deepCopyEvent,
   defaultErrorScoreHandler: () => defaultErrorScoreHandler,
-  defineDurableEval: () => defineDurableEval,
+  defineWorkflowEval: () => defineWorkflowEval,
   deserializePlainStringAsJSON: () => deserializePlainStringAsJSON,
   devNullWritableStream: () => devNullWritableStream,
   evaluatorDefinitionSchema: () => evaluatorDefinitionSchema,
   evaluatorDefinitionsSchema: () => evaluatorDefinitionsSchema,
   extractTraceContextFromHeaders: () => extractTraceContextFromHeaders,
+  failOpenAIAgentsTrace: () => failOpenAIAgentsTrace,
   flush: () => flush,
   getContextManager: () => getContextManager,
   getIdGenerator: () => getIdGenerator,
@@ -317,12 +325,14 @@ __export(exports_exports, {
   setFetch: () => setFetch,
   setMaskingFunction: () => setMaskingFunction,
   spanComponentsToObjectId: () => spanComponentsToObjectId,
+  startOpenAIAgentsTrace: () => startOpenAIAgentsTrace,
   startSpan: () => startSpan,
   summarize: () => summarize,
   templateRegistry: () => templateRegistry,
   toolFunctionDefinitionSchema: () => ToolFunctionDefinition,
   traceable: () => traceable,
   traced: () => traced,
+  updateOpenAIAgentsTrace: () => updateOpenAIAgentsTrace,
   updateSpan: () => updateSpan,
   uploadLogs3OverflowPayload: () => uploadLogs3OverflowPayload,
   utf8ByteLength: () => utf8ByteLength,
@@ -343,12 +353,15 @@ __export(exports_exports, {
   wrapCohere: () => wrapCohere,
   wrapCopilotClient: () => wrapCopilotClient,
   wrapCursorSDK: () => wrapCursorSDK,
+  wrapElevenLabs: () => wrapElevenLabs,
   wrapGenkit: () => wrapGenkit,
   wrapGoogleADK: () => wrapGoogleADK,
   wrapGoogleGenAI: () => wrapGoogleGenAI,
+  wrapGoogleGenerativeAI: () => wrapGoogleGenerativeAI,
   wrapGroq: () => wrapGroq,
   wrapHuggingFace: () => wrapHuggingFace,
   wrapHuggingFaceTransformers: () => wrapHuggingFaceTransformers,
+  wrapLangGraphSDK: () => wrapLangGraphSDK,
   wrapLangSmithClient: () => wrapLangSmithClient,
   wrapLangSmithRunTrees: () => wrapLangSmithRunTrees,
   wrapLangSmithTraceable: () => wrapLangSmithTraceable,
@@ -715,6 +728,143 @@ async function completeOpenAIBatchTrace(args) {
   );
 }
 
+// src/openai-agents-api.ts
+function isRecord(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function validateStartArgs(value) {
+  if (!isRecord(value)) {
+    throw new TypeError(
+      "startOpenAIAgentsTrace expected an OpenAI Agents parameters object"
+    );
+  }
+  const agent = value.agent;
+  if (agent !== void 0 && agent !== null && !isRecord(agent)) {
+    throw new TypeError(
+      "startOpenAIAgentsTrace expected agent to be an object or null"
+    );
+  }
+  if (isRecord(agent) && (agent.id !== void 0 && typeof agent.id !== "string" || agent.model !== void 0 && typeof agent.model !== "string" || agent.name !== void 0 && agent.name !== null && typeof agent.name !== "string")) {
+    throw new TypeError(
+      "startOpenAIAgentsTrace received invalid agent identity fields"
+    );
+  }
+  if (value.agent_id !== void 0 && typeof value.agent_id !== "string") {
+    throw new TypeError(
+      "startOpenAIAgentsTrace expected agent_id to be a string"
+    );
+  }
+  if (value.metadata !== void 0 && value.metadata !== null && !isRecord(value.metadata)) {
+    throw new TypeError(
+      "startOpenAIAgentsTrace expected metadata to be an object or null"
+    );
+  }
+  return value;
+}
+function validateEvent(event) {
+  if (!isRecord(event) || typeof event.type !== "string" || event.type.length === 0) {
+    throw new TypeError(
+      "updateOpenAIAgentsTrace expected an OpenAI Agents event object"
+    );
+  }
+  return event;
+}
+function createTraceState() {
+  const parent = _internalExportParentSynchronously(getSpanParentObject());
+  if (!parent) {
+    return null;
+  }
+  const parentComponents = SpanComponentsV4.fromStr(parent);
+  const rowId = newId();
+  const useLegacyIds = resolveUseLegacyUuidIds();
+  const spanId = useLegacyIds ? newId() : newId().replaceAll("-", "").slice(0, 16);
+  const rootSpanId = useLegacyIds ? spanId : newId().replaceAll("-", "");
+  const root = new SpanComponentsV4({
+    object_type: parentComponents.data.object_type,
+    ...parentComponents.data.object_id ? { object_id: parentComponents.data.object_id } : {
+      compute_object_metadata_args: parentComponents.data.compute_object_metadata_args ?? {}
+    },
+    propagated_event: parentComponents.data.propagated_event,
+    root_span_id: rootSpanId,
+    row_id: rowId,
+    span_id: spanId
+  }).toStr();
+  return {
+    callItems: {},
+    ended: false,
+    eventIds: [],
+    openTools: {},
+    root,
+    rootKey: spanId,
+    rootParent: parent,
+    startTime: getCurrentUnixTimestamp(),
+    subagents: {},
+    turnSubagents: {},
+    version: 1
+  };
+}
+function startOpenAIAgentsTrace(args) {
+  const validatedArgs = validateStartArgs(args);
+  const state = createTraceState();
+  if (!state) {
+    return null;
+  }
+  try {
+    void openAIChannels.agentsTraceStart.invoke(
+      async () => state,
+      void 0,
+      [{ args: validatedArgs, state }],
+      {}
+    ).catch((error) => {
+      debugLogger.debug(
+        "OpenAI Agents API instrumentation could not start:",
+        error
+      );
+    });
+  } catch (error) {
+    debugLogger.debug(
+      "OpenAI Agents API instrumentation could not start:",
+      error
+    );
+  }
+  return state;
+}
+async function updateOpenAIAgentsTrace(token, event) {
+  const state = token;
+  const validatedEvent = validateEvent(event);
+  try {
+    return await openAIChannels.agentsTraceCapture.invoke(
+      async () => state,
+      void 0,
+      [{ state, event: validatedEvent }],
+      {}
+    );
+  } catch (error) {
+    debugLogger.debug(
+      "OpenAI Agents API instrumentation could not update:",
+      error
+    );
+    return token;
+  }
+}
+async function failOpenAIAgentsTrace(token, error) {
+  const state = token;
+  try {
+    return await openAIChannels.agentsTraceFail.invoke(
+      async () => state,
+      void 0,
+      [{ state, error }],
+      {}
+    );
+  } catch (instrumentationError) {
+    debugLogger.debug(
+      "OpenAI Agents API instrumentation could not record failure:",
+      instrumentationError
+    );
+    return token;
+  }
+}
+
 // src/functions/invoke.ts
 async function invoke(args) {
   const {
@@ -2164,7 +2314,31 @@ var MAX_STORED_LLM_INPUTS = 100;
 var MAX_STORED_REASONING_BLOCKS = 100;
 var MAX_STORED_SPAN_REFERENCES = 1e4;
 var MAX_STORED_STEP_STARTS = 1e4;
+var EVE_HANDLED_EVENT_TYPES = /* @__PURE__ */ new Set([
+  "action.result",
+  "actions.requested",
+  "message.completed",
+  "message.received",
+  "reasoning.completed",
+  "result.completed",
+  "session.completed",
+  "session.failed",
+  "session.started",
+  "step.completed",
+  "step.failed",
+  "step.started",
+  "subagent.called",
+  "subagent.completed",
+  "turn.completed",
+  "turn.failed",
+  "turn.started"
+]);
 function braintrustEveHook(options) {
+  if (!options || typeof options.defineState !== "function") {
+    throw new TypeError(
+      "braintrustEveHook requires Eve's defineState function"
+    );
+  }
   const state = options.defineState(EVE_TRACE_STATE_KEY, emptyEveTraceState);
   const bridge = new EveBridge(state);
   return {
@@ -2176,15 +2350,27 @@ function braintrustEveHook(options) {
   };
 }
 function createLegacyEveInstrumentation(options) {
+  if (!options || typeof options.defineState !== "function") {
+    throw new TypeError(
+      "braintrustEveInstrumentation requires Eve's defineState function"
+    );
+  }
   const state = options.defineState(EVE_TRACE_STATE_KEY, emptyEveTraceState);
   return {
     events: {
       "step.started": (input) => {
         try {
+          if (!isEveInstrumentationStepStartedEventInput(input)) {
+            debugLogger.warn(
+              "Ignoring malformed Eve instrumentation step.started event"
+            );
+            return void 0;
+          }
           captureEveModelInput(state, input);
         } catch (error) {
           debugLogger.warn("Error in Eve LLM input capture:", error);
         }
+        return void 0;
       }
     },
     recordInputs: false,
@@ -2193,7 +2379,77 @@ function createLegacyEveInstrumentation(options) {
   };
 }
 function isEveHandleMessageStreamEvent(event) {
-  return isObject(event) && typeof event["type"] === "string";
+  if (!isObject(event) || typeof event["type"] !== "string") {
+    return false;
+  }
+  const type = event["type"];
+  if (type === "session.completed") {
+    return true;
+  }
+  if (!isObject(event["data"])) {
+    return false;
+  }
+  const data = event["data"];
+  if (type === "session.started") {
+    return true;
+  }
+  if (type === "session.failed") {
+    return typeof data["sessionId"] === "string" && typeof data["code"] === "string" && typeof data["message"] === "string";
+  }
+  if (type === "session.waiting") {
+    return data["wait"] === "next-user-message";
+  }
+  if (typeof data["turnId"] !== "string" || typeof data["sequence"] !== "number" || !Number.isFinite(data["sequence"])) {
+    return false;
+  }
+  if (type === "turn.started" || type === "turn.completed" || type === "subagent.called" || type === "subagent.completed") {
+    if (type === "subagent.called") {
+      return typeof data["callId"] === "string" && typeof data["childSessionId"] === "string" && typeof data["name"] === "string";
+    }
+    if (type === "subagent.completed") {
+      return typeof data["callId"] === "string" && typeof data["subagentName"] === "string";
+    }
+    return true;
+  }
+  if (type === "turn.failed") {
+    return typeof data["code"] === "string" && typeof data["message"] === "string";
+  }
+  if (type === "message.received") {
+    return typeof data["message"] === "string";
+  }
+  if (typeof data["stepIndex"] !== "number" || !Number.isFinite(data["stepIndex"]) || data["stepIndex"] < 0) {
+    return false;
+  }
+  switch (type) {
+    case "step.started":
+      return true;
+    case "message.completed":
+      return typeof data["finishReason"] === "string" && (typeof data["message"] === "string" || data["message"] === null);
+    case "reasoning.completed":
+      return typeof data["reasoning"] === "string";
+    case "result.completed":
+      return true;
+    case "step.completed":
+      return typeof data["finishReason"] === "string";
+    case "step.failed":
+      return typeof data["code"] === "string" && typeof data["message"] === "string";
+    case "actions.requested":
+      return Array.isArray(data["actions"]);
+    case "action.result":
+      return isObject(data["result"]) && (data["status"] === "completed" || data["status"] === "failed" || data["status"] === "rejected");
+    default:
+      return false;
+  }
+}
+function isEveInstrumentationStepStartedEventInput(input) {
+  if (!isObject(input)) {
+    return false;
+  }
+  const modelInput = input["modelInput"];
+  const session = input["session"];
+  const step = input["step"];
+  const turn = input["turn"];
+  return isObject(modelInput) && Array.isArray(modelInput["messages"]) && isObject(session) && typeof session["id"] === "string" && isObject(step) && typeof step["index"] === "number" && Number.isFinite(step["index"]) && step["index"] >= 0 && isObject(turn) && typeof turn["id"] === "string" && typeof turn["sequence"] === "number" && Number.isFinite(turn["sequence"]);
 }
 var ResumedEveSpan = class {
   constructor(reference) {
@@ -2362,7 +2618,17 @@ var EveBridge = class {
     });
   }
   async handle(event, ctx, hookMetadata) {
+    try {
+      await this.handleUnchecked(event, ctx, hookMetadata);
+    } catch (error) {
+      debugLogger.warn("Error in Eve hook instrumentation:", error);
+    }
+  }
+  async handleUnchecked(event, ctx, hookMetadata) {
     if (!isEveHandleMessageStreamEvent(event)) {
+      if (isObject(event) && typeof event["type"] === "string" && EVE_HANDLED_EVENT_TYPES.has(event["type"])) {
+        debugLogger.warn(`Ignoring malformed Eve hook ${event["type"]} event`);
+      }
       return;
     }
     const run = async () => {
@@ -2688,9 +2954,14 @@ var EveBridge = class {
     }
     for (const action of traceActions) {
       const name = action.kind === "tool-call" ? action.toolName : action.subagentName ?? action.name ?? "agent";
+      let args = "null";
+      try {
+        args = JSON.stringify(action.input) ?? "null";
+      } catch {
+      }
       toolCallsById.set(action.callId, {
         function: {
-          arguments: JSON.stringify(action.input),
+          arguments: args,
           name
         },
         id: action.callId,
@@ -3559,17 +3830,33 @@ function consumeCapturedEveModelInput(state, sessionId, turnId, stepIndex) {
   }
 }
 function capturedModelInput(modelInput) {
-  const { instructions, messages } = modelInput;
-  const value = [];
-  if (typeof instructions === "string") {
-    value.push({ content: instructions, role: "system" });
-  } else if (Array.isArray(instructions)) {
-    value.push(...instructions.map(capturedEveModelMessage));
-  } else if (instructions) {
-    value.push(capturedEveModelMessage(instructions));
-  }
-  value.push(...messages.map(capturedEveModelMessage));
   try {
+    if (!isObject(modelInput) || !Array.isArray(modelInput["messages"])) {
+      return void 0;
+    }
+    const value = [];
+    const instructions = modelInput["instructions"];
+    if (typeof instructions === "string") {
+      value.push({ content: instructions, role: "system" });
+    } else if (Array.isArray(instructions)) {
+      for (const instruction of instructions) {
+        const captured = capturedEveModelMessage(instruction);
+        if (captured?.role === "system") {
+          value.push(captured);
+        }
+      }
+    } else {
+      const captured = capturedEveModelMessage(instructions);
+      if (captured?.role === "system") {
+        value.push(captured);
+      }
+    }
+    for (const message of modelInput["messages"]) {
+      const captured = capturedEveModelMessage(message);
+      if (captured) {
+        value.push(captured);
+      }
+    }
     const cloned = JSON.parse(JSON.stringify(value));
     if (!Array.isArray(cloned)) {
       return void 0;
@@ -3580,116 +3867,202 @@ function capturedModelInput(modelInput) {
   }
 }
 function capturedEveModelMessage(message) {
-  const { content, role } = message;
+  if (!isObject(message)) {
+    return void 0;
+  }
+  const role = message["role"];
+  if (role !== "system" && role !== "user" && role !== "assistant" && role !== "tool") {
+    return void 0;
+  }
+  const content = message["content"];
   if (typeof content === "string") {
     return { content, role };
   }
-  return { content: content.map(capturedEveModelContentPart), role };
+  if (!Array.isArray(content)) {
+    return void 0;
+  }
+  const capturedContent = [];
+  for (const part of content) {
+    const captured = capturedEveModelContentPart(part);
+    if (captured) {
+      capturedContent.push(captured);
+    }
+  }
+  return { content: capturedContent, role };
 }
 function capturedEveModelContentPart(part) {
-  switch (part.type) {
+  if (!isObject(part) || typeof part["type"] !== "string") {
+    return void 0;
+  }
+  switch (part["type"]) {
     case "text":
-    case "reasoning":
-      return { text: part.text, type: part.type };
+    case "reasoning": {
+      const text = part["text"];
+      return typeof text === "string" ? { text, type: part["type"] } : void 0;
+    }
     case "image":
+      if (!Object.hasOwn(part, "image")) {
+        return void 0;
+      }
       return {
-        image: part.image,
-        ...part.mediaType !== void 0 ? { mediaType: part.mediaType } : {},
+        image: part["image"],
+        ...typeof part["mediaType"] === "string" ? { mediaType: part["mediaType"] } : {},
         type: "image"
       };
     case "file":
-    case "reasoning-file":
+    case "reasoning-file": {
+      if (!Object.hasOwn(part, "data") || typeof part["mediaType"] !== "string") {
+        return void 0;
+      }
       return {
-        data: part.data,
-        ...part.type === "file" && part.filename !== void 0 ? { filename: part.filename } : {},
-        mediaType: part.mediaType,
-        type: part.type
+        data: part["data"],
+        ...part["type"] === "file" && typeof part["filename"] === "string" ? { filename: part["filename"] } : {},
+        mediaType: part["mediaType"],
+        type: part["type"]
       };
+    }
     case "custom":
       return {
-        ..."kind" in part ? { kind: part.kind } : {},
+        ...typeof part["kind"] === "string" ? { kind: part["kind"] } : {},
         type: "custom"
       };
-    case "tool-call":
+    case "tool-call": {
+      if (typeof part["toolCallId"] !== "string" || typeof part["toolName"] !== "string") {
+        return void 0;
+      }
       return {
-        input: part.input,
-        ...part.providerExecuted !== void 0 ? { providerExecuted: part.providerExecuted } : {},
-        toolCallId: part.toolCallId,
-        toolName: part.toolName,
+        input: part["input"],
+        ...typeof part["providerExecuted"] === "boolean" ? { providerExecuted: part["providerExecuted"] } : {},
+        toolCallId: part["toolCallId"],
+        toolName: part["toolName"],
         type: "tool-call"
       };
+    }
     case "tool-result": {
-      const output = part.output;
+      const output = part["output"];
+      if (typeof part["toolCallId"] !== "string" || typeof part["toolName"] !== "string" || !isObject(output) || typeof output["type"] !== "string") {
+        return void 0;
+      }
       let capturedOutput;
-      switch (output.type) {
+      switch (output["type"]) {
         case "text":
-        case "error-text":
-          capturedOutput = { type: output.type, value: output.value };
+        case "error-text": {
+          if (typeof output["value"] !== "string") {
+            return void 0;
+          }
+          capturedOutput = {
+            type: output["type"],
+            value: output["value"]
+          };
           break;
+        }
         case "json":
         case "error-json":
-          capturedOutput = { type: output.type, value: output.value };
+          capturedOutput = {
+            type: output["type"],
+            value: output["value"]
+          };
           break;
         case "execution-denied":
           capturedOutput = {
-            ...output.reason !== void 0 ? { reason: output.reason } : {},
+            ...typeof output["reason"] === "string" ? { reason: output["reason"] } : {},
             type: "execution-denied"
           };
           break;
-        case "content":
+        case "content": {
+          if (!Array.isArray(output["value"])) {
+            return void 0;
+          }
+          const value = [];
+          for (const outputPart of output["value"]) {
+            const captured = capturedEveModelContentPart(outputPart);
+            if (captured) {
+              value.push(captured);
+            }
+          }
           capturedOutput = {
             type: "content",
-            value: output.value.map(capturedEveModelContentPart)
+            value
           };
           break;
+        }
+        default:
+          return void 0;
       }
       return {
         output: capturedOutput,
-        toolCallId: part.toolCallId,
-        toolName: part.toolName,
+        toolCallId: part["toolCallId"],
+        toolName: part["toolName"],
         type: "tool-result"
       };
     }
-    case "tool-approval-request":
+    case "tool-approval-request": {
+      if (typeof part["approvalId"] !== "string" || typeof part["toolCallId"] !== "string") {
+        return void 0;
+      }
       return {
-        approvalId: part.approvalId,
-        ...part.isAutomatic !== void 0 ? { isAutomatic: part.isAutomatic } : {},
-        ...part.signature !== void 0 ? { signature: part.signature } : {},
-        toolCallId: part.toolCallId,
+        approvalId: part["approvalId"],
+        ...typeof part["isAutomatic"] === "boolean" ? { isAutomatic: part["isAutomatic"] } : {},
+        ...typeof part["signature"] === "string" ? { signature: part["signature"] } : {},
+        toolCallId: part["toolCallId"],
         type: "tool-approval-request"
       };
-    case "tool-approval-response":
+    }
+    case "tool-approval-response": {
+      if (typeof part["approvalId"] !== "string" || typeof part["approved"] !== "boolean") {
+        return void 0;
+      }
       return {
-        approvalId: part.approvalId,
-        approved: part.approved,
-        ...part.providerExecuted !== void 0 ? { providerExecuted: part.providerExecuted } : {},
-        ...part.reason !== void 0 ? { reason: part.reason } : {},
+        approvalId: part["approvalId"],
+        approved: part["approved"],
+        ...typeof part["providerExecuted"] === "boolean" ? { providerExecuted: part["providerExecuted"] } : {},
+        ...typeof part["reason"] === "string" ? { reason: part["reason"] } : {},
         type: "tool-approval-response"
       };
+    }
     case "file-data":
-    case "image-data":
+    case "image-data": {
+      if (typeof part["data"] !== "string" || typeof part["mediaType"] !== "string") {
+        return void 0;
+      }
       return {
-        data: part.data,
-        ...part.type === "file-data" && part.filename !== void 0 ? { filename: part.filename } : {},
-        mediaType: part.mediaType,
-        type: part.type
+        data: part["data"],
+        ...part["type"] === "file-data" && typeof part["filename"] === "string" ? { filename: part["filename"] } : {},
+        mediaType: part["mediaType"],
+        type: part["type"]
       };
+    }
     case "file-url":
-    case "image-url":
+    case "image-url": {
+      if (typeof part["url"] !== "string") {
+        return void 0;
+      }
       return {
-        ...part.type === "file-url" && part.mediaType !== void 0 ? { mediaType: part.mediaType } : {},
-        type: part.type,
-        url: part.url
+        ...part["type"] === "file-url" && typeof part["mediaType"] === "string" ? { mediaType: part["mediaType"] } : {},
+        type: part["type"],
+        url: part["url"]
       };
+    }
     case "file-id":
-    case "image-file-id":
-      return { fileId: part.fileId, type: part.type };
+    case "image-file-id": {
+      const fileId = part["fileId"];
+      if (typeof fileId !== "string" && !isObject(fileId)) {
+        return void 0;
+      }
+      return { fileId, type: part["type"] };
+    }
     case "file-reference":
-    case "image-file-reference":
+    case "image-file-reference": {
+      if (!isObject(part["providerReference"])) {
+        return void 0;
+      }
       return {
-        providerReference: part.providerReference,
-        type: part.type
+        providerReference: part["providerReference"],
+        type: part["type"]
       };
+    }
+    default:
+      return void 0;
   }
 }
 function isCapturedModelInput(input) {
@@ -3825,18 +4198,54 @@ function createEveInstrumentationProvider(options = {}) {
       recordOutputs: true
     }),
     events: {
-      "action.completed": (event, context) => bridge.handleActionTerminal(event, context),
-      "action.failed": (event, context) => bridge.handleActionTerminal(event, context),
-      "action.started": (event, context) => bridge.handleActionStarted(event, context),
-      "model.call.completed": (event, context) => bridge.handleModelTerminal(event, context),
-      "model.call.failed": (event, context) => bridge.handleModelTerminal(event, context),
-      "model.call.started": (event, context) => bridge.handleModelStarted(event, context),
-      "step.attempt.completed": (event) => bridge.handleStepAttemptTerminal(event),
-      "step.attempt.failed": (event) => bridge.handleStepAttemptTerminal(event),
-      "turn.cancelled": (event, context) => bridge.handleTurnTerminal(event, context),
-      "turn.completed": (event, context) => bridge.handleTurnTerminal(event, context),
-      "turn.failed": (event, context) => bridge.handleTurnTerminal(event, context),
-      "turn.started": (event, context) => bridge.handleTurnStarted(event, context)
+      "action.completed": providerHandler(
+        "action.completed",
+        bridge.handleActionTerminal.bind(bridge)
+      ),
+      "action.failed": providerHandler(
+        "action.failed",
+        bridge.handleActionTerminal.bind(bridge)
+      ),
+      "action.started": providerHandler(
+        "action.started",
+        bridge.handleActionStarted.bind(bridge)
+      ),
+      "model.call.completed": providerHandler(
+        "model.call.completed",
+        bridge.handleModelTerminal.bind(bridge)
+      ),
+      "model.call.failed": providerHandler(
+        "model.call.failed",
+        bridge.handleModelTerminal.bind(bridge)
+      ),
+      "model.call.started": providerHandler(
+        "model.call.started",
+        bridge.handleModelStarted.bind(bridge)
+      ),
+      "step.attempt.completed": providerHandler(
+        "step.attempt.completed",
+        (event) => bridge.handleStepAttemptTerminal(event)
+      ),
+      "step.attempt.failed": providerHandler(
+        "step.attempt.failed",
+        (event) => bridge.handleStepAttemptTerminal(event)
+      ),
+      "turn.cancelled": providerHandler(
+        "turn.cancelled",
+        bridge.handleTurnTerminal.bind(bridge)
+      ),
+      "turn.completed": providerHandler(
+        "turn.completed",
+        bridge.handleTurnTerminal.bind(bridge)
+      ),
+      "turn.failed": providerHandler(
+        "turn.failed",
+        bridge.handleTurnTerminal.bind(bridge)
+      ),
+      "turn.started": providerHandler(
+        "turn.started",
+        bridge.handleTurnStarted.bind(bridge)
+      )
     },
     flush,
     setup: options.setup
@@ -4192,17 +4601,20 @@ function modelOutput(event) {
   const reasoning = [];
   const toolCalls2 = [];
   for (const part of content) {
-    if (part.type === "text") {
-      text += part.text;
-    } else if (part.type === "reasoning" && part.text.trim().length > 0) {
-      reasoning.push({ content: part.text });
-    } else if (part.type === "tool-call") {
+    if (!isObject(part)) {
+      continue;
+    }
+    if (part["type"] === "text" && typeof part["text"] === "string") {
+      text += part["text"];
+    } else if (part["type"] === "reasoning" && typeof part["text"] === "string" && part["text"].trim().length > 0) {
+      reasoning.push({ content: part["text"] });
+    } else if (part["type"] === "tool-call" && typeof part["toolName"] === "string" && typeof part["callId"] === "string") {
       toolCalls2.push({
         function: {
-          arguments: safeJsonStringify(part.input),
-          name: part.toolName
+          arguments: safeJsonStringify(part["input"]),
+          name: part["toolName"]
         },
-        id: part.callId,
+        id: part["callId"],
         type: "function"
       });
     }
@@ -4221,13 +4633,13 @@ function modelOutput(event) {
   ];
 }
 function usageMetrics(usage) {
-  const promptTokens = nonNegativeNumber(usage.inputTokens);
-  const completionTokens = nonNegativeNumber(usage.outputTokens);
+  const promptTokens = nonNegativeNumber(usage?.inputTokens);
+  const completionTokens = nonNegativeNumber(usage?.outputTokens);
   const cachedTokens = nonNegativeNumber(
-    usage.inputTokenDetails?.cacheReadTokens
+    usage?.inputTokenDetails?.cacheReadTokens
   );
   const cacheCreationTokens = nonNegativeNumber(
-    usage.inputTokenDetails?.cacheWriteTokens
+    usage?.inputTokenDetails?.cacheWriteTokens
   );
   return {
     ...promptTokens !== void 0 ? { prompt_tokens: promptTokens } : {},
@@ -4264,10 +4676,76 @@ function turnIdempotencyKey(sessionId, turnId) {
 function actionIdempotencyKey(sessionId, turnId, callId) {
   return `action:${sessionId}:${turnId}:${callId}`;
 }
+function providerHandler(type, handler) {
+  return async (event, context) => {
+    try {
+      if (!isEveProviderEvent(event, type) || !isEveProviderContext(context)) {
+        debugLogger.warn(`Ignoring malformed Eve provider ${type} event`);
+        return;
+      }
+      await handler(event, context);
+    } catch (error) {
+      debugLogger.warn(`Error in Eve provider ${type}:`, error);
+    }
+  };
+}
+function isEveProviderEvent(event, type) {
+  if (!isObject(event) || event["type"] !== type || typeof event["idempotencyKey"] !== "string" || event["idempotencyKey"].length === 0) {
+    return false;
+  }
+  if (type === "turn.started") {
+    const parentLineage = event["parentLineage"];
+    return typeof event["rootSessionId"] === "string" && typeof event["sessionId"] === "string" && typeof event["turnId"] === "string" && typeof event["sequence"] === "number" && Number.isFinite(event["sequence"]) && (parentLineage === void 0 || isEveProviderParentLineage(parentLineage));
+  }
+  if (type === "turn.cancelled" || type === "turn.completed" || type === "turn.failed") {
+    return typeof event["sessionId"] === "string" && typeof event["turnId"] === "string";
+  }
+  if (!isEveProviderScope(event["scope"])) {
+    return false;
+  }
+  if (type === "model.call.started") {
+    const model = event["model"];
+    return isObject(model) && typeof model["modelId"] === "string" && typeof model["provider"] === "string";
+  }
+  if (type === "model.call.completed") {
+    return typeof event["finishReason"] === "string" && isObject(event["usage"]) && (event["content"] === void 0 || Array.isArray(event["content"]));
+  }
+  if (type === "model.call.failed") {
+    return true;
+  }
+  if (type === "action.started") {
+    return typeof event["callId"] === "string" && typeof event["name"] === "string" && (event["kind"] === "load-skill" || event["kind"] === "remote-agent-call" || event["kind"] === "subagent-call" || event["kind"] === "tool-call");
+  }
+  if (type === "action.completed") {
+    const output = event["output"];
+    return event["outcome"] === "completed" && isObject(output) && (output["type"] === "result" || output["type"] === "error");
+  }
+  if (type === "action.failed") {
+    return event["outcome"] === "abandoned" || event["outcome"] === "cancelled" || event["outcome"] === "failed" || event["outcome"] === "rejected";
+  }
+  return type === "step.attempt.completed" || type === "step.attempt.failed";
+}
+function isEveProviderContext(value) {
+  if (!isObject(value) || !isObject(value["state"])) {
+    return false;
+  }
+  return typeof value["state"]["get"] === "function" && typeof value["state"]["set"] === "function";
+}
+function isEveProviderScope(value) {
+  return isObject(value) && typeof value["attemptId"] === "string" && typeof value["attemptIndex"] === "number" && Number.isFinite(value["attemptIndex"]) && typeof value["sessionId"] === "string" && typeof value["stepIndex"] === "number" && Number.isFinite(value["stepIndex"]) && typeof value["turnId"] === "string" && (value["rootSessionId"] === void 0 || typeof value["rootSessionId"] === "string");
+}
+function isEveProviderParentLineage(value) {
+  return isObject(value) && typeof value["callId"] === "string" && typeof value["sessionId"] === "string" && typeof value["turnId"] === "string";
+}
 
 // src/instrumentation/plugins/eve-instrumentation.ts
 var EVE_INSTRUMENTATION_PROVIDER = /* @__PURE__ */ Symbol.for("eve.instrumentation.provider");
 function braintrustEveInstrumentation(options) {
+  if (!options || typeof options !== "object") {
+    throw new TypeError(
+      "braintrustEveInstrumentation requires an options object"
+    );
+  }
   const definition = "defineState" in options ? createLegacyEveInstrumentation(options) : createEveInstrumentationProvider(options);
   const declaration = {
     ...definition,
@@ -5214,7 +5692,77 @@ function isModuleNamespace5(value) {
   }
 }
 
+// src/wrappers/google-generative-ai.ts
+function wrapGoogleGenerativeAI(client) {
+  if (!isObject(client) || typeof client.getGenerativeModel !== "function")
+    return client;
+  return wrapClient(client);
+}
+var proxies = /* @__PURE__ */ new WeakMap();
+function wrapClient(client) {
+  const cached = proxies.get(client);
+  if (cached) return cached;
+  const proxy = new Proxy(client, {
+    get(target, prop, receiver) {
+      const value = Reflect.get(target, prop, receiver);
+      if (typeof value !== "function") return value;
+      if (prop === "getGenerativeModel" || prop === "getGenerativeModelFromCachedContent" || prop === "startChat") {
+        return (...args) => wrapClient(Reflect.apply(value, target, args));
+      }
+      switch (prop) {
+        case "generateContent":
+          return (...args) => googleGenerativeAIChannels.generateContent.invoke(
+            value,
+            target,
+            args,
+            {}
+          );
+        case "generateContentStream":
+          return (...args) => googleGenerativeAIChannels.generateContentStream.invoke(
+            value,
+            target,
+            args,
+            {}
+          );
+        case "embedContent":
+          return (...args) => googleGenerativeAIChannels.embedContent.invoke(
+            value,
+            target,
+            args,
+            {}
+          );
+        case "batchEmbedContents":
+          return (...args) => googleGenerativeAIChannels.batchEmbedContents.invoke(
+            value,
+            target,
+            args,
+            {}
+          );
+        case "sendMessage":
+          return (...args) => googleGenerativeAIChannels.sendMessage.invoke(
+            value,
+            target,
+            args,
+            {}
+          );
+        case "sendMessageStream":
+          return (...args) => googleGenerativeAIChannels.sendMessageStream.invoke(
+            value,
+            target,
+            args,
+            {}
+          );
+      }
+      return value.bind(target);
+    }
+  });
+  proxies.set(client, proxy);
+  proxies.set(proxy, proxy);
+  return proxy;
+}
+
 // src/wrappers/google-genai.ts
+var patchedHttpResponses = /* @__PURE__ */ new WeakSet();
 function wrapGoogleGenAI(googleGenAI) {
   if (!googleGenAI || typeof googleGenAI !== "object") {
     console.warn("Invalid Google GenAI module. Not wrapping.");
@@ -5226,6 +5774,21 @@ function wrapGoogleGenAI(googleGenAI) {
     );
     return googleGenAI;
   }
+  const httpResponse = googleGenAI.HttpResponse;
+  if (typeof httpResponse === "function" && isObject(httpResponse.prototype) && typeof httpResponse.prototype.json === "function" && !patchedHttpResponses.has(httpResponse.prototype)) {
+    const prototype = httpResponse.prototype;
+    const originalJson = prototype.json;
+    if (Reflect.set(prototype, "json", function() {
+      return googleGenAIChannels.httpResponseJson.invoke(
+        originalJson,
+        this,
+        [],
+        {}
+      );
+    })) {
+      patchedHttpResponses.add(prototype);
+    }
+  }
   return new Proxy(googleGenAI, {
     get(target, prop, receiver) {
       if (prop === "GoogleGenAI") {
@@ -5476,17 +6039,17 @@ function wrapGenkit(genkit) {
   console.warn("Unsupported Genkit object. Not wrapping.");
   return genkit;
 }
-function isRecord(value) {
+function isRecord2(value) {
   return typeof value === "object" && value !== null;
 }
 function isPropertyBag(value) {
-  return isRecord(value) || typeof value === "function";
+  return isRecord2(value) || typeof value === "function";
 }
 function hasFunction(value, methodName) {
   return isPropertyBag(value) && methodName in value && typeof value[methodName] === "function";
 }
 function isGenkitInstance(value) {
-  return isRecord(value) && (hasFunction(value, "generate") || hasFunction(value, "generateStream") || hasFunction(value, "defineFlow") || hasFunction(value, "defineTool"));
+  return isRecord2(value) && (hasFunction(value, "generate") || hasFunction(value, "generateStream") || hasFunction(value, "defineFlow") || hasFunction(value, "defineTool"));
 }
 function isGenkitModule(value) {
   return hasFunction(value, "genkit");
@@ -5540,7 +6103,7 @@ function patchGenkitRegistry(instance) {
   patchGenkitRegistryConstructor(registry);
 }
 function patchGenkitRegistryLookup(registry) {
-  if (!isRecord(registry) || hasRegistryPatchedFlag(registry) || !hasFunction(registry, "lookupAction")) {
+  if (!isRecord2(registry) || hasRegistryPatchedFlag(registry) || !hasFunction(registry, "lookupAction")) {
     return;
   }
   const originalLookupAction = registry.lookupAction;
@@ -5566,7 +6129,7 @@ function patchGenkitRegistryLookup(registry) {
   }
 }
 function patchGenkitRegistryConstructor(registry) {
-  if (!isRecord(registry)) {
+  if (!isRecord2(registry)) {
     return;
   }
   const constructor = registry.constructor;
@@ -5582,7 +6145,7 @@ function patchGenkitRegistryConstructor(registry) {
       configurable: true,
       value: (...args) => {
         const childRegistry = originalWithParent.apply(constructor, args);
-        if (args.some((arg) => isRecord(arg) && hasRegistryPatchedFlag(arg))) {
+        if (args.some((arg) => isRecord2(arg) && hasRegistryPatchedFlag(arg))) {
           patchGenkitRegistryLookup(childRegistry);
           patchGenkitRegistryConstructor(childRegistry);
         }
@@ -5681,7 +6244,7 @@ function hasRegistryConstructorPatchedFlag(value) {
   );
 }
 function isPromiseLike(value) {
-  return isRecord(value) && "then" in value && typeof value.then === "function";
+  return isRecord2(value) && "then" in value && typeof value.then === "function";
 }
 
 // src/wrappers/huggingface.ts
@@ -6158,14 +6721,14 @@ function wrapMistral(mistral) {
   console.warn("Unsupported Mistral library. Not wrapping.");
   return mistral;
 }
-function isRecord2(value) {
+function isRecord3(value) {
   return typeof value === "object" && value !== null;
 }
 function hasFunction3(value, methodName) {
-  return isRecord2(value) && methodName in value && typeof value[methodName] === "function";
+  return isRecord3(value) && methodName in value && typeof value[methodName] === "function";
 }
 function isSupportedMistralClient(value) {
-  if (!isRecord2(value)) {
+  if (!isRecord3(value)) {
     return false;
   }
   return value.chat !== void 0 && hasChat(value.chat) || value.embeddings !== void 0 && hasEmbeddings(value.embeddings) || value.fim !== void 0 && hasFim(value.fim) || value.agents !== void 0 && hasAgents(value.agents) || value.classifiers !== void 0 && hasClassifiers(value.classifiers);
@@ -6340,17 +6903,53 @@ function wrapAgentsStream(stream) {
   });
 }
 
-// src/wrappers/ollama.ts
-function wrapOllama(ollama) {
-  if (isSupportedOllamaClient(ollama)) {
-    return ollamaProxy(ollama);
+// src/wrappers/langgraph-sdk.ts
+var clients = /* @__PURE__ */ new WeakMap();
+function wrapLangGraphSDK(client) {
+  const runsClient = isObject(client) ? client.runs : void 0;
+  if (!isObject(client) || !isObject(runsClient) || !["wait", "stream"].every((key) => typeof runsClient[key] === "function")) {
+    debugLogger.warn("Unsupported LangGraph SDK client. Not wrapping.");
+    return client;
   }
-  debugLogger.warn("Unsupported Ollama library. Not wrapping.");
-  return ollama;
-}
-var ollamaProxyCache = /* @__PURE__ */ new WeakMap();
-function isSupportedOllamaClient(value) {
-  return isObject(value) && ["chat", "generate", "embed"].some(
+  const sdk = client;
+  const cached = clients.get(sdk);
+  if (cached) return cached;
+  const runs = new Proxy(sdk.runs, {
+    get(target, key) {
+      switch (key) {
+        case "wait":
+          return (...args) => langGraphSDKChannels.wait.invoke(target.wait, target, args, {});
+        case "stream":
+          return (...args) => langGraphSDKChannels.stream.invoke(target.stream, target, args, {});
+        default:
+          const value = Reflect.get(target, key, target);
+          return typeof value === "function" ? value.bind(target) : value;
+      }
+    }
+  });
+  const wrapped = new Proxy(sdk, {
+    get(target, key) {
+      if (key === "runs") return runs;
+      const value = Reflect.get(target, key, target);
+      return typeof value === "function" ? value.bind(target) : value;
+    }
+  });
+  clients.set(sdk, wrapped);
+  clients.set(wrapped, wrapped);
+  return wrapped;
+}
+
+// src/wrappers/ollama.ts
+function wrapOllama(ollama) {
+  if (isSupportedOllamaClient(ollama)) {
+    return ollamaProxy(ollama);
+  }
+  debugLogger.warn("Unsupported Ollama library. Not wrapping.");
+  return ollama;
+}
+var ollamaProxyCache = /* @__PURE__ */ new WeakMap();
+function isSupportedOllamaClient(value) {
+  return isObject(value) && ["chat", "generate", "embed"].some(
     (name) => typeof value[name] === "function"
   );
 }
@@ -6426,14 +7025,14 @@ function wrapCohere(cohere) {
   return cohere;
 }
 var cohereProxyCache = /* @__PURE__ */ new WeakMap();
-function isRecord3(value) {
+function isRecord4(value) {
   return typeof value === "object" && value !== null;
 }
 function hasFunction4(value, methodName) {
-  return isRecord3(value) && methodName in value && typeof value[methodName] === "function";
+  return isRecord4(value) && methodName in value && typeof value[methodName] === "function";
 }
 function isSupportedCohereClient(value) {
-  if (!isRecord3(value)) {
+  if (!isRecord4(value)) {
     return false;
   }
   return hasFunction4(value, "chat") || hasFunction4(value, "chatStream") || hasFunction4(value, "embed") || hasFunction4(value, "rerank");
@@ -6571,20 +7170,20 @@ function wrapGroq(groq) {
   console.warn("Unsupported Groq library. Not wrapping.");
   return groq;
 }
-function isRecord4(value) {
+function isRecord5(value) {
   return typeof value === "object" && value !== null;
 }
 function hasFunction6(value, methodName) {
-  return isRecord4(value) && methodName in value && typeof value[methodName] === "function";
+  return isRecord5(value) && methodName in value && typeof value[methodName] === "function";
 }
 function hasChat2(value) {
-  return isRecord4(value) && isRecord4(value.completions) && hasFunction6(value.completions, "create");
+  return isRecord5(value) && isRecord5(value.completions) && hasFunction6(value.completions, "create");
 }
 function hasEmbeddings2(value) {
   return hasFunction6(value, "create");
 }
 function isSupportedGroqClient(value) {
-  return isRecord4(value) && (value.chat !== void 0 && hasChat2(value.chat) || value.embeddings !== void 0 && hasEmbeddings2(value.embeddings));
+  return isRecord5(value) && (value.chat !== void 0 && hasChat2(value.chat) || value.embeddings !== void 0 && hasEmbeddings2(value.embeddings));
 }
 function groqProxy(groq) {
   const privateMethodWorkaroundCache = /* @__PURE__ */ new WeakMap();
@@ -6667,11 +7266,11 @@ var BEDROCK_RUNTIME_OPERATION_METHODS = /* @__PURE__ */ new Set([
   "invokeModelWithBidirectionalStream",
   "invokeModelWithResponseStream"
 ]);
-function isRecord5(value) {
+function isRecord6(value) {
   return typeof value === "object" && value !== null;
 }
 function isSupportedBedrockRuntimeClient(value) {
-  return isRecord5(value) && typeof value.send === "function";
+  return isRecord6(value) && typeof value.send === "function";
 }
 function bedrockRuntimeProxy(client) {
   const cached = bedrockRuntimeProxyCache.get(client);
@@ -8956,12 +9555,272 @@ function waterfall(tasks, callback) {
 }
 var waterfall$1 = awaitify(waterfall);
 
+// src/framework.ts
+import { v5 as uuidv5 } from "uuid";
+
 // src/trace.ts
+var spanFilterFields = /* @__PURE__ */ new Set([
+  "spanType",
+  "name",
+  "hasError",
+  "metadata",
+  "duration"
+]);
+function isRecord7(value) {
+  if (value === null || typeof value !== "object" || Array.isArray(value)) {
+    return false;
+  }
+  const prototype = Object.getPrototypeOf(value);
+  return prototype === Object.prototype || prototype === null;
+}
+function validateMetadataValue(value, ancestors) {
+  if (value === null || typeof value === "string" || typeof value === "boolean") {
+    return;
+  }
+  if (typeof value === "number") {
+    if (Number.isFinite(value)) {
+      return;
+    }
+    throw new Error("filters.metadata numbers must be finite");
+  }
+  if (typeof value !== "object" || value === null) {
+    throw new Error("filters.metadata values must be JSON-serializable");
+  }
+  if (!Array.isArray(value) && !isRecord7(value)) {
+    throw new Error("filters.metadata values must be JSON-serializable");
+  }
+  if (ancestors.has(value)) {
+    throw new Error("filters.metadata must not contain cycles");
+  }
+  ancestors.add(value);
+  if (Array.isArray(value)) {
+    for (let index = 0; index < value.length; index++) {
+      validateMetadataValue(value[index], ancestors);
+    }
+  } else {
+    if (Object.getOwnPropertySymbols(value).length > 0) {
+      throw new Error("filters.metadata keys must be strings");
+    }
+    for (const nested of Object.values(value)) {
+      validateMetadataValue(nested, ancestors);
+    }
+  }
+  ancestors.delete(value);
+}
+function compileMetadataLeaves(metadata, path = [], ancestors = /* @__PURE__ */ new WeakSet()) {
+  if (Object.getOwnPropertySymbols(metadata).length > 0) {
+    throw new Error("filters.metadata keys must be strings");
+  }
+  if (ancestors.has(metadata)) {
+    throw new Error("filters.metadata must not contain cycles");
+  }
+  ancestors.add(metadata);
+  const leaves = [];
+  for (const [key, value] of Object.entries(metadata)) {
+    const childPath = [...path, key];
+    if (isRecord7(value)) {
+      leaves.push(...compileMetadataLeaves(value, childPath, ancestors));
+    } else {
+      validateMetadataValue(value, ancestors);
+      leaves.push([childPath, value]);
+    }
+  }
+  ancestors.delete(metadata);
+  return leaves;
+}
+function normalizeSpanFilters(filters, spanType) {
+  if (filters != null && !isRecord7(filters)) {
+    throw new Error("filters must be an object");
+  }
+  const values = { ...filters ?? {} };
+  if (spanType !== void 0) {
+    if (!Array.isArray(spanType) || !spanType.every((item) => typeof item === "string")) {
+      throw new Error("spanType must be an array of strings");
+    }
+    if (Object.hasOwn(values, "spanType")) {
+      throw new Error(
+        "spanType cannot be provided both directly and in filters"
+      );
+    }
+    if (spanType.length > 0) {
+      values.spanType = spanType;
+    }
+  }
+  if (Object.keys(values).some((field) => !spanFilterFields.has(field))) {
+    throw new Error("Unsupported span filter fields");
+  }
+  for (const field of ["spanType", "name"]) {
+    if (Object.hasOwn(values, field)) {
+      const items = values[field];
+      if (!Array.isArray(items) || !items.every((item) => typeof item === "string")) {
+        throw new Error(`filters.${field} must be an array of strings`);
+      }
+    }
+  }
+  if (Object.hasOwn(values, "hasError") && typeof values.hasError !== "boolean") {
+    throw new Error("filters.hasError must be a boolean");
+  }
+  let compiledMetadataLeaves = [];
+  if (Object.hasOwn(values, "metadata")) {
+    if (!isRecord7(values.metadata)) {
+      throw new Error("filters.metadata must be an object");
+    }
+    compiledMetadataLeaves = compileMetadataLeaves(values.metadata);
+  }
+  if (Object.hasOwn(values, "duration")) {
+    if (!isRecord7(values.duration) || Object.keys(values.duration).some(
+      (bound) => bound !== "min" && bound !== "max"
+    )) {
+      throw new Error("filters.duration must contain only min and/or max");
+    }
+    if (Object.values(values.duration).some(
+      (bound) => typeof bound !== "number" || !Number.isFinite(bound)
+    )) {
+      throw new Error("filters.duration bounds must be finite numbers");
+    }
+  }
+  const normalized = {};
+  if (Array.isArray(values.spanType)) {
+    normalized.spanType = values.spanType;
+  }
+  if (Array.isArray(values.name)) {
+    normalized.name = values.name;
+  }
+  if (typeof values.hasError === "boolean") {
+    normalized.hasError = values.hasError;
+  }
+  if (isRecord7(values.metadata)) {
+    normalized.metadata = values.metadata;
+  }
+  if (isRecord7(values.duration)) {
+    normalized.duration = {};
+    if (typeof values.duration.min === "number") {
+      normalized.duration.min = values.duration.min;
+    }
+    if (typeof values.duration.max === "number") {
+      normalized.duration.max = values.duration.max;
+    }
+  }
+  return { filters: normalized, metadataLeaves: compiledMetadataLeaves };
+}
+function metadataEqual(actual, expected) {
+  if (expected === null) {
+    return actual === null || actual === void 0;
+  }
+  if (Array.isArray(expected)) {
+    return Array.isArray(actual) && actual.length === expected.length && expected.every((value, index) => metadataEqual(actual[index], value));
+  }
+  if (isRecord7(expected)) {
+    if (!isRecord7(actual)) {
+      return false;
+    }
+    const expectedKeys = Object.keys(expected);
+    const actualKeys = Object.keys(actual);
+    return actualKeys.length === expectedKeys.length && expectedKeys.every(
+      (key) => Object.hasOwn(actual, key) && metadataEqual(actual[key], expected[key])
+    );
+  }
+  return actual === expected;
+}
+function matchesSpanFilters(span, compiledFilters) {
+  const { filters, metadataLeaves } = compiledFilters;
+  if (filters.spanType !== void 0 && !filters.spanType.includes(span.span_attributes?.type ?? "")) {
+    return false;
+  }
+  if (filters.name !== void 0 && !filters.name.includes(span.span_attributes?.name ?? "")) {
+    return false;
+  }
+  if (filters.hasError !== void 0 && (span.error !== null && span.error !== void 0) !== filters.hasError) {
+    return false;
+  }
+  for (const [path, expected] of metadataLeaves) {
+    let actual = span.metadata;
+    for (const key of path) {
+      actual = isRecord7(actual) && Object.hasOwn(actual, key) ? actual[key] : void 0;
+    }
+    if (!metadataEqual(actual, expected)) {
+      return false;
+    }
+  }
+  if (filters.duration && Object.keys(filters.duration).length > 0) {
+    const metrics = span.metrics;
+    const start = isRecord7(metrics) ? metrics.start : void 0;
+    const end = isRecord7(metrics) ? metrics.end : void 0;
+    if (typeof start !== "number" || !Number.isFinite(start) || typeof end !== "number" || !Number.isFinite(end)) {
+      return false;
+    }
+    const elapsed = end - start;
+    if (filters.duration.min !== void 0 && elapsed < filters.duration.min) {
+      return false;
+    }
+    if (filters.duration.max !== void 0 && elapsed > filters.duration.max) {
+      return false;
+    }
+  }
+  return true;
+}
+function btqlComparison(op, name, value) {
+  return {
+    op,
+    left: { op: "ident", name },
+    right: { op: "literal", value }
+  };
+}
+function btqlNullCheck(op, name) {
+  return { op, expr: { op: "ident", name } };
+}
+function spanFilterClauses(compiledFilters) {
+  const { filters, metadataLeaves } = compiledFilters;
+  const children = [];
+  for (const [field, attribute] of [
+    ["spanType", "type"],
+    ["name", "name"]
+  ]) {
+    const values = filters[field];
+    if (values !== void 0) {
+      children.push(
+        values.length > 0 ? btqlComparison("in", ["span_attributes", attribute], values) : { op: "literal", value: false }
+      );
+    }
+  }
+  if (filters.hasError !== void 0) {
+    children.push(
+      btqlNullCheck(filters.hasError ? "isnotnull" : "isnull", ["error"])
+    );
+  }
+  for (const [path, value] of metadataLeaves) {
+    const name = ["metadata", ...path];
+    children.push(
+      value === null ? btqlNullCheck("isnull", name) : btqlComparison("eq", name, value)
+    );
+  }
+  const elapsed = {
+    op: "sub",
+    left: { op: "ident", name: ["metrics", "end"] },
+    right: { op: "ident", name: ["metrics", "start"] }
+  };
+  if (filters.duration?.min !== void 0) {
+    children.push({
+      op: "ge",
+      left: elapsed,
+      right: { op: "literal", value: filters.duration.min }
+    });
+  }
+  if (filters.duration?.max !== void 0) {
+    children.push({
+      op: "le",
+      left: elapsed,
+      right: { op: "literal", value: filters.duration.max }
+    });
+  }
+  return children;
+}
 var SpanFetcher = class _SpanFetcher extends ObjectFetcher {
-  constructor(objectType, _objectId, rootSpanId, _state, spanTypeFilter, includeScorers = false, brainstoreRealtime = true) {
+  constructor(objectType, _objectId, rootSpanId, _state, spanTypeFilter, includeScorers = false, brainstoreRealtime = true, filters) {
+    const normalizedFilters = normalizeSpanFilters(filters, spanTypeFilter);
     const filterExpr = _SpanFetcher.buildFilter(
       rootSpanId,
-      spanTypeFilter,
+      normalizedFilters,
       includeScorers
     );
     super(
@@ -8982,42 +9841,22 @@ var SpanFetcher = class _SpanFetcher extends ObjectFetcher {
   rootSpanId;
   _state;
   spanTypeFilter;
-  static buildFilter(rootSpanId, spanTypeFilter, includeScorers = false) {
+  static buildFilter(rootSpanId, filters, includeScorers = false) {
+    const purpose = ["span_attributes", "purpose"];
     const children = [
-      // Base filter: root_span_id = 'value'
-      {
-        op: "eq",
-        left: { op: "ident", name: ["root_span_id"] },
-        right: { op: "literal", value: rootSpanId }
-      }
+      btqlComparison("eq", ["root_span_id"], rootSpanId)
     ];
     if (!includeScorers) {
       children.push({
         op: "or",
         children: [
-          {
-            op: "isnull",
-            expr: { op: "ident", name: ["span_attributes", "purpose"] }
-          },
-          {
-            op: "ne",
-            left: { op: "ident", name: ["span_attributes", "purpose"] },
-            right: { op: "literal", value: "scorer" }
-          }
+          btqlNullCheck("isnull", purpose),
+          btqlComparison("ne", purpose, "scorer")
         ]
       });
     }
-    if (spanTypeFilter && spanTypeFilter.length > 0) {
-      children.push({
-        op: "in",
-        left: { op: "ident", name: ["span_attributes", "type"] },
-        right: { op: "literal", value: spanTypeFilter }
-      });
-    }
-    return {
-      op: "and",
-      children
-    };
+    children.push(...spanFilterClauses(filters));
+    return { op: "and", children };
   }
   get id() {
     return Promise.resolve(this._objectId);
@@ -9032,67 +9871,80 @@ var CachedSpanFetcher = class {
   fetchFn;
   constructor(objectTypeOrFetchFn, objectId, rootSpanId, getState, brainstoreRealtime = true) {
     if (typeof objectTypeOrFetchFn === "function") {
-      this.fetchFn = (spanType) => objectTypeOrFetchFn(spanType);
+      this.fetchFn = async (filters) => (await objectTypeOrFetchFn(filters.filters.spanType)).filter(
+        (span) => matchesSpanFilters(span, filters)
+      );
     } else {
       const objectType = objectTypeOrFetchFn;
-      this.fetchFn = async (spanType, includeScorers) => {
+      this.fetchFn = async (filters, includeScorers) => {
         const state = await getState();
         const fetcher = new SpanFetcher(
           objectType,
           objectId,
           rootSpanId,
           state,
-          spanType,
+          void 0,
           includeScorers,
-          brainstoreRealtime
+          brainstoreRealtime,
+          filters.filters
         );
-        const rows = await fetcher.fetchedData();
-        return rows.map((row) => ({
-          input: row.input,
-          output: row.output,
-          expected: row.expected,
-          error: row.error,
-          scores: row.scores,
-          metrics: row.metrics,
-          metadata: row.metadata,
-          span_id: row.span_id,
-          span_parents: row.span_parents,
-          is_root: row.is_root,
-          span_attributes: row.span_attributes,
-          id: row.id,
-          _xact_id: row._xact_id,
-          _pagination_key: row._pagination_key,
-          root_span_id: row.root_span_id,
-          created: row.created,
-          tags: row.tags
-        }));
+        let spans = (await fetcher.fetchedData()).map(
+          (row) => ({ ...row })
+        );
+        if (filters.filters.metadata !== void 0) {
+          spans = spans.filter((span) => matchesSpanFilters(span, filters));
+        }
+        return spans;
       };
     }
   }
   async getSpans({
-    spanType,
+    spanType: requestedSpanType,
+    filters,
     includeScorers = false
   } = {}) {
+    const normalizedFilters = normalizeSpanFilters(filters, requestedSpanType);
+    const spanType = normalizedFilters.filters.spanType;
+    const hasAdvancedFilters = Object.keys(normalizedFilters.filters).some(
+      (field) => field !== "spanType"
+    );
+    if (spanType?.length === 0) {
+      return [];
+    }
     if (includeScorers) {
-      return this.fetchFn(spanType, true);
+      return this.fetchFn(normalizedFilters, true);
     }
     if (this.allFetched) {
-      return this.getFromCache(spanType);
+      const spans = this.getFromCache(spanType);
+      return hasAdvancedFilters ? spans.filter((span) => matchesSpanFilters(span, normalizedFilters)) : spans;
     }
-    if (!spanType || spanType.length === 0) {
+    if (hasAdvancedFilters && spanType && spanType.every((type) => this.spanCache.has(type))) {
+      return this.getFromCache(spanType).filter(
+        (span) => matchesSpanFilters(span, normalizedFilters)
+      );
+    }
+    if (hasAdvancedFilters) {
+      return this.fetchFn(normalizedFilters, false);
+    }
+    if (!spanType) {
+      this.spanCache.clear();
       await this.fetchSpans(void 0);
-      this.allFetched = true;
+      if (this.spanCache.size > 0) {
+        this.allFetched = true;
+      }
       return this.getFromCache(void 0);
     }
-    const missingTypes = spanType.filter((t) => !this.spanCache.has(t));
-    if (missingTypes.length === 0) {
-      return this.getFromCache(spanType);
+    const missingTypes = spanType.filter((type) => !this.spanCache.has(type));
+    if (missingTypes.length > 0) {
+      await this.fetchSpans(missingTypes);
     }
-    await this.fetchSpans(missingTypes);
     return this.getFromCache(spanType);
   }
   async fetchSpans(spanType) {
-    const spans = await this.fetchFn(spanType, false);
+    const spans = await this.fetchFn(
+      normalizeSpanFilters(spanType ? { spanType } : void 0),
+      false
+    );
     for (const span of spans) {
       const type = span.span_attributes?.type ?? "";
       const existing = this.spanCache.get(type) ?? [];
@@ -9172,34 +10024,20 @@ var LocalTrace = class {
    */
   async getSpans({
     spanType,
+    filters,
     includeScorers = false
   } = {}) {
+    const normalizedFilters = normalizeSpanFilters(filters, spanType);
     const cachedSpans = this.state.spanCache.getByRootSpanId(this.rootSpanId);
     if (cachedSpans && cachedSpans.length > 0) {
-      let spans = includeScorers ? cachedSpans : cachedSpans.filter(
-        (span) => span.span_attributes?.purpose !== "scorer"
-      );
-      if (spanType && spanType.length > 0) {
-        spans = spans.filter(
-          (span) => spanType.includes(span.span_attributes?.type ?? "")
-        );
-      }
-      return spans.map((span) => ({
-        input: span.input,
-        output: span.output,
-        expected: span.expected,
-        error: span.error,
-        scores: span.scores,
-        metrics: span.metrics,
-        metadata: span.metadata,
-        span_id: span.span_id,
-        span_parents: span.span_parents,
-        is_root: span.is_root,
-        span_attributes: span.span_attributes,
-        tags: span.tags
-      }));
-    }
-    return this.cachedFetcher.getSpans({ spanType, includeScorers });
+      return cachedSpans.filter(
+        (span) => (includeScorers || span.span_attributes?.purpose !== "scorer") && matchesSpanFilters({ ...span }, normalizedFilters)
+      ).map((span) => ({ ...span }));
+    }
+    return this.cachedFetcher.getSpans({
+      filters: normalizedFilters.filters,
+      includeScorers
+    });
   }
   /**
    * Get the thread (preprocessed messages) for this trace.
@@ -9619,7 +10457,7 @@ async function Eval(name, evaluator, reporterOrOpts) {
       data,
       { disabled: Boolean(options.parent || options.noSendLogs) }
     );
-    if (experiment && typeof process !== "undefined" && globalThis.BRAINTRUST_CONTEXT_MANAGER !== void 0) {
+    if (experiment) {
       await experiment._waitForId();
     }
     if (experiment && options.onStart) {
@@ -9944,6 +10782,10 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
         } : void 0;
         const parsedDatumOrigin = ObjectReference.safeParse(datum.origin);
         const origin = inlineDatasetOrigin ?? (parsedDatumOrigin?.success ? parsedDatumOrigin.data : void 0);
+        const upsertId = datum.upsert_id && trialIndex > 0 ? uuidv5(
+          `braintrust:eval:${datum.upsert_id}:trial:${trialIndex}`,
+          uuidv5.URL
+        ) : datum.upsert_id;
         const baseEvent = {
           name: "eval",
           spanAttributes: {
@@ -9954,7 +10796,7 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
             expected: "expected" in datum ? datum.expected : void 0,
             tags: datum.tags,
             origin,
-            ...datum.upsert_id ? { id: datum.upsert_id } : {}
+            ...upsertId ? { id: upsertId } : {}
           }
         };
         const callback = async (rootSpan) => {
@@ -10463,14 +11305,14 @@ var defaultReporter = {
   }
 };
 
-// src/durable-eval.ts
+// src/workflow-eval.ts
 var encoder = new TextEncoder();
 var decoder = new TextDecoder();
-var BATCH_TASK_KIND = "braintrust.durable.batch-task";
-var BATCH_SCORER_KIND = "braintrust.durable.batch-scorer";
-var DEFAULT_BATCH_SIZE = 1e3;
-var DurableEvalMemoryStore = class {
+var WORKFLOW_TASK_KIND = "braintrust.workflow.task";
+var WORKFLOW_SCORER_KIND = "braintrust.workflow.scorer";
+var WorkflowEvalMemoryStore = class {
   values = /* @__PURE__ */ new Map();
+  sets = /* @__PURE__ */ new Map();
   async read(key) {
     return this.values.get(key)?.slice();
   }
@@ -10483,8 +11325,16 @@ var DurableEvalMemoryStore = class {
     this.values.set(key, value.slice());
     return { value: value.slice(), created: true };
   }
+  async addToSet(key, member) {
+    let members = this.sets.get(key);
+    if (!members) this.sets.set(key, members = /* @__PURE__ */ new Set());
+    members.add(member);
+  }
+  async getSetSize(key) {
+    return this.sets.get(key)?.size ?? 0;
+  }
 };
-var DurableEvalRedisStore = class {
+var WorkflowEvalRedisStore = class {
   client;
   keyPrefix;
   ttlMs;
@@ -10493,14 +11343,16 @@ var DurableEvalRedisStore = class {
     this.keyPrefix = options.keyPrefix ?? "braintrust-eval:";
     this.ttlMs = options.ttlMs ?? 1e3 * 60 * 60 * 24 * 7;
     if (!Number.isInteger(this.ttlMs) || this.ttlMs < 1) {
-      throw new Error("DurableEvalRedisStore ttlMs must be a positive integer");
+      throw new Error(
+        "WorkflowEvalRedisStore ttlMs must be a positive integer"
+      );
     }
   }
   async read(key) {
     const value = await this.client.get(`${this.keyPrefix}${key}`);
     if (value == null) return void 0;
     if (typeof value !== "string") {
-      throw new Error("DurableEvalRedisStore expected GET to return a string");
+      throw new Error("WorkflowEvalRedisStore expected GET to return a string");
     }
     return base64ToUint8Array(value);
   }
@@ -10517,7 +11369,7 @@ var DurableEvalRedisStore = class {
       await set.call(client, redisKey, encoded, { px: this.ttlMs });
     } else {
       throw new Error(
-        "DurableEvalRedisStore requires a node-redis, ioredis, or @upstash/redis client"
+        "WorkflowEvalRedisStore requires a node-redis, ioredis, or @upstash/redis client"
       );
     }
   }
@@ -10535,49 +11387,73 @@ var DurableEvalRedisStore = class {
       setOptions = [{ px: this.ttlMs, nx: true, get: true }];
     } else {
       throw new Error(
-        "DurableEvalRedisStore getOrSet requires a node-redis, ioredis, or @upstash/redis client"
+        "WorkflowEvalRedisStore getOrSet requires a node-redis, ioredis, or @upstash/redis client"
       );
     }
     const existing = await set.call(client, redisKey, encoded, ...setOptions);
     if (existing === null) return { value: value.slice(), created: true };
     if (typeof existing !== "string") {
       throw new Error(
-        "DurableEvalRedisStore expected atomic SET to return a string or null"
+        "WorkflowEvalRedisStore expected atomic SET to return a string or null"
       );
     }
     return { value: base64ToUint8Array(existing), created: false };
   }
+  async addToSet(key, member) {
+    await this.evalSet(
+      "redis.call('SADD', KEYS[1], ARGV[1]); redis.call('PEXPIRE', KEYS[1], ARGV[2]); return 1",
+      key,
+      [member, String(this.ttlMs)]
+    );
+  }
+  async getSetSize(key) {
+    return this.evalSet("return redis.call('SCARD', KEYS[1])", key, []);
+  }
+  async evalSet(script, key, args) {
+    const client = this.client;
+    const redisKey = `${this.keyPrefix}${key}`;
+    const result = typeof client.defineCommand === "function" ? await client.eval(script, 1, redisKey, ...args) : typeof client.sendCommand === "function" ? await client.eval(script, { keys: [redisKey], arguments: args }) : await client.eval(script, [redisKey], args);
+    if (typeof result !== "number") {
+      throw new Error(
+        "WorkflowEvalRedisStore expected EVAL to return a number"
+      );
+    }
+    return result;
+  }
 };
-var BatchTask = class {
+var WorkflowTask = class {
   constructor(processor) {
     this.processor = processor;
   }
   processor;
-  kind = BATCH_TASK_KIND;
+  kind = WORKFLOW_TASK_KIND;
 };
-var BatchScorer = class {
+var WorkflowScorer = class {
   constructor(processor) {
     this.processor = processor;
     this.name = processor.name;
   }
   processor;
-  kind = BATCH_SCORER_KIND;
+  kind = WORKFLOW_SCORER_KIND;
   name;
 };
-function defineDurableEval(projectName, evaluator) {
+function defineWorkflowEval(projectName, evaluator) {
+  if (evaluator.maxConcurrency !== void 0 && (!Number.isInteger(evaluator.maxConcurrency) || evaluator.maxConcurrency < 1)) {
+    throw new Error("maxConcurrency must be a positive integer");
+  }
   const definition = {
     projectName,
     evalName: evaluator.experimentName ?? projectName,
     evaluator
   };
   return {
-    start: (options = {}) => startDurableEval(definition, options),
-    status: (options) => getDurableEvalStatus(definition, options),
-    poll: (options) => pollDurableEval(definition, options),
-    processBatchResult: (result) => processDurableBatchResult(definition, result)
+    start: (options = {}) => startWorkflowEval(definition, options),
+    status: (options) => getWorkflowEvalStatus(definition, options),
+    poll: (options) => pollWorkflowEval(definition, options),
+    processSubmissionResult: (result) => processWorkflowSubmissionResult(definition, result)
   };
 }
-async function startDurableEval(definition, options) {
+async function startWorkflowEval(definition, options) {
   const store = definition.evaluator.store;
   const runId = newId();
   const key = runKey(definition.projectName, definition.evalName, runId);
@@ -10597,7 +11473,7 @@ async function startDurableEval(definition, options) {
       update: true
     }
   );
-  if (!isBatchTask(definition.evaluator.task) && !(definition.evaluator.scores ?? []).some(isBatchScorer)) {
+  if (!isWorkflowTask(definition.evaluator.task) && !(definition.evaluator.scores ?? []).some(isWorkflowScorer)) {
     const result = await runEvaluator(
       experiment,
       {
@@ -10624,67 +11500,91 @@ async function startDurableEval(definition, options) {
       parameters: assertJsonValue(parameters, "eval parameters"),
       status: "completed",
       summary: result.summary,
+      caseCount: 0,
       cases: [],
-      batches: []
+      submissions: []
     };
     await experiment?.flush();
     await writeRunRecord(store, key, state2);
     return currentStatus(definition, state2);
   }
+  const cases = await materializeCases(definition, data, experiment);
   const state = {
     runId,
     experimentName,
     noSendLogs: options.noSendLogs ?? false,
     parameters: assertJsonValue(parameters, "eval parameters"),
     status: "running",
-    cases: await materializeCases(definition, data, experiment),
-    batches: []
+    caseCount: cases.length,
+    cases,
+    submissions: []
   };
+  await writeJson(
+    store,
+    `${key}/case-ids`,
+    cases.map(({ id }) => id)
+  );
   await writeCaseBaseRecords(store, key, state.cases);
   await writeRunRecord(store, key, state);
-  return advanceDurableEval(definition, state, store, key, experiment);
+  return advanceWorkflowEval(definition, state, store, key, experiment);
 }
-async function getDurableEvalStatus(definition, options) {
+async function getWorkflowEvalStatus(definition, options) {
   const store = definition.evaluator.store;
-  const state = await readRunState(
-    definition,
-    store,
-    runKey(definition.projectName, definition.evalName, options.runId)
+  const key = runKey(
+    definition.projectName,
+    definition.evalName,
+    options.runId
   );
-  if (!state) throw new Error(`Durable eval run ${options.runId} is missing`);
+  const state = await readJson(store, key);
+  if (!state) throw new Error(`Workflow eval run ${options.runId} is missing`);
   return currentStatus(definition, state);
 }
-async function processDurableBatchResult(definition, result) {
-  if (!result.batchId && !result.externalId) {
-    throw new Error("Batch results require batchId or externalId");
+async function processWorkflowSubmissionResult(definition, result) {
+  if (!result.submissionId && !result.externalId) {
+    throw new Error("Submission results require submissionId or externalId");
   }
   const store = definition.evaluator.store;
   const key = runKey(definition.projectName, definition.evalName, result.runId);
-  const state = await readRunState(definition, store, key);
-  if (!state) throw new Error(`Durable eval run ${result.runId} is missing`);
-  const byBatch = result.batchId ? state.batches.find((candidate) => candidate.id === result.batchId) : void 0;
-  const byExternal = result.externalId ? state.batches.find(
-    (candidate) => candidate.externalId === result.externalId
+  const run = await readJson(store, key);
+  if (!run) throw new Error(`Workflow eval run ${result.runId} is missing`);
+  const externalSubmissionId = result.externalId ? await readJson(
+    store,
+    `${key}/external/${encodedKeyPart(result.externalId)}`
   ) : void 0;
-  if (byBatch && byExternal && byBatch.id !== byExternal.id) {
-    throw new Error("batchId and externalId identify different batches");
-  }
-  const batch = byBatch ?? byExternal;
-  if (!batch) throw new Error("No submitted batch matches this result");
-  if (batch.status !== "complete") {
-    const records = await collectBatch(definition, state, batch);
-    batch.status = "complete";
-    await writeCaseRecords(store, key, records);
-    await writeBatchRecords(store, key, [batch]);
-  }
-  return advanceDurableEval(
+  if (result.submissionId && externalSubmissionId && result.submissionId !== externalSubmissionId) {
+    throw new Error(
+      "submissionId and externalId identify different submissions"
+    );
+  }
+  const submissionId = result.submissionId ?? externalSubmissionId;
+  const submission = submissionId ? await readJson(
+    store,
+    submissionRecordKey(key, submissionId)
+  ) : void 0;
+  if (!submission) throw new Error("No submission matches this result");
+  if (result.externalId && submission.externalId !== result.externalId) {
+    throw new Error(
+      "submissionId and externalId identify different submissions"
+    );
+  }
+  if (run.status === "completed") return currentStatus(definition, run);
+  const state = await readRunState(definition, store, key, [
+    submission.itemId
+  ]);
+  if (submission.status !== "complete") {
+    const record = await collectSubmission(definition, state, submission);
+    await writeCaseRecords(store, key, [record]);
+    submission.status = "complete";
+  }
+  await writeSubmissionRecords(store, key, [submission]);
+  return advanceWorkflowEval(
     definition,
-    await readRunState(definition, store, key),
+    await readRunState(definition, store, key, [submission.itemId]),
     store,
     key
   );
 }
-async function pollDurableEval(definition, options) {
+async function pollWorkflowEval(definition, options) {
   const store = definition.evaluator.store;
   const key = runKey(
     definition.projectName,
@@ -10692,39 +11592,53 @@ async function pollDurableEval(definition, options) {
     options.runId
   );
   const state = await readRunState(definition, store, key);
-  if (!state) throw new Error(`Durable eval run ${options.runId} is missing`);
-  const batches = state.batches.filter((batch) => {
-    if (batch.status === "complete") return false;
-    return processorForStage(definition, batch.kind, batch.scorerName).completion.mode === "poll";
-  });
-  const results = await Promise.all(
-    batches.map(async (batch) => ({
-      batch,
-      result: await processorForStage(definition, batch.kind, batch.scorerName).completion.poll(batch.submissionData, {
-        runId: state.runId,
-        batchId: batch.id
-      })
-    }))
-  );
-  const changedCases = /* @__PURE__ */ new Map();
-  const changedBatches = [];
-  for (const { batch, result } of results) {
+  if (!state) throw new Error(`Workflow eval run ${options.runId} is missing`);
+  const submissions = state.submissions.filter((submission) => {
+    if (submission.status === "complete") return false;
+    return processorForStage(definition, submission.kind, submission.scorerName).completion.mode === "poll";
+  });
+  const workers = queue(async (submission) => {
+    const completion = processorForStage(
+      definition,
+      submission.kind,
+      submission.scorerName
+    ).completion;
+    if (completion.mode !== "poll") return;
+    const result = await completion.poll(submission.submissionData, {
+      runId: state.runId,
+      submissionId: submission.id
+    });
     if (result.status === "failed") throw asError(result.error);
-    if (result.status !== "complete") continue;
-    for (const record of await collectBatch(definition, state, batch)) {
-      changedCases.set(record.id, record);
-    }
-    batch.status = "complete";
-    changedBatches.push(batch);
-  }
-  if (changedBatches.length > 0) {
-    await writeCaseRecords(store, key, [...changedCases.values()]);
-    await writeBatchRecords(store, key, changedBatches);
+    if (result.status === "complete") {
+      const record = await collectSubmission(definition, state, submission);
+      await writeCaseRecords(store, key, [record]);
+      submission.status = "complete";
+      await writeSubmissionRecords(store, key, [submission]);
+    }
+  }, definition.evaluator.maxConcurrency ?? 10);
+  const results = await Promise.allSettled(
+    submissions.map((submission) => workers.pushAsync(submission))
+  );
+  let status;
+  const errors = results.flatMap(
+    (result) => result.status === "rejected" ? [asError(result.reason)] : []
+  );
+  try {
+    status = await advanceWorkflowEval(
+      definition,
+      await readRunState(definition, store, key),
+      store,
+      key
+    );
+  } catch (error) {
+    errors.push(asError(error));
   }
-  const currentState = changedBatches.length > 0 ? await readRunState(definition, store, key) : state;
-  return advanceDurableEval(definition, currentState, store, key);
+  if (errors.length === 1) throw errors[0];
+  if (errors.length > 1)
+    throw new AggregateError(errors, "Workflow submission callbacks failed");
+  return status;
 }
-async function openDurableExperiment(definition, state) {
+async function openWorkflowExperiment(definition, state) {
   const data = [];
   return await _internalInitEvaluatorExperiment(
     definition.projectName,
@@ -10737,45 +11651,49 @@ async function openDurableExperiment(definition, state) {
     }
   );
 }
-async function advanceDurableEval(definition, state, store, key, existingExperiment) {
+async function advanceWorkflowEval(definition, state, store, key, existingExperiment) {
   if (state.status === "completed") return currentStatus(definition, state);
-  const experiment = existingExperiment === void 0 ? await openDurableExperiment(definition, state) : existingExperiment;
+  const experiment = existingExperiment === void 0 ? await openWorkflowExperiment(definition, state) : existingExperiment;
+  const caseIds = state.cases.map(({ id }) => id);
   await runTaskStage(definition, state, store, key, experiment);
   await logCompletedTasks(definition, state, store, key, experiment);
-  state = await readRunState(definition, store, key) ?? state;
-  if (state.cases.some((record) => !record.taskComplete || !record.taskLogged)) {
-    return currentStatus(definition, state);
-  }
+  state = await readRunState(definition, store, key, caseIds) ?? state;
   await runScoreStages(definition, state, store, key, experiment);
-  state = await readRunState(definition, store, key) ?? state;
+  state = await readRunState(definition, store, key, caseIds) ?? state;
   const scorerNames = resolveScorers(definition.evaluator.scores ?? []).map(
     ({ name }) => name
   );
   const classifierNames = (definition.evaluator.classifiers ?? []).map(
     classifierName
   );
-  if (state.cases.some(
-    (record) => scorerNames.some(
-      (name) => !Object.hasOwn(record.scores, name) || !Object.hasOwn(record.loggedScores, name)
-    ) || classifierNames.some(
-      (name) => !Object.hasOwn(record.loggedClassifications, name)
-    )
-  )) {
+  await Promise.all(
+    state.cases.map(async (record) => {
+      if (record.taskComplete && record.taskLogged && scorerNames.every(
+        (name) => Object.hasOwn(record.scores, name) && Object.hasOwn(record.loggedScores, name)
+      ) && classifierNames.every(
+        (name) => Object.hasOwn(record.loggedClassifications, name)
+      )) {
+        await store.addToSet(`${key}/progress/cases`, record.id);
+      }
+    })
+  );
+  if (await store.getSetSize(`${key}/progress/cases`) !== state.caseCount) {
     return currentStatus(definition, state);
   }
   if (!await claimAction(store, key, "finish")) {
-    const latest = await readRunState(definition, store, key);
+    const latest = await readJson(store, key);
     return currentStatus(definition, latest ?? state);
   }
+  state = await readRunState(definition, store, key);
   state.summary = await finishExperiment(definition, state, experiment);
   state.status = "completed";
   await writeRunRecord(store, key, state);
   return currentStatus(definition, state);
 }
-function currentStatus(definition, state) {
+async function currentStatus(definition, state) {
   if (state.status === "completed") {
     if (!state.summary) {
-      throw new Error(`Durable eval run ${state.runId} has no saved summary`);
+      throw new Error(`Workflow eval run ${state.runId} has no saved summary`);
     }
     return {
       status: "completed",
@@ -10784,11 +11702,20 @@ function currentStatus(definition, state) {
       summary: state.summary
     };
   }
+  const key = runKey(definition.projectName, definition.evalName, state.runId);
+  const store = definition.evaluator.store;
   const pending = { poll: 0, webhook: 0 };
-  for (const batch of state.batches) {
-    if (batch.status === "complete") continue;
-    pending[processorForStage(definition, batch.kind, batch.scorerName).completion.mode]++;
-  }
+  await Promise.all(
+    ["poll", "webhook"].map(async (mode) => {
+      const completed = await store.getSetSize(
+        `${key}/progress/${mode}/complete`
+      );
+      const submitted = await store.getSetSize(
+        `${key}/progress/${mode}/submitted`
+      );
+      pending[mode] = submitted - completed;
+    })
+  );
   return { status: "waiting", runId: state.runId, pending };
 }
 async function startCaseRoot(definition, state, record, experiment) {
@@ -10849,7 +11776,7 @@ async function logTaskResult(definition, state, record, experiment, task) {
       expected: "expected" in datum ? datum.expected : void 0,
       metadata: {
         ...record.metadata,
-        durable_eval: {
+        workflow_eval: {
           run_id: state.runId,
           case_id: record.caseId,
           trial_index: record.trialIndex
@@ -10880,8 +11807,8 @@ async function logCompletedTasks(definition, state, store, key, experiment) {
   }
 }
 async function runTaskStage(definition, state, store, key, experiment) {
-  if (isBatchTask(definition.evaluator.task)) {
-    await ensureBatches(definition, state, store, key, "task");
+  if (isWorkflowTask(definition.evaluator.task)) {
+    await ensureSubmissions(definition, state, store, key, "task");
     return;
   }
   const task = definition.evaluator.task;
@@ -10907,8 +11834,9 @@ async function runScoreStages(definition, state, store, key, experiment) {
     changed.clear();
   };
   for (const { name, scorer } of scorers) {
-    if (isBatchScorer(scorer)) {
+    if (isWorkflowScorer(scorer)) {
       for (const record of state.cases) {
+        if (!record.taskComplete || !record.taskLogged) continue;
         if (Object.hasOwn(record.scores, name) && !Object.hasOwn(record.loggedScores, name)) {
           if (!await claimAction(store, key, "score-log", record.id, name)) {
             continue;
@@ -10924,10 +11852,11 @@ async function runScoreStages(definition, state, store, key, experiment) {
         }
       }
       await persistChangedCases();
-      await ensureBatches(definition, state, store, key, "score", name);
+      await ensureSubmissions(definition, state, store, key, "score", name);
       continue;
     }
     for (const record of state.cases) {
+      if (!record.taskComplete || !record.taskLogged) continue;
       if (Object.hasOwn(record.loggedScores, name)) continue;
       if (!await claimAction(store, key, "score", record.id, name)) continue;
       await evaluateAndLogScore(
@@ -10944,6 +11873,7 @@ async function runScoreStages(definition, state, store, key, experiment) {
   for (const [index, classifier] of (definition.evaluator.classifiers ?? []).entries()) {
     const name = classifierName(classifier, index);
     for (const record of state.cases) {
+      if (!record.taskComplete || !record.taskLogged) continue;
       if (Object.hasOwn(record.loggedClassifications, name)) continue;
       if (!await claimAction(store, key, "classification", record.id, name)) {
         continue;
@@ -10966,13 +11896,14 @@ function scorerArgs(record) {
   return {
     ...datum,
     metadata: record.metadata,
+    tags: record.tags,
     output: record.output
   };
 }
 function resumeCaseRoot(definition, record, experiment) {
   if (!experiment) return NOOP_SPAN;
   if (!record.rootSpan) {
-    throw new Error(`Durable eval case ${record.caseId} has no root span`);
+    throw new Error(`Workflow eval case ${record.caseId} has no root span`);
   }
   return _internalResumeSpan({
     exported: record.rootSpan,
@@ -11060,117 +11991,110 @@ async function evaluateAndLogClassification(definition, state, record, name, cla
     root.end();
   }
 }
-async function ensureBatches(definition, state, store, key, kind, scorerName2) {
+async function ensureSubmissions(definition, state, store, key, kind, scorerName2) {
   const processor = processorForStage(definition, kind, scorerName2);
-  const plans = plannedBatches(
+  const plans = plannedSubmissions(
     definition,
     state.runId,
     state.cases.map(({ id }) => id)
   );
   const casesById = new Map(state.cases.map((record) => [record.id, record]));
-  for (const plan of plans) {
-    if (plan.kind !== kind || plan.scorerName !== scorerName2) continue;
-    if (state.batches.some(({ id }) => id === plan.id)) continue;
-    const records = plan.itemIds.map((id) => casesById.get(id));
-    const ready = records.every(
-      (record) => kind === "task" ? !record.taskComplete : !Object.hasOwn(record.scores, scorerName2)
-    );
-    if (!ready) continue;
-    const batchId = plan.id;
+  const existingIds = new Set(state.submissions.map(({ id }) => id));
+  const workers = queue(async (plan) => {
+    if (plan.kind !== kind || plan.scorerName !== scorerName2) return;
+    if (existingIds.has(plan.id)) return;
+    const record = casesById.get(plan.itemId);
+    const ready = kind === "task" ? !record.taskComplete : record.taskComplete && record.taskLogged && !Object.hasOwn(record.scores, scorerName2);
+    if (!ready) return;
+    const submissionId = plan.id;
     const claim = await store.getOrSet(
-      claimRecordKey(key, "batch", batchId),
-      encoder.encode(batchId)
-    );
-    if (!claim.created) continue;
-    const context = { runId: state.runId, batchId };
-    const items = records.map(
-      (record) => kind === "task" ? taskBatchItem(record, state.parameters) : scorerBatchItem(record)
+      claimRecordKey(key, "submission", submissionId),
+      encoder.encode(submissionId)
     );
+    if (!claim.created) return;
+    const context = { runId: state.runId, submissionId };
+    const item = kind === "task" ? taskSubmissionItem(record, state.parameters) : scorerSubmissionItem(record);
     const submissionData = assertJsonValue(
-      await processor.submit(items, context),
-      `submission data for batch ${batchId}`
+      await processor.submit(item, context),
+      `submission data for submission ${submissionId}`
     );
     const externalId = processor.completion.mode === "webhook" ? processor.completion.getExternalId(submissionData, context) : void 0;
     if (externalId !== void 0 && !externalId.trim()) {
-      throw new Error(`Batch ${batchId} produced an empty externalId`);
+      throw new Error(
+        `Submission ${submissionId} produced an empty externalId`
+      );
     }
-    const batch = {
-      id: batchId,
+    const submission = {
+      id: submissionId,
       kind,
       scorerName: scorerName2,
-      itemIds: records.map((record) => record.id),
+      itemId: record.id,
       submissionData,
       externalId,
-      status: "submitted"
+      status: "submitted",
+      completionMode: processor.completion.mode
     };
-    state.batches.push(batch);
-    await writeBatchRecords(store, key, [batch]);
-  }
+    state.submissions.push(submission);
+    await writeSubmissionRecords(store, key, [submission]);
+  }, definition.evaluator.maxConcurrency ?? 10);
+  const results = await Promise.allSettled(
+    plans.map((plan) => workers.pushAsync(plan))
+  );
+  const errors = results.flatMap(
+    (result) => result.status === "rejected" ? [asError(result.reason)] : []
+  );
+  if (errors.length === 1) throw errors[0];
+  if (errors.length > 1)
+    throw new AggregateError(errors, "Workflow submissions failed");
 }
-async function collectBatch(definition, state, batch) {
-  const processor = processorForStage(definition, batch.kind, batch.scorerName);
-  const context = { runId: state.runId, batchId: batch.id };
-  const results = await processor.collect(batch.submissionData, context);
-  if (!Array.isArray(results)) {
-    throw new Error(`collect for batch ${batch.id} must return an array`);
+async function collectSubmission(definition, state, submission) {
+  const processor = processorForStage(
+    definition,
+    submission.kind,
+    submission.scorerName
+  );
+  const context = { runId: state.runId, submissionId: submission.id };
+  const result = await processor.collect(submission.submissionData, context);
+  if (typeof result !== "object" || result === null || Array.isArray(result)) {
+    throw new Error(
+      `collect for submission ${submission.id} must return a result object`
+    );
   }
-  const expectedIds = new Set(batch.itemIds);
-  const seen = /* @__PURE__ */ new Set();
-  const records = [];
-  for (const result of results) {
-    const id = resultItemId(result);
-    if (!expectedIds.has(id)) {
-      throw new Error(`Batch ${batch.id} returned unknown item ${id}`);
-    }
-    if (seen.has(id)) {
-      throw new Error(`Batch ${batch.id} returned item ${id} more than once`);
-    }
-    seen.add(id);
-    const record = state.cases.find((candidate) => candidate.id === id);
-    records.push(record);
-    if (batch.kind === "task") {
-      record.output = assertJsonValue(
-        result.output,
-        `task output for item ${id}`
-      );
-      if ("metadata" in result && result.metadata !== void 0) {
-        record.metadata = assertJsonValue(
-          {
-            ...record.metadata,
-            ...result.metadata
-          },
-          `metadata for ${id}`
-        );
-      }
-      if ("tags" in result && result.tags !== void 0)
-        record.tags = result.tags;
-      record.taskComplete = true;
-    } else {
-      record.scores[batch.scorerName] = assertJsonValue(
-        result.score,
-        `score output for item ${id}`
+  const record = state.cases.find(
+    (candidate) => candidate.id === submission.itemId
+  );
+  if (submission.kind === "task") {
+    record.output = assertJsonValue(
+      result.output,
+      `task output for item ${record.id}`
+    );
+    if (result.metadata !== void 0) {
+      record.metadata = assertJsonValue(
+        { ...record.metadata, ...result.metadata },
+        `metadata for ${record.id}`
       );
     }
-  }
-  const missing = batch.itemIds.filter((id) => !seen.has(id));
-  if (missing.length > 0) {
-    throw new Error(
-      `Batch ${batch.id} did not return results for: ${missing.join(", ")}`
+    if (result.tags !== void 0) record.tags = result.tags;
+    record.taskComplete = true;
+  } else {
+    record.scores[submission.scorerName] = assertJsonValue(
+      result.score,
+      `score output for item ${record.id}`
     );
   }
-  return records;
+  return record;
 }
 function processorForStage(definition, kind, scorerName2) {
   if (kind === "task") {
-    if (!isBatchTask(definition.evaluator.task)) {
-      throw new Error("Definition no longer contains the batch task");
+    if (!isWorkflowTask(definition.evaluator.task)) {
+      throw new Error("Definition no longer contains the submission task");
     }
     return definition.evaluator.task.processor;
   }
   const scorer = resolveScorers(definition.evaluator.scores ?? []).find(
     ({ name }) => name === scorerName2
   )?.scorer;
-  if (!isBatchScorer(scorer)) {
+  if (!isWorkflowScorer(scorer)) {
     throw new Error(`Definition no longer contains scorer ${scorerName2}`);
   }
   return scorer.processor;
@@ -11192,15 +12116,15 @@ async function materializeCases(definition, data, experiment) {
     const caseId = datum.id ?? datum.upsert_id ?? (evaluator.caseId ? await evaluator.caseId(datum) : void 0);
     if (!caseId) {
       throw new Error(
-        "Every durable eval case requires id, upsert_id, or caseId"
+        "Every workflow eval case requires id, upsert_id, or caseId"
       );
     }
     if (seen.has(caseId))
-      throw new Error(`Duplicate durable eval case id: ${caseId}`);
+      throw new Error(`Duplicate workflow eval case id: ${caseId}`);
     seen.add(caseId);
     const trialCount = datum.trialCount ?? evaluator.trialCount ?? 1;
     if (!Number.isInteger(trialCount) || trialCount < 1) {
-      throw new Error(`Invalid trialCount for durable eval case ${caseId}`);
+      throw new Error(`Invalid trialCount for workflow eval case ${caseId}`);
     }
     for (let trialIndex = 0; trialIndex < trialCount; trialIndex++) {
       records.push({
@@ -11224,7 +12148,7 @@ async function materializeCases(definition, data, experiment) {
   }
   return records;
 }
-function taskBatchItem(record, parameters) {
+function taskSubmissionItem(record, parameters) {
   const datum = record.datum;
   return {
     id: record.id,
@@ -11236,7 +12160,7 @@ function taskBatchItem(record, parameters) {
     trialIndex: record.trialIndex
   };
 }
-function scorerBatchItem(record) {
+function scorerSubmissionItem(record) {
   const datum = record.datum;
   return {
     id: record.id,
@@ -11307,12 +12231,12 @@ async function finishExperiment(definition, state, experiment) {
 }
 function resolveScorers(scorers) {
   return scorers.map((scorer, index) => ({
-    name: isBatchScorer(scorer) ? scorer.name : scorer.name || `scorer_${index}`,
+    name: isWorkflowScorer(scorer) ? scorer.name : scorer.name || `scorer_${index}`,
     scorer
   }));
 }
 function runKey(projectName, evalName, runId) {
-  return `durable-eval/v1/runs/${contentVersion(encoder.encode(`${projectName}\0${evalName}\0${runId}`))}`;
+  return `workflow-eval/v1/runs/${contentVersion(encoder.encode(`${projectName}\0${evalName}\0${runId}`))}`;
 }
 function encodedKeyPart(value) {
   return uint8ArrayToBase64(encoder.encode(value)).replaceAll("+", "-").replaceAll("/", "_").replace(/=+$/, "");
@@ -11321,8 +12245,8 @@ function caseRecordKey(key, caseId, kind, ...names) {
   const suffix = names.map(encodedKeyPart).join("/");
   return `${key}/cases/${encodedKeyPart(caseId)}/${kind}${suffix ? `/${suffix}` : ""}`;
 }
-function batchRecordKey(key, batchId) {
-  return `${key}/batches/${encodedKeyPart(batchId)}`;
+function submissionRecordKey(key, submissionId) {
+  return `${key}/submissions/${encodedKeyPart(submissionId)}`;
 }
 function claimRecordKey(key, kind, ...parts) {
   const identity = stableStringify([kind, parts]);
@@ -11334,33 +12258,24 @@ async function claimAction(store, key, kind, ...parts) {
     encoder.encode("claimed")
   )).created;
 }
-function plannedBatches(definition, runId, caseIds) {
+function plannedSubmissions(definition, runId, caseIds) {
   const stages = [];
-  if (isBatchTask(definition.evaluator.task)) stages.push({ kind: "task" });
+  if (isWorkflowTask(definition.evaluator.task)) stages.push({ kind: "task" });
   for (const { name, scorer } of resolveScorers(
     definition.evaluator.scores ?? []
   )) {
-    if (isBatchScorer(scorer)) {
+    if (isWorkflowScorer(scorer)) {
       stages.push({ kind: "score", scorerName: name });
     }
   }
   const plans = [];
   for (const { kind, scorerName: scorerName2 } of stages) {
-    const batchSize = processorForStage(definition, kind, scorerName2).batchSize ?? DEFAULT_BATCH_SIZE;
-    if (!Number.isInteger(batchSize) || batchSize < 1) {
-      throw new Error(
-        `Invalid batchSize for ${scorerName2 ?? "task"}: ${batchSize}`
-      );
-    }
-    for (let offset = 0; offset < caseIds.length; offset += batchSize) {
-      const itemIds = caseIds.slice(offset, offset + batchSize);
+    for (const itemId of caseIds) {
       plans.push({
-        id: deterministicId(
-          stableStringify([runId, kind, scorerName2, itemIds])
-        ),
+        id: deterministicId(stableStringify([runId, kind, scorerName2, itemId])),
         kind,
         scorerName: scorerName2,
-        itemIds
+        itemId
       });
     }
   }
@@ -11420,7 +12335,7 @@ async function readCaseRecord(definition, store, key, id) {
       }))
     )
   ]);
-  if (!base) throw new Error(`Durable eval case ${id} is missing`);
+  if (!base) throw new Error(`Workflow eval case ${id} is missing`);
   const scores = /* @__PURE__ */ Object.create(null);
   for (const { name, value } of scoreValues) {
     if (value !== void 0) scores[name] = value;
@@ -11451,32 +12366,34 @@ async function readCaseRecord(definition, store, key, id) {
     loggedClassifications
   };
 }
-async function readRunState(definition, store, key) {
+async function readRunState(definition, store, key, caseIds) {
   const record = await readJson(store, key);
   if (!record) return void 0;
-  const plans = plannedBatches(definition, record.runId, record.caseIds);
-  const [cases, batchRecords] = await Promise.all([
+  const selectedIds = caseIds ?? (record.caseCount === 0 ? [] : await readJson(store, `${key}/case-ids`));
+  if (!selectedIds)
+    throw new Error(`Workflow eval run ${record.runId} has no case index`);
+  const plans = plannedSubmissions(definition, record.runId, selectedIds);
+  const [cases, submissionRecords] = await Promise.all([
     Promise.all(
-      record.caseIds.map((id) => readCaseRecord(definition, store, key, id))
+      selectedIds.map((id) => readCaseRecord(definition, store, key, id))
     ),
     Promise.all(
       plans.map(async ({ id }) => {
-        return readJson(store, batchRecordKey(key, id));
+        return readJson(
+          store,
+          submissionRecordKey(key, id)
+        );
       })
     )
   ]);
-  const batches = batchRecords.filter(
+  const submissions = submissionRecords.filter(
     (value) => value !== void 0
   );
-  const { caseIds: _caseIds, ...state } = record;
-  return { ...state, cases, batches };
+  return { ...record, cases, submissions };
 }
 async function writeRunRecord(store, key, state) {
-  const { cases, batches: _batches, ...record } = state;
-  await writeJson(store, key, {
-    ...record,
-    caseIds: cases.map(({ id }) => id)
-  });
+  const { cases: _cases, submissions: _submissions, ...record } = state;
+  await writeJson(store, key, record);
 }
 async function writeCaseBaseRecords(store, key, records) {
   await Promise.all(
@@ -11548,11 +12465,25 @@ async function writeCaseRecords(store, key, records) {
   }
   await Promise.all(writes);
 }
-async function writeBatchRecords(store, key, records) {
+async function writeSubmissionRecords(store, key, records) {
   await Promise.all(
-    records.map(
-      (record) => writeJson(store, batchRecordKey(key, record.id), record)
-    )
+    records.map(async (record) => {
+      if (record.externalId !== void 0) {
+        const locator = await store.getOrSet(
+          `${key}/external/${encodedKeyPart(record.externalId)}`,
+          encoder.encode(JSON.stringify(record.id))
+        );
+        if (JSON.parse(decoder.decode(locator.value)) !== record.id) {
+          throw new Error(`Duplicate externalId: ${record.externalId}`);
+        }
+      }
+      const progressKey = `${key}/progress/${record.completionMode}`;
+      await store.addToSet(`${progressKey}/submitted`, record.id);
+      if (record.status === "complete") {
+        await store.addToSet(`${progressKey}/complete`, record.id);
+      }
+      await writeJson(store, submissionRecordKey(key, record.id), record);
+    })
   );
 }
 function deterministicId(value) {
@@ -11597,17 +12528,11 @@ function assertJsonValue(value, label) {
     throw new Error(`${label} must be JSON serializable`, { cause: error });
   }
 }
-function resultItemId(value) {
-  if (typeof value !== "object" || value === null || !("id" in value) || typeof value.id !== "string") {
-    throw new Error("Batch results must contain a string id");
-  }
-  return value.id;
-}
-function isBatchTask(value) {
-  return typeof value === "object" && value !== null && "kind" in value && value.kind === BATCH_TASK_KIND;
+function isWorkflowTask(value) {
+  return typeof value === "object" && value !== null && "kind" in value && value.kind === WORKFLOW_TASK_KIND;
 }
-function isBatchScorer(value) {
-  return typeof value === "object" && value !== null && "kind" in value && value.kind === BATCH_SCORER_KIND;
+function isWorkflowScorer(value) {
+  return typeof value === "object" && value !== null && "kind" in value && value.kind === WORKFLOW_SCORER_KIND;
 }
 function asError(error) {
   return error instanceof Error ? error : new Error(String(error));
@@ -12534,6 +13459,65 @@ var evaluatorDefinitionsSchema = z5.record(
   evaluatorDefinitionSchema
 );
 
+// src/wrappers/elevenlabs.ts
+var clients2 = /* @__PURE__ */ new WeakMap();
+function wrapElevenLabs(client) {
+  if (!isObject(client) || !isObject(client.textToSpeech) || typeof client.textToSpeech.convert !== "function") {
+    debugLogger.warn("Unsupported ElevenLabs client. Not wrapping.");
+    return client;
+  }
+  const cached = clients2.get(client);
+  if (cached) return cached;
+  const sdk = client;
+  const speech = new Proxy(sdk.textToSpeech, {
+    get(target, prop, receiver) {
+      switch (prop) {
+        case "convert":
+        case "stream":
+          return (...args) => elevenLabsChannels[prop].invoke(target[prop], target, args, {});
+        case "convertWithTimestamps":
+          return (...args) => elevenLabsChannels.convertWithTimestamps.invoke(
+            target.convertWithTimestamps,
+            target,
+            args,
+            {}
+          );
+        case "streamWithTimestamps":
+          return (...args) => elevenLabsChannels.streamWithTimestamps.invoke(
+            target.streamWithTimestamps,
+            target,
+            args,
+            {}
+          );
+        default:
+          return Reflect.get(target, prop, receiver);
+      }
+    }
+  });
+  const transcription = new Proxy(sdk.speechToText, {
+    get(target, prop, receiver) {
+      if (prop === "convert")
+        return (request, options) => elevenLabsChannels.transcribe.invoke(
+          target.convert,
+          target,
+          [request, options],
+          {}
+        );
+      return Reflect.get(target, prop, receiver);
+    }
+  });
+  const proxy = new Proxy(sdk, {
+    get(target, prop, receiver) {
+      if (prop === "textToSpeech") return speech;
+      if (prop === "speechToText") return transcription;
+      return Reflect.get(target, prop, receiver);
+    }
+  });
+  clients2.set(client, proxy);
+  clients2.set(proxy, proxy);
+  return proxy;
+}
+
 // src/node/index.ts
 configureNode();
 export {
@@ -12545,8 +13529,6 @@ export {
   BRAINTRUST_PARENT_KEY,
   BaseAttachment,
   BaseExperiment,
-  BatchScorer,
-  BatchTask,
   BraintrustLangChainCallbackHandler,
   BraintrustMiddleware,
   BraintrustObservabilityExporter,
@@ -12560,8 +13542,6 @@ export {
   DEFAULT_MAX_REQUEST_SIZE,
   Dataset,
   DatasetPipeline,
-  DurableEvalMemoryStore,
-  DurableEvalRedisStore,
   ERR_PERMALINK,
   Eval,
   EvalResultWithSummary,
@@ -12596,6 +13576,10 @@ export {
   TestBackgroundLogger,
   ToolBuilder,
   UUIDGenerator,
+  WorkflowEvalMemoryStore,
+  WorkflowEvalRedisStore,
+  WorkflowScorer,
+  WorkflowTask,
   X_CACHED_HEADER,
   _exportsForTestingOnly,
   _internalGetGlobalState,
@@ -12621,12 +13605,13 @@ export {
   deepCopyEvent,
   exports_exports as default,
   defaultErrorScoreHandler,
-  defineDurableEval,
+  defineWorkflowEval,
   deserializePlainStringAsJSON,
   devNullWritableStream,
   evaluatorDefinitionSchema,
   evaluatorDefinitionsSchema,
   extractTraceContextFromHeaders,
+  failOpenAIAgentsTrace,
   flush,
   getContextManager,
   getIdGenerator,
@@ -12673,12 +13658,14 @@ export {
   setFetch,
   setMaskingFunction,
   spanComponentsToObjectId,
+  startOpenAIAgentsTrace,
   startSpan,
   summarize,
   templateRegistry,
   ToolFunctionDefinition as toolFunctionDefinitionSchema,
   traceable,
   traced,
+  updateOpenAIAgentsTrace,
   updateSpan,
   uploadLogs3OverflowPayload,
   utf8ByteLength,
@@ -12699,12 +13686,15 @@ export {
   wrapCohere,
   wrapCopilotClient,
   wrapCursorSDK,
+  wrapElevenLabs,
   wrapGenkit,
   wrapGoogleADK,
   wrapGoogleGenAI,
+  wrapGoogleGenerativeAI,
   wrapGroq,
   wrapHuggingFace,
   wrapHuggingFaceTransformers,
+  wrapLangGraphSDK,
   wrapLangSmithClient,
   wrapLangSmithRunTrees,
   wrapLangSmithTraceable,