@wix/evalforge-evaluator 0.111.0 → 0.112.0

package/build/index.js

@@ -24,7 +24,7 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
 ));
 
 // src/index.ts
-var import_evalforge_types10 = require("@wix/evalforge-types");
+var import_evalforge_types11 = require("@wix/evalforge-types");
 
 // src/config.ts
 function loadConfig() {
@@ -509,7 +509,7 @@ async function fetchEvaluationData(api, projectId2, evalRunId2) {
 }
 
 // src/run-scenario/index.ts
-var import_evalforge_types8 = require("@wix/evalforge-types");
+var import_evalforge_types9 = require("@wix/evalforge-types");
 var import_eval_assertions = require("@wix/eval-assertions");
 
 // src/run-scenario/environment.ts
@@ -1261,7 +1261,10 @@ async function executeWithClaudeCode(skills, scenario, options) {
     "Edit",
     "Bash",
     "Glob",
-    "Grep"
+    "Grep",
+    "Agent",
+    "WebFetch",
+    "WebSearch"
   ];
   const allowedTools = (options.mcps?.length ?? 0) > 0 ? [...baseAllowedTools, "mcp__*"] : baseAllowedTools;
   const queryOptions = {
@@ -1896,13 +1899,15 @@ function extractTotalUsage(result) {
 }
 function buildLLMTraceFromSteps(steps, totalDurationMs, usage, model) {
   const totalCost = usage.costUsd ?? 0;
-  const totalStepInputTokens = steps.reduce(
-    (sum, s) => sum + s.usage.inputTokens,
+  const effectiveInput = (s) => s.usage.inputTokens + (s.usage.cacheReadTokens ?? 0) + (s.usage.cacheWriteTokens ?? 0);
+  const totalStepEffectiveInput = steps.reduce(
+    (sum, s) => sum + effectiveInput(s),
     0
   );
   const totalStepDuration = steps.reduce((sum, s) => sum + s.durationMs, 0);
-  const inputTokensDuplicated = usage.inputTokens > 0 && totalStepInputTokens > usage.inputTokens * 1.2;
-  const traceSteps = steps.map((step, index) => {
+  const authoritativeEffectiveInput = usage.inputTokens + (usage.cacheReadTokens ?? 0) + (usage.cacheWriteTokens ?? 0);
+  const inputTokensDuplicated = authoritativeEffectiveInput > 0 && totalStepEffectiveInput > authoritativeEffectiveInput * 1.2;
+  const traceSteps = steps.flatMap((step, turnIndex) => {
     let stepPromptTokens;
     let stepOutputTokens;
     let proportion;
@@ -1911,34 +1916,128 @@ function buildLLMTraceFromSteps(steps, totalDurationMs, usage, model) {
       stepPromptTokens = Math.round(usage.inputTokens * proportion);
       stepOutputTokens = Math.round(usage.outputTokens * proportion);
     } else {
-      proportion = totalStepInputTokens > 0 ? step.usage.inputTokens / totalStepInputTokens : 0;
-      stepPromptTokens = step.usage.inputTokens;
+      const stepEffective = effectiveInput(step);
+      proportion = totalStepEffectiveInput > 0 ? stepEffective / totalStepEffectiveInput : 0;
+      stepPromptTokens = Math.round(usage.inputTokens * proportion);
       stepOutputTokens = Math.round(usage.outputTokens * proportion);
     }
-    const stepTotalTokens = stepPromptTokens + stepOutputTokens;
     const costProportion = proportion;
-    const stepType = step.toolCalls?.length ? import_evalforge_types4.LLMStepType.TOOL_USE : step.thinking && !step.text ? import_evalforge_types4.LLMStepType.THINKING : import_evalforge_types4.LLMStepType.COMPLETION;
-    return {
-      id: (0, import_crypto.randomUUID)(),
-      stepNumber: index + 1,
-      type: stepType,
-      model,
-      provider: "anthropic",
-      startedAt: step.startedAt.toISOString(),
-      durationMs: step.durationMs,
-      tokenUsage: {
-        prompt: stepPromptTokens,
-        completion: stepOutputTokens,
-        total: stepTotalTokens
-      },
-      costUsd: totalCost * costProportion,
-      toolName: step.toolCalls?.[0]?.toolName,
-      toolArguments: step.toolCalls?.[0] ? JSON.stringify(step.toolCalls[0].args) : void 0,
-      outputPreview: (step.text || step.thinking)?.slice(0, 200),
-      success: step.finishReason !== "error" && !step.hasToolError,
-      error: step.finishReason === "error" ? "Generation failed" : step.hasToolError ? step.toolErrorContent ?? "Tool call failed" : void 0
-    };
-  });
+    const toolCallCount = step.toolCalls?.length ?? 0;
+    const isSuccess = step.finishReason !== "error" && !step.hasToolError;
+    const errorMsg = step.hasToolError ? step.toolErrorContent ?? "Tool call failed" : step.finishReason === "error" ? "Generation failed" : void 0;
+    const subSteps = [];
+    const stepCost = totalCost * costProportion;
+    const hasThinking = !!step.thinking;
+    const hasText = !!step.text;
+    const thinkingSubSteps = hasThinking ? 1 : 0;
+    const toolSubSteps = toolCallCount > 0 ? toolCallCount : 0;
+    const textSubSteps = hasText && toolCallCount > 0 ? 1 : 0;
+    const totalSubSteps = thinkingSubSteps + toolSubSteps + textSubSteps || 1;
+    if (hasThinking && (hasText || toolCallCount > 0)) {
+      subSteps.push({
+        id: (0, import_crypto.randomUUID)(),
+        stepNumber: 0,
+        // renumbered below
+        turnIndex,
+        type: import_evalforge_types4.LLMStepType.THINKING,
+        model,
+        provider: "anthropic",
+        startedAt: step.startedAt.toISOString(),
+        durationMs: Math.round(step.durationMs / totalSubSteps),
+        tokenUsage: {
+          prompt: Math.round(stepPromptTokens / totalSubSteps),
+          completion: Math.round(stepOutputTokens / totalSubSteps),
+          total: Math.round(
+            (stepPromptTokens + stepOutputTokens) / totalSubSteps
+          )
+        },
+        costUsd: stepCost / totalSubSteps,
+        outputPreview: step.thinking?.slice(0, 200),
+        success: isSuccess,
+        error: errorMsg
+      });
+    }
+    if (toolCallCount > 0) {
+      for (let tcIdx = 0; tcIdx < toolCallCount; tcIdx++) {
+        const tc = step.toolCalls[tcIdx];
+        const isLast = tcIdx === toolCallCount - 1 && textSubSteps === 0;
+        const toolBudgetSteps = toolSubSteps + textSubSteps;
+        const toolFraction = toolBudgetSteps > 0 ? 1 / toolBudgetSteps : 1;
+        const remainingFraction = (totalSubSteps - thinkingSubSteps) / totalSubSteps;
+        subSteps.push({
+          id: (0, import_crypto.randomUUID)(),
+          stepNumber: 0,
+          turnIndex,
+          type: import_evalforge_types4.LLMStepType.TOOL_USE,
+          model,
+          provider: "anthropic",
+          startedAt: step.startedAt.toISOString(),
+          durationMs: isLast ? step.durationMs - subSteps.reduce((s, ss) => s + ss.durationMs, 0) : Math.round(step.durationMs * remainingFraction * toolFraction),
+          tokenUsage: {
+            prompt: Math.round(
+              stepPromptTokens * remainingFraction * toolFraction
+            ),
+            completion: Math.round(
+              stepOutputTokens * remainingFraction * toolFraction
+            ),
+            total: Math.round(
+              (stepPromptTokens + stepOutputTokens) * remainingFraction * toolFraction
+            )
+          },
+          costUsd: stepCost * remainingFraction * toolFraction,
+          toolName: tc.toolName,
+          toolArguments: JSON.stringify(tc.args),
+          outputPreview: tcIdx === 0 && !hasText ? (step.text || step.thinking)?.slice(0, 200) : void 0,
+          success: isSuccess,
+          error: errorMsg
+        });
+      }
+    }
+    if (hasText && toolCallCount > 0) {
+      subSteps.push({
+        id: (0, import_crypto.randomUUID)(),
+        stepNumber: 0,
+        turnIndex,
+        type: import_evalforge_types4.LLMStepType.COMPLETION,
+        model,
+        provider: "anthropic",
+        startedAt: step.startedAt.toISOString(),
+        durationMs: step.durationMs - subSteps.reduce((s, ss) => s + ss.durationMs, 0),
+        tokenUsage: {
+          prompt: stepPromptTokens - subSteps.reduce((s, ss) => s + ss.tokenUsage.prompt, 0),
+          completion: stepOutputTokens - subSteps.reduce((s, ss) => s + ss.tokenUsage.completion, 0),
+          total: stepPromptTokens + stepOutputTokens - subSteps.reduce((s, ss) => s + ss.tokenUsage.total, 0)
+        },
+        costUsd: stepCost - subSteps.reduce((s, ss) => s + ss.costUsd, 0),
+        outputPreview: step.text?.slice(0, 200),
+        success: isSuccess,
+        error: errorMsg
+      });
+    }
+    if (subSteps.length === 0) {
+      const stepType = hasThinking && !hasText ? import_evalforge_types4.LLMStepType.THINKING : import_evalforge_types4.LLMStepType.COMPLETION;
+      subSteps.push({
+        id: (0, import_crypto.randomUUID)(),
+        stepNumber: 0,
+        turnIndex,
+        type: stepType,
+        model,
+        provider: "anthropic",
+        startedAt: step.startedAt.toISOString(),
+        durationMs: step.durationMs,
+        tokenUsage: {
+          prompt: stepPromptTokens,
+          completion: stepOutputTokens,
+          total: stepPromptTokens + stepOutputTokens
+        },
+        costUsd: stepCost,
+        outputPreview: (step.text || step.thinking)?.slice(0, 200),
+        success: isSuccess,
+        error: errorMsg
+      });
+    }
+    return subSteps;
+  }).map((s, i) => ({ ...s, stepNumber: i + 1 }));
   const finalTokens = {
     prompt: usage.inputTokens,
     completion: usage.outputTokens,
@@ -1960,6 +2059,7 @@ function buildLLMTraceFromSteps(steps, totalDurationMs, usage, model) {
   }
   const summary = {
     totalSteps: traceSteps.length,
+    totalTurns: steps.length,
     totalDurationMs,
     totalTokens: finalTokens,
     totalCostUsd: totalCost,
@@ -2049,7 +2149,7 @@ defaultRegistry.register(claudeCodeAdapter);
 var import_ai = require("ai");
 var import_anthropic = require("@ai-sdk/anthropic");
 var import_openai = require("@ai-sdk/openai");
-var import_evalforge_types6 = require("@wix/evalforge-types");
+var import_evalforge_types7 = require("@wix/evalforge-types");
 var import_crypto2 = require("crypto");
 
 // src/run-scenario/agents/simple-agent/mcp-tools.ts
@@ -2145,48 +2245,35 @@ function extractErrorText(content) {
 }
 
 // src/run-scenario/agents/simple-agent/cost-calculation.ts
+var import_evalforge_types6 = require("@wix/evalforge-types");
 var PROVIDER_ANTHROPIC = "anthropic";
 var MODEL_PRICING = {
+  // Anthropic — Claude 4.6
+  "claude-sonnet-4-6": { input: 3, output: 15 },
+  "claude-opus-4-6": { input: 15, output: 75 },
   // Anthropic — Claude 4.5
-  CLAUDE_4_5_OPUS_1_0: { input: 5, output: 25 },
-  CLAUDE_4_5_SONNET_1_0: { input: 3, output: 15 },
-  CLAUDE_4_5_HAIKU_1_0: { input: 1, output: 5 },
-  // Anthropic — Claude 4 / 4.1
-  CLAUDE_4_1_OPUS_1_0: { input: 15, output: 75 },
-  CLAUDE_4_OPUS_1_0: { input: 15, output: 75 },
-  CLAUDE_4_SONNET_1_0: { input: 3, output: 15 },
-  // Anthropic — Claude 3.x
-  CLAUDE_3_5_SONNET_2_0: { input: 3, output: 15 },
-  CLAUDE_3_5_SONNET_1_0: { input: 3, output: 15 },
-  CLAUDE_3_HAIKU_1_0: { input: 0.25, output: 1.25 },
+  "claude-opus-4-5": { input: 5, output: 25 },
+  "claude-sonnet-4-5": { input: 3, output: 15 },
+  "claude-haiku-4-5": { input: 1, output: 5 },
+  // Anthropic — Claude 4
+  "claude-opus-4": { input: 15, output: 75 },
+  "claude-sonnet-4": { input: 3, output: 15 },
   // OpenAI — GPT-5
-  GPT_5_2_2025_12_11: { input: 1.75, output: 14 },
-  GPT_5_2025_08_07: { input: 1.25, output: 10 },
-  GPT_5_MINI_2025_08_07: { input: 0.25, output: 2 },
-  GPT_5_NANO_2025_08_07: { input: 0.05, output: 0.4 },
+  "gpt-5": { input: 1.25, output: 10 },
+  "gpt-5-mini": { input: 0.25, output: 2 },
+  "gpt-5-nano": { input: 0.05, output: 0.4 },
   // OpenAI — GPT-4.1
-  GPT_4_1_2025_04_14: { input: 2, output: 8 },
-  GPT_4_1_MINI_2025_04_14: { input: 0.4, output: 1.6 },
-  GPT_4_1_NANO_2025_04_14: { input: 0.1, output: 0.4 },
+  "gpt-4.1": { input: 2, output: 8 },
+  "gpt-4.1-mini": { input: 0.4, output: 1.6 },
+  "gpt-4.1-nano": { input: 0.1, output: 0.4 },
   // OpenAI — GPT-4o
-  GPT_4O_2024_05_13: { input: 2.5, output: 10 },
-  GPT_4O_2024_08_06: { input: 2.5, output: 10 },
-  GPT_4O_2024_11_20: { input: 2.5, output: 10 },
-  GPT_4O_MINI_2024_07_18: { input: 0.15, output: 0.6 },
+  "gpt-4o": { input: 2.5, output: 10 },
+  "gpt-4o-mini": { input: 0.15, output: 0.6 },
   // OpenAI — Reasoning
-  O3_2025_04_16: { input: 2, output: 8 },
-  O4_MINI_2025_04_16: { input: 1.1, output: 4.4 },
-  O3_MINI_2025_01_31: { input: 1.1, output: 4.4 },
-  O1_2024_12_17: { input: 15, output: 60 },
-  O1_MINI: { input: 1.1, output: 4.4 },
-  O1_MINI_2024_09_12: { input: 1.1, output: 4.4 },
-  O1_PREVIEW: { input: 15, output: 60 },
-  O1_PREVIEW_2024_09_12: { input: 15, output: 60 },
-  // OpenAI — Legacy
-  GPT_4_TURBO_2024_04_09: { input: 10, output: 30 },
-  GPT_4_1106_PREVIEW: { input: 10, output: 30 },
-  GPT_3_5_TURBO: { input: 0.5, output: 1.5 },
-  GPT_3_5_TURBO_0125: { input: 0.5, output: 1.5 }
+  o3: { input: 2, output: 8 },
+  "o4-mini": { input: 1.1, output: 4.4 },
+  "o3-mini": { input: 1.1, output: 4.4 },
+  o1: { input: 15, output: 60 }
 };
 function extractGatewayCost(step, provider) {
   try {
@@ -2205,7 +2292,8 @@ function extractGatewayCost(step, provider) {
   }
 }
 function calculateFromPricing(modelId, tokenUsage) {
-  const pricing = MODEL_PRICING[modelId];
+  const normalized = (0, import_evalforge_types6.normalizeModelId)(modelId);
+  const pricing = MODEL_PRICING[normalized] ?? MODEL_PRICING[Object.keys(MODEL_PRICING).find((key) => normalized.startsWith(key)) ?? ""];
   if (!pricing) return 0;
   return tokenUsage.prompt / 1e6 * pricing.input + tokenUsage.completion / 1e6 * pricing.output;
 }
@@ -2280,9 +2368,7 @@ var PROVIDER_ANTHROPIC2 = "anthropic";
 var PROVIDER_OPENAI = "openai";
 var DEFAULT_MAX_TOOL_STEPS = 25;
 function createModel(modelId, baseUrl, headers) {
-  const isClaudeModel = import_evalforge_types6.AVAILABLE_CLAUDE_MODEL_IDS.includes(
-    modelId
-  );
+  const isClaudeModel = isClaudeModelId(modelId);
   if (isClaudeModel) {
     const anthropic = (0, import_anthropic.createAnthropic)({
       baseURL: `${baseUrl}/proxy/anthropic`,
@@ -2296,13 +2382,17 @@ function createModel(modelId, baseUrl, headers) {
     apiKey: "proxy-auth",
     headers
   });
-  if (import_evalforge_types6.OPENAI_RESPONSES_MODEL_IDS.has(modelId)) {
+  if ([...import_evalforge_types7.OPENAI_RESPONSES_MODEL_IDS].some(
+    (id) => modelId === id || modelId.startsWith(id)
+  )) {
     return openai.responses(modelId);
   }
   return openai.chat(modelId);
 }
 function isClaudeModelId(modelId) {
-  return import_evalforge_types6.AVAILABLE_CLAUDE_MODEL_IDS.includes(modelId);
+  return import_evalforge_types7.AVAILABLE_CLAUDE_MODEL_IDS.some(
+    (id) => modelId === id || modelId.startsWith(id)
+  );
 }
 function extractSkillContent(files) {
   if (!files || files.length === 0) return void 0;
@@ -2336,7 +2426,9 @@ async function executeWithAiSdk(context) {
   }
   try {
     const isAnthropic = provider === PROVIDER_ANTHROPIC2;
-    const isResponsesAPI = import_evalforge_types6.OPENAI_RESPONSES_MODEL_IDS.has(modelConfig.model);
+    const isResponsesAPI = [...import_evalforge_types7.OPENAI_RESPONSES_MODEL_IDS].some(
+      (id) => modelConfig.model === id || modelConfig.model.startsWith(id)
+    );
     const supportsThinking = isAnthropic || isResponsesAPI;
     const providerOpts = {
       ...isAnthropic && {
@@ -2446,7 +2538,8 @@ function buildLLMTrace(steps, totalDurationMs, totalUsage, modelId, provider, ex
     return {
       id: (0, import_crypto2.randomUUID)(),
       stepNumber: i + 1,
-      type: step.toolCalls.length > 0 ? import_evalforge_types6.LLMStepType.TOOL_USE : import_evalforge_types6.LLMStepType.COMPLETION,
+      turnIndex: i,
+      type: step.toolCalls.length > 0 ? import_evalforge_types7.LLMStepType.TOOL_USE : import_evalforge_types7.LLMStepType.COMPLETION,
       model: modelId,
       provider,
       startedAt: new Date(
@@ -2473,6 +2566,7 @@ function buildLLMTrace(steps, totalDurationMs, totalUsage, modelId, provider, ex
     steps: traceSteps,
     summary: {
       totalSteps: traceSteps.length,
+      totalTurns: traceSteps.length,
       totalDurationMs,
       totalTokens: finalTokens,
       totalCostUsd,
@@ -2497,7 +2591,7 @@ function emitStartEvent(traceContext, startTime) {
       targetId: traceContext.targetId,
       targetName: traceContext.targetName,
       stepNumber: 0,
-      type: import_evalforge_types6.LiveTraceEventType.PROGRESS,
+      type: import_evalforge_types7.LiveTraceEventType.PROGRESS,
       outputPreview: "Starting Simple Agent execution...",
       elapsedMs: Date.now() - startTime,
       timestamp: (/* @__PURE__ */ new Date()).toISOString(),
@@ -2521,7 +2615,7 @@ function emitStepEvents(traceContext, steps, startTime) {
         targetId: traceContext.targetId,
         targetName: traceContext.targetName,
         stepNumber: i + 1,
-        type: isToolStep ? import_evalforge_types6.LiveTraceEventType.TOOL_USE : import_evalforge_types6.LiveTraceEventType.COMPLETION,
+        type: isToolStep ? import_evalforge_types7.LiveTraceEventType.TOOL_USE : import_evalforge_types7.LiveTraceEventType.COMPLETION,
         toolName: firstToolCall?.toolName,
         toolArgs: firstToolCall ? (JSON.stringify(firstToolCall.input) ?? "").slice(0, 500) : void 0,
         outputPreview: step.text?.slice(0, 500),
@@ -2544,7 +2638,7 @@ function emitCompletionEvent(traceContext, stepNumber) {
       targetId: traceContext.targetId,
       targetName: traceContext.targetName,
       stepNumber,
-      type: import_evalforge_types6.LiveTraceEventType.COMPLETION,
+      type: import_evalforge_types7.LiveTraceEventType.COMPLETION,
       outputPreview: "Scenario execution completed",
       timestamp: (/* @__PURE__ */ new Date()).toISOString(),
       isComplete: true
@@ -3296,17 +3390,11 @@ function extractTemplateFiles(before, after) {
 }
 
 // src/run-scenario/run-agent-with-context.ts
-var import_evalforge_types7 = require("@wix/evalforge-types");
-var DEFAULT_AGENT_COMMAND = import_evalforge_types7.AgentRunCommand.CLAUDE;
+var import_evalforge_types8 = require("@wix/evalforge-types");
+var DEFAULT_AGENT_COMMAND = import_evalforge_types8.AgentRunCommand.CLAUDE;
 async function runAgentWithContext(config, evalRunId2, scenario, evalData, workDir) {
-  const hasEntities = evalData.skills.length > 0 || evalData.mcps.length > 0 || evalData.subAgents.length > 0 || (evalData.rules?.length ?? 0) > 0;
-  if (!hasEntities) {
-    throw new Error(
-      `Eval run ${evalRunId2} has no entities configured: at least one skill, MCP, sub-agent, or rule is required.`
-    );
-  }
   const agent = evalData.agent ?? void 0;
-  const isSDK = agent?.agentType === import_evalforge_types7.AgentType.SDK;
+  const isSDK = agent?.agentType === import_evalforge_types8.AgentType.SDK;
   const identifier = isSDK ? simpleAgentAdapter.id : agent?.runCommand ?? DEFAULT_AGENT_COMMAND;
   const adapter = getAdapter(identifier);
   const startedAt = (/* @__PURE__ */ new Date()).toISOString();
@@ -3392,7 +3480,7 @@ async function runScenario(config, evalRunId2, scenario, evalData, template, res
     })),
     durationMs: partialResult.duration
   };
-  const defaultJudgeModel = import_evalforge_types8.DEFAULT_JUDGE_MODEL;
+  const defaultJudgeModel = import_evalforge_types9.DEFAULT_JUDGE_MODEL;
   const assertionContext = {
     workDir,
     defaultJudgeModel,
@@ -3407,10 +3495,10 @@ async function runScenario(config, evalRunId2, scenario, evalData, template, res
     assertionContext
   ) : [];
   const passed = assertionResults.filter(
-    (r) => r.status === import_evalforge_types8.AssertionResultStatus.PASSED
+    (r) => r.status === import_evalforge_types9.AssertionResultStatus.PASSED
   ).length;
   const failed = assertionResults.filter(
-    (r) => r.status === import_evalforge_types8.AssertionResultStatus.FAILED
+    (r) => r.status === import_evalforge_types9.AssertionResultStatus.FAILED
   ).length;
   const total = assertionResults.length;
   const passRate = total > 0 ? Math.round(passed / total * 100) : 100;
@@ -3424,7 +3512,7 @@ async function runScenario(config, evalRunId2, scenario, evalData, template, res
 }
 
 // src/error-reporter.ts
-var import_evalforge_types9 = require("@wix/evalforge-types");
+var import_evalforge_types10 = require("@wix/evalforge-types");
 function formatError(error, phase, context) {
   const timestamp = (/* @__PURE__ */ new Date()).toISOString();
   if (error instanceof Error) {
@@ -3598,13 +3686,7 @@ async function runEvaluation(projectId2, evalRunId2) {
     presetId: evalData.evalRun.presetId,
     skillIds: evalData.evalRun.skillIds
   };
-  const hasEntities = skills.length > 0 || evalData.mcps.length > 0 || evalData.subAgents.length > 0 || evalData.rules.length > 0;
-  if (scenarioItems.length > 0 && !hasEntities) {
-    throw new Error(
-      `[${ExecutionPhase.VALIDATION}] Eval run has no entities configured: at least one skill, MCP, sub-agent, or rule is required.`
-    );
-  }
-  if (scenarioItems.length > 0 && hasEntities && !agent) {
+  if (scenarioItems.length > 0 && !agent) {
     throw new Error(
       `[${ExecutionPhase.VALIDATION}] Eval run has no agent: set agentId for evaluation runs. (agentId: ${evalData.evalRun.agentId || "not set"})`
     );
@@ -3675,7 +3757,7 @@ async function runEvaluation(projectId2, evalRunId2) {
   };
   try {
     await api.updateEvalRun(projectId2, evalRunId2, {
-      status: import_evalforge_types10.EvalStatus.COMPLETED,
+      status: import_evalforge_types11.EvalStatus.COMPLETED,
       completedAt: (/* @__PURE__ */ new Date()).toISOString()
     });
   } catch (updateErr) {
@@ -3716,7 +3798,7 @@ runEvaluation(projectId, evalRunId).then(() => {
       authToken: config.authToken
     });
     await api.updateEvalRun(projectId, evalRunId, {
-      status: import_evalforge_types10.EvalStatus.FAILED,
+      status: import_evalforge_types11.EvalStatus.FAILED,
       completedAt: (/* @__PURE__ */ new Date()).toISOString(),
       jobError,
       jobStatus: "FAILED"
@@ -3739,7 +3821,7 @@ runEvaluation(projectId, evalRunId).then(() => {
           authToken
         });
         await api.updateEvalRun(projectId, evalRunId, {
-          status: import_evalforge_types10.EvalStatus.FAILED,
+          status: import_evalforge_types11.EvalStatus.FAILED,
           completedAt: (/* @__PURE__ */ new Date()).toISOString(),
           jobError: `Config load failed, then: ${jobError}`,
           jobStatus: "FAILED"