@wix/evalforge-evaluator 0.85.0 → 0.86.0

package/build/index.mjs

@@ -379,7 +379,10 @@ async function fetchEvaluationData(api, projectId2, evalRunId2) {
 }
 
 // src/run-scenario/index.ts
-import { AssertionResultStatus } from "@wix/evalforge-types";
+import {
+  AssertionResultStatus,
+  AVAILABLE_MODEL_IDS
+} from "@wix/evalforge-types";
 import {
   evaluateAssertions as evaluateAssertionsBase
 } from "@wix/eval-assertions";
@@ -618,15 +621,12 @@ function getAdapter(runCommand) {
   return adapter;
 }
 
-// src/run-scenario/agents/claude-code/claude-code-adapter.ts
-import { AVAILABLE_MODELS_MAP } from "@wix/evalforge-types";
-
 // src/run-scenario/agents/claude-code/execute.ts
 import {
+  ClaudeModel,
   LLMStepType,
   LiveTraceEventType,
-  TRACE_EVENT_PREFIX,
-  AVAILABLE_MODELS
+  TRACE_EVENT_PREFIX
 } from "@wix/evalforge-types";
 import { randomUUID } from "crypto";
 
@@ -682,21 +682,7 @@ async function writeSubAgentsToFilesystem(cwd, subAgents) {
 }
 
 // src/run-scenario/agents/claude-code/execute.ts
-var DEFAULT_MODEL = "claude-3-5-sonnet-latest";
-function calculateStepCost(inputTokens, outputTokens, modelName) {
-  const model = AVAILABLE_MODELS.find(
-    (m) => m.name === modelName || m.providerModelId === modelName || // Handle model aliases like "claude-3-5-sonnet-latest" -> "claude-3-5-sonnet-20241022"
-    modelName.includes("claude-3-5-sonnet") ? m.providerModelId.includes("claude-3-5-sonnet") : modelName.includes("claude-4-sonnet") ? m.providerModelId.includes("claude-4-sonnet") : modelName.includes("claude-4-opus") ? m.providerModelId.includes("claude-4-opus") : false
-  );
-  if (!model) {
-    const inputCost2 = inputTokens / 1e6 * 3;
-    const outputCost2 = outputTokens / 1e6 * 15;
-    return inputCost2 + outputCost2;
-  }
-  const inputCost = inputTokens / 1e6 * model.pricing.inputPer1M;
-  const outputCost = outputTokens / 1e6 * model.pricing.outputPer1M;
-  return inputCost + outputCost;
-}
+var DEFAULT_MODEL = ClaudeModel.CLAUDE_4_5_SONNET_1_0;
 function emitTraceEvent(event, tracePushUrl, routeHeader, authToken) {
   console.log(`${TRACE_EVENT_PREFIX}${JSON.stringify(event)}`);
   if (tracePushUrl) {
@@ -909,6 +895,13 @@ async function executeWithClaudeCode(skills, scenario, options) {
   }
   const startTime = /* @__PURE__ */ new Date();
   const allMessages = [];
+  const { mkdir: mkdirAsync, writeFile: writeFile4 } = await import("fs/promises");
+  const claudeDir = `${options.cwd}/.claude`;
+  await mkdirAsync(claudeDir, { recursive: true });
+  await writeFile4(`${claudeDir}/settings.json`, "{}", {
+    flag: "wx"
+  }).catch(() => {
+  });
   if (options.mcps && options.mcps.length > 0) {
     await writeMcpToFilesystem(options.cwd, options.mcps);
   }
@@ -1435,7 +1428,14 @@ Stack: ${errorStack.split("\n").slice(0, 5).join("\n")}` : "")
     startTime,
     endTime
   );
-  const outputText = extractFinalOutput(allMessages);
+  const rawOutput = extractFinalOutput(allMessages);
+  const isError = sdkResult?.subtype !== "success";
+  let outputText = rawOutput;
+  if (!rawOutput && isError) {
+    const hasErrors = sdkResult && "errors" in sdkResult && sdkResult.errors?.length;
+    const errorDetails = hasErrors ? sdkResult.errors.join("; ") : sdkResult?.subtype ?? "unknown";
+    outputText = `[ERROR] Agent execution failed: ${errorDetails}`;
+  }
   const usage = extractTotalUsage(sdkResult);
   const llmTrace = buildLLMTraceFromSteps(
     steps,
@@ -1577,12 +1577,13 @@ function extractTotalUsage(result) {
   };
 }
 function buildLLMTraceFromSteps(steps, totalDurationMs, usage, model) {
+  const totalCost = usage.costUsd ?? 0;
+  const totalStepTokens = steps.reduce(
+    (sum, s) => sum + s.usage.totalTokens,
+    0
+  );
   const traceSteps = steps.map((step, index) => {
-    const stepCost = calculateStepCost(
-      step.usage.inputTokens,
-      step.usage.outputTokens,
-      model
-    );
+    const proportion = totalStepTokens > 0 ? step.usage.totalTokens / totalStepTokens : 0;
     return {
       id: randomUUID(),
       stepNumber: index + 1,
@@ -1596,7 +1597,7 @@ function buildLLMTraceFromSteps(steps, totalDurationMs, usage, model) {
         completion: step.usage.outputTokens,
         total: step.usage.totalTokens
       },
-      costUsd: stepCost,
+      costUsd: totalCost * proportion,
       toolName: step.toolCalls?.[0]?.toolName,
       toolArguments: step.toolCalls?.[0] ? JSON.stringify(step.toolCalls[0].args) : void 0,
       outputPreview: step.text?.slice(0, 200),
@@ -1609,13 +1610,12 @@ function buildLLMTraceFromSteps(steps, totalDurationMs, usage, model) {
     completion: traceSteps.reduce((sum, s) => sum + s.tokenUsage.completion, 0),
     total: traceSteps.reduce((sum, s) => sum + s.tokenUsage.total, 0)
   };
-  const stepsTotalCost = traceSteps.reduce((sum, s) => sum + s.costUsd, 0);
   const finalTokens = {
     prompt: usage.inputTokens > 0 ? usage.inputTokens : stepsTokens.prompt,
     completion: usage.outputTokens > 0 ? usage.outputTokens : stepsTokens.completion,
     total: usage.totalTokens > 0 ? usage.totalTokens : stepsTokens.total
   };
-  const finalCost = usage.costUsd !== void 0 && usage.costUsd > 0 ? usage.costUsd : stepsTotalCost;
+  const finalCost = totalCost;
   const summary = {
     totalSteps: traceSteps.length,
     totalDurationMs,
@@ -1661,7 +1661,7 @@ var ClaudeCodeAdapter = class {
       mcps,
       subAgents
     } = context;
-    const modelForSdk = modelConfig?.model ? AVAILABLE_MODELS_MAP[modelConfig.model]?.providerModelId ?? modelConfig.model : void 0;
+    const modelForSdk = modelConfig?.model;
     const options = {
       cwd,
       model: modelForSdk,
@@ -2510,8 +2510,10 @@ async function runScenario(config, evalRunId2, scenario, evalData, template, res
     }))
   };
   const { "x-wix-ai-gateway-stream": _stream, ...judgeHeaders } = config.aiGatewayHeaders;
+  const defaultJudgeModel = AVAILABLE_MODEL_IDS[0];
   const assertionContext = {
     workDir,
+    defaultJudgeModel,
     llmConfig: {
       baseUrl: config.aiGatewayUrl,
       headers: judgeHeaders