@wix/evalforge-evaluator 0.85.0 → 0.86.0

package/build/index.js

@@ -24,7 +24,7 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
 ));
 
 // src/index.ts
-var import_evalforge_types7 = require("@wix/evalforge-types");
+var import_evalforge_types6 = require("@wix/evalforge-types");
 
 // src/config.ts
 function loadConfig() {
@@ -399,7 +399,7 @@ async function fetchEvaluationData(api, projectId2, evalRunId2) {
 }
 
 // src/run-scenario/index.ts
-var import_evalforge_types5 = require("@wix/evalforge-types");
+var import_evalforge_types4 = require("@wix/evalforge-types");
 var import_eval_assertions = require("@wix/eval-assertions");
 
 // src/run-scenario/environment.ts
@@ -636,9 +636,6 @@ function getAdapter(runCommand) {
   return adapter;
 }
 
-// src/run-scenario/agents/claude-code/claude-code-adapter.ts
-var import_evalforge_types4 = require("@wix/evalforge-types");
-
 // src/run-scenario/agents/claude-code/execute.ts
 var import_evalforge_types3 = require("@wix/evalforge-types");
 var import_crypto = require("crypto");
@@ -695,21 +692,7 @@ async function writeSubAgentsToFilesystem(cwd, subAgents) {
 }
 
 // src/run-scenario/agents/claude-code/execute.ts
-var DEFAULT_MODEL = "claude-3-5-sonnet-latest";
-function calculateStepCost(inputTokens, outputTokens, modelName) {
-  const model = import_evalforge_types3.AVAILABLE_MODELS.find(
-    (m) => m.name === modelName || m.providerModelId === modelName || // Handle model aliases like "claude-3-5-sonnet-latest" -> "claude-3-5-sonnet-20241022"
-    modelName.includes("claude-3-5-sonnet") ? m.providerModelId.includes("claude-3-5-sonnet") : modelName.includes("claude-4-sonnet") ? m.providerModelId.includes("claude-4-sonnet") : modelName.includes("claude-4-opus") ? m.providerModelId.includes("claude-4-opus") : false
-  );
-  if (!model) {
-    const inputCost2 = inputTokens / 1e6 * 3;
-    const outputCost2 = outputTokens / 1e6 * 15;
-    return inputCost2 + outputCost2;
-  }
-  const inputCost = inputTokens / 1e6 * model.pricing.inputPer1M;
-  const outputCost = outputTokens / 1e6 * model.pricing.outputPer1M;
-  return inputCost + outputCost;
-}
+var DEFAULT_MODEL = import_evalforge_types3.ClaudeModel.CLAUDE_4_5_SONNET_1_0;
 function emitTraceEvent(event, tracePushUrl, routeHeader, authToken) {
   console.log(`${import_evalforge_types3.TRACE_EVENT_PREFIX}${JSON.stringify(event)}`);
   if (tracePushUrl) {
@@ -922,6 +905,13 @@ async function executeWithClaudeCode(skills, scenario, options) {
   }
   const startTime = /* @__PURE__ */ new Date();
   const allMessages = [];
+  const { mkdir: mkdirAsync, writeFile: writeFile4 } = await import("fs/promises");
+  const claudeDir = `${options.cwd}/.claude`;
+  await mkdirAsync(claudeDir, { recursive: true });
+  await writeFile4(`${claudeDir}/settings.json`, "{}", {
+    flag: "wx"
+  }).catch(() => {
+  });
   if (options.mcps && options.mcps.length > 0) {
     await writeMcpToFilesystem(options.cwd, options.mcps);
   }
@@ -1448,7 +1438,14 @@ Stack: ${errorStack.split("\n").slice(0, 5).join("\n")}` : "")
     startTime,
     endTime
   );
-  const outputText = extractFinalOutput(allMessages);
+  const rawOutput = extractFinalOutput(allMessages);
+  const isError = sdkResult?.subtype !== "success";
+  let outputText = rawOutput;
+  if (!rawOutput && isError) {
+    const hasErrors = sdkResult && "errors" in sdkResult && sdkResult.errors?.length;
+    const errorDetails = hasErrors ? sdkResult.errors.join("; ") : sdkResult?.subtype ?? "unknown";
+    outputText = `[ERROR] Agent execution failed: ${errorDetails}`;
+  }
   const usage = extractTotalUsage(sdkResult);
   const llmTrace = buildLLMTraceFromSteps(
     steps,
@@ -1590,12 +1587,13 @@ function extractTotalUsage(result) {
   };
 }
 function buildLLMTraceFromSteps(steps, totalDurationMs, usage, model) {
+  const totalCost = usage.costUsd ?? 0;
+  const totalStepTokens = steps.reduce(
+    (sum, s) => sum + s.usage.totalTokens,
+    0
+  );
   const traceSteps = steps.map((step, index) => {
-    const stepCost = calculateStepCost(
-      step.usage.inputTokens,
-      step.usage.outputTokens,
-      model
-    );
+    const proportion = totalStepTokens > 0 ? step.usage.totalTokens / totalStepTokens : 0;
     return {
       id: (0, import_crypto.randomUUID)(),
       stepNumber: index + 1,
@@ -1609,7 +1607,7 @@ function buildLLMTraceFromSteps(steps, totalDurationMs, usage, model) {
         completion: step.usage.outputTokens,
         total: step.usage.totalTokens
       },
-      costUsd: stepCost,
+      costUsd: totalCost * proportion,
       toolName: step.toolCalls?.[0]?.toolName,
       toolArguments: step.toolCalls?.[0] ? JSON.stringify(step.toolCalls[0].args) : void 0,
       outputPreview: step.text?.slice(0, 200),
@@ -1622,13 +1620,12 @@ function buildLLMTraceFromSteps(steps, totalDurationMs, usage, model) {
     completion: traceSteps.reduce((sum, s) => sum + s.tokenUsage.completion, 0),
     total: traceSteps.reduce((sum, s) => sum + s.tokenUsage.total, 0)
   };
-  const stepsTotalCost = traceSteps.reduce((sum, s) => sum + s.costUsd, 0);
   const finalTokens = {
     prompt: usage.inputTokens > 0 ? usage.inputTokens : stepsTokens.prompt,
     completion: usage.outputTokens > 0 ? usage.outputTokens : stepsTokens.completion,
     total: usage.totalTokens > 0 ? usage.totalTokens : stepsTokens.total
   };
-  const finalCost = usage.costUsd !== void 0 && usage.costUsd > 0 ? usage.costUsd : stepsTotalCost;
+  const finalCost = totalCost;
   const summary = {
     totalSteps: traceSteps.length,
     totalDurationMs,
@@ -1674,7 +1671,7 @@ var ClaudeCodeAdapter = class {
       mcps,
       subAgents
     } = context;
-    const modelForSdk = modelConfig?.model ? import_evalforge_types4.AVAILABLE_MODELS_MAP[modelConfig.model]?.providerModelId ?? modelConfig.model : void 0;
+    const modelForSdk = modelConfig?.model;
     const options = {
       cwd,
       model: modelForSdk,
@@ -2523,8 +2520,10 @@ async function runScenario(config, evalRunId2, scenario, evalData, template, res
     }))
   };
   const { "x-wix-ai-gateway-stream": _stream, ...judgeHeaders } = config.aiGatewayHeaders;
+  const defaultJudgeModel = import_evalforge_types4.AVAILABLE_MODEL_IDS[0];
   const assertionContext = {
     workDir,
+    defaultJudgeModel,
     llmConfig: {
       baseUrl: config.aiGatewayUrl,
       headers: judgeHeaders
@@ -2536,10 +2535,10 @@ async function runScenario(config, evalRunId2, scenario, evalData, template, res
     assertionContext
   ) : [];
   const passed = assertionResults.filter(
-    (r) => r.status === import_evalforge_types5.AssertionResultStatus.PASSED
+    (r) => r.status === import_evalforge_types4.AssertionResultStatus.PASSED
   ).length;
   const failed = assertionResults.filter(
-    (r) => r.status === import_evalforge_types5.AssertionResultStatus.FAILED
+    (r) => r.status === import_evalforge_types4.AssertionResultStatus.FAILED
   ).length;
   const total = assertionResults.length;
   const passRate = total > 0 ? Math.round(passed / total * 100) : 100;
@@ -2553,7 +2552,7 @@ async function runScenario(config, evalRunId2, scenario, evalData, template, res
 }
 
 // src/error-reporter.ts
-var import_evalforge_types6 = require("@wix/evalforge-types");
+var import_evalforge_types5 = require("@wix/evalforge-types");
 function formatError(error, phase, context) {
   const timestamp = (/* @__PURE__ */ new Date()).toISOString();
   if (error instanceof Error) {
@@ -2802,7 +2801,7 @@ async function runEvaluation(projectId2, evalRunId2) {
   };
   try {
     await api.updateEvalRun(projectId2, evalRunId2, {
-      status: import_evalforge_types7.EvalStatus.COMPLETED,
+      status: import_evalforge_types6.EvalStatus.COMPLETED,
       completedAt: (/* @__PURE__ */ new Date()).toISOString()
     });
   } catch (updateErr) {
@@ -2843,7 +2842,7 @@ runEvaluation(projectId, evalRunId).then(() => {
       authToken: config.authToken
     });
     await api.updateEvalRun(projectId, evalRunId, {
-      status: import_evalforge_types7.EvalStatus.FAILED,
+      status: import_evalforge_types6.EvalStatus.FAILED,
       completedAt: (/* @__PURE__ */ new Date()).toISOString(),
       jobError,
       jobStatus: "FAILED"
@@ -2866,7 +2865,7 @@ runEvaluation(projectId, evalRunId).then(() => {
           authToken
         });
         await api.updateEvalRun(projectId, evalRunId, {
-          status: import_evalforge_types7.EvalStatus.FAILED,
+          status: import_evalforge_types6.EvalStatus.FAILED,
           completedAt: (/* @__PURE__ */ new Date()).toISOString(),
           jobError: `Config load failed, then: ${jobError}`,
           jobStatus: "FAILED"