@wix/evalforge-evaluator 0.32.0 → 0.34.0

package/build/index.js

@@ -6591,6 +6591,20 @@ var import_crypto = require("crypto");
 var import_promises3 = require("fs/promises");
 var import_path5 = require("path");
 var DEFAULT_MODEL = "claude-3-5-sonnet-latest";
+function calculateStepCost(inputTokens, outputTokens, modelName) {
+  const model = import_evalforge_types.AVAILABLE_MODELS.find(
+    (m) => m.name === modelName || m.providerModelId === modelName || // Handle model aliases like "claude-3-5-sonnet-latest" -> "claude-3-5-sonnet-20241022"
+    modelName.includes("claude-3-5-sonnet") ? m.providerModelId.includes("claude-3-5-sonnet") : modelName.includes("claude-4-sonnet") ? m.providerModelId.includes("claude-4-sonnet") : modelName.includes("claude-4-opus") ? m.providerModelId.includes("claude-4-opus") : false
+  );
+  if (!model) {
+    const inputCost2 = inputTokens / 1e6 * 3;
+    const outputCost2 = outputTokens / 1e6 * 15;
+    return inputCost2 + outputCost2;
+  }
+  const inputCost = inputTokens / 1e6 * model.pricing.inputPer1M;
+  const outputCost = outputTokens / 1e6 * model.pricing.outputPer1M;
+  return inputCost + outputCost;
+}
 function emitTraceEvent(event, tracePushUrl, routeHeader, authToken) {
   console.log(`${import_evalforge_types.TRACE_EVENT_PREFIX}${JSON.stringify(event)}`);
   if (tracePushUrl) {
@@ -7022,15 +7036,22 @@ async function executeWithClaudeCode(skill, scenario, options) {
     const sdkPromise = (async () => {
       const evaluatorPromptSuffix = `
 
-IMPORTANT: This is an automated evaluation run. Execute the requested changes immediately without asking for confirmation. Do not ask "would you like me to proceed?" or similar questions - just implement the solution directly.`;
+IMPORTANT: This is an automated evaluation run. Follow these guidelines:
+1. Execute the requested changes immediately without asking for confirmation.
+2. Do NOT ask "would you like me to proceed?" or similar questions.
+3. Do NOT use the Task tool to delegate simple operations - do them directly yourself.
+4. Keep your approach simple and direct - avoid excessive planning.
+5. Make targeted edits using Read and Edit tools rather than exploring the entire codebase.
+6. If you encounter an error, fix it directly rather than starting over.`;
       const fullPrompt = scenario.triggerPrompt + evaluatorPromptSuffix;
       for await (const message of query({
         prompt: fullPrompt,
         options: queryOptions
       })) {
         messageCount++;
+        const receivedAt = /* @__PURE__ */ new Date();
         console.log("[SDK Message]", JSON.stringify(message, null, 2));
-        allMessages.push(message);
+        allMessages.push({ message, receivedAt });
         if (messageCount <= 3) {
           console.error(
             "[DEBUG-H5] SDK message received",
@@ -7297,7 +7318,11 @@ Stack: ${errorStack.split("\n").slice(0, 5).join("\n")}` : "")
   }
   const endTime = /* @__PURE__ */ new Date();
   const totalDurationMs = endTime.getTime() - startTime.getTime();
-  const { steps, result: sdkResult } = processMessages(allMessages, startTime);
+  const { steps, result: sdkResult } = processMessages(
+    allMessages,
+    startTime,
+    endTime
+  );
   const outputText = extractFinalOutput(allMessages);
   const usage = extractTotalUsage(sdkResult);
   const llmTrace = buildLLMTraceFromSteps(
@@ -7348,25 +7373,36 @@ function isAssistantMessage(message) {
 function isResultMessage(message) {
   return message.type === "result";
 }
-function processMessages(messages, startTime) {
+function processMessages(timestampedMessages, startTime, endTime) {
   const steps = [];
   let result;
-  let stepIndex = 0;
-  const assistantMessages = /* @__PURE__ */ new Map();
-  for (const message of messages) {
+  const assistantMessageGroups = /* @__PURE__ */ new Map();
+  for (const { message, receivedAt } of timestampedMessages) {
     if (isAssistantMessage(message)) {
       const uuid3 = message.uuid;
-      if (!assistantMessages.has(uuid3)) {
-        assistantMessages.set(uuid3, []);
+      if (!assistantMessageGroups.has(uuid3)) {
+        assistantMessageGroups.set(uuid3, {
+          messages: [],
+          firstReceivedAt: receivedAt,
+          lastReceivedAt: receivedAt
+        });
       }
-      assistantMessages.get(uuid3).push(message);
+      const group = assistantMessageGroups.get(uuid3);
+      group.messages.push(message);
+      group.lastReceivedAt = receivedAt;
     } else if (isResultMessage(message)) {
       result = message;
     }
   }
-  for (const groupedMessages of assistantMessages.values()) {
-    const lastMessage = groupedMessages[groupedMessages.length - 1];
-    const stepStartTime = new Date(startTime.getTime() + stepIndex * 100);
+  const sortedGroups = Array.from(assistantMessageGroups.values()).sort(
+    (a, b) => a.firstReceivedAt.getTime() - b.firstReceivedAt.getTime()
+  );
+  for (let i = 0; i < sortedGroups.length; i++) {
+    const group = sortedGroups[i];
+    const lastMessage = group.messages[group.messages.length - 1];
+    const stepStartTime = group.firstReceivedAt;
+    const nextStepStartTime = i < sortedGroups.length - 1 ? sortedGroups[i + 1].firstReceivedAt : endTime;
+    const durationMs = nextStepStartTime.getTime() - stepStartTime.getTime();
     const usage = lastMessage.message.usage;
     const inputTokens = usage.input_tokens;
     const outputTokens = usage.output_tokens;
@@ -7392,17 +7428,9 @@ function processMessages(messages, startTime) {
       finishReason: mapStopReason(lastMessage.message.stop_reason),
       toolCalls: toolCalls.length > 0 ? toolCalls : void 0,
       startedAt: stepStartTime,
-      durationMs: 0
-      // Will be updated from result
+      durationMs: Math.max(0, durationMs)
+      // Ensure non-negative
     });
-    stepIndex++;
-  }
-  if (result && steps.length > 0) {
-    const totalDuration = result.duration_ms;
-    const durationPerStep = Math.floor(totalDuration / steps.length);
-    for (const step of steps) {
-      step.durationMs = durationPerStep;
-    }
   }
   return { steps, result };
 }
@@ -7419,9 +7447,9 @@ function mapStopReason(stopReason) {
       return "unknown";
   }
 }
-function extractFinalOutput(messages) {
-  for (let i = messages.length - 1; i >= 0; i--) {
-    const message = messages[i];
+function extractFinalOutput(timestampedMessages) {
+  for (let i = timestampedMessages.length - 1; i >= 0; i--) {
+    const { message } = timestampedMessages[i];
     if (isAssistantMessage(message)) {
       for (const block of message.message.content) {
         if (block.type === "text" && block.text) {
@@ -7445,42 +7473,56 @@ function extractTotalUsage(result) {
   };
 }
 function buildLLMTraceFromSteps(steps, totalDurationMs, usage, model) {
-  const traceSteps = steps.map((step, index) => ({
-    id: (0, import_crypto.randomUUID)(),
-    stepNumber: index + 1,
-    type: step.toolCalls?.length ? import_evalforge_types.LLMStepType.TOOL_USE : import_evalforge_types.LLMStepType.COMPLETION,
-    model,
-    provider: "anthropic",
-    startedAt: step.startedAt.toISOString(),
-    durationMs: step.durationMs,
-    tokenUsage: {
-      prompt: step.usage.inputTokens,
-      completion: step.usage.outputTokens,
-      total: step.usage.totalTokens
-    },
-    costUsd: 0,
-    // Individual step costs not available
-    toolName: step.toolCalls?.[0]?.toolName,
-    toolArguments: step.toolCalls?.[0] ? JSON.stringify(step.toolCalls[0].args) : void 0,
-    outputPreview: step.text?.slice(0, 200),
-    success: step.finishReason !== "error",
-    error: step.finishReason === "error" ? "Generation failed" : void 0
-  }));
+  const traceSteps = steps.map((step, index) => {
+    const stepCost = calculateStepCost(
+      step.usage.inputTokens,
+      step.usage.outputTokens,
+      model
+    );
+    return {
+      id: (0, import_crypto.randomUUID)(),
+      stepNumber: index + 1,
+      type: step.toolCalls?.length ? import_evalforge_types.LLMStepType.TOOL_USE : import_evalforge_types.LLMStepType.COMPLETION,
+      model,
+      provider: "anthropic",
+      startedAt: step.startedAt.toISOString(),
+      durationMs: step.durationMs,
+      tokenUsage: {
+        prompt: step.usage.inputTokens,
+        completion: step.usage.outputTokens,
+        total: step.usage.totalTokens
+      },
+      costUsd: stepCost,
+      toolName: step.toolCalls?.[0]?.toolName,
+      toolArguments: step.toolCalls?.[0] ? JSON.stringify(step.toolCalls[0].args) : void 0,
+      outputPreview: step.text?.slice(0, 200),
+      success: step.finishReason !== "error",
+      error: step.finishReason === "error" ? "Generation failed" : void 0
+    };
+  });
+  const stepsTokens = {
+    prompt: traceSteps.reduce((sum, s) => sum + s.tokenUsage.prompt, 0),
+    completion: traceSteps.reduce((sum, s) => sum + s.tokenUsage.completion, 0),
+    total: traceSteps.reduce((sum, s) => sum + s.tokenUsage.total, 0)
+  };
+  const stepsTotalCost = traceSteps.reduce((sum, s) => sum + s.costUsd, 0);
+  const finalTokens = {
+    prompt: usage.inputTokens > 0 ? usage.inputTokens : stepsTokens.prompt,
+    completion: usage.outputTokens > 0 ? usage.outputTokens : stepsTokens.completion,
+    total: usage.totalTokens > 0 ? usage.totalTokens : stepsTokens.total
+  };
+  const finalCost = usage.costUsd !== void 0 && usage.costUsd > 0 ? usage.costUsd : stepsTotalCost;
   const summary = {
     totalSteps: traceSteps.length,
     totalDurationMs,
-    totalTokens: {
-      prompt: usage.inputTokens,
-      completion: usage.outputTokens,
-      total: usage.totalTokens
-    },
-    totalCostUsd: usage.costUsd || 0,
+    totalTokens: finalTokens,
+    totalCostUsd: finalCost,
     modelBreakdown: {
       [model]: {
         count: traceSteps.length,
         durationMs: totalDurationMs,
-        tokens: usage.totalTokens,
-        costUsd: usage.costUsd || 0
+        tokens: finalTokens.total,
+        costUsd: finalCost
       }
     },
     modelsUsed: [model]
@@ -7495,21 +7537,7 @@ function buildLLMTraceFromSteps(steps, totalDurationMs, usage, model) {
 // src/run-scenario/file-diff.ts
 var import_fs6 = require("fs");
 var import_path6 = require("path");
-var IGNORED_PATTERNS = [
-  "node_modules",
-  ".git",
-  ".claude",
-  ".cursor",
-  "dist",
-  "build",
-  ".next",
-  ".turbo",
-  "__pycache__",
-  ".pytest_cache",
-  ".venv",
-  "venv",
-  ".DS_Store"
-];
+var IGNORED_PATTERNS = [];
 var BINARY_EXTENSIONS = [
   ".png",
   ".jpg",