@wix/evalforge-evaluator 0.107.0 → 0.109.0

package/build/index.js

@@ -1769,6 +1769,7 @@ function createTraceEventFromMessage(message, context2, stepNumber, isComplete)
   let toolArgs;
   let outputPreview;
   let filePath;
+  let thinking;
   for (const block of message.message.content) {
     if (block.type === "tool_use") {
       type = import_evalforge_types4.LiveTraceEventType.TOOL_USE;
@@ -1785,6 +1786,15 @@ function createTraceEventFromMessage(message, context2, stepNumber, isComplete)
       }
     } else if (block.type === "text") {
       outputPreview = block.text.slice(0, 500);
+      if (!toolName) {
+        type = import_evalforge_types4.LiveTraceEventType.COMPLETION;
+      }
+    } else if (block.type === "thinking") {
+      const thinkingBlock = block;
+      thinking = thinkingBlock.thinking.slice(0, 500);
+      if (!outputPreview && !toolName) {
+        type = import_evalforge_types4.LiveTraceEventType.THINKING;
+      }
     }
   }
   return {
@@ -1799,6 +1809,7 @@ function createTraceEventFromMessage(message, context2, stepNumber, isComplete)
     toolArgs,
     outputPreview,
     filePath,
+    thinking,
     timestamp: (/* @__PURE__ */ new Date()).toISOString(),
     isComplete
   };
@@ -2424,28 +2435,53 @@ function processMessages(timestampedMessages, startTime, endTime) {
   const sortedGroups = Array.from(assistantMessageGroups.values()).sort(
     (a, b) => a.firstReceivedAt.getTime() - b.firstReceivedAt.getTime()
   );
-  for (let i = 0; i < sortedGroups.length; i++) {
-    const group = sortedGroups[i];
-    const lastMessage = group.messages[group.messages.length - 1];
-    const stepStartTime = group.firstReceivedAt;
-    const nextStepStartTime = i < sortedGroups.length - 1 ? sortedGroups[i + 1].firstReceivedAt : endTime;
+  const mergedTurns = [];
+  for (const group of sortedGroups) {
+    const lastMsg = group.messages[group.messages.length - 1];
+    const inputTokens = lastMsg.message.usage.input_tokens;
+    const prev = mergedTurns[mergedTurns.length - 1];
+    const prevLastMsg = prev?.groups[prev.groups.length - 1].messages.at(-1);
+    const prevInputTokens = prevLastMsg?.message.usage.input_tokens;
+    if (prev && prevInputTokens === inputTokens) {
+      prev.groups.push(group);
+      prev.lastReceivedAt = group.lastReceivedAt;
+    } else {
+      mergedTurns.push({
+        groups: [group],
+        firstReceivedAt: group.firstReceivedAt,
+        lastReceivedAt: group.lastReceivedAt
+      });
+    }
+  }
+  for (let i = 0; i < mergedTurns.length; i++) {
+    const turn = mergedTurns[i];
+    const stepStartTime = turn.firstReceivedAt;
+    const nextStepStartTime = i < mergedTurns.length - 1 ? mergedTurns[i + 1].firstReceivedAt : endTime;
     const durationMs = nextStepStartTime.getTime() - stepStartTime.getTime();
-    const usage = lastMessage.message.usage;
-    const inputTokens = usage.input_tokens;
-    const outputTokens = usage.output_tokens;
     let text2 = "";
     let thinking = "";
     const toolCalls = [];
-    for (const block of lastMessage.message.content) {
-      if (block.type === "text") {
-        text2 += block.text;
-      } else if (block.type === "thinking") {
-        thinking += block.thinking;
-      } else if (block.type === "tool_use") {
-        toolCalls.push({
-          toolName: block.name,
-          args: block.input
-        });
+    let lastStopReason = null;
+    let inputTokens = 0;
+    let outputTokens = 0;
+    let cacheReadTokens;
+    let cacheWriteTokens;
+    for (const group of turn.groups) {
+      const lastMessage = group.messages[group.messages.length - 1];
+      lastStopReason = lastMessage.message.stop_reason;
+      const msgUsage = lastMessage.message.usage;
+      inputTokens = msgUsage.input_tokens;
+      outputTokens = msgUsage.output_tokens;
+      cacheReadTokens = msgUsage.cache_read_input_tokens ?? void 0;
+      cacheWriteTokens = msgUsage.cache_creation_input_tokens ?? void 0;
+      for (const block of lastMessage.message.content) {
+        if (block.type === "text") {
+          text2 += block.text;
+        } else if (block.type === "thinking") {
+          thinking += block.thinking;
+        } else if (block.type === "tool_use") {
+          toolCalls.push({ toolName: block.name, args: block.input });
+        }
       }
     }
     steps.push({
@@ -2454,13 +2490,14 @@ function processMessages(timestampedMessages, startTime, endTime) {
       usage: {
         inputTokens,
         outputTokens,
-        totalTokens: inputTokens + outputTokens
+        totalTokens: inputTokens + outputTokens,
+        cacheReadTokens: cacheReadTokens || void 0,
+        cacheWriteTokens: cacheWriteTokens || void 0
       },
-      finishReason: mapStopReason(lastMessage.message.stop_reason),
+      finishReason: mapStopReason(lastStopReason),
       toolCalls: toolCalls.length > 0 ? toolCalls : void 0,
       startedAt: stepStartTime,
       durationMs: Math.max(0, durationMs)
-      // Ensure non-negative
     });
   }
   return { steps, result };
@@ -2500,35 +2537,35 @@ function extractTotalUsage(result) {
     inputTokens: usage.input_tokens,
     outputTokens: usage.output_tokens,
     totalTokens: usage.input_tokens + usage.output_tokens,
-    costUsd: result.total_cost_usd
+    costUsd: result.total_cost_usd,
+    cacheReadTokens: usage.cache_read_input_tokens || void 0,
+    cacheWriteTokens: usage.cache_creation_input_tokens || void 0,
+    durationApiMs: result.duration_api_ms || void 0
   };
 }
 function buildLLMTraceFromSteps(steps, totalDurationMs, usage, model) {
   const totalCost = usage.costUsd ?? 0;
-  const totalStepTokens = steps.reduce(
-    (sum, s) => sum + s.usage.totalTokens,
+  const totalStepInputTokens = steps.reduce(
+    (sum, s) => sum + s.usage.inputTokens,
     0
   );
   const totalStepDuration = steps.reduce((sum, s) => sum + s.durationMs, 0);
-  const hasPerStepTokens = totalStepTokens > 0;
+  const inputTokensDuplicated = usage.inputTokens > 0 && totalStepInputTokens > usage.inputTokens * 1.2;
   const traceSteps = steps.map((step, index) => {
-    let tokenUsage;
+    let stepPromptTokens;
+    let stepOutputTokens;
     let proportion;
-    if (hasPerStepTokens) {
-      proportion = step.usage.totalTokens / totalStepTokens;
-      tokenUsage = {
-        prompt: step.usage.inputTokens,
-        completion: step.usage.outputTokens,
-        total: step.usage.totalTokens
-      };
-    } else {
+    if (inputTokensDuplicated) {
       proportion = totalStepDuration > 0 ? step.durationMs / totalStepDuration : 0;
-      tokenUsage = {
-        prompt: Math.round(usage.inputTokens * proportion),
-        completion: Math.round(usage.outputTokens * proportion),
-        total: Math.round(usage.totalTokens * proportion)
-      };
+      stepPromptTokens = Math.round(usage.inputTokens * proportion);
+      stepOutputTokens = Math.round(usage.outputTokens * proportion);
+    } else {
+      proportion = totalStepInputTokens > 0 ? step.usage.inputTokens / totalStepInputTokens : 0;
+      stepPromptTokens = step.usage.inputTokens;
+      stepOutputTokens = Math.round(usage.outputTokens * proportion);
     }
+    const stepTotalTokens = stepPromptTokens + stepOutputTokens;
+    const costProportion = proportion;
     const stepType = step.toolCalls?.length ? import_evalforge_types4.LLMStepType.TOOL_USE : step.thinking && !step.text ? import_evalforge_types4.LLMStepType.THINKING : import_evalforge_types4.LLMStepType.COMPLETION;
     return {
       id: (0, import_crypto.randomUUID)(),
@@ -2538,11 +2575,15 @@ function buildLLMTraceFromSteps(steps, totalDurationMs, usage, model) {
       provider: "anthropic",
       startedAt: step.startedAt.toISOString(),
       durationMs: step.durationMs,
-      tokenUsage,
-      costUsd: totalCost * proportion,
+      tokenUsage: {
+        prompt: stepPromptTokens,
+        completion: stepOutputTokens,
+        total: stepTotalTokens
+      },
+      costUsd: totalCost * costProportion,
       toolName: step.toolCalls?.[0]?.toolName,
       toolArguments: step.toolCalls?.[0] ? JSON.stringify(step.toolCalls[0].args) : void 0,
-      outputPreview: step.text?.slice(0, 200),
+      outputPreview: (step.text || step.thinking)?.slice(0, 200),
       success: step.finishReason !== "error",
       error: step.finishReason === "error" ? "Generation failed" : void 0
     };
@@ -2552,21 +2593,35 @@ function buildLLMTraceFromSteps(steps, totalDurationMs, usage, model) {
     completion: usage.outputTokens,
     total: usage.totalTokens
   };
-  const finalCost = totalCost;
+  const stepTypeBreakdown = {};
+  for (const ts of traceSteps) {
+    const entry = stepTypeBreakdown[ts.type] ?? {
+      count: 0,
+      durationMs: 0,
+      tokens: 0,
+      costUsd: 0
+    };
+    entry.count += 1;
+    entry.durationMs += ts.durationMs;
+    entry.tokens += ts.tokenUsage.total;
+    entry.costUsd += ts.costUsd;
+    stepTypeBreakdown[ts.type] = entry;
+  }
   const summary = {
     totalSteps: traceSteps.length,
     totalDurationMs,
     totalTokens: finalTokens,
-    totalCostUsd: finalCost,
+    totalCostUsd: totalCost,
     modelBreakdown: {
       [model]: {
         count: traceSteps.length,
         durationMs: totalDurationMs,
         tokens: finalTokens.total,
-        costUsd: finalCost
+        costUsd: totalCost
       }
     },
-    modelsUsed: [model]
+    modelsUsed: [model],
+    stepTypeBreakdown
   };
   return {
     id: (0, import_crypto.randomUUID)(),
@@ -2606,6 +2661,7 @@ var ClaudeCodeAdapter = class {
       model: modelForSdk,
       temperature: modelConfig?.temperature,
       maxTokens: modelConfig?.maxTokens,
+      maxTurns: modelConfig?.maxTurns,
       aiGatewayUrl,
       aiGatewayHeaders,
       traceContext,
@@ -42638,7 +42694,6 @@ var MODEL_PRICING = {
   CLAUDE_4_OPUS_1_0: { input: 15, output: 75 },
   CLAUDE_4_SONNET_1_0: { input: 3, output: 15 },
   // Anthropic — Claude 3.x
-  CLAUDE_3_7_SONNET_1_0: { input: 3, output: 15 },
   CLAUDE_3_5_SONNET_2_0: { input: 3, output: 15 },
   CLAUDE_3_5_SONNET_1_0: { input: 3, output: 15 },
   CLAUDE_3_HAIKU_1_0: { input: 0.25, output: 1.25 },
@@ -42756,7 +42811,7 @@ function estimateStepTimestamp(startMs, stepIndex, totalSteps) {
 // src/run-scenario/agents/simple-agent/execute.ts
 var PROVIDER_ANTHROPIC2 = "anthropic";
 var PROVIDER_OPENAI = "openai";
-var MAX_TOOL_STEPS = 25;
+var DEFAULT_MAX_TOOL_STEPS = 25;
 function createModel(modelId, baseUrl, headers) {
   const isClaudeModel = import_evalforge_types6.AVAILABLE_CLAUDE_MODEL_IDS.includes(
     modelId
@@ -42774,6 +42829,9 @@ function createModel(modelId, baseUrl, headers) {
     apiKey: "proxy-auth",
     headers
   });
+  if (import_evalforge_types6.OPENAI_RESPONSES_MODEL_IDS.has(modelId)) {
+    return openai2.responses(modelId);
+  }
   return openai2.chat(modelId);
 }
 function isClaudeModelId(modelId) {
@@ -42810,14 +42868,34 @@ async function executeWithAiSdk(context2) {
     emitStartEvent(traceContext, startTime);
   }
   try {
+    const isAnthropic = provider === PROVIDER_ANTHROPIC2;
+    const isResponsesAPI = import_evalforge_types6.OPENAI_RESPONSES_MODEL_IDS.has(modelConfig.model);
+    const supportsThinking = isAnthropic || isResponsesAPI;
+    const providerOpts = {
+      ...isAnthropic && {
+        anthropic: {
+          thinking: { type: "enabled", budgetTokens: 1e4 }
+        }
+      },
+      ...isResponsesAPI && {
+        openai: {
+          // Prevent the SDK from sending item_reference inputs that the proxy can't forward
+          store: false,
+          forceReasoning: true,
+          reasoningEffort: "high",
+          reasoningSummary: "detailed"
+        }
+      }
+    };
     const result = await generateText({
       model,
       system: systemPrompt,
       prompt: scenario.triggerPrompt,
-      temperature: modelConfig.temperature,
+      temperature: supportsThinking ? void 0 : modelConfig.temperature,
       maxOutputTokens: modelConfig.maxTokens,
       tools: mcpTools,
-      stopWhen: mcpTools ? stepCountIs(MAX_TOOL_STEPS) : stepCountIs(1)
+      stopWhen: mcpTools ? stepCountIs(modelConfig.maxTurns ?? DEFAULT_MAX_TOOL_STEPS) : stepCountIs(1),
+      providerOptions: providerOpts
     });
     const durationMs = Date.now() - startTime;
     const usage = {