@wix/evalforge-evaluator 0.106.0 → 0.108.0

package/build/index.js

@@ -1505,6 +1505,7 @@ async function writeMcpToFilesystem(cwd, mcps) {
 // src/run-scenario/agents/claude-code/write-sub-agents.ts
 var import_promises5 = require("fs/promises");
 var import_path6 = require("path");
+var import_evalforge_github_client3 = require("@wix/evalforge-github-client");
 var AGENTS_DIR = ".claude/agents";
 function toAgentFilename(name26, index, nameCount) {
   const base = (name26 || "").toLowerCase().replace(/\s+/g, "-").replace(/[^a-z0-9-]/g, "").replace(/^-+|-+$/g, "") || `sub-agent-${index}`;
@@ -1512,7 +1513,34 @@ function toAgentFilename(name26, index, nameCount) {
   nameCount.set(base, count + 1);
   return count === 0 ? base : `${base}-${count + 1}`;
 }
-async function writeSubAgentsToFilesystem(cwd, subAgents) {
+async function resolveSubAgentContent(agent, fetchFn) {
+  if (agent.source) {
+    try {
+      const content = await fetchFn(agent.source, {
+        userAgent: "EvalForge-Evaluator"
+      });
+      console.log(
+        `[SubAgents] Fetched "${agent.name}" from ${agent.source.owner}/${agent.source.repo}/${agent.source.path}@${agent.source.ref}`
+      );
+      return content;
+    } catch (error48) {
+      const message = error48 instanceof Error ? error48.message : "Unknown error";
+      console.error(
+        `[SubAgents] "${agent.name}": GitHub fetch failed: ${message}`
+      );
+      throw new Error(
+        `Failed to fetch sub-agent "${agent.name}" from GitHub: ${message}`
+      );
+    }
+  }
+  if (!agent.subAgentMd) {
+    console.warn(
+      `[SubAgents] "${agent.name}" has empty inline content \u2013 the agent file will be blank`
+    );
+  }
+  return agent.subAgentMd;
+}
+async function writeSubAgentsToFilesystem(cwd, subAgents, fetchFn = import_evalforge_github_client3.fetchGitHubFile) {
   if (subAgents.length === 0) return;
   const agentsDir = (0, import_path6.join)(cwd, AGENTS_DIR);
   await (0, import_promises5.mkdir)(agentsDir, { recursive: true });
@@ -1520,7 +1548,8 @@ async function writeSubAgentsToFilesystem(cwd, subAgents) {
   for (const [i, agent] of subAgents.entries()) {
     const filename = toAgentFilename(agent.name, i, nameCount);
     const filePath = (0, import_path6.join)(agentsDir, `${filename}.md`);
-    await (0, import_promises5.writeFile)(filePath, agent.subAgentMd, "utf8");
+    const content = await resolveSubAgentContent(agent, fetchFn);
+    await (0, import_promises5.writeFile)(filePath, content, "utf8");
   }
   console.log(`[SubAgents] Written to ${agentsDir}`);
 }
@@ -1740,6 +1769,7 @@ function createTraceEventFromMessage(message, context2, stepNumber, isComplete)
   let toolArgs;
   let outputPreview;
   let filePath;
+  let thinking;
   for (const block of message.message.content) {
     if (block.type === "tool_use") {
       type = import_evalforge_types4.LiveTraceEventType.TOOL_USE;
@@ -1756,6 +1786,15 @@ function createTraceEventFromMessage(message, context2, stepNumber, isComplete)
       }
     } else if (block.type === "text") {
       outputPreview = block.text.slice(0, 500);
+      if (!toolName) {
+        type = import_evalforge_types4.LiveTraceEventType.COMPLETION;
+      }
+    } else if (block.type === "thinking") {
+      const thinkingBlock = block;
+      thinking = thinkingBlock.thinking.slice(0, 500);
+      if (!outputPreview && !toolName) {
+        type = import_evalforge_types4.LiveTraceEventType.THINKING;
+      }
     }
   }
   return {
@@ -1770,6 +1809,7 @@ function createTraceEventFromMessage(message, context2, stepNumber, isComplete)
     toolArgs,
     outputPreview,
     filePath,
+    thinking,
     timestamp: (/* @__PURE__ */ new Date()).toISOString(),
     isComplete
   };
@@ -2395,28 +2435,53 @@ function processMessages(timestampedMessages, startTime, endTime) {
   const sortedGroups = Array.from(assistantMessageGroups.values()).sort(
     (a, b) => a.firstReceivedAt.getTime() - b.firstReceivedAt.getTime()
   );
-  for (let i = 0; i < sortedGroups.length; i++) {
-    const group = sortedGroups[i];
-    const lastMessage = group.messages[group.messages.length - 1];
-    const stepStartTime = group.firstReceivedAt;
-    const nextStepStartTime = i < sortedGroups.length - 1 ? sortedGroups[i + 1].firstReceivedAt : endTime;
+  const mergedTurns = [];
+  for (const group of sortedGroups) {
+    const lastMsg = group.messages[group.messages.length - 1];
+    const inputTokens = lastMsg.message.usage.input_tokens;
+    const prev = mergedTurns[mergedTurns.length - 1];
+    const prevLastMsg = prev?.groups[prev.groups.length - 1].messages.at(-1);
+    const prevInputTokens = prevLastMsg?.message.usage.input_tokens;
+    if (prev && prevInputTokens === inputTokens) {
+      prev.groups.push(group);
+      prev.lastReceivedAt = group.lastReceivedAt;
+    } else {
+      mergedTurns.push({
+        groups: [group],
+        firstReceivedAt: group.firstReceivedAt,
+        lastReceivedAt: group.lastReceivedAt
+      });
+    }
+  }
+  for (let i = 0; i < mergedTurns.length; i++) {
+    const turn = mergedTurns[i];
+    const stepStartTime = turn.firstReceivedAt;
+    const nextStepStartTime = i < mergedTurns.length - 1 ? mergedTurns[i + 1].firstReceivedAt : endTime;
     const durationMs = nextStepStartTime.getTime() - stepStartTime.getTime();
-    const usage = lastMessage.message.usage;
-    const inputTokens = usage.input_tokens;
-    const outputTokens = usage.output_tokens;
     let text2 = "";
     let thinking = "";
     const toolCalls = [];
-    for (const block of lastMessage.message.content) {
-      if (block.type === "text") {
-        text2 += block.text;
-      } else if (block.type === "thinking") {
-        thinking += block.thinking;
-      } else if (block.type === "tool_use") {
-        toolCalls.push({
-          toolName: block.name,
-          args: block.input
-        });
+    let lastStopReason = null;
+    let inputTokens = 0;
+    let outputTokens = 0;
+    let cacheReadTokens;
+    let cacheWriteTokens;
+    for (const group of turn.groups) {
+      const lastMessage = group.messages[group.messages.length - 1];
+      lastStopReason = lastMessage.message.stop_reason;
+      const msgUsage = lastMessage.message.usage;
+      inputTokens = msgUsage.input_tokens;
+      outputTokens = msgUsage.output_tokens;
+      cacheReadTokens = msgUsage.cache_read_input_tokens ?? void 0;
+      cacheWriteTokens = msgUsage.cache_creation_input_tokens ?? void 0;
+      for (const block of lastMessage.message.content) {
+        if (block.type === "text") {
+          text2 += block.text;
+        } else if (block.type === "thinking") {
+          thinking += block.thinking;
+        } else if (block.type === "tool_use") {
+          toolCalls.push({ toolName: block.name, args: block.input });
+        }
       }
     }
     steps.push({
@@ -2425,13 +2490,14 @@ function processMessages(timestampedMessages, startTime, endTime) {
       usage: {
         inputTokens,
         outputTokens,
-        totalTokens: inputTokens + outputTokens
+        totalTokens: inputTokens + outputTokens,
+        cacheReadTokens: cacheReadTokens || void 0,
+        cacheWriteTokens: cacheWriteTokens || void 0
       },
-      finishReason: mapStopReason(lastMessage.message.stop_reason),
+      finishReason: mapStopReason(lastStopReason),
       toolCalls: toolCalls.length > 0 ? toolCalls : void 0,
       startedAt: stepStartTime,
       durationMs: Math.max(0, durationMs)
-      // Ensure non-negative
     });
   }
   return { steps, result };
@@ -2471,35 +2537,35 @@ function extractTotalUsage(result) {
     inputTokens: usage.input_tokens,
     outputTokens: usage.output_tokens,
     totalTokens: usage.input_tokens + usage.output_tokens,
-    costUsd: result.total_cost_usd
+    costUsd: result.total_cost_usd,
+    cacheReadTokens: usage.cache_read_input_tokens || void 0,
+    cacheWriteTokens: usage.cache_creation_input_tokens || void 0,
+    durationApiMs: result.duration_api_ms || void 0
   };
 }
 function buildLLMTraceFromSteps(steps, totalDurationMs, usage, model) {
   const totalCost = usage.costUsd ?? 0;
-  const totalStepTokens = steps.reduce(
-    (sum, s) => sum + s.usage.totalTokens,
+  const totalStepInputTokens = steps.reduce(
+    (sum, s) => sum + s.usage.inputTokens,
     0
   );
   const totalStepDuration = steps.reduce((sum, s) => sum + s.durationMs, 0);
-  const hasPerStepTokens = totalStepTokens > 0;
+  const inputTokensDuplicated = usage.inputTokens > 0 && totalStepInputTokens > usage.inputTokens * 1.2;
   const traceSteps = steps.map((step, index) => {
-    let tokenUsage;
+    let stepPromptTokens;
+    let stepOutputTokens;
     let proportion;
-    if (hasPerStepTokens) {
-      proportion = step.usage.totalTokens / totalStepTokens;
-      tokenUsage = {
-        prompt: step.usage.inputTokens,
-        completion: step.usage.outputTokens,
-        total: step.usage.totalTokens
-      };
-    } else {
+    if (inputTokensDuplicated) {
       proportion = totalStepDuration > 0 ? step.durationMs / totalStepDuration : 0;
-      tokenUsage = {
-        prompt: Math.round(usage.inputTokens * proportion),
-        completion: Math.round(usage.outputTokens * proportion),
-        total: Math.round(usage.totalTokens * proportion)
-      };
+      stepPromptTokens = Math.round(usage.inputTokens * proportion);
+      stepOutputTokens = Math.round(usage.outputTokens * proportion);
+    } else {
+      proportion = totalStepInputTokens > 0 ? step.usage.inputTokens / totalStepInputTokens : 0;
+      stepPromptTokens = step.usage.inputTokens;
+      stepOutputTokens = Math.round(usage.outputTokens * proportion);
     }
+    const stepTotalTokens = stepPromptTokens + stepOutputTokens;
+    const costProportion = proportion;
     const stepType = step.toolCalls?.length ? import_evalforge_types4.LLMStepType.TOOL_USE : step.thinking && !step.text ? import_evalforge_types4.LLMStepType.THINKING : import_evalforge_types4.LLMStepType.COMPLETION;
     return {
       id: (0, import_crypto.randomUUID)(),
@@ -2509,11 +2575,15 @@ function buildLLMTraceFromSteps(steps, totalDurationMs, usage, model) {
       provider: "anthropic",
       startedAt: step.startedAt.toISOString(),
       durationMs: step.durationMs,
-      tokenUsage,
-      costUsd: totalCost * proportion,
+      tokenUsage: {
+        prompt: stepPromptTokens,
+        completion: stepOutputTokens,
+        total: stepTotalTokens
+      },
+      costUsd: totalCost * costProportion,
       toolName: step.toolCalls?.[0]?.toolName,
       toolArguments: step.toolCalls?.[0] ? JSON.stringify(step.toolCalls[0].args) : void 0,
-      outputPreview: step.text?.slice(0, 200),
+      outputPreview: (step.text || step.thinking)?.slice(0, 200),
       success: step.finishReason !== "error",
       error: step.finishReason === "error" ? "Generation failed" : void 0
     };
@@ -2523,21 +2593,35 @@ function buildLLMTraceFromSteps(steps, totalDurationMs, usage, model) {
     completion: usage.outputTokens,
     total: usage.totalTokens
   };
-  const finalCost = totalCost;
+  const stepTypeBreakdown = {};
+  for (const ts of traceSteps) {
+    const entry = stepTypeBreakdown[ts.type] ?? {
+      count: 0,
+      durationMs: 0,
+      tokens: 0,
+      costUsd: 0
+    };
+    entry.count += 1;
+    entry.durationMs += ts.durationMs;
+    entry.tokens += ts.tokenUsage.total;
+    entry.costUsd += ts.costUsd;
+    stepTypeBreakdown[ts.type] = entry;
+  }
   const summary = {
     totalSteps: traceSteps.length,
     totalDurationMs,
     totalTokens: finalTokens,
-    totalCostUsd: finalCost,
+    totalCostUsd: totalCost,
     modelBreakdown: {
       [model]: {
         count: traceSteps.length,
         durationMs: totalDurationMs,
         tokens: finalTokens.total,
-        costUsd: finalCost
+        costUsd: totalCost
       }
     },
-    modelsUsed: [model]
+    modelsUsed: [model],
+    stepTypeBreakdown
   };
   return {
     id: (0, import_crypto.randomUUID)(),
@@ -42745,6 +42829,9 @@ function createModel(modelId, baseUrl, headers) {
     apiKey: "proxy-auth",
     headers
   });
+  if (import_evalforge_types6.OPENAI_RESPONSES_MODEL_IDS.has(modelId)) {
+    return openai2.responses(modelId);
+  }
   return openai2.chat(modelId);
 }
 function isClaudeModelId(modelId) {
@@ -42781,14 +42868,34 @@ async function executeWithAiSdk(context2) {
     emitStartEvent(traceContext, startTime);
   }
   try {
+    const isAnthropic = provider === PROVIDER_ANTHROPIC2;
+    const isResponsesAPI = import_evalforge_types6.OPENAI_RESPONSES_MODEL_IDS.has(modelConfig.model);
+    const supportsThinking = isAnthropic || isResponsesAPI;
+    const providerOpts = {
+      ...isAnthropic && {
+        anthropic: {
+          thinking: { type: "enabled", budgetTokens: 1e4 }
+        }
+      },
+      ...isResponsesAPI && {
+        openai: {
+          // Prevent the SDK from sending item_reference inputs that the proxy can't forward
+          store: false,
+          forceReasoning: true,
+          reasoningEffort: "high",
+          reasoningSummary: "detailed"
+        }
+      }
+    };
     const result = await generateText({
       model,
       system: systemPrompt,
       prompt: scenario.triggerPrompt,
-      temperature: modelConfig.temperature,
+      temperature: supportsThinking ? void 0 : modelConfig.temperature,
       maxOutputTokens: modelConfig.maxTokens,
       tools: mcpTools,
-      stopWhen: mcpTools ? stepCountIs(MAX_TOOL_STEPS) : stepCountIs(1)
+      stopWhen: mcpTools ? stepCountIs(MAX_TOOL_STEPS) : stepCountIs(1),
+      providerOptions: providerOpts
     });
     const durationMs = Date.now() - startTime;
     const usage = {