@wix/evalforge-evaluator 0.102.0 → 0.103.0

package/build/index.js

@@ -879,6 +879,83 @@ async function writeRulesToFilesystem(cwd, rules) {
   console.log(`[Rules] Written ${rules.length} rule(s) to ${cwd}`);
 }
 
+// src/run-scenario/agents/claude-code/build-conversation.ts
+function isAssistantMessage(message) {
+  return message.type === "assistant";
+}
+function buildConversation(timestampedMessages) {
+  const messages = [];
+  for (const { message, receivedAt } of timestampedMessages) {
+    const timestamp = receivedAt.toISOString();
+    if (isAssistantMessage(message)) {
+      const content = [];
+      for (const block of message.message.content) {
+        if (block.type === "text") {
+          content.push({ type: "text", text: block.text });
+        } else if (block.type === "thinking") {
+          content.push({
+            type: "thinking",
+            thinking: block.thinking
+          });
+        } else if (block.type === "tool_use") {
+          content.push({
+            type: "tool_use",
+            toolName: block.name,
+            toolId: block.id,
+            input: block.input
+          });
+        }
+      }
+      if (content.length > 0) {
+        messages.push({ role: "assistant", content, timestamp });
+      }
+    } else if (message.type === "user") {
+      const userMsg = message;
+      const content = [];
+      const msgContent = userMsg.message?.content;
+      if (typeof msgContent === "string") {
+        content.push({ type: "text", text: msgContent });
+      } else if (Array.isArray(msgContent)) {
+        for (const block of msgContent) {
+          if (typeof block === "object" && block !== null) {
+            const b = block;
+            if (b.type === "tool_result") {
+              const rawContent = b.content;
+              let text = "";
+              if (typeof rawContent === "string") {
+                text = rawContent;
+              } else if (Array.isArray(rawContent)) {
+                text = rawContent.filter(
+                  (c) => typeof c === "object" && c !== null && c.type === "text"
+                ).map((c) => c.text).join("\n");
+              }
+              content.push({
+                type: "tool_result",
+                toolUseId: String(b.tool_use_id ?? ""),
+                content: text,
+                isError: b.is_error === true ? true : void 0
+              });
+            } else if (b.type === "text" && typeof b.text === "string") {
+              content.push({ type: "text", text: b.text });
+            }
+          }
+        }
+      }
+      if (content.length > 0) {
+        messages.push({ role: "user", content, timestamp });
+      }
+    } else if (message.type === "system") {
+      const sysMsg = message;
+      messages.push({
+        role: "system",
+        content: [{ type: "text", text: sysMsg.subtype || "system" }],
+        timestamp
+      });
+    }
+  }
+  return messages;
+}
+
 // src/run-scenario/agents/claude-code/execute.ts
 var DEFAULT_MODEL = import_evalforge_types3.ClaudeModel.CLAUDE_4_5_SONNET_1_0;
 function emitTraceEvent(event, tracePushUrl, routeHeader, authToken) {
@@ -1009,7 +1086,7 @@ function createTraceEventFromAnyMessage(message, context, stepNumber, isComplete
     timestamp: (/* @__PURE__ */ new Date()).toISOString(),
     isComplete
   };
-  if (isAssistantMessage(message)) {
+  if (isAssistantMessage2(message)) {
     return createTraceEventFromMessage(
       message,
       context,
@@ -1025,15 +1102,25 @@ function createTraceEventFromAnyMessage(message, context, stepNumber, isComplete
       outputPreview = content.slice(0, 500);
     } else if (Array.isArray(content)) {
       for (const block of content) {
-        if (typeof block === "object" && "text" in block && block.text) {
-          outputPreview = String(block.text).slice(0, 500);
+        if (typeof block !== "object" || block === null) continue;
+        const b = block;
+        if (b.type === "text" && typeof b.text === "string") {
+          outputPreview = b.text.slice(0, 500);
+          break;
+        }
+        if (b.type === "tool_result") {
+          const raw = b.content;
+          const preview = typeof raw === "string" ? raw : Array.isArray(raw) ? raw.filter(
+            (c) => c.type === "text" && typeof c.text === "string"
+          ).map((c) => c.text).join("\n") : "";
+          outputPreview = preview.slice(0, 500);
           break;
         }
       }
     }
     return {
       ...baseEvent,
-      type: import_evalforge_types3.LiveTraceEventType.USER,
+      type: import_evalforge_types3.LiveTraceEventType.TOOL_RESULT,
       outputPreview: outputPreview || "(tool result)"
     };
   }
@@ -1647,6 +1734,7 @@ Stack: ${errorStack.split("\n").slice(0, 5).join("\n")}` : "")
     usage,
     options.model || DEFAULT_MODEL
   );
+  const conversation = buildConversation(allMessages);
   return {
     result: {
       outputText,
@@ -1658,7 +1746,8 @@ Stack: ${errorStack.split("\n").slice(0, 5).join("\n")}` : "")
       },
       costUsd: usage.costUsd
     },
-    llmTrace
+    llmTrace,
+    conversation
   };
 }
 function buildSdkEnvironment(options) {
@@ -1675,7 +1764,7 @@ function buildSdkEnvironment(options) {
   }
   return env;
 }
-function isAssistantMessage(message) {
+function isAssistantMessage2(message) {
   return message.type === "assistant";
 }
 function isResultMessage(message) {
@@ -1686,7 +1775,7 @@ function processMessages(timestampedMessages, startTime, endTime) {
   let result;
   const assistantMessageGroups = /* @__PURE__ */ new Map();
   for (const { message, receivedAt } of timestampedMessages) {
-    if (isAssistantMessage(message)) {
+    if (isAssistantMessage2(message)) {
       const uuid = message.uuid;
       if (!assistantMessageGroups.has(uuid)) {
         assistantMessageGroups.set(uuid, {
@@ -1715,10 +1804,13 @@ function processMessages(timestampedMessages, startTime, endTime) {
     const inputTokens = usage.input_tokens;
     const outputTokens = usage.output_tokens;
     let text = "";
+    let thinking = "";
     const toolCalls = [];
     for (const block of lastMessage.message.content) {
       if (block.type === "text") {
         text += block.text;
+      } else if (block.type === "thinking") {
+        thinking += block.thinking;
       } else if (block.type === "tool_use") {
         toolCalls.push({
           toolName: block.name,
@@ -1728,6 +1820,7 @@ function processMessages(timestampedMessages, startTime, endTime) {
     }
     steps.push({
       text,
+      thinking: thinking || void 0,
       usage: {
         inputTokens,
         outputTokens,
@@ -1758,7 +1851,7 @@ function mapStopReason(stopReason) {
 function extractFinalOutput(timestampedMessages) {
   for (let i = timestampedMessages.length - 1; i >= 0; i--) {
     const { message } = timestampedMessages[i];
-    if (isAssistantMessage(message)) {
+    if (isAssistantMessage2(message)) {
       for (const block of message.message.content) {
         if (block.type === "text" && block.text) {
           return block.text;
@@ -1788,10 +1881,11 @@ function buildLLMTraceFromSteps(steps, totalDurationMs, usage, model) {
   );
   const traceSteps = steps.map((step, index) => {
     const proportion = totalStepTokens > 0 ? step.usage.totalTokens / totalStepTokens : 0;
+    const stepType = step.toolCalls?.length ? import_evalforge_types3.LLMStepType.TOOL_USE : step.thinking && !step.text ? import_evalforge_types3.LLMStepType.THINKING : import_evalforge_types3.LLMStepType.COMPLETION;
     return {
       id: (0, import_crypto.randomUUID)(),
       stepNumber: index + 1,
-      type: step.toolCalls?.length ? import_evalforge_types3.LLMStepType.TOOL_USE : import_evalforge_types3.LLMStepType.COMPLETION,
+      type: stepType,
       model,
       provider: "anthropic",
       startedAt: step.startedAt.toISOString(),
@@ -1881,7 +1975,7 @@ var ClaudeCodeAdapter = class {
       rules,
       systemPrompt
     };
-    const { result, llmTrace } = await executeWithClaudeCode(
+    const { result, llmTrace, conversation } = await executeWithClaudeCode(
       skills,
       scenario,
       options
@@ -1895,7 +1989,8 @@ var ClaudeCodeAdapter = class {
         totalTokens: result.usage.totalTokens
       },
       costUsd: result.costUsd,
-      llmTrace
+      llmTrace,
+      conversation
     };
   }
 };
@@ -2668,7 +2763,7 @@ async function runAgentWithContext(config, evalRunId2, scenario, evalData, workD
     rules: evalData.rules?.length > 0 ? evalData.rules : void 0,
     systemPrompt: agent?.systemPrompt
   };
-  const { outputText, durationMs, llmTrace } = await adapter.execute(executionContext);
+  const { outputText, durationMs, llmTrace, conversation } = await adapter.execute(executionContext);
   const completedAt = (/* @__PURE__ */ new Date()).toISOString();
   const afterSnapshot = workDir ? snapshotDirectory(workDir) : {};
   const fileDiffs = diffSnapshots(beforeSnapshot, afterSnapshot);
@@ -2686,7 +2781,8 @@ async function runAgentWithContext(config, evalRunId2, scenario, evalData, workD
     templateFiles: templateFiles && templateFiles.length > 0 ? templateFiles : void 0,
     startedAt,
     completedAt,
-    llmTrace
+    llmTrace,
+    conversation
   };
 }