@wix/evalforge-evaluator 0.27.0 → 0.29.0

package/build/index.js

@@ -6630,13 +6630,23 @@ function createTraceEventFromMessage(message, context, stepNumber, isComplete) {
   let toolName;
   let toolArgs;
   let outputPreview;
+  let filePath;
   for (const block of message.message.content) {
     if (block.type === "tool_use") {
       type = import_evalforge_types.LiveTraceEventType.TOOL_USE;
       toolName = block.name;
-      toolArgs = JSON.stringify(block.input).slice(0, 200);
+      toolArgs = JSON.stringify(block.input).slice(0, 500);
+      const input = block.input;
+      if (input.file_path || input.path || input.target_file) {
+        filePath = String(input.file_path || input.path || input.target_file);
+        if (block.name === "Write" || block.name === "Edit" || block.name === "write" || block.name === "edit") {
+          type = import_evalforge_types.LiveTraceEventType.FILE_WRITE;
+        } else if (block.name === "Read" || block.name === "read" || block.name === "View") {
+          type = import_evalforge_types.LiveTraceEventType.FILE_READ;
+        }
+      }
     } else if (block.type === "text") {
-      outputPreview = block.text.slice(0, 200);
+      outputPreview = block.text.slice(0, 500);
     }
   }
   return {
@@ -6650,9 +6660,63 @@ function createTraceEventFromMessage(message, context, stepNumber, isComplete) {
     toolName,
     toolArgs,
     outputPreview,
+    filePath,
+    timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+    isComplete
+  };
+}
+function createTraceEventFromAnyMessage(message, context, stepNumber, isComplete) {
+  const baseEvent = {
+    evalRunId: context.evalRunId,
+    scenarioId: context.scenarioId,
+    scenarioName: context.scenarioName,
+    targetId: context.targetId,
+    targetName: context.targetName,
+    stepNumber,
     timestamp: (/* @__PURE__ */ new Date()).toISOString(),
     isComplete
   };
+  if (isAssistantMessage(message)) {
+    return createTraceEventFromMessage(
+      message,
+      context,
+      stepNumber,
+      isComplete
+    );
+  }
+  if (message.type === "user") {
+    const userMsg = message;
+    let outputPreview = "";
+    if (userMsg.message?.content) {
+      for (const block of userMsg.message.content) {
+        if ("text" in block && block.text) {
+          outputPreview = block.text.slice(0, 500);
+          break;
+        }
+      }
+    }
+    return {
+      ...baseEvent,
+      type: import_evalforge_types.LiveTraceEventType.USER,
+      outputPreview: outputPreview || "(tool result)"
+    };
+  }
+  if (message.type === "system") {
+    const sysMsg = message;
+    return {
+      ...baseEvent,
+      type: import_evalforge_types.LiveTraceEventType.SYSTEM,
+      outputPreview: sysMsg.message?.slice(0, 500) || sysMsg.subtype || "system"
+    };
+  }
+  if (message.type === "result") {
+    return null;
+  }
+  return {
+    ...baseEvent,
+    type: import_evalforge_types.LiveTraceEventType.PROGRESS,
+    outputPreview: `Message type: ${message.type}`
+  };
 }
 async function executeWithClaudeCode(skill, scenario, options) {
   console.log("[executeWithClaudeCode] Starting execution", {
@@ -6850,6 +6914,9 @@ async function executeWithClaudeCode(skill, scenario, options) {
   const SDK_TIMEOUT_MS = Math.max(3e5, maxTurns * 6e4);
   let timeoutHandle;
   let timedOut = false;
+  const HEARTBEAT_INTERVAL_MS = 1e4;
+  let heartbeatHandle;
+  const executionStartTime = Date.now();
   try {
     const timeoutPromise = new Promise((_, reject) => {
       timeoutHandle = setTimeout(() => {
@@ -6861,9 +6928,37 @@ async function executeWithClaudeCode(skill, scenario, options) {
         );
       }, SDK_TIMEOUT_MS);
     });
+    if (traceContext) {
+      heartbeatHandle = setInterval(() => {
+        const elapsedMs = Date.now() - executionStartTime;
+        const progressEvent = {
+          evalRunId: traceContext.evalRunId,
+          scenarioId: traceContext.scenarioId,
+          scenarioName: traceContext.scenarioName,
+          targetId: traceContext.targetId,
+          targetName: traceContext.targetName,
+          stepNumber: traceStepNumber,
+          type: import_evalforge_types.LiveTraceEventType.PROGRESS,
+          outputPreview: `Executing... (${Math.round(elapsedMs / 1e3)}s elapsed, ${messageCount} messages)`,
+          elapsedMs,
+          timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+          isComplete: false
+        };
+        emitTraceEvent(
+          progressEvent,
+          traceContext.tracePushUrl,
+          traceContext.routeHeader,
+          traceContext.authToken
+        );
+      }, HEARTBEAT_INTERVAL_MS);
+    }
     const sdkPromise = (async () => {
+      const evaluatorPromptSuffix = `
+
+IMPORTANT: This is an automated evaluation run. Execute the requested changes immediately without asking for confirmation. Do not ask "would you like me to proceed?" or similar questions - just implement the solution directly.`;
+      const fullPrompt = scenario.triggerPrompt + evaluatorPromptSuffix;
       for await (const message of query({
-        prompt: scenario.triggerPrompt,
+        prompt: fullPrompt,
         options: queryOptions
       })) {
         messageCount++;
@@ -6879,21 +6974,23 @@ async function executeWithClaudeCode(skill, scenario, options) {
             })
           );
         }
-        if (traceContext && isAssistantMessage(message)) {
+        if (traceContext) {
           traceStepNumber++;
-          const traceEvent = createTraceEventFromMessage(
+          const traceEvent = createTraceEventFromAnyMessage(
             message,
             traceContext,
             traceStepNumber,
             false
             // Not complete yet
           );
-          emitTraceEvent(
-            traceEvent,
-            traceContext.tracePushUrl,
-            traceContext.routeHeader,
-            traceContext.authToken
-          );
+          if (traceEvent) {
+            emitTraceEvent(
+              traceEvent,
+              traceContext.tracePushUrl,
+              traceContext.routeHeader,
+              traceContext.authToken
+            );
+          }
         }
       }
     })();
@@ -6901,6 +6998,9 @@ async function executeWithClaudeCode(skill, scenario, options) {
     if (timeoutHandle) {
       clearTimeout(timeoutHandle);
     }
+    if (heartbeatHandle) {
+      clearInterval(heartbeatHandle);
+    }
     console.log(
       "[executeWithClaudeCode] Claude Agent SDK query completed, received",
       allMessages.length,
@@ -6910,6 +7010,9 @@ async function executeWithClaudeCode(skill, scenario, options) {
     if (timeoutHandle) {
       clearTimeout(timeoutHandle);
     }
+    if (heartbeatHandle) {
+      clearInterval(heartbeatHandle);
+    }
     if (timedOut) {
       console.error("[SDK-TIMEOUT] Execution timed out:", sdkError);
     }
@@ -7453,6 +7556,32 @@ function diffSnapshots(before, after) {
   diffs.sort((a, b) => a.path.localeCompare(b.path));
   return diffs;
 }
+function extractTemplateFiles(before, after) {
+  const files = [];
+  const allPaths = /* @__PURE__ */ new Set([...Object.keys(before), ...Object.keys(after)]);
+  for (const path9 of allPaths) {
+    const beforeContent = before[path9];
+    const afterContent = after[path9];
+    if (afterContent === void 0) {
+      continue;
+    }
+    let status;
+    if (beforeContent === void 0) {
+      status = "new";
+    } else if (beforeContent !== afterContent) {
+      status = "modified";
+    } else {
+      status = "unchanged";
+    }
+    files.push({
+      path: path9,
+      content: afterContent,
+      status
+    });
+  }
+  files.sort((a, b) => a.path.localeCompare(b.path));
+  return files;
+}
 
 // src/run-scenario/callSkill.ts
 async function callSkill(config2, evalRunId2, scenario, skill, agent, workDir) {
@@ -7485,6 +7614,7 @@ async function callSkill(config2, evalRunId2, scenario, skill, agent, workDir) {
   const completedAt = (/* @__PURE__ */ new Date()).toISOString();
   const afterSnapshot = workDir ? snapshotDirectory(workDir) : {};
   const fileDiffs = diffSnapshots(beforeSnapshot, afterSnapshot);
+  const templateFiles = workDir ? extractTemplateFiles(beforeSnapshot, afterSnapshot) : void 0;
   return {
     id: (0, import_crypto2.randomUUID)(),
     targetId: skill.id,
@@ -7495,6 +7625,7 @@ async function callSkill(config2, evalRunId2, scenario, skill, agent, workDir) {
     duration: result.durationMs,
     outputText: result.outputText,
     fileDiffs: fileDiffs.length > 0 ? fileDiffs : void 0,
+    templateFiles: templateFiles && templateFiles.length > 0 ? templateFiles : void 0,
     startedAt,
     completedAt,
     llmTrace