@wix/evalforge-evaluator 0.9.0 → 0.11.0

package/build/index.mjs

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 
 // src/index.ts
-import { EvalStatus } from "@wix/evalforge-types";
+import { EvalStatus as EvalStatus2 } from "@wix/evalforge-types";
 
 // src/config.ts
 function loadConfig() {
@@ -6251,45 +6251,94 @@ async function executeWithClaudeCode(skill, scenario, options) {
   if (options.maxTokens !== void 0) {
     queryOptions.maxTokens = options.maxTokens;
   }
-  for await (const message of query({
-    prompt: scenario.triggerPrompt,
-    options: queryOptions
-  })) {
-    messageCount++;
-    console.log("[SDK Message]", JSON.stringify(message, null, 2));
-    allMessages.push(message);
-    if (messageCount <= 3) {
-      console.error(
-        "[DEBUG-H5] SDK message received",
-        JSON.stringify({
-          messageCount,
-          type: message.type,
-          timestamp: Date.now()
-        })
-      );
+  try {
+    for await (const message of query({
+      prompt: scenario.triggerPrompt,
+      options: queryOptions
+    })) {
+      messageCount++;
+      console.log("[SDK Message]", JSON.stringify(message, null, 2));
+      allMessages.push(message);
+      if (messageCount <= 3) {
+        console.error(
+          "[DEBUG-H5] SDK message received",
+          JSON.stringify({
+            messageCount,
+            type: message.type,
+            timestamp: Date.now()
+          })
+        );
+      }
+      if (traceContext && isAssistantMessage(message)) {
+        traceStepNumber++;
+        const traceEvent = createTraceEventFromMessage(
+          message,
+          traceContext,
+          traceStepNumber,
+          false
+          // Not complete yet
+        );
+        emitTraceEvent(
+          traceEvent,
+          traceContext.tracePushUrl,
+          traceContext.routeHeader,
+          traceContext.authToken
+        );
+      }
     }
-    if (traceContext && isAssistantMessage(message)) {
-      traceStepNumber++;
-      const traceEvent = createTraceEventFromMessage(
-        message,
-        traceContext,
-        traceStepNumber,
-        false
-        // Not complete yet
-      );
-      emitTraceEvent(
-        traceEvent,
-        traceContext.tracePushUrl,
-        traceContext.routeHeader,
-        traceContext.authToken
-      );
+    console.log(
+      "[executeWithClaudeCode] Claude Agent SDK query completed, received",
+      allMessages.length,
+      "messages"
+    );
+  } catch (sdkError) {
+    const errorMessage = sdkError instanceof Error ? sdkError.message : String(sdkError);
+    const errorStack = sdkError instanceof Error ? sdkError.stack : void 0;
+    console.error("[executeWithClaudeCode] Claude SDK execution FAILED");
+    console.error("[executeWithClaudeCode] Error message:", errorMessage);
+    if (errorStack) {
+      console.error("[executeWithClaudeCode] Stack trace:", errorStack);
+    }
+    if (sdkError && typeof sdkError === "object") {
+      const errObj = sdkError;
+      const extraInfo = {};
+      for (const key of [
+        "code",
+        "status",
+        "stderr",
+        "stdout",
+        "exitCode",
+        "signal",
+        "cause"
+      ]) {
+        if (key in errObj && errObj[key] !== void 0) {
+          extraInfo[key] = errObj[key];
+        }
+      }
+      if (Object.keys(extraInfo).length > 0) {
+        console.error(
+          "[executeWithClaudeCode] Additional error info:",
+          JSON.stringify(extraInfo)
+        );
+      }
     }
+    console.error(
+      "[executeWithClaudeCode] Context:",
+      JSON.stringify({
+        skillId: skill.id,
+        skillName: skill.name,
+        scenarioId: scenario.id,
+        scenarioName: scenario.name,
+        messagesReceived: messageCount,
+        cwd: options.cwd,
+        model: options.model || DEFAULT_MODEL
+      })
+    );
+    throw new Error(
+      `Claude SDK execution failed after ${messageCount} messages: ${errorMessage}` + (errorStack ? `
+Stack: ${errorStack.split("\n").slice(0, 3).join("\n")}` : "")
+    );
   }
-  console.log(
-    "[executeWithClaudeCode] Claude Agent SDK query completed, received",
-    allMessages.length,
-    "messages"
-  );
   if (traceContext) {
     emitTraceEvent(
       {
@@ -6603,6 +6652,67 @@ async function runScenario(config, evalRunId2, scenario, target, template) {
   };
 }
 
+// src/error-reporter.ts
+import { EvalStatus } from "@wix/evalforge-types";
+function formatError(error, phase, context) {
+  const timestamp = (/* @__PURE__ */ new Date()).toISOString();
+  if (error instanceof Error) {
+    return {
+      message: error.message,
+      stack: error.stack,
+      errorType: error.constructor.name,
+      phase,
+      context,
+      timestamp
+    };
+  }
+  return {
+    message: String(error),
+    errorType: typeof error,
+    phase,
+    context,
+    timestamp
+  };
+}
+function formatErrorForJobError(details) {
+  const parts = [];
+  if (details.phase) {
+    parts.push(`[Phase: ${details.phase}]`);
+  }
+  if (details.errorType && details.errorType !== "Error") {
+    parts.push(`${details.errorType}: ${details.message}`);
+  } else {
+    parts.push(details.message);
+  }
+  if (details.context && Object.keys(details.context).length > 0) {
+    parts.push(`
+Context: ${JSON.stringify(details.context)}`);
+  }
+  if (details.stack) {
+    const stackLines = details.stack.split("\n").slice(0, 6);
+    parts.push(`
+Stack:
+${stackLines.join("\n")}`);
+  }
+  return parts.join(" ");
+}
+var ExecutionPhase = {
+  CONFIG: "config-loading",
+  API_CLIENT: "api-client-creation",
+  FETCH_EVAL_RUN: "fetch-eval-run",
+  FETCH_SKILLS: "fetch-skills",
+  FETCH_AGENT: "fetch-agent",
+  FETCH_SCENARIOS: "fetch-scenarios",
+  VALIDATION: "validation",
+  PREPARE_WORKSPACE: "prepare-workspace",
+  EXECUTE_SKILL: "execute-skill",
+  EXECUTE_AGENT: "execute-agent",
+  CLAUDE_SDK_IMPORT: "claude-sdk-import",
+  CLAUDE_SDK_EXECUTION: "claude-sdk-execution",
+  ADD_RESULT: "add-result",
+  UPDATE_STATUS: "update-status"
+};
+
 // src/index.ts
 console.error(
   "[EVALUATOR-BOOT] Module loading started",
@@ -6610,13 +6720,22 @@ console.error(
 );
 console.error("[EVALUATOR-BOOT] All static imports successful");
 async function runEvaluation(projectId2, evalRunId2) {
+  const state = {
+    config: null,
+    api: null,
+    currentPhase: ExecutionPhase.CONFIG,
+    currentContext: { projectId: projectId2, evalRunId: evalRunId2 }
+  };
   console.error(
     "[DEBUG-H1] runEvaluation entry",
     JSON.stringify({ projectId: projectId2, evalRunId: evalRunId2, timestamp: Date.now() })
   );
+  state.currentPhase = ExecutionPhase.CONFIG;
+  state.currentContext = { projectId: projectId2, evalRunId: evalRunId2 };
   let config;
   try {
     config = loadConfig();
+    state.config = config;
     console.error(
       "[DEBUG-H1] loadConfig SUCCESS",
       JSON.stringify({
@@ -6632,10 +6751,13 @@ async function runEvaluation(projectId2, evalRunId2) {
       "[DEBUG-H1] loadConfig FAILED",
       JSON.stringify({
         error: configErr instanceof Error ? configErr.message : String(configErr),
+        stack: configErr instanceof Error ? configErr.stack : void 0,
         timestamp: Date.now()
       })
     );
-    throw configErr;
+    throw new Error(
+      `[${ExecutionPhase.CONFIG}] ${configErr instanceof Error ? configErr.message : String(configErr)}`
+    );
   }
   console.log("[Evaluator] Config loaded", {
     serverUrl: config.serverUrl,
@@ -6644,11 +6766,22 @@ async function runEvaluation(projectId2, evalRunId2) {
     hasAiGatewayHeaders: Object.keys(config.aiGatewayHeaders).length > 0,
     hasRouteHeader: !!config.routeHeader
   });
-  const api = createApiClient(config.serverUrl, {
-    apiPrefix: config.apiPrefix,
-    routeHeader: config.routeHeader,
-    authToken: config.authToken
-  });
+  state.currentPhase = ExecutionPhase.API_CLIENT;
+  let api;
+  try {
+    api = createApiClient(config.serverUrl, {
+      apiPrefix: config.apiPrefix,
+      routeHeader: config.routeHeader,
+      authToken: config.authToken
+    });
+    state.api = api;
+  } catch (apiErr) {
+    throw new Error(
+      `[${ExecutionPhase.API_CLIENT}] Failed to create API client: ${apiErr instanceof Error ? apiErr.message : String(apiErr)}`
+    );
+  }
+  state.currentPhase = ExecutionPhase.FETCH_EVAL_RUN;
+  state.currentContext = { projectId: projectId2, evalRunId: evalRunId2, serverUrl: config.serverUrl };
   console.error(
     "[DEBUG-H2] fetchEvaluationData START",
     JSON.stringify({ serverUrl: config.serverUrl, timestamp: Date.now() })
@@ -6667,32 +6800,61 @@ async function runEvaluation(projectId2, evalRunId2) {
       })
     );
   } catch (fetchErr) {
+    const errorMsg = fetchErr instanceof Error ? fetchErr.message : String(fetchErr);
     console.error(
       "[DEBUG-H2] fetchEvaluationData FAILED",
       JSON.stringify({
-        error: fetchErr instanceof Error ? fetchErr.message : String(fetchErr),
+        error: errorMsg,
+        stack: fetchErr instanceof Error ? fetchErr.stack : void 0,
         timestamp: Date.now()
       })
     );
-    throw fetchErr;
+    throw new Error(
+      `[${ExecutionPhase.FETCH_EVAL_RUN}] Failed to fetch evaluation data: ${errorMsg}`
+    );
   }
   const { codeAgent, skills, scenarioItems } = evalData;
+  state.currentPhase = ExecutionPhase.VALIDATION;
+  state.currentContext = {
+    projectId: projectId2,
+    evalRunId: evalRunId2,
+    scenarioCount: scenarioItems.length,
+    skillCount: skills.length,
+    hasAgent: !!codeAgent,
+    agentId: evalData.evalRun.agentId,
+    skillsGroupId: evalData.evalRun.skillsGroupId
+  };
   if (scenarioItems.length > 0 && skills.length === 0) {
     throw new Error(
-      "Eval run has no skills: set skillsGroupId and ensure the group has skills"
+      `[${ExecutionPhase.VALIDATION}] Eval run has no skills: set skillsGroupId and ensure the group has skills. (skillsGroupId: ${evalData.evalRun.skillsGroupId || "not set"})`
     );
   }
   if (scenarioItems.length > 0 && skills.length > 0 && !codeAgent) {
     throw new Error(
-      "Eval run has no code agent: set agentId for skill-based runs"
+      `[${ExecutionPhase.VALIDATION}] Eval run has no code agent: set agentId for skill-based runs. (agentId: ${evalData.evalRun.agentId || "not set"})`
     );
   }
+  let completedScenarios = 0;
+  const totalScenarios = scenarioItems.length * skills.length;
   for (const { scenario, template } of scenarioItems) {
     for (const skill of skills) {
+      state.currentPhase = ExecutionPhase.EXECUTE_SKILL;
+      state.currentContext = {
+        projectId: projectId2,
+        evalRunId: evalRunId2,
+        scenarioId: scenario.id,
+        scenarioName: scenario.name,
+        skillId: skill.id,
+        skillName: skill.name,
+        agentId: codeAgent?.id,
+        agentName: codeAgent?.name,
+        progress: `${completedScenarios + 1}/${totalScenarios}`
+      };
       console.log(
         "[Evaluator] Running skill:",
         skill.name,
-        codeAgent ? `with agent: ${codeAgent.name}` : ""
+        codeAgent ? `with agent: ${codeAgent.name}` : "",
+        `(${completedScenarios + 1}/${totalScenarios})`
       );
       try {
         const result = await runScenario(
@@ -6703,17 +6865,48 @@ async function runEvaluation(projectId2, evalRunId2) {
           template
         );
         console.log("[Evaluator] Skill completed, adding result");
+        state.currentPhase = ExecutionPhase.ADD_RESULT;
+        state.currentContext = {
+          ...state.currentContext,
+          resultId: result.id
+        };
         await api.addResult(projectId2, evalRunId2, result);
+        completedScenarios++;
       } catch (err) {
-        console.error("[Evaluator] Failed to run skill:", skill.name, err);
-        throw err;
+        const errorMsg = err instanceof Error ? err.message : String(err);
+        const errorStack = err instanceof Error ? err.stack : void 0;
+        console.error(
+          "[Evaluator] Failed to run skill:",
+          skill.name,
+          "Error:",
+          errorMsg
+        );
+        if (errorStack) {
+          console.error("[Evaluator] Stack trace:", errorStack);
+        }
+        throw new Error(
+          `[${state.currentPhase}] Failed to execute skill "${skill.name}" on scenario "${scenario.name}": ${errorMsg}`
+        );
       }
     }
   }
-  await api.updateEvalRun(projectId2, evalRunId2, {
-    status: EvalStatus.COMPLETED,
-    completedAt: (/* @__PURE__ */ new Date()).toISOString()
-  });
+  state.currentPhase = ExecutionPhase.UPDATE_STATUS;
+  state.currentContext = {
+    projectId: projectId2,
+    evalRunId: evalRunId2,
+    completedScenarios,
+    totalScenarios
+  };
+  try {
+    await api.updateEvalRun(projectId2, evalRunId2, {
+      status: EvalStatus2.COMPLETED,
+      completedAt: (/* @__PURE__ */ new Date()).toISOString()
+    });
+  } catch (updateErr) {
+    throw new Error(
+      `[${ExecutionPhase.UPDATE_STATUS}] Failed to update eval run status to COMPLETED: ${updateErr instanceof Error ? updateErr.message : String(updateErr)}`
+    );
+  }
 }
 var projectId = process.argv[2];
 var evalRunId = process.argv[3];
@@ -6729,7 +6922,16 @@ runEvaluation(projectId, evalRunId).then(() => {
   console.error("[EVALUATOR-BOOT] runEvaluation completed successfully");
   process.exit(0);
 }).catch(async (err) => {
-  console.error("[EVALUATOR-BOOT] runEvaluation FAILED:", err);
+  const errorDetails = formatError(err, "main-execution", {
+    projectId,
+    evalRunId
+  });
+  const jobError = formatErrorForJobError(errorDetails);
+  console.error("[EVALUATOR-BOOT] runEvaluation FAILED");
+  console.error(
+    "[EVALUATOR-BOOT] Error details:",
+    JSON.stringify(errorDetails, null, 2)
+  );
   try {
     const config = loadConfig();
     const api = createApiClient(config.serverUrl, {
@@ -6738,15 +6940,42 @@ runEvaluation(projectId, evalRunId).then(() => {
       authToken: config.authToken
     });
     await api.updateEvalRun(projectId, evalRunId, {
-      status: EvalStatus.FAILED,
-      completedAt: (/* @__PURE__ */ new Date()).toISOString()
+      status: EvalStatus2.FAILED,
+      completedAt: (/* @__PURE__ */ new Date()).toISOString(),
+      jobError,
+      jobStatus: "FAILED"
     });
-    console.error("[EVALUATOR-BOOT] Updated eval run status to FAILED");
+    console.error(
+      "[EVALUATOR-BOOT] Updated eval run status to FAILED with error details"
+    );
   } catch (updateErr) {
     console.error(
       "[EVALUATOR-BOOT] Failed to update eval run status:",
-      updateErr
+      updateErr instanceof Error ? updateErr.message : String(updateErr)
     );
+    try {
+      const serverUrl = process.env.EVAL_SERVER_URL;
+      const authToken = process.env.EVAL_AUTH_TOKEN;
+      const routeHeader = process.env.EVAL_ROUTE_HEADER;
+      if (serverUrl) {
+        const api = createApiClient(serverUrl, {
+          routeHeader,
+          authToken
+        });
+        await api.updateEvalRun(projectId, evalRunId, {
+          status: EvalStatus2.FAILED,
+          completedAt: (/* @__PURE__ */ new Date()).toISOString(),
+          jobError: `Config load failed, then: ${jobError}`,
+          jobStatus: "FAILED"
+        });
+        console.error("[EVALUATOR-BOOT] Fallback: Updated status to FAILED");
+      }
+    } catch (fallbackErr) {
+      console.error(
+        "[EVALUATOR-BOOT] Fallback also failed:",
+        fallbackErr instanceof Error ? fallbackErr.message : String(fallbackErr)
+      );
+    }
   }
   process.exit(1);
 });