@wix/evalforge-evaluator 0.12.0 → 0.14.0

package/build/index.mjs

@@ -6216,10 +6216,45 @@ async function executeWithClaudeCode(skill, scenario, options) {
     );
   }
   const sdkEnv = buildSdkEnvironment(options);
-  console.log("[executeWithClaudeCode] SDK environment built", {
-    hasAnthropicBaseUrl: !!sdkEnv.ANTHROPIC_BASE_URL,
-    hasAnthropicCustomHeaders: !!sdkEnv.ANTHROPIC_CUSTOM_HEADERS
-  });
+  console.log("[SDK-DEBUG] ====== CLAUDE SDK INITIALIZATION ======");
+  console.log("[SDK-DEBUG] Timestamp:", (/* @__PURE__ */ new Date()).toISOString());
+  console.log("[SDK-DEBUG] Node version:", process.version);
+  console.log("[SDK-DEBUG] Platform:", process.platform, process.arch);
+  console.log("[SDK-DEBUG] CWD:", options.cwd);
+  console.log("[SDK-DEBUG] Process CWD:", process.cwd());
+  console.log("[SDK-DEBUG] SDK Environment:");
+  console.log(
+    "[SDK-DEBUG]   ANTHROPIC_API_KEY:",
+    sdkEnv.ANTHROPIC_API_KEY ? `${sdkEnv.ANTHROPIC_API_KEY.substring(0, 15)}...` : "NOT SET"
+  );
+  console.log(
+    "[SDK-DEBUG]   ANTHROPIC_AUTH_TOKEN:",
+    sdkEnv.ANTHROPIC_AUTH_TOKEN ? `${sdkEnv.ANTHROPIC_AUTH_TOKEN.substring(0, 15)}...` : "NOT SET"
+  );
+  console.log(
+    "[SDK-DEBUG]   ANTHROPIC_BASE_URL:",
+    sdkEnv.ANTHROPIC_BASE_URL || "NOT SET"
+  );
+  console.log(
+    "[SDK-DEBUG]   ANTHROPIC_CUSTOM_HEADERS present:",
+    !!sdkEnv.ANTHROPIC_CUSTOM_HEADERS
+  );
+  if (sdkEnv.ANTHROPIC_CUSTOM_HEADERS) {
+    const headerNames = sdkEnv.ANTHROPIC_CUSTOM_HEADERS.split("\n").map((h) => {
+      const colonIdx = h.indexOf(":");
+      return colonIdx > 0 ? h.substring(0, colonIdx) : h;
+    }).join(", ");
+    console.log("[SDK-DEBUG]   ANTHROPIC_CUSTOM_HEADERS keys:", headerNames);
+  }
+  console.log("[SDK-DEBUG]   PATH available:", !!sdkEnv.PATH);
+  console.log("[SDK-DEBUG]   HOME:", sdkEnv.HOME || "NOT SET");
+  console.log("[SDK-DEBUG] Skill:", skill.id, "-", skill.name);
+  console.log("[SDK-DEBUG] Scenario:", scenario.id, "-", scenario.name);
+  console.log(
+    "[SDK-DEBUG] Prompt preview:",
+    scenario.triggerPrompt.substring(0, 100) + "..."
+  );
+  console.log("[SDK-DEBUG] ============================================");
   let traceStepNumber = 0;
   const traceContext = options.traceContext;
   const maxTurns = options.maxTurns ?? 10;
@@ -6251,6 +6286,27 @@ async function executeWithClaudeCode(skill, scenario, options) {
   if (options.maxTokens !== void 0) {
     queryOptions.maxTokens = options.maxTokens;
   }
+  console.log("[SDK-DEBUG] Query options:");
+  console.log("[SDK-DEBUG]   model:", queryOptions.model);
+  console.log("[SDK-DEBUG]   maxTurns:", queryOptions.maxTurns);
+  console.log(
+    "[SDK-DEBUG]   maxThinkingTokens:",
+    queryOptions.maxThinkingTokens
+  );
+  console.log("[SDK-DEBUG]   temperature:", queryOptions.temperature);
+  console.log("[SDK-DEBUG]   maxTokens:", queryOptions.maxTokens);
+  console.log("[SDK-DEBUG]   permissionMode:", queryOptions.permissionMode);
+  console.log(
+    "[SDK-DEBUG]   allowDangerouslySkipPermissions:",
+    queryOptions.allowDangerouslySkipPermissions
+  );
+  console.log("[SDK-DEBUG]   settingSources:", queryOptions.settingSources);
+  console.log("[SDK-DEBUG]   allowedTools:", queryOptions.allowedTools);
+  console.log(
+    "[SDK-DEBUG]   mcpServers:",
+    queryOptions.mcpServers ? Object.keys(queryOptions.mcpServers) : "none"
+  );
+  console.log("[SDK-DEBUG] Calling SDK query()...");
   try {
     for await (const message of query({
       prompt: scenario.triggerPrompt,
@@ -6292,51 +6348,149 @@ async function executeWithClaudeCode(skill, scenario, options) {
       "messages"
     );
   } catch (sdkError) {
+    console.error("[SDK-ERROR] ====== CLAUDE SDK EXECUTION FAILED ======");
+    console.error("[SDK-ERROR] Timestamp:", (/* @__PURE__ */ new Date()).toISOString());
+    console.error(
+      "[SDK-ERROR] Messages received before failure:",
+      messageCount
+    );
     const errorMessage = sdkError instanceof Error ? sdkError.message : String(sdkError);
     const errorStack = sdkError instanceof Error ? sdkError.stack : void 0;
-    console.error("[executeWithClaudeCode] Claude SDK execution FAILED");
-    console.error("[executeWithClaudeCode] Error message:", errorMessage);
+    const errorName = sdkError instanceof Error ? sdkError.name : "Unknown";
+    console.error("[SDK-ERROR] Error name:", errorName);
+    console.error("[SDK-ERROR] Error message:", errorMessage);
     if (errorStack) {
-      console.error("[executeWithClaudeCode] Stack trace:", errorStack);
+      console.error("[SDK-ERROR] Full stack trace:");
+      console.error(errorStack);
     }
     if (sdkError && typeof sdkError === "object") {
-      const errObj = sdkError;
-      const extraInfo = {};
-      for (const key of [
+      const errObj2 = sdkError;
+      console.error("[SDK-ERROR] Error object properties:");
+      for (const key of Object.keys(errObj2)) {
+        const value = errObj2[key];
+        if (value !== void 0 && key !== "stack") {
+          try {
+            const valueStr = typeof value === "object" ? JSON.stringify(value, null, 2) : String(value);
+            console.error(`[SDK-ERROR]   ${key}:`, valueStr.substring(0, 1e3));
+          } catch {
+            console.error(`[SDK-ERROR]   ${key}: [cannot serialize]`);
+          }
+        }
+      }
+      const sdkErrorKeys2 = [
         "code",
         "status",
+        "statusCode",
+        "statusText",
         "stderr",
         "stdout",
+        "output",
         "exitCode",
         "signal",
-        "cause"
-      ]) {
-        if (key in errObj && errObj[key] !== void 0) {
-          extraInfo[key] = errObj[key];
+        "killed",
+        "cause",
+        "reason",
+        "details",
+        "response",
+        "request",
+        "config",
+        "errno",
+        "syscall",
+        "path",
+        "spawnargs"
+      ];
+      const extraInfo = {};
+      for (const key of sdkErrorKeys2) {
+        if (key in errObj2 && errObj2[key] !== void 0) {
+          extraInfo[key] = errObj2[key];
         }
       }
       if (Object.keys(extraInfo).length > 0) {
-        console.error(
-          "[executeWithClaudeCode] Additional error info:",
-          JSON.stringify(extraInfo)
-        );
+        console.error("[SDK-ERROR] SDK-specific error details:");
+        console.error(JSON.stringify(extraInfo, null, 2));
+      }
+      if (errObj2.cause && typeof errObj2.cause === "object") {
+        console.error("[SDK-ERROR] Error cause:");
+        try {
+          console.error(JSON.stringify(errObj2.cause, null, 2));
+        } catch {
+          console.error("[SDK-ERROR] Error cause: [cannot serialize]");
+        }
       }
     }
+    console.error("[SDK-ERROR] Execution context:");
+    console.error("[SDK-ERROR]   skillId:", skill.id);
+    console.error("[SDK-ERROR]   skillName:", skill.name);
+    console.error("[SDK-ERROR]   scenarioId:", scenario.id);
+    console.error("[SDK-ERROR]   scenarioName:", scenario.name);
+    console.error("[SDK-ERROR]   cwd:", options.cwd);
+    console.error("[SDK-ERROR]   model:", options.model || DEFAULT_MODEL);
+    console.error("[SDK-ERROR]   aiGatewayUrl:", options.aiGatewayUrl);
     console.error(
-      "[executeWithClaudeCode] Context:",
-      JSON.stringify({
-        skillId: skill.id,
-        skillName: skill.name,
-        scenarioId: scenario.id,
-        scenarioName: scenario.name,
-        messagesReceived: messageCount,
-        cwd: options.cwd,
-        model: options.model || DEFAULT_MODEL
-      })
+      "[SDK-ERROR]   hasAiGatewayHeaders:",
+      !!options.aiGatewayHeaders
     );
+    console.error("[SDK-ERROR] Environment info:");
+    console.error("[SDK-ERROR]   NODE_ENV:", process.env.NODE_ENV);
+    console.error("[SDK-ERROR]   HOME:", process.env.HOME);
+    console.error("[SDK-ERROR]   USER:", process.env.USER);
+    console.error("[SDK-ERROR]   SHELL:", process.env.SHELL);
+    console.error("[SDK-ERROR] ==========================================");
+    const errObj = sdkError;
+    const sdkSpecificInfo = {};
+    const sdkErrorKeys = [
+      "exitCode",
+      "stderr",
+      "stdout",
+      "signal",
+      "killed",
+      "code",
+      "status",
+      "errno",
+      "syscall",
+      "spawnargs"
+    ];
+    for (const key of sdkErrorKeys) {
+      if (errObj && key in errObj && errObj[key] !== void 0) {
+        const val = errObj[key];
+        if (typeof val === "string" && val.length > 500) {
+          sdkSpecificInfo[key] = val.substring(0, 500) + "... [truncated]";
+        } else {
+          sdkSpecificInfo[key] = val;
+        }
+      }
+    }
+    let causeInfo;
+    if (errObj?.cause && typeof errObj.cause === "object") {
+      try {
+        const causeStr = JSON.stringify(errObj.cause, null, 2);
+        causeInfo = causeStr.length > 500 ? causeStr.substring(0, 500) + "... [truncated]" : causeStr;
+      } catch {
+        causeInfo = "[cannot serialize cause]";
+      }
+    }
+    const sdkEnvDebug = {
+      ANTHROPIC_BASE_URL: sdkEnv.ANTHROPIC_BASE_URL,
+      hasANTHROPIC_API_KEY: !!sdkEnv.ANTHROPIC_API_KEY,
+      hasANTHROPIC_AUTH_TOKEN: !!sdkEnv.ANTHROPIC_AUTH_TOKEN,
+      hasANTHROPIC_CUSTOM_HEADERS: !!sdkEnv.ANTHROPIC_CUSTOM_HEADERS,
+      ANTHROPIC_CUSTOM_HEADERS_preview: sdkEnv.ANTHROPIC_CUSTOM_HEADERS ? sdkEnv.ANTHROPIC_CUSTOM_HEADERS.split("\n").map((h) => h.split(":")[0]).join(", ") : void 0
+    };
+    const errorDetails = {
+      messageCount,
+      errorName,
+      errorMessage,
+      skillId: skill.id,
+      scenarioId: scenario.id,
+      model: options.model || DEFAULT_MODEL,
+      sdkEnv: sdkEnvDebug,
+      sdkError: Object.keys(sdkSpecificInfo).length > 0 ? sdkSpecificInfo : void 0,
+      cause: causeInfo
+    };
     throw new Error(
-      `Claude SDK execution failed after ${messageCount} messages: ${errorMessage}` + (errorStack ? `
-Stack: ${errorStack.split("\n").slice(0, 3).join("\n")}` : "")
+      `Claude SDK execution failed after ${messageCount} messages: ${errorMessage}
+Details: ${JSON.stringify(errorDetails, null, 2)}` + (errorStack ? `
+Stack: ${errorStack.split("\n").slice(0, 5).join("\n")}` : "")
     );
   }
   if (traceContext) {
@@ -6393,7 +6547,9 @@ async function writeSkillToFilesystem(cwd, skill) {
 }
 function buildSdkEnvironment(options) {
   const env = { ...process.env };
-  env.ANTHROPIC_AUTH_TOKEN = "fake-api-token";
+  const placeholderApiKey = "sk-ant-api03-placeholder-auth-handled-by-gateway-000000000000000000000000";
+  env.ANTHROPIC_API_KEY = placeholderApiKey;
+  env.ANTHROPIC_AUTH_TOKEN = placeholderApiKey;
   if (options.aiGatewayUrl) {
     env.ANTHROPIC_BASE_URL = options.aiGatewayUrl;
   }