@wix/evalforge-evaluator 0.22.0 → 0.23.0

package/build/index.mjs

@@ -6343,40 +6343,59 @@ async function executeWithClaudeCode(skill, scenario, options) {
       traceContext.authToken
     );
   }
+  const SDK_TIMEOUT_MS = Math.max(3e5, maxTurns * 6e4);
+  let timeoutHandle;
+  let timedOut = false;
   try {
-    for await (const message of query({
-      prompt: scenario.triggerPrompt,
-      options: queryOptions
-    })) {
-      messageCount++;
-      console.log("[SDK Message]", JSON.stringify(message, null, 2));
-      allMessages.push(message);
-      if (messageCount <= 3) {
-        console.error(
-          "[DEBUG-H5] SDK message received",
-          JSON.stringify({
-            messageCount,
-            type: message.type,
-            timestamp: Date.now()
-          })
-        );
-      }
-      if (traceContext && isAssistantMessage(message)) {
-        traceStepNumber++;
-        const traceEvent = createTraceEventFromMessage(
-          message,
-          traceContext,
-          traceStepNumber,
-          false
-          // Not complete yet
-        );
-        emitTraceEvent(
-          traceEvent,
-          traceContext.tracePushUrl,
-          traceContext.routeHeader,
-          traceContext.authToken
+    const timeoutPromise = new Promise((_, reject) => {
+      timeoutHandle = setTimeout(() => {
+        timedOut = true;
+        reject(
+          new Error(
+            `SDK execution timed out after ${SDK_TIMEOUT_MS}ms. Skill: ${skill.name}, Scenario: ${scenario.name}, Messages received: ${messageCount}, MaxTurns: ${maxTurns}`
+          )
         );
+      }, SDK_TIMEOUT_MS);
+    });
+    const sdkPromise = (async () => {
+      for await (const message of query({
+        prompt: scenario.triggerPrompt,
+        options: queryOptions
+      })) {
+        messageCount++;
+        console.log("[SDK Message]", JSON.stringify(message, null, 2));
+        allMessages.push(message);
+        if (messageCount <= 3) {
+          console.error(
+            "[DEBUG-H5] SDK message received",
+            JSON.stringify({
+              messageCount,
+              type: message.type,
+              timestamp: Date.now()
+            })
+          );
+        }
+        if (traceContext && isAssistantMessage(message)) {
+          traceStepNumber++;
+          const traceEvent = createTraceEventFromMessage(
+            message,
+            traceContext,
+            traceStepNumber,
+            false
+            // Not complete yet
+          );
+          emitTraceEvent(
+            traceEvent,
+            traceContext.tracePushUrl,
+            traceContext.routeHeader,
+            traceContext.authToken
+          );
+        }
       }
+    })();
+    await Promise.race([sdkPromise, timeoutPromise]);
+    if (timeoutHandle) {
+      clearTimeout(timeoutHandle);
     }
     console.log(
       "[executeWithClaudeCode] Claude Agent SDK query completed, received",
@@ -6384,6 +6403,12 @@ async function executeWithClaudeCode(skill, scenario, options) {
       "messages"
     );
   } catch (sdkError) {
+    if (timeoutHandle) {
+      clearTimeout(timeoutHandle);
+    }
+    if (timedOut) {
+      console.error("[SDK-TIMEOUT] Execution timed out:", sdkError);
+    }
     console.error("[SDK-ERROR] ====== CLAUDE SDK EXECUTION FAILED ======");
     console.error("[SDK-ERROR] Timestamp:", (/* @__PURE__ */ new Date()).toISOString());
     console.error(
@@ -7425,33 +7450,14 @@ async function testClaudeDirectExecution(config) {
   const versionResult = await runAndLog(
     "claude --version",
     `${envExports} && "${claudePath}" --version 2>&1`,
-    15e3
-  );
-  const fullCmdResult = await runAndLog(
-    "claude -p (with json output)",
-    `${envExports} && "${claudePath}" -p "Say hello" --output-format json 2>&1`,
-    45e3
-  );
-  const simpleCmdResult = await runAndLog(
-    "claude -p (simple)",
-    `${envExports} && "${claudePath}" -p "Hello" 2>&1`,
-    45e3
+    1e4
+    // Short timeout - should complete in <2s
   );
-  const printFlagResult = await runAndLog(
-    "claude --print (long flag)",
-    `${envExports} && "${claudePath}" --print "Hi" 2>&1`,
-    45e3
-  );
-  const positionalResult = await runAndLog(
-    'claude "prompt" (positional)',
-    `${envExports} && "${claudePath}" "Hello world" 2>&1`,
-    45e3
-  );
-  await runAndLog("claude --help", `"${claudePath}" --help 2>&1`, 15e3);
+  await runAndLog("claude --help", `"${claudePath}" --help 2>&1`, 1e4);
   await runAndLog(
     "claude --version (no custom env)",
     `"${claudePath}" --version 2>&1`,
-    15e3
+    1e4
   );
   const homeDir = process.env.HOME || "/tmp";
   const claudeConfigDir = path9.join(homeDir, ".claude");
@@ -7461,35 +7467,110 @@ async function testClaudeDirectExecution(config) {
     try {
       const configContents = fs11.readdirSync(claudeConfigDir);
       details.claudeConfigContents = configContents;
-      for (const file of configContents) {
-        if (file.includes("log") || file.includes("error")) {
-          const logPath = path9.join(claudeConfigDir, file);
-          const catCmd = `cat "${logPath}" 2>&1 | tail -50`;
-          const logContent = await execCommand(catCmd);
-          details[`claudeLogFile_${file}`] = logContent.stdout.slice(0, 1e3);
-        }
-      }
     } catch (e) {
       details.claudeConfigError = e instanceof Error ? e.message : String(e);
     }
   }
   details.commandResults = commandResults;
-  const anyPromptWorked = fullCmdResult.exitCode === 0 || simpleCmdResult.exitCode === 0 || printFlagResult.exitCode === 0 || positionalResult.exitCode === 0;
   const versionWorked = versionResult.exitCode === 0;
-  const passed = anyPromptWorked;
-  let errorMsg;
-  if (!passed) {
-    const failedCmds = commandResults.filter((r) => r.exitCode !== 0).map((r) => `${r.name}: exit=${r.exitCode}`).join(", ");
-    errorMsg = `All Claude CLI commands failed. ${failedCmds}. Version works: ${versionWorked}`;
-  }
+  const passed = versionWorked;
   return {
-    name: "claude-direct-execution",
+    name: "claude-cli-basic",
     passed,
     details,
-    error: errorMsg,
+    error: passed ? void 0 : `Claude CLI --version failed with exit code ${versionResult.exitCode}`,
     durationMs: Date.now() - start
   };
 }
+async function testClaudeSdkExecution(config) {
+  const start = Date.now();
+  const details = {};
+  const SDK_TIMEOUT_MS = 3e4;
+  try {
+    console.error("[SDK-DIAG] Importing Claude Agent SDK...");
+    const sdk = await import("@anthropic-ai/claude-agent-sdk");
+    details.sdkImported = true;
+    const env = { ...process.env };
+    const placeholderApiKey = "sk-ant-api03-placeholder-auth-handled-by-gateway-000000000000000000000000";
+    env.ANTHROPIC_API_KEY = placeholderApiKey;
+    env.ANTHROPIC_AUTH_TOKEN = placeholderApiKey;
+    if (config.aiGatewayUrl) {
+      env.ANTHROPIC_BASE_URL = config.aiGatewayUrl;
+    }
+    if (config.aiGatewayHeaders) {
+      const headerLines = Object.entries(config.aiGatewayHeaders).map(([key, value]) => `${key}:${value}`).join("\n");
+      env.ANTHROPIC_CUSTOM_HEADERS = headerLines;
+    }
+    details.envConfigured = true;
+    details.hasBaseUrl = !!env.ANTHROPIC_BASE_URL;
+    details.hasCustomHeaders = !!env.ANTHROPIC_CUSTOM_HEADERS;
+    const canUseTool = async () => {
+      console.error("[SDK-DIAG] canUseTool callback called - returning allow");
+      return { behavior: "allow" };
+    };
+    const queryOptions = {
+      env,
+      cwd: "/tmp",
+      settingSources: ["project"],
+      allowedTools: ["Read", "Write", "Edit", "Bash", "Glob", "Grep"],
+      model: "claude-3-5-sonnet-latest",
+      maxTurns: 1,
+      // Just one turn for this test
+      permissionMode: "default",
+      canUseTool
+    };
+    details.queryOptionsConfigured = true;
+    console.error("[SDK-DIAG] Starting SDK query with canUseTool...");
+    const timeoutPromise = new Promise((_, reject) => {
+      setTimeout(
+        () => reject(new Error(`SDK test timed out after ${SDK_TIMEOUT_MS}ms`)),
+        SDK_TIMEOUT_MS
+      );
+    });
+    const sdkPromise = (async () => {
+      let messageCount2 = 0;
+      let resultMessage2 = null;
+      for await (const message of sdk.query({
+        prompt: 'Say "SDK test successful" and nothing else.',
+        ...queryOptions
+      })) {
+        messageCount2++;
+        console.error(`[SDK-DIAG] Received message ${messageCount2}:`, message);
+        if (message.type === "result" || message.type === "assistant") {
+          resultMessage2 = message;
+        }
+      }
+      return { messageCount: messageCount2, resultMessage: resultMessage2 };
+    })();
+    const { messageCount, resultMessage } = await Promise.race([
+      sdkPromise,
+      timeoutPromise
+    ]);
+    details.messageCount = messageCount;
+    details.hasResultMessage = !!resultMessage;
+    details.resultPreview = resultMessage && typeof resultMessage === "object" ? JSON.stringify(resultMessage).slice(0, 500) : null;
+    const passed = messageCount > 0;
+    return {
+      name: "claude-sdk-execution",
+      passed,
+      details,
+      error: passed ? void 0 : "SDK query completed but returned no messages",
+      durationMs: Date.now() - start
+    };
+  } catch (err) {
+    const error = err instanceof Error ? err.message : String(err);
+    details.error = error;
+    details.errorStack = err instanceof Error ? err.stack?.split("\n").slice(0, 5) : void 0;
+    console.error("[SDK-DIAG] SDK test failed:", error);
+    return {
+      name: "claude-sdk-execution",
+      passed: false,
+      details,
+      error: `SDK execution failed: ${error}`,
+      durationMs: Date.now() - start
+    };
+  }
+}
 async function testChildProcessSpawning() {
   const start = Date.now();
   const details = {};
@@ -7694,15 +7775,13 @@ async function runDiagnostics(config, evalRunId2) {
   await runTest("claude-cli-execution", testClaudeExecution);
   await runTest("environment-dump", testEnvironmentDump);
   await runTest("file-system-structure", testFileSystemStructure);
-  await runTest("network-connectivity", () => testNetworkConnectivity(config));
-  await runTest("ai-gateway-api-call", () => testAiGatewayApiCall(config));
-  await runTest(
-    "claude-direct-execution",
-    () => testClaudeDirectExecution(config)
-  );
   await runTest("child-process-spawning", testChildProcessSpawning);
-  await runTest("sdk-import", testSdkImport);
   await runTest("file-system-write", testFileSystemWrite);
+  await runTest("sdk-import", testSdkImport);
+  await runTest("network-connectivity", () => testNetworkConnectivity(config));
+  await runTest("ai-gateway-api-call", () => testAiGatewayApiCall(config));
+  await runTest("claude-cli-basic", () => testClaudeDirectExecution(config));
+  await runTest("claude-sdk-execution", () => testClaudeSdkExecution(config));
   const completedAt = (/* @__PURE__ */ new Date()).toISOString();
   const totalDurationMs = Date.now() - startTime;
   const report = {