@wix/evalforge-evaluator 0.20.0 → 0.22.0

package/build/index.js

@@ -6285,6 +6285,9 @@ async function executeWithClaudeCode(skill, scenario, options) {
     })
   );
   let messageCount = 0;
+  const canUseTool = async () => {
+    return { behavior: "allow" };
+  };
   const queryOptions = {
     env: sdkEnv,
     cwd: options.cwd,
@@ -6294,8 +6297,10 @@ async function executeWithClaudeCode(skill, scenario, options) {
     maxTurns,
     maxThinkingTokens: options.maxThinkingTokens,
     mcpServers: options.mcpServers,
-    permissionMode: "bypassPermissions",
-    allowDangerouslySkipPermissions: true
+    // Use 'default' permission mode with custom canUseTool handler
+    // instead of 'bypassPermissions' which fails on root
+    permissionMode: "default",
+    canUseTool
   };
   if (options.temperature !== void 0) {
     queryOptions.temperature = options.temperature;
@@ -6314,8 +6319,8 @@ async function executeWithClaudeCode(skill, scenario, options) {
   console.log("[SDK-DEBUG]   maxTokens:", queryOptions.maxTokens);
   console.log("[SDK-DEBUG]   permissionMode:", queryOptions.permissionMode);
   console.log(
-    "[SDK-DEBUG]   allowDangerouslySkipPermissions:",
-    queryOptions.allowDangerouslySkipPermissions
+    "[SDK-DEBUG]   canUseTool:",
+    queryOptions.canUseTool ? "custom handler (auto-allow)" : "not set"
   );
   console.log("[SDK-DEBUG]   settingSources:", queryOptions.settingSources);
   console.log("[SDK-DEBUG]   allowedTools:", queryOptions.allowedTools);
@@ -7365,6 +7370,35 @@ async function testAiGatewayApiCall(config) {
 async function testClaudeDirectExecution(config) {
   const start = Date.now();
   const details = {};
+  const commandResults = [];
+  const runAndLog = async (name2, command, timeoutMs = 3e4) => {
+    console.error(`
+[CLAUDE-DIAG] ========== ${name2} ==========`);
+    const cmdPreview = command.length > 500 ? command.slice(0, 500) + "..." : command;
+    console.error(`[CLAUDE-DIAG] Command: ${cmdPreview}`);
+    const cmdStart = Date.now();
+    const result = await execCommand(command, timeoutMs);
+    const cmdDuration = Date.now() - cmdStart;
+    console.error(`[CLAUDE-DIAG] Exit code: ${result.exitCode}`);
+    console.error(`[CLAUDE-DIAG] Duration: ${cmdDuration}ms`);
+    console.error(`[CLAUDE-DIAG] Stdout (${result.stdout.length} chars):`);
+    console.error(result.stdout || "(empty)");
+    if (result.stderr) {
+      console.error(`[CLAUDE-DIAG] Stderr (${result.stderr.length} chars):`);
+      console.error(result.stderr);
+    }
+    console.error(`[CLAUDE-DIAG] ========== END ${name2} ==========
+`);
+    commandResults.push({
+      name: name2,
+      command: command.slice(0, 300),
+      exitCode: result.exitCode,
+      stdout: result.stdout.slice(0, 1500),
+      stderr: result.stderr.slice(0, 500),
+      durationMs: cmdDuration
+    });
+    return result;
+  };
   const npmRootResult = await execCommand("npm root -g");
   const npmRoot = npmRootResult.stdout;
   const claudePath = path9.join(
@@ -7378,6 +7412,7 @@ async function testClaudeDirectExecution(config) {
   details.claudePath = claudePath;
   details.claudeExists = fs11.existsSync(claudePath);
   if (!details.claudeExists) {
+    details.commandResults = commandResults;
     return {
       name: "claude-direct-execution",
       passed: false,
@@ -7390,34 +7425,51 @@ async function testClaudeDirectExecution(config) {
   const headers = config.aiGatewayHeaders;
   details.gatewayUrl = gatewayUrl;
   details.hasHeaders = !!headers;
-  const headerLines = headers ? Object.entries(headers).map(([key, value]) => `${key}:${value}`).join("\n") : "";
-  const envVars = [
-    `ANTHROPIC_API_KEY=sk-ant-api03-placeholder-auth-via-gateway-000000000000`,
-    `ANTHROPIC_AUTH_TOKEN=sk-ant-api03-placeholder-auth-via-gateway-000000000000`,
-    `ANTHROPIC_BASE_URL=${gatewayUrl || ""}`,
-    `ANTHROPIC_CUSTOM_HEADERS="${headerLines.replace(/"/g, '\\"')}"`,
-    `HOME=${process.env.HOME || "/tmp"}`,
-    `PATH=${process.env.PATH || ""}`
+  const headerLinesEscaped = headers ? Object.entries(headers).map(([key, value]) => `${key}:${value}`).join("\\n") : "";
+  const envExportParts = [
+    `export ANTHROPIC_API_KEY="sk-ant-api03-placeholder-auth-via-gateway-000000000000"`,
+    `export ANTHROPIC_AUTH_TOKEN="sk-ant-api03-placeholder-auth-via-gateway-000000000000"`,
+    `export ANTHROPIC_BASE_URL="${gatewayUrl || ""}"`,
+    `export ANTHROPIC_CUSTOM_HEADERS="$(printf '${headerLinesEscaped}')"`,
+    // Use printf!
+    `export HOME="${process.env.HOME || "/tmp"}"`,
+    `export PATH="${process.env.PATH || ""}"`
   ];
-  const envExports = envVars.map((v) => `export ${v}`).join(" && ");
-  console.error("[DIAG] Testing claude --version with SDK environment...");
-  const versionCmd = `${envExports} && "${claudePath}" --version 2>&1`;
-  const versionResult = await execCommand(versionCmd, 15e3);
-  details.versionTest = {
-    exitCode: versionResult.exitCode,
-    stdout: versionResult.stdout.slice(0, 1e3),
-    stderr: versionResult.stderr.slice(0, 1e3)
-  };
-  console.error("[DIAG] Testing claude with simple prompt (like SDK does)...");
-  const promptCmd = `${envExports} && "${claudePath}" -p "Say hello" --output-format json --dangerously-skip-permissions 2>&1`;
-  const promptResult = await execCommand(promptCmd, 3e4);
-  details.promptTest = {
-    command: 'claude -p "Say hello" --output-format json --dangerously-skip-permissions',
-    exitCode: promptResult.exitCode,
-    stdout: promptResult.stdout.slice(0, 2e3),
-    stderr: promptResult.stderr.slice(0, 1e3)
-  };
-  console.error("[DIAG] Checking for claude error logs or state...");
+  const envExports = envExportParts.join(" && ");
+  details.envExportsPreview = envExportParts.map(
+    (e) => e.includes("SECRET") || e.includes("secret") ? e.replace(/:.+/, ":[REDACTED]") : e
+  ).join("\n");
+  const versionResult = await runAndLog(
+    "claude --version",
+    `${envExports} && "${claudePath}" --version 2>&1`,
+    15e3
+  );
+  const fullCmdResult = await runAndLog(
+    "claude -p (with json output)",
+    `${envExports} && "${claudePath}" -p "Say hello" --output-format json 2>&1`,
+    45e3
+  );
+  const simpleCmdResult = await runAndLog(
+    "claude -p (simple)",
+    `${envExports} && "${claudePath}" -p "Hello" 2>&1`,
+    45e3
+  );
+  const printFlagResult = await runAndLog(
+    "claude --print (long flag)",
+    `${envExports} && "${claudePath}" --print "Hi" 2>&1`,
+    45e3
+  );
+  const positionalResult = await runAndLog(
+    'claude "prompt" (positional)',
+    `${envExports} && "${claudePath}" "Hello world" 2>&1`,
+    45e3
+  );
+  await runAndLog("claude --help", `"${claudePath}" --help 2>&1`, 15e3);
+  await runAndLog(
+    "claude --version (no custom env)",
+    `"${claudePath}" --version 2>&1`,
+    15e3
+  );
   const homeDir = process.env.HOME || "/tmp";
   const claudeConfigDir = path9.join(homeDir, ".claude");
   details.claudeConfigDir = claudeConfigDir;
@@ -7426,24 +7478,32 @@ async function testClaudeDirectExecution(config) {
     try {
       const configContents = fs11.readdirSync(claudeConfigDir);
       details.claudeConfigContents = configContents;
+      for (const file of configContents) {
+        if (file.includes("log") || file.includes("error")) {
+          const logPath = path9.join(claudeConfigDir, file);
+          const catCmd = `cat "${logPath}" 2>&1 | tail -50`;
+          const logContent = await execCommand(catCmd);
+          details[`claudeLogFile_${file}`] = logContent.stdout.slice(0, 1e3);
+        }
+      }
     } catch (e) {
       details.claudeConfigError = e instanceof Error ? e.message : String(e);
     }
   }
-  console.error("[DIAG] Testing claude with potential debug flags...");
-  const debugCmd = `${envExports} && "${claudePath}" -p "hi" --verbose 2>&1 || echo "VERBOSE_NOT_SUPPORTED"`;
-  const debugResult = await execCommand(debugCmd, 15e3);
-  details.debugTest = {
-    exitCode: debugResult.exitCode,
-    stdout: debugResult.stdout.slice(0, 1500),
-    stderr: debugResult.stderr.slice(0, 500)
-  };
-  const passed = promptResult.exitCode === 0;
+  details.commandResults = commandResults;
+  const anyPromptWorked = fullCmdResult.exitCode === 0 || simpleCmdResult.exitCode === 0 || printFlagResult.exitCode === 0 || positionalResult.exitCode === 0;
+  const versionWorked = versionResult.exitCode === 0;
+  const passed = anyPromptWorked;
+  let errorMsg;
+  if (!passed) {
+    const failedCmds = commandResults.filter((r) => r.exitCode !== 0).map((r) => `${r.name}: exit=${r.exitCode}`).join(", ");
+    errorMsg = `All Claude CLI commands failed. ${failedCmds}. Version works: ${versionWorked}`;
+  }
   return {
     name: "claude-direct-execution",
     passed,
     details,
-    error: passed ? void 0 : `Claude CLI failed with exit code ${promptResult.exitCode}. Output: ${promptResult.stdout.slice(0, 300)}`,
+    error: errorMsg,
     durationMs: Date.now() - start
   };
 }
@@ -7653,7 +7713,10 @@ async function runDiagnostics(config, evalRunId2) {
   await runTest("file-system-structure", testFileSystemStructure);
   await runTest("network-connectivity", () => testNetworkConnectivity(config));
   await runTest("ai-gateway-api-call", () => testAiGatewayApiCall(config));
-  await runTest("claude-direct-execution", () => testClaudeDirectExecution(config));
+  await runTest(
+    "claude-direct-execution",
+    () => testClaudeDirectExecution(config)
+  );
   await runTest("child-process-spawning", testChildProcessSpawning);
   await runTest("sdk-import", testSdkImport);
   await runTest("file-system-write", testFileSystemWrite);