@wix/evalforge-evaluator 0.18.0 → 0.20.0

package/build/index.js

@@ -6103,7 +6103,7 @@ var import_evalforge_types = require("@wix/evalforge-types");
 var import_crypto = require("crypto");
 var import_promises3 = require("fs/promises");
 var import_path5 = require("path");
-var DEFAULT_MODEL = "claude-sonnet-4-20250514";
+var DEFAULT_MODEL = "claude-3-5-sonnet-latest";
 function emitTraceEvent(event, tracePushUrl, routeHeader, authToken) {
   console.log(`${import_evalforge_types.TRACE_EVENT_PREFIX}${JSON.stringify(event)}`);
   if (tracePushUrl) {
@@ -7362,6 +7362,91 @@ async function testAiGatewayApiCall(config) {
     durationMs: Date.now() - start
   };
 }
+async function testClaudeDirectExecution(config) {
+  const start = Date.now();
+  const details = {};
+  const npmRootResult = await execCommand("npm root -g");
+  const npmRoot = npmRootResult.stdout;
+  const claudePath = path9.join(
+    npmRoot,
+    "@wix",
+    "evalforge-evaluator",
+    "node_modules",
+    ".bin",
+    "claude"
+  );
+  details.claudePath = claudePath;
+  details.claudeExists = fs11.existsSync(claudePath);
+  if (!details.claudeExists) {
+    return {
+      name: "claude-direct-execution",
+      passed: false,
+      details,
+      error: `Claude binary not found at ${claudePath}`,
+      durationMs: Date.now() - start
+    };
+  }
+  const gatewayUrl = config.aiGatewayUrl;
+  const headers = config.aiGatewayHeaders;
+  details.gatewayUrl = gatewayUrl;
+  details.hasHeaders = !!headers;
+  const headerLines = headers ? Object.entries(headers).map(([key, value]) => `${key}:${value}`).join("\n") : "";
+  const envVars = [
+    `ANTHROPIC_API_KEY=sk-ant-api03-placeholder-auth-via-gateway-000000000000`,
+    `ANTHROPIC_AUTH_TOKEN=sk-ant-api03-placeholder-auth-via-gateway-000000000000`,
+    `ANTHROPIC_BASE_URL=${gatewayUrl || ""}`,
+    `ANTHROPIC_CUSTOM_HEADERS="${headerLines.replace(/"/g, '\\"')}"`,
+    `HOME=${process.env.HOME || "/tmp"}`,
+    `PATH=${process.env.PATH || ""}`
+  ];
+  const envExports = envVars.map((v) => `export ${v}`).join(" && ");
+  console.error("[DIAG] Testing claude --version with SDK environment...");
+  const versionCmd = `${envExports} && "${claudePath}" --version 2>&1`;
+  const versionResult = await execCommand(versionCmd, 15e3);
+  details.versionTest = {
+    exitCode: versionResult.exitCode,
+    stdout: versionResult.stdout.slice(0, 1e3),
+    stderr: versionResult.stderr.slice(0, 1e3)
+  };
+  console.error("[DIAG] Testing claude with simple prompt (like SDK does)...");
+  const promptCmd = `${envExports} && "${claudePath}" -p "Say hello" --output-format json --dangerously-skip-permissions 2>&1`;
+  const promptResult = await execCommand(promptCmd, 3e4);
+  details.promptTest = {
+    command: 'claude -p "Say hello" --output-format json --dangerously-skip-permissions',
+    exitCode: promptResult.exitCode,
+    stdout: promptResult.stdout.slice(0, 2e3),
+    stderr: promptResult.stderr.slice(0, 1e3)
+  };
+  console.error("[DIAG] Checking for claude error logs or state...");
+  const homeDir = process.env.HOME || "/tmp";
+  const claudeConfigDir = path9.join(homeDir, ".claude");
+  details.claudeConfigDir = claudeConfigDir;
+  details.claudeConfigExists = fs11.existsSync(claudeConfigDir);
+  if (fs11.existsSync(claudeConfigDir)) {
+    try {
+      const configContents = fs11.readdirSync(claudeConfigDir);
+      details.claudeConfigContents = configContents;
+    } catch (e) {
+      details.claudeConfigError = e instanceof Error ? e.message : String(e);
+    }
+  }
+  console.error("[DIAG] Testing claude with potential debug flags...");
+  const debugCmd = `${envExports} && "${claudePath}" -p "hi" --verbose 2>&1 || echo "VERBOSE_NOT_SUPPORTED"`;
+  const debugResult = await execCommand(debugCmd, 15e3);
+  details.debugTest = {
+    exitCode: debugResult.exitCode,
+    stdout: debugResult.stdout.slice(0, 1500),
+    stderr: debugResult.stderr.slice(0, 500)
+  };
+  const passed = promptResult.exitCode === 0;
+  return {
+    name: "claude-direct-execution",
+    passed,
+    details,
+    error: passed ? void 0 : `Claude CLI failed with exit code ${promptResult.exitCode}. Output: ${promptResult.stdout.slice(0, 300)}`,
+    durationMs: Date.now() - start
+  };
+}
 async function testChildProcessSpawning() {
   const start = Date.now();
   const details = {};
@@ -7568,6 +7653,7 @@ async function runDiagnostics(config, evalRunId2) {
   await runTest("file-system-structure", testFileSystemStructure);
   await runTest("network-connectivity", () => testNetworkConnectivity(config));
   await runTest("ai-gateway-api-call", () => testAiGatewayApiCall(config));
+  await runTest("claude-direct-execution", () => testClaudeDirectExecution(config));
   await runTest("child-process-spawning", testChildProcessSpawning);
   await runTest("sdk-import", testSdkImport);
   await runTest("file-system-write", testFileSystemWrite);