@wix/evalforge-evaluator 0.16.0 → 0.17.0

package/build/index.js

@@ -7289,6 +7289,79 @@ async function testNetworkConnectivity(config) {
     durationMs: Date.now() - start
   };
 }
+async function testAiGatewayApiCall(config) {
+  const start = Date.now();
+  const details = {};
+  const gatewayUrl = config.aiGatewayUrl;
+  const headers = config.aiGatewayHeaders;
+  details.gatewayUrl = gatewayUrl;
+  details.hasHeaders = !!headers;
+  details.headerKeys = headers ? Object.keys(headers) : [];
+  if (!gatewayUrl) {
+    return {
+      name: "ai-gateway-api-call",
+      passed: false,
+      details,
+      error: "No AI_GATEWAY_URL configured",
+      durationMs: Date.now() - start
+    };
+  }
+  if (!headers) {
+    return {
+      name: "ai-gateway-api-call",
+      passed: false,
+      details,
+      error: "No AI_GATEWAY_HEADERS configured",
+      durationMs: Date.now() - start
+    };
+  }
+  const headerFlags = Object.entries(headers).map(([k, v]) => `-H "${k}: ${v}"`).join(" ");
+  const requestBody = JSON.stringify({
+    model: "claude-3-5-sonnet-latest",
+    max_tokens: 10,
+    messages: [{ role: "user", content: "Say hi" }]
+  }).replace(/"/g, '\\"');
+  const messagesUrl = `${gatewayUrl}/v1/messages`;
+  const curlCmd = `curl -s --max-time 15 ${headerFlags} -H "Content-Type: application/json" -H "anthropic-version: 2023-06-01" -d "${requestBody}" "${messagesUrl}" 2>&1`;
+  const redactedCmd = curlCmd.replace(/app-secret:[^"]+/g, "app-secret:[REDACTED]").replace(
+    /-H "x-wix-ai-gateway-app-secret: [^"]+"/g,
+    '-H "x-wix-ai-gateway-app-secret: [REDACTED]"'
+  );
+  details.curlCommand = redactedCmd;
+  console.error("[DIAG] Making actual API call to AI Gateway...");
+  console.error("[DIAG] URL:", messagesUrl);
+  const result = await execCommand(curlCmd, 2e4);
+  details.responseRaw = result.stdout.slice(0, 1500);
+  details.exitCode = result.exitCode;
+  let responseJson = null;
+  try {
+    responseJson = JSON.parse(result.stdout);
+    details.responseParsed = true;
+  } catch {
+    details.responseParsed = false;
+    details.parseError = "Response is not valid JSON";
+  }
+  const isError = result.stdout.includes('"type":"error"') || result.stdout.includes('"error":{') || result.stdout.includes("authentication_error") || result.stdout.includes("permission_error");
+  const isSuccess = result.stdout.includes('"type":"message"') || result.stdout.includes('"content":');
+  details.isError = isError;
+  details.isSuccess = isSuccess;
+  if (isError && responseJson && typeof responseJson === "object") {
+    const errorObj = responseJson;
+    if (errorObj.error && typeof errorObj.error === "object") {
+      const error = errorObj.error;
+      details.errorType = error.type;
+      details.errorMessage = error.message;
+    }
+  }
+  const passed = result.exitCode === 0 && isSuccess && !isError;
+  return {
+    name: "ai-gateway-api-call",
+    passed,
+    details,
+    error: passed ? void 0 : isError ? `API returned error: ${details.errorType || "unknown"} - ${details.errorMessage || result.stdout.slice(0, 200)}` : `API call failed: exit=${result.exitCode}, response=${result.stdout.slice(0, 200)}`,
+    durationMs: Date.now() - start
+  };
+}
 async function testChildProcessSpawning() {
   const start = Date.now();
   const details = {};
@@ -7494,6 +7567,7 @@ async function runDiagnostics(config, evalRunId2) {
   await runTest("environment-dump", testEnvironmentDump);
   await runTest("file-system-structure", testFileSystemStructure);
   await runTest("network-connectivity", () => testNetworkConnectivity(config));
+  await runTest("ai-gateway-api-call", () => testAiGatewayApiCall(config));
   await runTest("child-process-spawning", testChildProcessSpawning);
   await runTest("sdk-import", testSdkImport);
   await runTest("file-system-write", testFileSystemWrite);