@wix/evalforge-evaluator 0.63.0 → 0.65.0

package/build/index.js

@@ -6451,6 +6451,7 @@ var import_crypto = require("crypto");
 
 // src/run-scenario/agents/claude-code/write-mcp.ts
 var import_promises4 = require("fs/promises");
+var import_child_process = require("child_process");
 var import_path6 = require("path");
 var import_evalforge_types2 = require("@wix/evalforge-types");
 async function writeMcpToFilesystem(cwd, mcps) {
@@ -6476,6 +6477,137 @@ async function writeMcpToFilesystem(cwd, mcps) {
   await (0, import_promises4.writeFile)(filePath, content, "utf8");
   console.log(`[MCP] Written to ${filePath}`);
 }
+async function probeMcpServers(mcps, probeMs = 5e3) {
+  const results = [];
+  for (const mcp of mcps) {
+    const config = mcp.config;
+    for (const [name2, value] of Object.entries(config)) {
+      if (typeof value !== "object" || value === null) continue;
+      const cfg = value;
+      const command = cfg.command;
+      const args = cfg.args;
+      if (typeof command !== "string" || !Array.isArray(args)) continue;
+      const result = await probeOneServer(
+        name2,
+        command,
+        args,
+        probeMs
+      );
+      results.push(result);
+    }
+  }
+  return results;
+}
+function probeOneServer(name2, command, args, probeMs) {
+  return new Promise((resolve2) => {
+    const startMs = Date.now();
+    let stdout = "";
+    let stderr = "";
+    let settled = false;
+    const finish = (exitCode, signal) => {
+      if (settled) return;
+      settled = true;
+      resolve2({
+        name: name2,
+        command,
+        args,
+        exitCode,
+        signal,
+        stdout: stdout.slice(-2e3),
+        stderr: stderr.slice(-2e3),
+        durationMs: Date.now() - startMs
+      });
+    };
+    const child = (0, import_child_process.spawn)(command, args, {
+      stdio: ["pipe", "pipe", "pipe"],
+      env: process.env
+    });
+    child.stdout.on("data", (chunk) => {
+      stdout += chunk.toString();
+    });
+    child.stderr.on("data", (chunk) => {
+      stderr += chunk.toString();
+    });
+    child.on("error", (err) => {
+      stderr += `
+spawn error: ${err.message}`;
+      finish(null, null);
+    });
+    child.on("close", (code2, sig) => {
+      finish(code2, sig);
+    });
+    setTimeout(() => {
+      if (!settled) {
+        child.kill("SIGTERM");
+        finish(null, "PROBE_TIMEOUT");
+      }
+    }, probeMs);
+  });
+}
+async function probeNpmConfig(cwd) {
+  const homedir = process.env.HOME ?? "~";
+  const [cwdNpmrc, homeNpmrc, npmConfigList, npmConfigGetRegistry] = await Promise.all([
+    (0, import_promises4.readFile)((0, import_path6.join)(cwd, ".npmrc"), "utf8").catch(
+      (e) => `[not found: ${e.message}]`
+    ),
+    (0, import_promises4.readFile)((0, import_path6.join)(homedir, ".npmrc"), "utf8").catch(
+      (e) => `[not found: ${e.message}]`
+    ),
+    runShellCapture("npm config list", cwd, 3e3),
+    runShellCapture("npm config get registry", cwd, 3e3)
+  ]);
+  const npmEnvVars = {};
+  for (const [key, value] of Object.entries(process.env)) {
+    if (key.toLowerCase().startsWith("npm_config")) {
+      npmEnvVars[key] = value;
+    }
+  }
+  return {
+    cwd,
+    cwdNpmrc: cwdNpmrc.slice(0, 1e3),
+    homeNpmrc: homeNpmrc.slice(0, 1e3),
+    npmConfigList: npmConfigList.slice(0, 2e3),
+    npmConfigGetRegistry: npmConfigGetRegistry.trim(),
+    npmEnvVars,
+    homedir
+  };
+}
+function runShellCapture(cmd, cwd, timeoutMs) {
+  return new Promise((resolve2) => {
+    let output = "";
+    let settled = false;
+    const child = (0, import_child_process.spawn)("sh", ["-c", cmd], {
+      stdio: ["pipe", "pipe", "pipe"],
+      cwd,
+      env: process.env
+    });
+    child.stdout.on("data", (chunk) => {
+      output += chunk.toString();
+    });
+    child.stderr.on("data", (chunk) => {
+      output += chunk.toString();
+    });
+    child.on("close", () => {
+      if (!settled) {
+        settled = true;
+        resolve2(output);
+      }
+    });
+    child.on("error", (err) => {
+      if (!settled) {
+        settled = true;
+        resolve2(`[error: ${err.message}]`);
+      }
+    });
+    setTimeout(() => {
+      if (!settled) {
+        settled = true;
+        child.kill("SIGTERM");
+        resolve2(output + "\n[timed out]");
+      }
+    }, timeoutMs);
+  });
+}
 
 // src/run-scenario/agents/claude-code/write-sub-agents.ts
 var import_promises5 = require("fs/promises");
@@ -6730,6 +6862,56 @@ async function executeWithClaudeCode(skills, scenario, options) {
   const allMessages = [];
   if (options.mcps && options.mcps.length > 0) {
     await writeMcpToFilesystem(options.cwd, options.mcps);
+    const probeResults = await probeMcpServers(options.mcps);
+    if (options.traceContext) {
+      emitTraceEvent(
+        {
+          evalRunId: options.traceContext.evalRunId,
+          scenarioId: options.traceContext.scenarioId,
+          scenarioName: options.traceContext.scenarioName,
+          targetId: options.traceContext.targetId,
+          targetName: options.traceContext.targetName,
+          stepNumber: 0,
+          type: import_evalforge_types3.LiveTraceEventType.DIAGNOSTIC,
+          outputPreview: JSON.stringify({
+            event: "mcp-probe",
+            results: probeResults
+          }).slice(0, 2e3),
+          timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+          isComplete: false
+        },
+        options.traceContext.tracePushUrl,
+        options.traceContext.routeHeader,
+        options.traceContext.authToken
+      );
+    }
+    const npmDiag = await probeNpmConfig(options.cwd);
+    console.log(
+      "[DEBUG-d744ca] npm-config-diagnostic",
+      JSON.stringify(npmDiag)
+    );
+    if (options.traceContext) {
+      emitTraceEvent(
+        {
+          evalRunId: options.traceContext.evalRunId,
+          scenarioId: options.traceContext.scenarioId,
+          scenarioName: options.traceContext.scenarioName,
+          targetId: options.traceContext.targetId,
+          targetName: options.traceContext.targetName,
+          stepNumber: 0,
+          type: import_evalforge_types3.LiveTraceEventType.DIAGNOSTIC,
+          outputPreview: JSON.stringify({
+            event: "npm-config-diagnostic",
+            ...npmDiag
+          }).slice(0, 4e3),
+          timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+          isComplete: false
+        },
+        options.traceContext.tracePushUrl,
+        options.traceContext.routeHeader,
+        options.traceContext.authToken
+      );
+    }
   }
   if (options.subAgents && options.subAgents.length > 0) {
     await writeSubAgentsToFilesystem(options.cwd, options.subAgents);