@wix/evalforge-evaluator 0.63.0 → 0.64.0

package/build/index.mjs

@@ -6438,6 +6438,7 @@ import { randomUUID } from "crypto";
 
 // src/run-scenario/agents/claude-code/write-mcp.ts
 import { writeFile as writeFile2 } from "fs/promises";
+import { spawn } from "child_process";
 import { join as join3 } from "path";
 import { MCP_SERVERS_JSON_KEY } from "@wix/evalforge-types";
 async function writeMcpToFilesystem(cwd, mcps) {
@@ -6463,6 +6464,73 @@ async function writeMcpToFilesystem(cwd, mcps) {
   await writeFile2(filePath, content, "utf8");
   console.log(`[MCP] Written to ${filePath}`);
 }
+async function probeMcpServers(mcps, probeMs = 5e3) {
+  const results = [];
+  for (const mcp of mcps) {
+    const config = mcp.config;
+    for (const [name2, value] of Object.entries(config)) {
+      if (typeof value !== "object" || value === null) continue;
+      const cfg = value;
+      const command = cfg.command;
+      const args = cfg.args;
+      if (typeof command !== "string" || !Array.isArray(args)) continue;
+      const result = await probeOneServer(
+        name2,
+        command,
+        args,
+        probeMs
+      );
+      results.push(result);
+    }
+  }
+  return results;
+}
+function probeOneServer(name2, command, args, probeMs) {
+  return new Promise((resolve2) => {
+    const startMs = Date.now();
+    let stdout = "";
+    let stderr = "";
+    let settled = false;
+    const finish = (exitCode, signal) => {
+      if (settled) return;
+      settled = true;
+      resolve2({
+        name: name2,
+        command,
+        args,
+        exitCode,
+        signal,
+        stdout: stdout.slice(-2e3),
+        stderr: stderr.slice(-2e3),
+        durationMs: Date.now() - startMs
+      });
+    };
+    const child = spawn(command, args, {
+      stdio: ["pipe", "pipe", "pipe"],
+      env: process.env
+    });
+    child.stdout.on("data", (chunk) => {
+      stdout += chunk.toString();
+    });
+    child.stderr.on("data", (chunk) => {
+      stderr += chunk.toString();
+    });
+    child.on("error", (err) => {
+      stderr += `
+spawn error: ${err.message}`;
+      finish(null, null);
+    });
+    child.on("close", (code2, sig) => {
+      finish(code2, sig);
+    });
+    setTimeout(() => {
+      if (!settled) {
+        child.kill("SIGTERM");
+        finish(null, "PROBE_TIMEOUT");
+      }
+    }, probeMs);
+  });
+}
 
 // src/run-scenario/agents/claude-code/write-sub-agents.ts
 import { mkdir as mkdir3, writeFile as writeFile3 } from "fs/promises";
@@ -6717,6 +6785,29 @@ async function executeWithClaudeCode(skills, scenario, options) {
   const allMessages = [];
   if (options.mcps && options.mcps.length > 0) {
     await writeMcpToFilesystem(options.cwd, options.mcps);
+    const probeResults = await probeMcpServers(options.mcps);
+    if (options.traceContext) {
+      emitTraceEvent(
+        {
+          evalRunId: options.traceContext.evalRunId,
+          scenarioId: options.traceContext.scenarioId,
+          scenarioName: options.traceContext.scenarioName,
+          targetId: options.traceContext.targetId,
+          targetName: options.traceContext.targetName,
+          stepNumber: 0,
+          type: LiveTraceEventType.DIAGNOSTIC,
+          outputPreview: JSON.stringify({
+            event: "mcp-probe",
+            results: probeResults
+          }).slice(0, 2e3),
+          timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+          isComplete: false
+        },
+        options.traceContext.tracePushUrl,
+        options.traceContext.routeHeader,
+        options.traceContext.authToken
+      );
+    }
   }
   if (options.subAgents && options.subAgents.length > 0) {
     await writeSubAgentsToFilesystem(options.cwd, options.subAgents);