@wix/evalforge-evaluator 0.68.0 → 0.69.0

package/build/index.mjs

@@ -6437,8 +6437,7 @@ async function writeSkillFiles(skillDir, files) {
 import { randomUUID } from "crypto";
 
 // src/run-scenario/agents/claude-code/write-mcp.ts
-import { writeFile as writeFile2, readFile } from "fs/promises";
-import { spawn } from "child_process";
+import { writeFile as writeFile2 } from "fs/promises";
 import { join as join3 } from "path";
 import { MCP_SERVERS_JSON_KEY } from "@wix/evalforge-types";
 async function writeMcpToFilesystem(cwd, mcps) {
@@ -6464,6 +6463,11 @@ async function writeMcpToFilesystem(cwd, mcps) {
   await writeFile2(filePath, content, "utf8");
   console.log(`[MCP] Written to ${filePath}`);
 }
+
+// src/run-scenario/agents/claude-code/mcp-setup.ts
+import { readFile } from "fs/promises";
+import { spawn } from "child_process";
+import { join as join4 } from "path";
 async function probeMcpServers(mcps, probeMs = 5e3) {
   const results = [];
   for (const mcp of mcps) {
@@ -6531,10 +6535,125 @@ spawn error: ${err.message}`;
     }, probeMs);
   });
 }
+async function preInstallMcpPackages(mcps, timeoutMs = 12e4) {
+  const packages = extractNpxPackages(mcps);
+  if (packages.length === 0) return [];
+  const results = [];
+  for (const pkg of packages) {
+    const startMs = Date.now();
+    const alreadyInstalled = await isPackageGloballyInstalled(pkg);
+    if (alreadyInstalled) {
+      const durationMs2 = Date.now() - startMs;
+      console.log(
+        `[MCP] Package already installed globally, skipping: ${pkg} (${durationMs2}ms)`
+      );
+      results.push({
+        package: pkg,
+        success: true,
+        skipped: true,
+        output: "already installed",
+        durationMs: durationMs2
+      });
+      continue;
+    }
+    console.log(`[MCP] Pre-installing npx package globally: ${pkg}`);
+    const result = await installPackageGlobally(pkg, timeoutMs);
+    const durationMs = Date.now() - startMs;
+    results.push({ package: pkg, skipped: false, durationMs, ...result });
+    console.log(
+      `[MCP] Install ${pkg}: ${result.success ? "SUCCESS" : "FAILED"} (${durationMs}ms)`
+    );
+  }
+  return results;
+}
+function isPackageGloballyInstalled(pkg) {
+  return new Promise((resolve2) => {
+    let settled = false;
+    const child = spawn("npm", ["ls", "-g", "--depth=0", pkg], {
+      stdio: ["pipe", "pipe", "pipe"],
+      env: process.env
+    });
+    child.on("close", (code2) => {
+      if (!settled) {
+        settled = true;
+        resolve2(code2 === 0);
+      }
+    });
+    child.on("error", () => {
+      if (!settled) {
+        settled = true;
+        resolve2(false);
+      }
+    });
+    setTimeout(() => {
+      if (!settled) {
+        settled = true;
+        child.kill("SIGTERM");
+        resolve2(false);
+      }
+    }, 5e3);
+  });
+}
+function extractNpxPackages(mcps) {
+  const packages = [];
+  for (const mcp of mcps) {
+    const config = mcp.config;
+    for (const [, value] of Object.entries(config)) {
+      if (typeof value !== "object" || value === null) continue;
+      const cfg = value;
+      if (cfg.command !== "npx" || !Array.isArray(cfg.args)) continue;
+      for (const arg of cfg.args) {
+        if (!arg.startsWith("-")) {
+          packages.push(arg);
+          break;
+        }
+      }
+    }
+  }
+  return [...new Set(packages)];
+}
+function installPackageGlobally(pkg, timeoutMs) {
+  return new Promise((resolve2) => {
+    let output = "";
+    let settled = false;
+    const child = spawn("sh", ["-c", `npm install -g ${pkg} 2>&1`], {
+      stdio: ["pipe", "pipe", "pipe"],
+      env: process.env
+    });
+    child.stdout.on("data", (chunk) => {
+      output += chunk.toString();
+    });
+    child.stderr.on("data", (chunk) => {
+      output += chunk.toString();
+    });
+    child.on("close", (code2) => {
+      if (!settled) {
+        settled = true;
+        resolve2({ success: code2 === 0, output: output.slice(-1e3) });
+      }
+    });
+    child.on("error", (err) => {
+      if (!settled) {
+        settled = true;
+        resolve2({ success: false, output: `spawn error: ${err.message}` });
+      }
+    });
+    setTimeout(() => {
+      if (!settled) {
+        settled = true;
+        child.kill("SIGTERM");
+        resolve2({
+          success: false,
+          output: output.slice(-1e3) + "\n[install timed out]"
+        });
+      }
+    }, timeoutMs);
+  });
+}
 async function probeNpmConfig(cwd) {
   const homedir = process.env.HOME ?? "~";
   const [wixAuthFile, npmInstallDryRun, mcpPackageInstalled] = await Promise.all([
-    readFile(join3(homedir, ".wix", "auth", "api-key.json"), "utf8").catch(
+    readFile(join4(homedir, ".wix", "auth", "api-key.json"), "utf8").catch(
       (e) => `[not found: ${e.message}]`
     ),
     runShellCapture(
@@ -6618,7 +6737,7 @@ function runShellCapture(cmd, cwd, timeoutMs) {
 
 // src/run-scenario/agents/claude-code/write-sub-agents.ts
 import { mkdir as mkdir3, writeFile as writeFile3 } from "fs/promises";
-import { join as join4 } from "path";
+import { join as join5 } from "path";
 var AGENTS_DIR = ".claude/agents";
 function toAgentFilename(name2, index, nameCount) {
   const base = (name2 || "").toLowerCase().replace(/\s+/g, "-").replace(/[^a-z0-9-]/g, "").replace(/^-+|-+$/g, "") || `sub-agent-${index}`;
@@ -6628,12 +6747,12 @@ function toAgentFilename(name2, index, nameCount) {
 }
 async function writeSubAgentsToFilesystem(cwd, subAgents) {
   if (subAgents.length === 0) return;
-  const agentsDir = join4(cwd, AGENTS_DIR);
+  const agentsDir = join5(cwd, AGENTS_DIR);
   await mkdir3(agentsDir, { recursive: true });
   const nameCount = /* @__PURE__ */ new Map();
   for (const [i, agent] of subAgents.entries()) {
     const filename = toAgentFilename(agent.name, i, nameCount);
-    const filePath = join4(agentsDir, `${filename}.md`);
+    const filePath = join5(agentsDir, `${filename}.md`);
     await writeFile3(filePath, agent.subAgentMd, "utf8");
   }
   console.log(`[SubAgents] Written to ${agentsDir}`);
@@ -6869,6 +6988,29 @@ async function executeWithClaudeCode(skills, scenario, options) {
   const allMessages = [];
   if (options.mcps && options.mcps.length > 0) {
     await writeMcpToFilesystem(options.cwd, options.mcps);
+    const installResults = await preInstallMcpPackages(options.mcps);
+    if (installResults.length > 0 && options.traceContext) {
+      emitTraceEvent(
+        {
+          evalRunId: options.traceContext.evalRunId,
+          scenarioId: options.traceContext.scenarioId,
+          scenarioName: options.traceContext.scenarioName,
+          targetId: options.traceContext.targetId,
+          targetName: options.traceContext.targetName,
+          stepNumber: 0,
+          type: LiveTraceEventType.DIAGNOSTIC,
+          outputPreview: JSON.stringify({
+            event: "mcp-pre-install",
+            results: installResults
+          }).slice(0, 2e3),
+          timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+          isComplete: false
+        },
+        options.traceContext.tracePushUrl,
+        options.traceContext.routeHeader,
+        options.traceContext.authToken
+      );
+    }
     const probeResults = await probeMcpServers(options.mcps);
     if (options.traceContext) {
       emitTraceEvent(
@@ -7739,7 +7881,7 @@ defaultRegistry.register(claudeCodeAdapter);
 
 // src/run-scenario/file-diff.ts
 import { readdirSync as readdirSync2, readFileSync, statSync, existsSync as existsSync2 } from "fs";
-import { join as join6, relative } from "path";
+import { join as join7, relative } from "path";
 
 // ../../node_modules/diff/lib/index.mjs
 function Diff() {
@@ -7915,7 +8057,7 @@ Diff.prototype = {
   tokenize: function tokenize(value) {
     return Array.from(value);
   },
-  join: function join5(chars) {
+  join: function join6(chars) {
     return chars.join("");
   },
   postProcess: function postProcess(changeObjects) {
@@ -8355,7 +8497,7 @@ function snapshotDirectory(dir, baseDir) {
   }
   const entries = readdirSync2(dir, { withFileTypes: true });
   for (const entry of entries) {
-    const fullPath = join6(dir, entry.name);
+    const fullPath = join7(dir, entry.name);
     const relativePath = relative(base, fullPath);
     if (shouldIgnore(entry.name)) {
       continue;