@wix/evalforge-evaluator 0.67.0 → 0.69.0

package/build/index.mjs

@@ -6437,8 +6437,7 @@ async function writeSkillFiles(skillDir, files) {
 import { randomUUID } from "crypto";
 
 // src/run-scenario/agents/claude-code/write-mcp.ts
-import { writeFile as writeFile2, readFile } from "fs/promises";
-import { spawn } from "child_process";
+import { writeFile as writeFile2 } from "fs/promises";
 import { join as join3 } from "path";
 import { MCP_SERVERS_JSON_KEY } from "@wix/evalforge-types";
 async function writeMcpToFilesystem(cwd, mcps) {
@@ -6464,6 +6463,11 @@ async function writeMcpToFilesystem(cwd, mcps) {
   await writeFile2(filePath, content, "utf8");
   console.log(`[MCP] Written to ${filePath}`);
 }
+
+// src/run-scenario/agents/claude-code/mcp-setup.ts
+import { readFile } from "fs/promises";
+import { spawn } from "child_process";
+import { join as join4 } from "path";
 async function probeMcpServers(mcps, probeMs = 5e3) {
   const results = [];
   for (const mcp of mcps) {
@@ -6531,56 +6535,165 @@ spawn error: ${err.message}`;
     }, probeMs);
   });
 }
+async function preInstallMcpPackages(mcps, timeoutMs = 12e4) {
+  const packages = extractNpxPackages(mcps);
+  if (packages.length === 0) return [];
+  const results = [];
+  for (const pkg of packages) {
+    const startMs = Date.now();
+    const alreadyInstalled = await isPackageGloballyInstalled(pkg);
+    if (alreadyInstalled) {
+      const durationMs2 = Date.now() - startMs;
+      console.log(
+        `[MCP] Package already installed globally, skipping: ${pkg} (${durationMs2}ms)`
+      );
+      results.push({
+        package: pkg,
+        success: true,
+        skipped: true,
+        output: "already installed",
+        durationMs: durationMs2
+      });
+      continue;
+    }
+    console.log(`[MCP] Pre-installing npx package globally: ${pkg}`);
+    const result = await installPackageGlobally(pkg, timeoutMs);
+    const durationMs = Date.now() - startMs;
+    results.push({ package: pkg, skipped: false, durationMs, ...result });
+    console.log(
+      `[MCP] Install ${pkg}: ${result.success ? "SUCCESS" : "FAILED"} (${durationMs}ms)`
+    );
+  }
+  return results;
+}
+function isPackageGloballyInstalled(pkg) {
+  return new Promise((resolve2) => {
+    let settled = false;
+    const child = spawn("npm", ["ls", "-g", "--depth=0", pkg], {
+      stdio: ["pipe", "pipe", "pipe"],
+      env: process.env
+    });
+    child.on("close", (code2) => {
+      if (!settled) {
+        settled = true;
+        resolve2(code2 === 0);
+      }
+    });
+    child.on("error", () => {
+      if (!settled) {
+        settled = true;
+        resolve2(false);
+      }
+    });
+    setTimeout(() => {
+      if (!settled) {
+        settled = true;
+        child.kill("SIGTERM");
+        resolve2(false);
+      }
+    }, 5e3);
+  });
+}
+function extractNpxPackages(mcps) {
+  const packages = [];
+  for (const mcp of mcps) {
+    const config = mcp.config;
+    for (const [, value] of Object.entries(config)) {
+      if (typeof value !== "object" || value === null) continue;
+      const cfg = value;
+      if (cfg.command !== "npx" || !Array.isArray(cfg.args)) continue;
+      for (const arg of cfg.args) {
+        if (!arg.startsWith("-")) {
+          packages.push(arg);
+          break;
+        }
+      }
+    }
+  }
+  return [...new Set(packages)];
+}
+function installPackageGlobally(pkg, timeoutMs) {
+  return new Promise((resolve2) => {
+    let output = "";
+    let settled = false;
+    const child = spawn("sh", ["-c", `npm install -g ${pkg} 2>&1`], {
+      stdio: ["pipe", "pipe", "pipe"],
+      env: process.env
+    });
+    child.stdout.on("data", (chunk) => {
+      output += chunk.toString();
+    });
+    child.stderr.on("data", (chunk) => {
+      output += chunk.toString();
+    });
+    child.on("close", (code2) => {
+      if (!settled) {
+        settled = true;
+        resolve2({ success: code2 === 0, output: output.slice(-1e3) });
+      }
+    });
+    child.on("error", (err) => {
+      if (!settled) {
+        settled = true;
+        resolve2({ success: false, output: `spawn error: ${err.message}` });
+      }
+    });
+    setTimeout(() => {
+      if (!settled) {
+        settled = true;
+        child.kill("SIGTERM");
+        resolve2({
+          success: false,
+          output: output.slice(-1e3) + "\n[install timed out]"
+        });
+      }
+    }, timeoutMs);
+  });
+}
 async function probeNpmConfig(cwd) {
   const homedir = process.env.HOME ?? "~";
-  const [
-    cwdNpmrc,
-    homeNpmrc,
-    projectRootNpmrc,
-    globalNpmrc,
-    npmConfigListFull,
-    npmConfigGetRegistry,
-    npmViewWixMcp
-  ] = await Promise.all([
-    readFile(join3(cwd, ".npmrc"), "utf8").catch(
-      (e) => `[not found: ${e.message}]`
-    ),
-    readFile(join3(homedir, ".npmrc"), "utf8").catch(
-      (e) => `[not found: ${e.message}]`
-    ),
-    readFile("/user-code/.npmrc", "utf8").catch(
+  const [wixAuthFile, npmInstallDryRun, mcpPackageInstalled] = await Promise.all([
+    readFile(join4(homedir, ".wix", "auth", "api-key.json"), "utf8").catch(
       (e) => `[not found: ${e.message}]`
     ),
     runShellCapture(
-      'echo "=== globalconfig ===" && npm config get globalconfig && cat "$(npm config get globalconfig)" 2>/dev/null || echo "[no global npmrc]" && echo "=== userconfig ===" && npm config get userconfig && cat "$(npm config get userconfig)" 2>/dev/null || echo "[no user npmrc]" && echo "=== prefix ===" && npm config get prefix && cat "$(npm config get prefix)/etc/npmrc" 2>/dev/null || echo "[no prefix npmrc]"',
+      "npm install --dry-run @wix/mcp 2>&1 | tail -15",
       cwd,
-      5e3
+      15e3
     ),
-    runShellCapture("npm config list -l 2>&1 | head -60", cwd, 5e3),
-    runShellCapture("npm config get registry", cwd, 3e3),
     runShellCapture(
-      "npm view @wix/mcp version --registry https://registry.npmjs.org 2>&1",
+      'ls -la $(npm root -g)/@wix/mcp/package.json 2>/dev/null || echo "[not globally installed]" && npm list -g @wix/mcp 2>&1 | tail -5',
       cwd,
-      1e4
+      5e3
     )
   ]);
+  const wixAuthParsed = (() => {
+    try {
+      const parsed = JSON.parse(wixAuthFile);
+      return JSON.stringify({
+        hasToken: typeof parsed.token === "string",
+        tokenLength: typeof parsed.token === "string" ? parsed.token.length : 0,
+        hasAccountId: typeof parsed.accountId === "string",
+        hasUserInfo: typeof parsed.userInfo === "object"
+      });
+    } catch {
+      return wixAuthFile.slice(0, 500);
+    }
+  })();
   const npmEnvVars = {};
   for (const [key, value] of Object.entries(process.env)) {
     const lk = key.toLowerCase();
-    if (lk.startsWith("npm_config") || lk.includes("token") || lk.includes("auth") || lk.includes("registry")) {
+    if (lk.startsWith("npm_config") || lk.includes("token") || lk.includes("auth") || lk.includes("registry") || lk.startsWith("wix_")) {
       npmEnvVars[key] = lk.includes("token") || lk.includes("auth") || lk.includes("secret") ? `[REDACTED, length=${(value ?? "").length}]` : value;
     }
   }
   return {
     cwd,
     processCwd: process.cwd(),
-    cwdNpmrc: cwdNpmrc.slice(0, 1e3),
-    homeNpmrc: homeNpmrc.slice(0, 1e3),
-    projectRootNpmrc: projectRootNpmrc.slice(0, 1e3),
-    globalNpmrc: globalNpmrc.slice(0, 2e3),
-    npmConfigListFull: npmConfigListFull.slice(0, 3e3),
-    npmConfigGetRegistry: npmConfigGetRegistry.trim(),
-    npmViewWixMcp: npmViewWixMcp.trim().slice(0, 500),
+    wixAuthFile: wixAuthParsed,
+    wixAuthFileExists: !wixAuthFile.startsWith("[not found"),
+    npmInstallDryRun: npmInstallDryRun.trim().slice(0, 1500),
+    mcpPackageInstalled: mcpPackageInstalled.trim().slice(0, 500),
     npmEnvVars,
     homedir
   };
@@ -6624,7 +6737,7 @@ function runShellCapture(cmd, cwd, timeoutMs) {
 
 // src/run-scenario/agents/claude-code/write-sub-agents.ts
 import { mkdir as mkdir3, writeFile as writeFile3 } from "fs/promises";
-import { join as join4 } from "path";
+import { join as join5 } from "path";
 var AGENTS_DIR = ".claude/agents";
 function toAgentFilename(name2, index, nameCount) {
   const base = (name2 || "").toLowerCase().replace(/\s+/g, "-").replace(/[^a-z0-9-]/g, "").replace(/^-+|-+$/g, "") || `sub-agent-${index}`;
@@ -6634,12 +6747,12 @@ function toAgentFilename(name2, index, nameCount) {
 }
 async function writeSubAgentsToFilesystem(cwd, subAgents) {
   if (subAgents.length === 0) return;
-  const agentsDir = join4(cwd, AGENTS_DIR);
+  const agentsDir = join5(cwd, AGENTS_DIR);
   await mkdir3(agentsDir, { recursive: true });
   const nameCount = /* @__PURE__ */ new Map();
   for (const [i, agent] of subAgents.entries()) {
     const filename = toAgentFilename(agent.name, i, nameCount);
-    const filePath = join4(agentsDir, `${filename}.md`);
+    const filePath = join5(agentsDir, `${filename}.md`);
     await writeFile3(filePath, agent.subAgentMd, "utf8");
   }
   console.log(`[SubAgents] Written to ${agentsDir}`);
@@ -6874,8 +6987,30 @@ async function executeWithClaudeCode(skills, scenario, options) {
   const startTime = /* @__PURE__ */ new Date();
   const allMessages = [];
   if (options.mcps && options.mcps.length > 0) {
-    process.env.npm_config__auth = "";
     await writeMcpToFilesystem(options.cwd, options.mcps);
+    const installResults = await preInstallMcpPackages(options.mcps);
+    if (installResults.length > 0 && options.traceContext) {
+      emitTraceEvent(
+        {
+          evalRunId: options.traceContext.evalRunId,
+          scenarioId: options.traceContext.scenarioId,
+          scenarioName: options.traceContext.scenarioName,
+          targetId: options.traceContext.targetId,
+          targetName: options.traceContext.targetName,
+          stepNumber: 0,
+          type: LiveTraceEventType.DIAGNOSTIC,
+          outputPreview: JSON.stringify({
+            event: "mcp-pre-install",
+            results: installResults
+          }).slice(0, 2e3),
+          timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+          isComplete: false
+        },
+        options.traceContext.tracePushUrl,
+        options.traceContext.routeHeader,
+        options.traceContext.authToken
+      );
+    }
     const probeResults = await probeMcpServers(options.mcps);
     if (options.traceContext) {
       emitTraceEvent(
@@ -7746,7 +7881,7 @@ defaultRegistry.register(claudeCodeAdapter);
 
 // src/run-scenario/file-diff.ts
 import { readdirSync as readdirSync2, readFileSync, statSync, existsSync as existsSync2 } from "fs";
-import { join as join6, relative } from "path";
+import { join as join7, relative } from "path";
 
 // ../../node_modules/diff/lib/index.mjs
 function Diff() {
@@ -7922,7 +8057,7 @@ Diff.prototype = {
   tokenize: function tokenize(value) {
     return Array.from(value);
   },
-  join: function join5(chars) {
+  join: function join6(chars) {
     return chars.join("");
   },
   postProcess: function postProcess(changeObjects) {
@@ -8362,7 +8497,7 @@ function snapshotDirectory(dir, baseDir) {
   }
   const entries = readdirSync2(dir, { withFileTypes: true });
   for (const entry of entries) {
-    const fullPath = join6(dir, entry.name);
+    const fullPath = join7(dir, entry.name);
     const relativePath = relative(base, fullPath);
     if (shouldIgnore(entry.name)) {
       continue;