@wix/evalforge-evaluator 0.97.0 → 0.99.0

package/build/index.js

@@ -207,6 +207,53 @@ function createApiClient(serverUrl, options = "") {
 
 // src/fetch-evaluation-data.ts
 var import_evalforge_types = require("@wix/evalforge-types");
+
+// src/resolve-placeholders.ts
+var PLACEHOLDER_PATTERN = /\{\{([^}]+)\}\}/g;
+function findPlaceholders(value) {
+  const keys = /* @__PURE__ */ new Set();
+  collectPlaceholders(value, keys);
+  return [...keys];
+}
+function collectPlaceholders(value, keys) {
+  if (typeof value === "string") {
+    for (const match of value.matchAll(PLACEHOLDER_PATTERN)) {
+      keys.add(match[1].trim());
+    }
+  } else if (Array.isArray(value)) {
+    for (const item of value) {
+      collectPlaceholders(item, keys);
+    }
+  } else if (typeof value === "object" && value !== null) {
+    for (const val of Object.values(value)) {
+      collectPlaceholders(val, keys);
+    }
+  }
+}
+function resolveValue(value, placeholders) {
+  if (typeof value === "string") {
+    return value.replace(PLACEHOLDER_PATTERN, (match, key) => {
+      const trimmed = key.trim();
+      return trimmed in placeholders ? placeholders[trimmed] : match;
+    });
+  }
+  if (Array.isArray(value)) {
+    return value.map((item) => resolveValue(item, placeholders));
+  }
+  if (typeof value === "object" && value !== null) {
+    const result = {};
+    for (const [k, v] of Object.entries(value)) {
+      result[k] = resolveValue(v, placeholders);
+    }
+    return result;
+  }
+  return value;
+}
+function resolvePlaceholdersInString(text, placeholders) {
+  return resolveValue(text, placeholders);
+}
+
+// src/fetch-evaluation-data.ts
 function parseSkillNamesFromParams(value) {
   if (typeof value !== "string") {
     return [];
@@ -222,13 +269,11 @@ function applyParamsToAssertion(assertion, params) {
     return assertion;
   }
   if (assertion.type === "llm_judge") {
-    let prompt = assertion.prompt;
+    const stringParams = {};
     for (const [key, value] of Object.entries(params)) {
-      const placeholder = `{{${key}}}`;
-      const escapedPlaceholder = placeholder.replace(/[{}]/g, "\\$&");
-      const replacement = String(value ?? "");
-      prompt = prompt.replace(new RegExp(escapedPlaceholder, "g"), replacement);
+      stringParams[key] = String(value ?? "");
     }
+    const prompt = resolvePlaceholdersInString(assertion.prompt, stringParams);
     return {
       ...assertion,
       prompt,
@@ -697,9 +742,50 @@ var import_evalforge_types3 = require("@wix/evalforge-types");
 var import_crypto = require("crypto");
 
 // src/run-scenario/agents/claude-code/write-mcp.ts
+var import_promises3 = require("fs/promises");
+var import_path4 = require("path");
+var import_evalforge_types2 = require("@wix/evalforge-types");
+
+// src/run-scenario/agents/claude-code/resolve-mcp-placeholders.ts
 var import_promises2 = require("fs/promises");
 var import_path3 = require("path");
-var import_evalforge_types2 = require("@wix/evalforge-types");
+var import_os2 = require("os");
+var WIX_AUTH_FILE = (0, import_path3.join)((0, import_os2.homedir)(), ".wix", "auth", "api-key.json");
+async function loadWixAuthPlaceholders(authFilePath = WIX_AUTH_FILE) {
+  try {
+    const content = await (0, import_promises2.readFile)(authFilePath, "utf-8");
+    const auth = JSON.parse(content);
+    if (!auth.token || !auth.userInfo?.userId) {
+      return {};
+    }
+    return {
+      "wix-auth-token": auth.token,
+      "wix-auth-user-id": auth.userInfo.userId
+    };
+  } catch (err) {
+    console.warn(
+      `[MCP] Could not load Wix auth file: ${err.message}`
+    );
+    return {};
+  }
+}
+async function resolveMcpPlaceholders(mcpServers, authFilePath) {
+  const needed = findPlaceholders(mcpServers);
+  if (needed.length === 0) {
+    return mcpServers;
+  }
+  const placeholders = await loadWixAuthPlaceholders(authFilePath);
+  const unresolved = needed.filter((key) => !(key in placeholders));
+  if (unresolved.length > 0) {
+    throw new Error(
+      `MCP config contains unresolvable placeholders: ${unresolved.map((k) => `{{${k}}}`).join(", ")}. Ensure ~/.wix/auth/api-key.json exists (run \`npx @wix/cli login\`).`
+    );
+  }
+  console.log(`[MCP] Resolved ${needed.length} placeholder(s)`);
+  return resolveValue(mcpServers, placeholders);
+}
+
+// src/run-scenario/agents/claude-code/write-mcp.ts
 async function writeMcpToFilesystem(cwd, mcps) {
   if (mcps.length === 0) return;
   const mcpServers = {};
@@ -714,19 +800,20 @@ async function writeMcpToFilesystem(cwd, mcps) {
       mcpServers[key] = value;
     }
   }
+  const resolvedServers = await resolveMcpPlaceholders(mcpServers);
   const content = JSON.stringify(
-    { [import_evalforge_types2.MCP_SERVERS_JSON_KEY]: mcpServers },
+    { [import_evalforge_types2.MCP_SERVERS_JSON_KEY]: resolvedServers },
     null,
     2
   );
-  const filePath = (0, import_path3.join)(cwd, ".mcp.json");
-  await (0, import_promises2.writeFile)(filePath, content, "utf8");
+  const filePath = (0, import_path4.join)(cwd, ".mcp.json");
+  await (0, import_promises3.writeFile)(filePath, content, "utf8");
   console.log(`[MCP] Written to ${filePath}`);
 }
 
 // src/run-scenario/agents/claude-code/write-sub-agents.ts
-var import_promises3 = require("fs/promises");
-var import_path4 = require("path");
+var import_promises4 = require("fs/promises");
+var import_path5 = require("path");
 var AGENTS_DIR = ".claude/agents";
 function toAgentFilename(name, index, nameCount) {
   const base = (name || "").toLowerCase().replace(/\s+/g, "-").replace(/[^a-z0-9-]/g, "").replace(/^-+|-+$/g, "") || `sub-agent-${index}`;
@@ -736,20 +823,20 @@ function toAgentFilename(name, index, nameCount) {
 }
 async function writeSubAgentsToFilesystem(cwd, subAgents) {
   if (subAgents.length === 0) return;
-  const agentsDir = (0, import_path4.join)(cwd, AGENTS_DIR);
-  await (0, import_promises3.mkdir)(agentsDir, { recursive: true });
+  const agentsDir = (0, import_path5.join)(cwd, AGENTS_DIR);
+  await (0, import_promises4.mkdir)(agentsDir, { recursive: true });
   const nameCount = /* @__PURE__ */ new Map();
   for (const [i, agent] of subAgents.entries()) {
     const filename = toAgentFilename(agent.name, i, nameCount);
-    const filePath = (0, import_path4.join)(agentsDir, `${filename}.md`);
-    await (0, import_promises3.writeFile)(filePath, agent.subAgentMd, "utf8");
+    const filePath = (0, import_path5.join)(agentsDir, `${filename}.md`);
+    await (0, import_promises4.writeFile)(filePath, agent.subAgentMd, "utf8");
   }
   console.log(`[SubAgents] Written to ${agentsDir}`);
 }
 
 // src/run-scenario/agents/claude-code/write-rules.ts
-var import_promises4 = require("fs/promises");
-var import_path5 = require("path");
+var import_promises5 = require("fs/promises");
+var import_path6 = require("path");
 var CURSOR_RULES_DIR = ".cursor/rules";
 function toRuleFilename(name, index, nameCount) {
   const base = (name || "").toLowerCase().replace(/\s+/g, "-").replace(/[^a-z0-9-]/g, "").replace(/^-+|-+$/g, "") || `rule-${index}`;
@@ -760,13 +847,13 @@ function toRuleFilename(name, index, nameCount) {
 async function appendToFile(filePath, content) {
   let existing = "";
   try {
-    existing = await (0, import_promises4.readFile)(filePath, "utf8");
+    existing = await (0, import_promises5.readFile)(filePath, "utf8");
   } catch {
   }
   const merged = existing ? `${existing.trimEnd()}
 
 ${content}` : content;
-  await (0, import_promises4.writeFile)(filePath, merged, "utf8");
+  await (0, import_promises5.writeFile)(filePath, merged, "utf8");
 }
 async function writeRulesToFilesystem(cwd, rules) {
   if (rules.length === 0) return;
@@ -775,21 +862,21 @@ async function writeRulesToFilesystem(cwd, rules) {
   for (const [i, rule] of rules.entries()) {
     switch (rule.ruleType) {
       case "claude-md": {
-        await appendToFile((0, import_path5.join)(cwd, "CLAUDE.md"), rule.content);
+        await appendToFile((0, import_path6.join)(cwd, "CLAUDE.md"), rule.content);
         break;
       }
       case "agents-md": {
-        await appendToFile((0, import_path5.join)(cwd, "AGENTS.md"), rule.content);
+        await appendToFile((0, import_path6.join)(cwd, "AGENTS.md"), rule.content);
         break;
       }
       case "cursor-rule": {
         if (!hasCursorRules) {
-          await (0, import_promises4.mkdir)((0, import_path5.join)(cwd, CURSOR_RULES_DIR), { recursive: true });
+          await (0, import_promises5.mkdir)((0, import_path6.join)(cwd, CURSOR_RULES_DIR), { recursive: true });
           hasCursorRules = true;
         }
         const filename = toRuleFilename(rule.name, i, nameCount);
-        const filePath = (0, import_path5.join)(cwd, CURSOR_RULES_DIR, `${filename}.md`);
-        await (0, import_promises4.writeFile)(filePath, rule.content, "utf8");
+        const filePath = (0, import_path6.join)(cwd, CURSOR_RULES_DIR, `${filename}.md`);
+        await (0, import_promises5.writeFile)(filePath, rule.content, "utf8");
         break;
       }
     }
@@ -1824,7 +1911,7 @@ defaultRegistry.register(claudeCodeAdapter);
 
 // src/run-scenario/file-diff.ts
 var import_fs2 = require("fs");
-var import_path6 = require("path");
+var import_path7 = require("path");
 
 // ../../node_modules/diff/lib/index.mjs
 function Diff() {
@@ -2000,7 +2087,7 @@ Diff.prototype = {
   tokenize: function tokenize(value) {
     return Array.from(value);
   },
-  join: function join5(chars) {
+  join: function join6(chars) {
     return chars.join("");
   },
   postProcess: function postProcess(changeObjects) {
@@ -2440,8 +2527,8 @@ function snapshotDirectory(dir, baseDir) {
   }
   const entries = (0, import_fs2.readdirSync)(dir, { withFileTypes: true });
   for (const entry of entries) {
-    const fullPath = (0, import_path6.join)(dir, entry.name);
-    const relativePath = (0, import_path6.relative)(base, fullPath);
+    const fullPath = (0, import_path7.join)(dir, entry.name);
+    const relativePath = (0, import_path7.relative)(base, fullPath);
     if (shouldIgnore(entry.name)) {
       continue;
     }