@wix/evalforge-evaluator 0.98.0 → 0.99.0

package/build/index.mjs

@@ -187,6 +187,53 @@ import {
   isSystemAssertionId,
   SYSTEM_ASSERTIONS
 } from "@wix/evalforge-types";
+
+// src/resolve-placeholders.ts
+var PLACEHOLDER_PATTERN = /\{\{([^}]+)\}\}/g;
+function findPlaceholders(value) {
+  const keys = /* @__PURE__ */ new Set();
+  collectPlaceholders(value, keys);
+  return [...keys];
+}
+function collectPlaceholders(value, keys) {
+  if (typeof value === "string") {
+    for (const match of value.matchAll(PLACEHOLDER_PATTERN)) {
+      keys.add(match[1].trim());
+    }
+  } else if (Array.isArray(value)) {
+    for (const item of value) {
+      collectPlaceholders(item, keys);
+    }
+  } else if (typeof value === "object" && value !== null) {
+    for (const val of Object.values(value)) {
+      collectPlaceholders(val, keys);
+    }
+  }
+}
+function resolveValue(value, placeholders) {
+  if (typeof value === "string") {
+    return value.replace(PLACEHOLDER_PATTERN, (match, key) => {
+      const trimmed = key.trim();
+      return trimmed in placeholders ? placeholders[trimmed] : match;
+    });
+  }
+  if (Array.isArray(value)) {
+    return value.map((item) => resolveValue(item, placeholders));
+  }
+  if (typeof value === "object" && value !== null) {
+    const result = {};
+    for (const [k, v] of Object.entries(value)) {
+      result[k] = resolveValue(v, placeholders);
+    }
+    return result;
+  }
+  return value;
+}
+function resolvePlaceholdersInString(text, placeholders) {
+  return resolveValue(text, placeholders);
+}
+
+// src/fetch-evaluation-data.ts
 function parseSkillNamesFromParams(value) {
   if (typeof value !== "string") {
     return [];
@@ -202,13 +249,11 @@ function applyParamsToAssertion(assertion, params) {
     return assertion;
   }
   if (assertion.type === "llm_judge") {
-    let prompt = assertion.prompt;
+    const stringParams = {};
     for (const [key, value] of Object.entries(params)) {
-      const placeholder = `{{${key}}}`;
-      const escapedPlaceholder = placeholder.replace(/[{}]/g, "\\$&");
-      const replacement = String(value ?? "");
-      prompt = prompt.replace(new RegExp(escapedPlaceholder, "g"), replacement);
+      stringParams[key] = String(value ?? "");
     }
+    const prompt = resolvePlaceholdersInString(assertion.prompt, stringParams);
     return {
       ...assertion,
       prompt,
@@ -689,8 +734,49 @@ import { randomUUID } from "crypto";
 
 // src/run-scenario/agents/claude-code/write-mcp.ts
 import { writeFile as writeFile2 } from "fs/promises";
-import { join as join2 } from "path";
+import { join as join3 } from "path";
 import { MCP_SERVERS_JSON_KEY } from "@wix/evalforge-types";
+
+// src/run-scenario/agents/claude-code/resolve-mcp-placeholders.ts
+import { readFile } from "fs/promises";
+import { join as join2 } from "path";
+import { homedir } from "os";
+var WIX_AUTH_FILE = join2(homedir(), ".wix", "auth", "api-key.json");
+async function loadWixAuthPlaceholders(authFilePath = WIX_AUTH_FILE) {
+  try {
+    const content = await readFile(authFilePath, "utf-8");
+    const auth = JSON.parse(content);
+    if (!auth.token || !auth.userInfo?.userId) {
+      return {};
+    }
+    return {
+      "wix-auth-token": auth.token,
+      "wix-auth-user-id": auth.userInfo.userId
+    };
+  } catch (err) {
+    console.warn(
+      `[MCP] Could not load Wix auth file: ${err.message}`
+    );
+    return {};
+  }
+}
+async function resolveMcpPlaceholders(mcpServers, authFilePath) {
+  const needed = findPlaceholders(mcpServers);
+  if (needed.length === 0) {
+    return mcpServers;
+  }
+  const placeholders = await loadWixAuthPlaceholders(authFilePath);
+  const unresolved = needed.filter((key) => !(key in placeholders));
+  if (unresolved.length > 0) {
+    throw new Error(
+      `MCP config contains unresolvable placeholders: ${unresolved.map((k) => `{{${k}}}`).join(", ")}. Ensure ~/.wix/auth/api-key.json exists (run \`npx @wix/cli login\`).`
+    );
+  }
+  console.log(`[MCP] Resolved ${needed.length} placeholder(s)`);
+  return resolveValue(mcpServers, placeholders);
+}
+
+// src/run-scenario/agents/claude-code/write-mcp.ts
 async function writeMcpToFilesystem(cwd, mcps) {
   if (mcps.length === 0) return;
   const mcpServers = {};
@@ -705,19 +791,20 @@ async function writeMcpToFilesystem(cwd, mcps) {
       mcpServers[key] = value;
     }
   }
+  const resolvedServers = await resolveMcpPlaceholders(mcpServers);
   const content = JSON.stringify(
-    { [MCP_SERVERS_JSON_KEY]: mcpServers },
+    { [MCP_SERVERS_JSON_KEY]: resolvedServers },
     null,
     2
   );
-  const filePath = join2(cwd, ".mcp.json");
+  const filePath = join3(cwd, ".mcp.json");
   await writeFile2(filePath, content, "utf8");
   console.log(`[MCP] Written to ${filePath}`);
 }
 
 // src/run-scenario/agents/claude-code/write-sub-agents.ts
 import { mkdir as mkdir2, writeFile as writeFile3 } from "fs/promises";
-import { join as join3 } from "path";
+import { join as join4 } from "path";
 var AGENTS_DIR = ".claude/agents";
 function toAgentFilename(name, index, nameCount) {
   const base = (name || "").toLowerCase().replace(/\s+/g, "-").replace(/[^a-z0-9-]/g, "").replace(/^-+|-+$/g, "") || `sub-agent-${index}`;
@@ -727,20 +814,20 @@ function toAgentFilename(name, index, nameCount) {
 }
 async function writeSubAgentsToFilesystem(cwd, subAgents) {
   if (subAgents.length === 0) return;
-  const agentsDir = join3(cwd, AGENTS_DIR);
+  const agentsDir = join4(cwd, AGENTS_DIR);
   await mkdir2(agentsDir, { recursive: true });
   const nameCount = /* @__PURE__ */ new Map();
   for (const [i, agent] of subAgents.entries()) {
     const filename = toAgentFilename(agent.name, i, nameCount);
-    const filePath = join3(agentsDir, `${filename}.md`);
+    const filePath = join4(agentsDir, `${filename}.md`);
     await writeFile3(filePath, agent.subAgentMd, "utf8");
   }
   console.log(`[SubAgents] Written to ${agentsDir}`);
 }
 
 // src/run-scenario/agents/claude-code/write-rules.ts
-import { mkdir as mkdir3, writeFile as writeFile4, readFile } from "fs/promises";
-import { join as join4 } from "path";
+import { mkdir as mkdir3, writeFile as writeFile4, readFile as readFile2 } from "fs/promises";
+import { join as join5 } from "path";
 var CURSOR_RULES_DIR = ".cursor/rules";
 function toRuleFilename(name, index, nameCount) {
   const base = (name || "").toLowerCase().replace(/\s+/g, "-").replace(/[^a-z0-9-]/g, "").replace(/^-+|-+$/g, "") || `rule-${index}`;
@@ -751,7 +838,7 @@ function toRuleFilename(name, index, nameCount) {
 async function appendToFile(filePath, content) {
   let existing = "";
   try {
-    existing = await readFile(filePath, "utf8");
+    existing = await readFile2(filePath, "utf8");
   } catch {
   }
   const merged = existing ? `${existing.trimEnd()}
@@ -766,20 +853,20 @@ async function writeRulesToFilesystem(cwd, rules) {
   for (const [i, rule] of rules.entries()) {
     switch (rule.ruleType) {
       case "claude-md": {
-        await appendToFile(join4(cwd, "CLAUDE.md"), rule.content);
+        await appendToFile(join5(cwd, "CLAUDE.md"), rule.content);
         break;
       }
       case "agents-md": {
-        await appendToFile(join4(cwd, "AGENTS.md"), rule.content);
+        await appendToFile(join5(cwd, "AGENTS.md"), rule.content);
         break;
       }
       case "cursor-rule": {
         if (!hasCursorRules) {
-          await mkdir3(join4(cwd, CURSOR_RULES_DIR), { recursive: true });
+          await mkdir3(join5(cwd, CURSOR_RULES_DIR), { recursive: true });
           hasCursorRules = true;
         }
         const filename = toRuleFilename(rule.name, i, nameCount);
-        const filePath = join4(cwd, CURSOR_RULES_DIR, `${filename}.md`);
+        const filePath = join5(cwd, CURSOR_RULES_DIR, `${filename}.md`);
         await writeFile4(filePath, rule.content, "utf8");
         break;
       }
@@ -1815,7 +1902,7 @@ defaultRegistry.register(claudeCodeAdapter);
 
 // src/run-scenario/file-diff.ts
 import { readdirSync, readFileSync as readFileSync2, statSync, existsSync as existsSync2 } from "fs";
-import { join as join6, relative } from "path";
+import { join as join7, relative } from "path";
 
 // ../../node_modules/diff/lib/index.mjs
 function Diff() {
@@ -1991,7 +2078,7 @@ Diff.prototype = {
   tokenize: function tokenize(value) {
     return Array.from(value);
   },
-  join: function join5(chars) {
+  join: function join6(chars) {
     return chars.join("");
   },
   postProcess: function postProcess(changeObjects) {
@@ -2431,7 +2518,7 @@ function snapshotDirectory(dir, baseDir) {
   }
   const entries = readdirSync(dir, { withFileTypes: true });
   for (const entry of entries) {
-    const fullPath = join6(dir, entry.name);
+    const fullPath = join7(dir, entry.name);
     const relativePath = relative(base, fullPath);
     if (shouldIgnore(entry.name)) {
       continue;