@wix/evalforge-evaluator 0.182.0 → 0.184.0

package/build/index.mjs

@@ -7487,42 +7487,117 @@ import {
 // src/run-scenario/agents/claude-code/write-skills.ts
 import { mkdir as mkdir3 } from "fs/promises";
 import { join } from "path";
-import { fetchGitHubFolder as fetchGitHubFolder2 } from "@wix/evalforge-github-client";
-async function writeSkillsToFilesystem(cwd, skills, fetchFn = fetchGitHubFolder2) {
+
+// src/run-scenario/agents/shared/resolve-capability-content.ts
+import {
+  fetchGitHubFile as fetchGitHubFile2,
+  fetchGitHubFolder as fetchGitHubFolder2
+} from "@wix/evalforge-github-client";
+import { MCP_SERVERS_JSON_KEY } from "@wix/evalforge-types";
+var USER_AGENT = "EvalForge-Evaluator";
+async function resolveSkillFiles(skill, fetchFn = fetchGitHubFolder2) {
+  const version = skill.latestVersion;
+  if (version?.files && version.files.length > 0) {
+    console.log(
+      `[Skill] ${skill.name}: using ${version.files.length} file(s) from snapshot`
+    );
+    return version.files;
+  }
+  if (skill.source) {
+    const files = await fetchFn(skill.source, { userAgent: USER_AGENT });
+    console.log(
+      `[Skill] ${skill.name}: fetched ${files.length} file(s) from GitHub (live)`
+    );
+    return files;
+  }
+  throw new Error(`Skill ${skill.name} has no files and no source configured`);
+}
+async function fetchSourceFile(label, noun, name, source, fetchFn) {
+  try {
+    const content = await fetchFn(source, { userAgent: USER_AGENT });
+    console.log(
+      `[${label}] Fetched "${name}" from ${source.owner}/${source.repo}/${source.path}@${source.ref}`
+    );
+    return content;
+  } catch (error) {
+    const message = error instanceof Error ? error.message : "Unknown error";
+    console.error(`[${label}] "${name}": GitHub fetch failed: ${message}`);
+    throw new Error(
+      `Failed to fetch ${noun} "${name}" from GitHub: ${message}`
+    );
+  }
+}
+async function resolveSubAgentMd(agent, fetchFn = fetchGitHubFile2) {
+  if (agent.source) {
+    return fetchSourceFile(
+      "SubAgents",
+      "sub-agent",
+      agent.name,
+      agent.source,
+      fetchFn
+    );
+  }
+  if (!agent.subAgentMd) {
+    console.warn(
+      `[SubAgents] "${agent.name}" has empty inline content \u2013 the agent file will be blank`
+    );
+  }
+  return agent.subAgentMd;
+}
+async function resolveRuleText(rule, fetchFn = fetchGitHubFile2) {
+  if (!rule.source) {
+    return rule.content;
+  }
+  return fetchSourceFile("Rules", "rule", rule.name, rule.source, fetchFn);
+}
+async function resolveMcpConfig(mcp, fetchFn = fetchGitHubFile2) {
+  if (!mcp.source) {
+    return mcp.config;
+  }
+  const raw = await fetchSourceFile(
+    "MCP",
+    "MCP",
+    mcp.name,
+    mcp.source,
+    fetchFn
+  );
+  let parsed;
+  try {
+    parsed = JSON.parse(raw);
+  } catch (error) {
+    const message = error instanceof Error ? error.message : "Unknown error";
+    throw new Error(
+      `MCP "${mcp.name}" GitHub source is not valid JSON: ${message}`
+    );
+  }
+  if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
+    throw new Error(`MCP "${mcp.name}" GitHub source must be a JSON object`);
+  }
+  const obj = parsed;
+  const servers = obj[MCP_SERVERS_JSON_KEY];
+  if (typeof servers === "object" && servers !== null && !Array.isArray(servers)) {
+    return servers;
+  }
+  return obj;
+}
+
+// src/run-scenario/agents/claude-code/write-skills.ts
+async function writeSkillsToFilesystem(cwd, skills, fetchFn) {
   await Promise.all(
     skills.map((skill) => writeSkillToFilesystem(cwd, skill, fetchFn))
   );
 }
-async function writeSkillToFilesystem(cwd, skill, fetchFn = fetchGitHubFolder2) {
-  const skillName = skill.name;
-  const skillDir = join(cwd, ".claude", "skills", skillName);
+async function writeSkillToFilesystem(cwd, skill, fetchFn) {
+  const skillDir = join(cwd, ".claude", "skills", skill.name);
   await mkdir3(skillDir, { recursive: true });
-  const version = skill.latestVersion;
-  if (version?.files && version.files.length > 0) {
-    await writeFilesToDirectory(skillDir, version.files);
-    console.log(
-      `[Skill] ${skillName}: wrote ${version.files.length} file(s) from snapshot`
+  try {
+    const files = await resolveSkillFiles(skill, fetchFn);
+    await writeFilesToDirectory(skillDir, files);
+  } catch (error) {
+    const message = error instanceof Error ? error.message : "Unknown error";
+    throw new Error(
+      `Failed to write skill ${skill.name} to filesystem: ${message}`
     );
-  } else if (skill.source) {
-    try {
-      const files = await fetchFn(skill.source, {
-        userAgent: "EvalForge-Evaluator"
-      });
-      await writeFilesToDirectory(skillDir, files);
-      console.log(
-        `[Skill] ${skillName}: wrote ${files.length} file(s) from GitHub (live)`
-      );
-    } catch (error) {
-      const message = error instanceof Error ? error.message : "Unknown error";
-      console.error(
-        `[Skill] ${skillName}: GitHub fetch failed: ${message}, no files to fall back to`
-      );
-      throw new Error(
-        `Failed to write skill ${skillName} to filesystem: ${message}`
-      );
-    }
-  } else {
-    throw new Error(`Skill ${skillName} has no files and no source configured`);
   }
 }
 
@@ -7540,7 +7615,7 @@ import { randomUUID } from "crypto";
 // src/run-scenario/agents/claude-code/write-mcp.ts
 import { writeFile as writeFile3 } from "fs/promises";
 import { join as join3 } from "path";
-import { MCP_SERVERS_JSON_KEY } from "@wix/evalforge-types";
+import { MCP_SERVERS_JSON_KEY as MCP_SERVERS_JSON_KEY2 } from "@wix/evalforge-types";
 
 // src/run-scenario/agents/shared/resolve-mcp-placeholders.ts
 import { readFile } from "fs/promises";
@@ -7585,11 +7660,11 @@ async function resolveMcpPlaceholders(mcpServers, options = {}) {
 }
 
 // src/run-scenario/agents/claude-code/write-mcp.ts
-async function writeMcpToFilesystem(cwd, mcps) {
+async function writeMcpToFilesystem(cwd, mcps, fetchFn) {
   if (mcps.length === 0) return;
   const mcpServers = {};
   for (const mcp of mcps) {
-    const config = mcp.config;
+    const config = await resolveMcpConfig(mcp, fetchFn);
     for (const [key, value] of Object.entries(config)) {
       if (typeof value !== "object" || value === null || Array.isArray(value)) {
         throw new Error(
@@ -7601,7 +7676,7 @@ async function writeMcpToFilesystem(cwd, mcps) {
   }
   const resolvedServers = await resolveMcpPlaceholders(mcpServers, { cwd });
   const content = JSON.stringify(
-    { [MCP_SERVERS_JSON_KEY]: resolvedServers },
+    { [MCP_SERVERS_JSON_KEY2]: resolvedServers },
     null,
     2
   );
@@ -7613,9 +7688,6 @@ async function writeMcpToFilesystem(cwd, mcps) {
 // src/run-scenario/agents/claude-code/write-sub-agents.ts
 import { mkdir as mkdir4, writeFile as writeFile4 } from "fs/promises";
 import { join as join4 } from "path";
-import {
-  fetchGitHubFile as fetchGitHubFile2
-} from "@wix/evalforge-github-client";
 var AGENTS_DIR = ".claude/agents";
 function toAgentFilename(name, index, nameCount) {
   const base = (name || "").toLowerCase().replace(/\s+/g, "-").replace(/[^a-z0-9-]/g, "").replace(/^-+|-+$/g, "") || `sub-agent-${index}`;
@@ -7623,34 +7695,7 @@ function toAgentFilename(name, index, nameCount) {
   nameCount.set(base, count + 1);
   return count === 0 ? base : `${base}-${count + 1}`;
 }
-async function resolveSubAgentContent(agent, fetchFn) {
-  if (agent.source) {
-    try {
-      const content = await fetchFn(agent.source, {
-        userAgent: "EvalForge-Evaluator"
-      });
-      console.log(
-        `[SubAgents] Fetched "${agent.name}" from ${agent.source.owner}/${agent.source.repo}/${agent.source.path}@${agent.source.ref}`
-      );
-      return content;
-    } catch (error) {
-      const message = error instanceof Error ? error.message : "Unknown error";
-      console.error(
-        `[SubAgents] "${agent.name}": GitHub fetch failed: ${message}`
-      );
-      throw new Error(
-        `Failed to fetch sub-agent "${agent.name}" from GitHub: ${message}`
-      );
-    }
-  }
-  if (!agent.subAgentMd) {
-    console.warn(
-      `[SubAgents] "${agent.name}" has empty inline content \u2013 the agent file will be blank`
-    );
-  }
-  return agent.subAgentMd;
-}
-async function writeSubAgentsToFilesystem(cwd, subAgents, fetchFn = fetchGitHubFile2) {
+async function writeSubAgentsToFilesystem(cwd, subAgents, fetchFn) {
   if (subAgents.length === 0) return;
   const agentsDir = join4(cwd, AGENTS_DIR);
   await mkdir4(agentsDir, { recursive: true });
@@ -7658,7 +7703,7 @@ async function writeSubAgentsToFilesystem(cwd, subAgents, fetchFn = fetchGitHubF
   for (const [i, agent] of subAgents.entries()) {
     const filename = toAgentFilename(agent.name, i, nameCount);
     const filePath = join4(agentsDir, `${filename}.md`);
-    const content = await resolveSubAgentContent(agent, fetchFn);
+    const content = await resolveSubAgentMd(agent, fetchFn);
     await writeFile4(filePath, content, "utf8");
   }
   console.log(`[SubAgents] Written to ${agentsDir}`);
@@ -7708,18 +7753,19 @@ function validateGenericDirectory(dir, cwd) {
   }
   return trimmed;
 }
-async function writeRulesToFilesystem(cwd, rules) {
+async function writeRulesToFilesystem(cwd, rules, fetchFn) {
   if (rules.length === 0) return;
   const nameCount = /* @__PURE__ */ new Map();
   let hasCursorRules = false;
   for (const [i, rule] of rules.entries()) {
+    const content = await resolveRuleText(rule, fetchFn);
     switch (rule.ruleType) {
       case "claude-md": {
-        await appendToFile(join5(cwd, "CLAUDE.md"), rule.content);
+        await appendToFile(join5(cwd, "CLAUDE.md"), content);
         break;
       }
       case "agents-md": {
-        await appendToFile(join5(cwd, "AGENTS.md"), rule.content);
+        await appendToFile(join5(cwd, "AGENTS.md"), content);
         break;
       }
       case "cursor-rule": {
@@ -7729,7 +7775,7 @@ async function writeRulesToFilesystem(cwd, rules) {
         }
         const filename = toRuleFilename(rule.name, i, nameCount);
         const filePath = join5(cwd, CURSOR_RULES_DIR, `${filename}.md`);
-        await writeFile5(filePath, rule.content, "utf8");
+        await writeFile5(filePath, content, "utf8");
         break;
       }
       case "generic": {
@@ -7740,7 +7786,7 @@ async function writeRulesToFilesystem(cwd, rules) {
         const dirPath = join5(cwd, directory);
         await mkdir5(dirPath, { recursive: true });
         const filename = toRuleFilename(rule.name, i, nameCount);
-        await writeFile5(join5(dirPath, `${filename}.md`), rule.content, "utf8");
+        await writeFile5(join5(dirPath, `${filename}.md`), content, "utf8");
         break;
       }
       default: {
@@ -8655,9 +8701,12 @@ function processMessages(timestampedMessages, startTime, endTime) {
       if (!step.toolCalls) continue;
       for (const tc of step.toolCalls) {
         if (tc.toolUseId && toolResultErrors.has(tc.toolUseId)) {
-          step.hasToolError = true;
-          step.toolErrorContent = toolResultErrors.get(tc.toolUseId);
-          break;
+          tc.isError = true;
+          tc.errorContent = toolResultErrors.get(tc.toolUseId);
+          if (!step.hasToolError) {
+            step.hasToolError = true;
+            step.toolErrorContent = tc.errorContent;
+          }
         }
       }
     }
@@ -8761,8 +8810,8 @@ function buildLLMTraceFromSteps(steps, totalDurationMs, usage, model) {
         },
         costUsd: stepCost / totalSubSteps,
         outputPreview: step.thinking?.slice(0, 200),
-        success: isSuccess,
-        error: errorMsg
+        success: true,
+        error: void 0
       });
     }
     if (toolCallCount > 0) {
@@ -8772,6 +8821,8 @@ function buildLLMTraceFromSteps(steps, totalDurationMs, usage, model) {
         const toolBudgetSteps = toolSubSteps + textSubSteps;
         const toolFraction = toolBudgetSteps > 0 ? 1 / toolBudgetSteps : 1;
         const remainingFraction = (totalSubSteps - thinkingSubSteps) / totalSubSteps;
+        const toolSuccess = !tc.isError;
+        const toolError = tc.isError ? tc.errorContent ?? "Tool call failed" : void 0;
         subSteps.push({
           id: randomUUID(),
           stepNumber: 0,
@@ -8796,8 +8847,8 @@ function buildLLMTraceFromSteps(steps, totalDurationMs, usage, model) {
           toolName: tc.toolName,
           toolArguments: JSON.stringify(tc.args),
           outputPreview: tcIdx === 0 && !hasText ? (step.text || step.thinking)?.slice(0, 200) : void 0,
-          success: isSuccess,
-          error: errorMsg
+          success: toolSuccess,
+          error: toolError
         });
       }
     }
@@ -8818,8 +8869,8 @@ function buildLLMTraceFromSteps(steps, totalDurationMs, usage, model) {
         },
         costUsd: stepCost - subSteps.reduce((s, ss) => s + ss.costUsd, 0),
         outputPreview: step.text?.slice(0, 200),
-        success: isSuccess,
-        error: errorMsg
+        success: true,
+        error: void 0
       });
     }
     if (subSteps.length === 0) {
@@ -9003,51 +9054,28 @@ function tryParseJson(text) {
 // src/run-scenario/agents/opencode/write-skills.ts
 import { mkdir as mkdir6 } from "fs/promises";
 import { join as join6 } from "path";
-import { fetchGitHubFolder as fetchGitHubFolder3 } from "@wix/evalforge-github-client";
-async function writeSkillsToFilesystem2(cwd, skills, fetchFn = fetchGitHubFolder3) {
+async function writeSkillsToFilesystem2(cwd, skills, fetchFn) {
   await Promise.all(
     skills.map((skill) => writeSkillToFilesystem2(cwd, skill, fetchFn))
   );
 }
 async function writeSkillToFilesystem2(cwd, skill, fetchFn) {
-  const skillName = skill.name;
-  const skillDir = join6(cwd, ".opencode", "skills", skillName);
+  const skillDir = join6(cwd, ".opencode", "skills", skill.name);
   await mkdir6(skillDir, { recursive: true });
-  const version = skill.latestVersion;
-  if (version?.files && version.files.length > 0) {
-    await writeFilesToDirectory(skillDir, version.files);
-    console.log(
-      `[Skill] ${skillName}: wrote ${version.files.length} file(s) from snapshot`
+  try {
+    const files = await resolveSkillFiles(skill, fetchFn);
+    await writeFilesToDirectory(skillDir, files);
+  } catch (error) {
+    const message = error instanceof Error ? error.message : "Unknown error";
+    throw new Error(
+      `Failed to write skill ${skill.name} to filesystem: ${message}`
     );
-  } else if (skill.source) {
-    try {
-      const files = await fetchFn(skill.source, {
-        userAgent: "EvalForge-Evaluator"
-      });
-      await writeFilesToDirectory(skillDir, files);
-      console.log(
-        `[Skill] ${skillName}: wrote ${files.length} file(s) from GitHub (live)`
-      );
-    } catch (error) {
-      const message = error instanceof Error ? error.message : "Unknown error";
-      console.error(
-        `[Skill] ${skillName}: GitHub fetch failed: ${message}, no files to fall back to`
-      );
-      throw new Error(
-        `Failed to write skill ${skillName} to filesystem: ${message}`
-      );
-    }
-  } else {
-    throw new Error(`Skill ${skillName} has no files and no source configured`);
   }
 }
 
 // src/run-scenario/agents/opencode/write-sub-agents.ts
 import { mkdir as mkdir7, writeFile as writeFile6 } from "fs/promises";
 import { join as join7 } from "path";
-import {
-  fetchGitHubFile as fetchGitHubFile3
-} from "@wix/evalforge-github-client";
 var AGENTS_DIR2 = ".opencode/agents";
 function toAgentFilename2(name, index, nameCount) {
   const base = (name || "").toLowerCase().replace(/\s+/g, "-").replace(/[^a-z0-9-]/g, "").replace(/^-+|-+$/g, "") || `sub-agent-${index}`;
@@ -9055,34 +9083,7 @@ function toAgentFilename2(name, index, nameCount) {
   nameCount.set(base, count + 1);
   return count === 0 ? base : `${base}-${count + 1}`;
 }
-async function resolveSubAgentContent2(agent, fetchFn) {
-  if (agent.source) {
-    try {
-      const content = await fetchFn(agent.source, {
-        userAgent: "EvalForge-Evaluator"
-      });
-      console.log(
-        `[SubAgents] Fetched "${agent.name}" from ${agent.source.owner}/${agent.source.repo}/${agent.source.path}@${agent.source.ref}`
-      );
-      return content;
-    } catch (error) {
-      const message = error instanceof Error ? error.message : "Unknown error";
-      console.error(
-        `[SubAgents] "${agent.name}": GitHub fetch failed: ${message}`
-      );
-      throw new Error(
-        `Failed to fetch sub-agent "${agent.name}" from GitHub: ${message}`
-      );
-    }
-  }
-  if (!agent.subAgentMd) {
-    console.warn(
-      `[SubAgents] "${agent.name}" has empty inline content \u2013 the agent file will be blank`
-    );
-  }
-  return agent.subAgentMd;
-}
-async function writeSubAgentsToFilesystem2(cwd, subAgents, fetchFn = fetchGitHubFile3) {
+async function writeSubAgentsToFilesystem2(cwd, subAgents, fetchFn) {
   if (subAgents.length === 0) return;
   const agentsDir = join7(cwd, AGENTS_DIR2);
   await mkdir7(agentsDir, { recursive: true });
@@ -9090,7 +9091,7 @@ async function writeSubAgentsToFilesystem2(cwd, subAgents, fetchFn = fetchGitHub
   for (const [i, agent] of subAgents.entries()) {
     const filename = toAgentFilename2(agent.name, i, nameCount);
     const filePath = join7(agentsDir, `${filename}.md`);
-    const content = await resolveSubAgentContent2(agent, fetchFn);
+    const content = await resolveSubAgentMd(agent, fetchFn);
     await writeFile6(filePath, content, "utf8");
   }
   console.log(`[SubAgents] Written to ${agentsDir}`);
@@ -9192,7 +9193,7 @@ async function buildOpenCodeEnv(options) {
   if (options.mcps && options.mcps.length > 0) {
     const mcpServers = {};
     for (const mcpEntity of options.mcps) {
-      const entityConfig = mcpEntity.config;
+      const entityConfig = await resolveMcpConfig(mcpEntity);
       for (const [key, value] of Object.entries(entityConfig)) {
         if (typeof value !== "object" || value === null || Array.isArray(value)) {
           throw new Error(