@wix/evalforge-evaluator 0.183.0 → 0.185.0

package/build/index.mjs

@@ -7487,117 +7487,42 @@ import {
 // src/run-scenario/agents/claude-code/write-skills.ts
 import { mkdir as mkdir3 } from "fs/promises";
 import { join } from "path";
-
-// src/run-scenario/agents/shared/resolve-capability-content.ts
-import {
-  fetchGitHubFile as fetchGitHubFile2,
-  fetchGitHubFolder as fetchGitHubFolder2
-} from "@wix/evalforge-github-client";
-import { MCP_SERVERS_JSON_KEY } from "@wix/evalforge-types";
-var USER_AGENT = "EvalForge-Evaluator";
-async function resolveSkillFiles(skill, fetchFn = fetchGitHubFolder2) {
-  const version = skill.latestVersion;
-  if (version?.files && version.files.length > 0) {
-    console.log(
-      `[Skill] ${skill.name}: using ${version.files.length} file(s) from snapshot`
-    );
-    return version.files;
-  }
-  if (skill.source) {
-    const files = await fetchFn(skill.source, { userAgent: USER_AGENT });
-    console.log(
-      `[Skill] ${skill.name}: fetched ${files.length} file(s) from GitHub (live)`
-    );
-    return files;
-  }
-  throw new Error(`Skill ${skill.name} has no files and no source configured`);
-}
-async function fetchSourceFile(label, noun, name, source, fetchFn) {
-  try {
-    const content = await fetchFn(source, { userAgent: USER_AGENT });
-    console.log(
-      `[${label}] Fetched "${name}" from ${source.owner}/${source.repo}/${source.path}@${source.ref}`
-    );
-    return content;
-  } catch (error) {
-    const message = error instanceof Error ? error.message : "Unknown error";
-    console.error(`[${label}] "${name}": GitHub fetch failed: ${message}`);
-    throw new Error(
-      `Failed to fetch ${noun} "${name}" from GitHub: ${message}`
-    );
-  }
-}
-async function resolveSubAgentMd(agent, fetchFn = fetchGitHubFile2) {
-  if (agent.source) {
-    return fetchSourceFile(
-      "SubAgents",
-      "sub-agent",
-      agent.name,
-      agent.source,
-      fetchFn
-    );
-  }
-  if (!agent.subAgentMd) {
-    console.warn(
-      `[SubAgents] "${agent.name}" has empty inline content \u2013 the agent file will be blank`
-    );
-  }
-  return agent.subAgentMd;
-}
-async function resolveRuleText(rule, fetchFn = fetchGitHubFile2) {
-  if (!rule.source) {
-    return rule.content;
-  }
-  return fetchSourceFile("Rules", "rule", rule.name, rule.source, fetchFn);
-}
-async function resolveMcpConfig(mcp, fetchFn = fetchGitHubFile2) {
-  if (!mcp.source) {
-    return mcp.config;
-  }
-  const raw = await fetchSourceFile(
-    "MCP",
-    "MCP",
-    mcp.name,
-    mcp.source,
-    fetchFn
-  );
-  let parsed;
-  try {
-    parsed = JSON.parse(raw);
-  } catch (error) {
-    const message = error instanceof Error ? error.message : "Unknown error";
-    throw new Error(
-      `MCP "${mcp.name}" GitHub source is not valid JSON: ${message}`
-    );
-  }
-  if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
-    throw new Error(`MCP "${mcp.name}" GitHub source must be a JSON object`);
-  }
-  const obj = parsed;
-  const servers = obj[MCP_SERVERS_JSON_KEY];
-  if (typeof servers === "object" && servers !== null && !Array.isArray(servers)) {
-    return servers;
-  }
-  return obj;
-}
-
-// src/run-scenario/agents/claude-code/write-skills.ts
-async function writeSkillsToFilesystem(cwd, skills, fetchFn) {
+import { fetchGitHubFolder as fetchGitHubFolder2 } from "@wix/evalforge-github-client";
+async function writeSkillsToFilesystem(cwd, skills, fetchFn = fetchGitHubFolder2) {
   await Promise.all(
     skills.map((skill) => writeSkillToFilesystem(cwd, skill, fetchFn))
   );
 }
-async function writeSkillToFilesystem(cwd, skill, fetchFn) {
-  const skillDir = join(cwd, ".claude", "skills", skill.name);
+async function writeSkillToFilesystem(cwd, skill, fetchFn = fetchGitHubFolder2) {
+  const skillName = skill.name;
+  const skillDir = join(cwd, ".claude", "skills", skillName);
   await mkdir3(skillDir, { recursive: true });
-  try {
-    const files = await resolveSkillFiles(skill, fetchFn);
-    await writeFilesToDirectory(skillDir, files);
-  } catch (error) {
-    const message = error instanceof Error ? error.message : "Unknown error";
-    throw new Error(
-      `Failed to write skill ${skill.name} to filesystem: ${message}`
+  const version = skill.latestVersion;
+  if (version?.files && version.files.length > 0) {
+    await writeFilesToDirectory(skillDir, version.files);
+    console.log(
+      `[Skill] ${skillName}: wrote ${version.files.length} file(s) from snapshot`
     );
+  } else if (skill.source) {
+    try {
+      const files = await fetchFn(skill.source, {
+        userAgent: "EvalForge-Evaluator"
+      });
+      await writeFilesToDirectory(skillDir, files);
+      console.log(
+        `[Skill] ${skillName}: wrote ${files.length} file(s) from GitHub (live)`
+      );
+    } catch (error) {
+      const message = error instanceof Error ? error.message : "Unknown error";
+      console.error(
+        `[Skill] ${skillName}: GitHub fetch failed: ${message}, no files to fall back to`
+      );
+      throw new Error(
+        `Failed to write skill ${skillName} to filesystem: ${message}`
+      );
+    }
+  } else {
+    throw new Error(`Skill ${skillName} has no files and no source configured`);
   }
 }
 
@@ -7615,7 +7540,7 @@ import { randomUUID } from "crypto";
 // src/run-scenario/agents/claude-code/write-mcp.ts
 import { writeFile as writeFile3 } from "fs/promises";
 import { join as join3 } from "path";
-import { MCP_SERVERS_JSON_KEY as MCP_SERVERS_JSON_KEY2 } from "@wix/evalforge-types";
+import { MCP_SERVERS_JSON_KEY } from "@wix/evalforge-types";
 
 // src/run-scenario/agents/shared/resolve-mcp-placeholders.ts
 import { readFile } from "fs/promises";
@@ -7660,11 +7585,11 @@ async function resolveMcpPlaceholders(mcpServers, options = {}) {
 }
 
 // src/run-scenario/agents/claude-code/write-mcp.ts
-async function writeMcpToFilesystem(cwd, mcps, fetchFn) {
+async function writeMcpToFilesystem(cwd, mcps) {
   if (mcps.length === 0) return;
   const mcpServers = {};
   for (const mcp of mcps) {
-    const config = await resolveMcpConfig(mcp, fetchFn);
+    const config = mcp.config;
     for (const [key, value] of Object.entries(config)) {
       if (typeof value !== "object" || value === null || Array.isArray(value)) {
         throw new Error(
@@ -7676,7 +7601,7 @@ async function writeMcpToFilesystem(cwd, mcps, fetchFn) {
   }
   const resolvedServers = await resolveMcpPlaceholders(mcpServers, { cwd });
   const content = JSON.stringify(
-    { [MCP_SERVERS_JSON_KEY2]: resolvedServers },
+    { [MCP_SERVERS_JSON_KEY]: resolvedServers },
     null,
     2
   );
@@ -7688,6 +7613,9 @@ async function writeMcpToFilesystem(cwd, mcps, fetchFn) {
 // src/run-scenario/agents/claude-code/write-sub-agents.ts
 import { mkdir as mkdir4, writeFile as writeFile4 } from "fs/promises";
 import { join as join4 } from "path";
+import {
+  fetchGitHubFile as fetchGitHubFile2
+} from "@wix/evalforge-github-client";
 var AGENTS_DIR = ".claude/agents";
 function toAgentFilename(name, index, nameCount) {
   const base = (name || "").toLowerCase().replace(/\s+/g, "-").replace(/[^a-z0-9-]/g, "").replace(/^-+|-+$/g, "") || `sub-agent-${index}`;
@@ -7695,7 +7623,34 @@ function toAgentFilename(name, index, nameCount) {
   nameCount.set(base, count + 1);
   return count === 0 ? base : `${base}-${count + 1}`;
 }
-async function writeSubAgentsToFilesystem(cwd, subAgents, fetchFn) {
+async function resolveSubAgentContent(agent, fetchFn) {
+  if (agent.source) {
+    try {
+      const content = await fetchFn(agent.source, {
+        userAgent: "EvalForge-Evaluator"
+      });
+      console.log(
+        `[SubAgents] Fetched "${agent.name}" from ${agent.source.owner}/${agent.source.repo}/${agent.source.path}@${agent.source.ref}`
+      );
+      return content;
+    } catch (error) {
+      const message = error instanceof Error ? error.message : "Unknown error";
+      console.error(
+        `[SubAgents] "${agent.name}": GitHub fetch failed: ${message}`
+      );
+      throw new Error(
+        `Failed to fetch sub-agent "${agent.name}" from GitHub: ${message}`
+      );
+    }
+  }
+  if (!agent.subAgentMd) {
+    console.warn(
+      `[SubAgents] "${agent.name}" has empty inline content \u2013 the agent file will be blank`
+    );
+  }
+  return agent.subAgentMd;
+}
+async function writeSubAgentsToFilesystem(cwd, subAgents, fetchFn = fetchGitHubFile2) {
   if (subAgents.length === 0) return;
   const agentsDir = join4(cwd, AGENTS_DIR);
   await mkdir4(agentsDir, { recursive: true });
@@ -7703,7 +7658,7 @@ async function writeSubAgentsToFilesystem(cwd, subAgents, fetchFn) {
   for (const [i, agent] of subAgents.entries()) {
     const filename = toAgentFilename(agent.name, i, nameCount);
     const filePath = join4(agentsDir, `${filename}.md`);
-    const content = await resolveSubAgentMd(agent, fetchFn);
+    const content = await resolveSubAgentContent(agent, fetchFn);
     await writeFile4(filePath, content, "utf8");
   }
   console.log(`[SubAgents] Written to ${agentsDir}`);
@@ -7753,19 +7708,18 @@ function validateGenericDirectory(dir, cwd) {
   }
   return trimmed;
 }
-async function writeRulesToFilesystem(cwd, rules, fetchFn) {
+async function writeRulesToFilesystem(cwd, rules) {
   if (rules.length === 0) return;
   const nameCount = /* @__PURE__ */ new Map();
   let hasCursorRules = false;
   for (const [i, rule] of rules.entries()) {
-    const content = await resolveRuleText(rule, fetchFn);
     switch (rule.ruleType) {
       case "claude-md": {
-        await appendToFile(join5(cwd, "CLAUDE.md"), content);
+        await appendToFile(join5(cwd, "CLAUDE.md"), rule.content);
         break;
       }
       case "agents-md": {
-        await appendToFile(join5(cwd, "AGENTS.md"), content);
+        await appendToFile(join5(cwd, "AGENTS.md"), rule.content);
         break;
       }
       case "cursor-rule": {
@@ -7775,7 +7729,7 @@ async function writeRulesToFilesystem(cwd, rules, fetchFn) {
         }
         const filename = toRuleFilename(rule.name, i, nameCount);
         const filePath = join5(cwd, CURSOR_RULES_DIR, `${filename}.md`);
-        await writeFile5(filePath, content, "utf8");
+        await writeFile5(filePath, rule.content, "utf8");
         break;
       }
       case "generic": {
@@ -7786,7 +7740,7 @@ async function writeRulesToFilesystem(cwd, rules, fetchFn) {
         const dirPath = join5(cwd, directory);
         await mkdir5(dirPath, { recursive: true });
         const filename = toRuleFilename(rule.name, i, nameCount);
-        await writeFile5(join5(dirPath, `${filename}.md`), content, "utf8");
+        await writeFile5(join5(dirPath, `${filename}.md`), rule.content, "utf8");
         break;
       }
       default: {
@@ -8701,9 +8655,12 @@ function processMessages(timestampedMessages, startTime, endTime) {
       if (!step.toolCalls) continue;
       for (const tc of step.toolCalls) {
         if (tc.toolUseId && toolResultErrors.has(tc.toolUseId)) {
-          step.hasToolError = true;
-          step.toolErrorContent = toolResultErrors.get(tc.toolUseId);
-          break;
+          tc.isError = true;
+          tc.errorContent = toolResultErrors.get(tc.toolUseId);
+          if (!step.hasToolError) {
+            step.hasToolError = true;
+            step.toolErrorContent = tc.errorContent;
+          }
         }
       }
     }
@@ -8807,8 +8764,8 @@ function buildLLMTraceFromSteps(steps, totalDurationMs, usage, model) {
         },
         costUsd: stepCost / totalSubSteps,
         outputPreview: step.thinking?.slice(0, 200),
-        success: isSuccess,
-        error: errorMsg
+        success: true,
+        error: void 0
       });
     }
     if (toolCallCount > 0) {
@@ -8818,6 +8775,8 @@ function buildLLMTraceFromSteps(steps, totalDurationMs, usage, model) {
         const toolBudgetSteps = toolSubSteps + textSubSteps;
         const toolFraction = toolBudgetSteps > 0 ? 1 / toolBudgetSteps : 1;
         const remainingFraction = (totalSubSteps - thinkingSubSteps) / totalSubSteps;
+        const toolSuccess = !tc.isError;
+        const toolError = tc.isError ? tc.errorContent ?? "Tool call failed" : void 0;
         subSteps.push({
           id: randomUUID(),
           stepNumber: 0,
@@ -8842,8 +8801,8 @@ function buildLLMTraceFromSteps(steps, totalDurationMs, usage, model) {
           toolName: tc.toolName,
           toolArguments: JSON.stringify(tc.args),
           outputPreview: tcIdx === 0 && !hasText ? (step.text || step.thinking)?.slice(0, 200) : void 0,
-          success: isSuccess,
-          error: errorMsg
+          success: toolSuccess,
+          error: toolError
         });
       }
     }
@@ -8864,8 +8823,8 @@ function buildLLMTraceFromSteps(steps, totalDurationMs, usage, model) {
         },
         costUsd: stepCost - subSteps.reduce((s, ss) => s + ss.costUsd, 0),
         outputPreview: step.text?.slice(0, 200),
-        success: isSuccess,
-        error: errorMsg
+        success: true,
+        error: void 0
       });
     }
     if (subSteps.length === 0) {
@@ -9049,28 +9008,51 @@ function tryParseJson(text) {
 // src/run-scenario/agents/opencode/write-skills.ts
 import { mkdir as mkdir6 } from "fs/promises";
 import { join as join6 } from "path";
-async function writeSkillsToFilesystem2(cwd, skills, fetchFn) {
+import { fetchGitHubFolder as fetchGitHubFolder3 } from "@wix/evalforge-github-client";
+async function writeSkillsToFilesystem2(cwd, skills, fetchFn = fetchGitHubFolder3) {
   await Promise.all(
     skills.map((skill) => writeSkillToFilesystem2(cwd, skill, fetchFn))
   );
 }
 async function writeSkillToFilesystem2(cwd, skill, fetchFn) {
-  const skillDir = join6(cwd, ".opencode", "skills", skill.name);
+  const skillName = skill.name;
+  const skillDir = join6(cwd, ".opencode", "skills", skillName);
   await mkdir6(skillDir, { recursive: true });
-  try {
-    const files = await resolveSkillFiles(skill, fetchFn);
-    await writeFilesToDirectory(skillDir, files);
-  } catch (error) {
-    const message = error instanceof Error ? error.message : "Unknown error";
-    throw new Error(
-      `Failed to write skill ${skill.name} to filesystem: ${message}`
+  const version = skill.latestVersion;
+  if (version?.files && version.files.length > 0) {
+    await writeFilesToDirectory(skillDir, version.files);
+    console.log(
+      `[Skill] ${skillName}: wrote ${version.files.length} file(s) from snapshot`
     );
+  } else if (skill.source) {
+    try {
+      const files = await fetchFn(skill.source, {
+        userAgent: "EvalForge-Evaluator"
+      });
+      await writeFilesToDirectory(skillDir, files);
+      console.log(
+        `[Skill] ${skillName}: wrote ${files.length} file(s) from GitHub (live)`
+      );
+    } catch (error) {
+      const message = error instanceof Error ? error.message : "Unknown error";
+      console.error(
+        `[Skill] ${skillName}: GitHub fetch failed: ${message}, no files to fall back to`
+      );
+      throw new Error(
+        `Failed to write skill ${skillName} to filesystem: ${message}`
+      );
+    }
+  } else {
+    throw new Error(`Skill ${skillName} has no files and no source configured`);
   }
 }
 
 // src/run-scenario/agents/opencode/write-sub-agents.ts
 import { mkdir as mkdir7, writeFile as writeFile6 } from "fs/promises";
 import { join as join7 } from "path";
+import {
+  fetchGitHubFile as fetchGitHubFile3
+} from "@wix/evalforge-github-client";
 var AGENTS_DIR2 = ".opencode/agents";
 function toAgentFilename2(name, index, nameCount) {
   const base = (name || "").toLowerCase().replace(/\s+/g, "-").replace(/[^a-z0-9-]/g, "").replace(/^-+|-+$/g, "") || `sub-agent-${index}`;
@@ -9078,7 +9060,34 @@ function toAgentFilename2(name, index, nameCount) {
   nameCount.set(base, count + 1);
   return count === 0 ? base : `${base}-${count + 1}`;
 }
-async function writeSubAgentsToFilesystem2(cwd, subAgents, fetchFn) {
+async function resolveSubAgentContent2(agent, fetchFn) {
+  if (agent.source) {
+    try {
+      const content = await fetchFn(agent.source, {
+        userAgent: "EvalForge-Evaluator"
+      });
+      console.log(
+        `[SubAgents] Fetched "${agent.name}" from ${agent.source.owner}/${agent.source.repo}/${agent.source.path}@${agent.source.ref}`
+      );
+      return content;
+    } catch (error) {
+      const message = error instanceof Error ? error.message : "Unknown error";
+      console.error(
+        `[SubAgents] "${agent.name}": GitHub fetch failed: ${message}`
+      );
+      throw new Error(
+        `Failed to fetch sub-agent "${agent.name}" from GitHub: ${message}`
+      );
+    }
+  }
+  if (!agent.subAgentMd) {
+    console.warn(
+      `[SubAgents] "${agent.name}" has empty inline content \u2013 the agent file will be blank`
+    );
+  }
+  return agent.subAgentMd;
+}
+async function writeSubAgentsToFilesystem2(cwd, subAgents, fetchFn = fetchGitHubFile3) {
   if (subAgents.length === 0) return;
   const agentsDir = join7(cwd, AGENTS_DIR2);
   await mkdir7(agentsDir, { recursive: true });
@@ -9086,7 +9095,7 @@ async function writeSubAgentsToFilesystem2(cwd, subAgents, fetchFn) {
   for (const [i, agent] of subAgents.entries()) {
     const filename = toAgentFilename2(agent.name, i, nameCount);
     const filePath = join7(agentsDir, `${filename}.md`);
-    const content = await resolveSubAgentMd(agent, fetchFn);
+    const content = await resolveSubAgentContent2(agent, fetchFn);
     await writeFile6(filePath, content, "utf8");
   }
   console.log(`[SubAgents] Written to ${agentsDir}`);
@@ -9188,7 +9197,7 @@ async function buildOpenCodeEnv(options) {
   if (options.mcps && options.mcps.length > 0) {
     const mcpServers = {};
     for (const mcpEntity of options.mcps) {
-      const entityConfig = await resolveMcpConfig(mcpEntity);
+      const entityConfig = mcpEntity.config;
       for (const [key, value] of Object.entries(entityConfig)) {
         if (typeof value !== "object" || value === null || Array.isArray(value)) {
           throw new Error(