@wix/evalforge-evaluator 0.184.0 → 0.186.0

package/build/index.mjs

@@ -6821,21 +6821,37 @@ function createApiClient(serverUrl, options = "") {
     // The legacy REST endpoint enriched the capability with its latest version
     // server-side; ambassador's GetCapability returns the bare entity, so we
     // compose it with GetLatestCapabilityVersion in parallel here.
+    //
+    // The latest-version fetch is BEST-EFFORT: a failure must not drop the whole
+    // capability. Otherwise one broken snapshot fetch makes the capability (e.g.
+    // an MCP) silently vanish from the run. Runs that pin a version still resolve
+    // their content via getCapabilityVersion downstream.
     async getCapability(projectId2, id) {
-      const [capRes, versionRes] = await Promise.all([
+      const [capResult, versionResult] = await Promise.allSettled([
         httpClient.request(getCapability({ projectId: projectId2, capabilityId: id })),
         httpClient.request(
           getLatestCapabilityVersion({ projectId: projectId2, capabilityId: id })
         )
       ]);
-      const capability = capRes.data.capability;
+      if (capResult.status === "rejected") {
+        throw capResult.reason;
+      }
+      const capability = capResult.value.data.capability;
       if (!capability) {
         throw new Error(`Capability ${id} not found in project ${projectId2}`);
       }
-      const latestVersion = versionRes.data.capabilityVersion ? capabilityVersionFromProto(
-        versionRes.data.capabilityVersion,
-        projectId2
-      ) : void 0;
+      let latestVersion;
+      if (versionResult.status === "fulfilled" && versionResult.value.data.capabilityVersion) {
+        latestVersion = capabilityVersionFromProto(
+          versionResult.value.data.capabilityVersion,
+          projectId2
+        );
+      } else if (versionResult.status === "rejected") {
+        const reason = versionResult.reason instanceof Error ? versionResult.reason.message : String(versionResult.reason);
+        console.warn(
+          `[Capabilities] getLatestCapabilityVersion(${id}) failed; loading capability without a snapshot (pinned versions still resolve): ${reason}`
+        );
+      }
       return { ...capabilityFromProto(capability), latestVersion };
     },
     async getCapabilityVersion(projectId2, capabilityId, versionId) {
@@ -7487,117 +7503,42 @@ import {
 // src/run-scenario/agents/claude-code/write-skills.ts
 import { mkdir as mkdir3 } from "fs/promises";
 import { join } from "path";
-
-// src/run-scenario/agents/shared/resolve-capability-content.ts
-import {
-  fetchGitHubFile as fetchGitHubFile2,
-  fetchGitHubFolder as fetchGitHubFolder2
-} from "@wix/evalforge-github-client";
-import { MCP_SERVERS_JSON_KEY } from "@wix/evalforge-types";
-var USER_AGENT = "EvalForge-Evaluator";
-async function resolveSkillFiles(skill, fetchFn = fetchGitHubFolder2) {
-  const version = skill.latestVersion;
-  if (version?.files && version.files.length > 0) {
-    console.log(
-      `[Skill] ${skill.name}: using ${version.files.length} file(s) from snapshot`
-    );
-    return version.files;
-  }
-  if (skill.source) {
-    const files = await fetchFn(skill.source, { userAgent: USER_AGENT });
-    console.log(
-      `[Skill] ${skill.name}: fetched ${files.length} file(s) from GitHub (live)`
-    );
-    return files;
-  }
-  throw new Error(`Skill ${skill.name} has no files and no source configured`);
-}
-async function fetchSourceFile(label, noun, name, source, fetchFn) {
-  try {
-    const content = await fetchFn(source, { userAgent: USER_AGENT });
-    console.log(
-      `[${label}] Fetched "${name}" from ${source.owner}/${source.repo}/${source.path}@${source.ref}`
-    );
-    return content;
-  } catch (error) {
-    const message = error instanceof Error ? error.message : "Unknown error";
-    console.error(`[${label}] "${name}": GitHub fetch failed: ${message}`);
-    throw new Error(
-      `Failed to fetch ${noun} "${name}" from GitHub: ${message}`
-    );
-  }
-}
-async function resolveSubAgentMd(agent, fetchFn = fetchGitHubFile2) {
-  if (agent.source) {
-    return fetchSourceFile(
-      "SubAgents",
-      "sub-agent",
-      agent.name,
-      agent.source,
-      fetchFn
-    );
-  }
-  if (!agent.subAgentMd) {
-    console.warn(
-      `[SubAgents] "${agent.name}" has empty inline content \u2013 the agent file will be blank`
-    );
-  }
-  return agent.subAgentMd;
-}
-async function resolveRuleText(rule, fetchFn = fetchGitHubFile2) {
-  if (!rule.source) {
-    return rule.content;
-  }
-  return fetchSourceFile("Rules", "rule", rule.name, rule.source, fetchFn);
-}
-async function resolveMcpConfig(mcp, fetchFn = fetchGitHubFile2) {
-  if (!mcp.source) {
-    return mcp.config;
-  }
-  const raw = await fetchSourceFile(
-    "MCP",
-    "MCP",
-    mcp.name,
-    mcp.source,
-    fetchFn
-  );
-  let parsed;
-  try {
-    parsed = JSON.parse(raw);
-  } catch (error) {
-    const message = error instanceof Error ? error.message : "Unknown error";
-    throw new Error(
-      `MCP "${mcp.name}" GitHub source is not valid JSON: ${message}`
-    );
-  }
-  if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
-    throw new Error(`MCP "${mcp.name}" GitHub source must be a JSON object`);
-  }
-  const obj = parsed;
-  const servers = obj[MCP_SERVERS_JSON_KEY];
-  if (typeof servers === "object" && servers !== null && !Array.isArray(servers)) {
-    return servers;
-  }
-  return obj;
-}
-
-// src/run-scenario/agents/claude-code/write-skills.ts
-async function writeSkillsToFilesystem(cwd, skills, fetchFn) {
+import { fetchGitHubFolder as fetchGitHubFolder2 } from "@wix/evalforge-github-client";
+async function writeSkillsToFilesystem(cwd, skills, fetchFn = fetchGitHubFolder2) {
   await Promise.all(
     skills.map((skill) => writeSkillToFilesystem(cwd, skill, fetchFn))
   );
 }
-async function writeSkillToFilesystem(cwd, skill, fetchFn) {
-  const skillDir = join(cwd, ".claude", "skills", skill.name);
+async function writeSkillToFilesystem(cwd, skill, fetchFn = fetchGitHubFolder2) {
+  const skillName = skill.name;
+  const skillDir = join(cwd, ".claude", "skills", skillName);
   await mkdir3(skillDir, { recursive: true });
-  try {
-    const files = await resolveSkillFiles(skill, fetchFn);
-    await writeFilesToDirectory(skillDir, files);
-  } catch (error) {
-    const message = error instanceof Error ? error.message : "Unknown error";
-    throw new Error(
-      `Failed to write skill ${skill.name} to filesystem: ${message}`
+  const version = skill.latestVersion;
+  if (version?.files && version.files.length > 0) {
+    await writeFilesToDirectory(skillDir, version.files);
+    console.log(
+      `[Skill] ${skillName}: wrote ${version.files.length} file(s) from snapshot`
     );
+  } else if (skill.source) {
+    try {
+      const files = await fetchFn(skill.source, {
+        userAgent: "EvalForge-Evaluator"
+      });
+      await writeFilesToDirectory(skillDir, files);
+      console.log(
+        `[Skill] ${skillName}: wrote ${files.length} file(s) from GitHub (live)`
+      );
+    } catch (error) {
+      const message = error instanceof Error ? error.message : "Unknown error";
+      console.error(
+        `[Skill] ${skillName}: GitHub fetch failed: ${message}, no files to fall back to`
+      );
+      throw new Error(
+        `Failed to write skill ${skillName} to filesystem: ${message}`
+      );
+    }
+  } else {
+    throw new Error(`Skill ${skillName} has no files and no source configured`);
   }
 }
 
@@ -7615,7 +7556,7 @@ import { randomUUID } from "crypto";
 // src/run-scenario/agents/claude-code/write-mcp.ts
 import { writeFile as writeFile3 } from "fs/promises";
 import { join as join3 } from "path";
-import { MCP_SERVERS_JSON_KEY as MCP_SERVERS_JSON_KEY2 } from "@wix/evalforge-types";
+import { MCP_SERVERS_JSON_KEY } from "@wix/evalforge-types";
 
 // src/run-scenario/agents/shared/resolve-mcp-placeholders.ts
 import { readFile } from "fs/promises";
@@ -7660,11 +7601,11 @@ async function resolveMcpPlaceholders(mcpServers, options = {}) {
 }
 
 // src/run-scenario/agents/claude-code/write-mcp.ts
-async function writeMcpToFilesystem(cwd, mcps, fetchFn) {
+async function writeMcpToFilesystem(cwd, mcps) {
   if (mcps.length === 0) return;
   const mcpServers = {};
   for (const mcp of mcps) {
-    const config = await resolveMcpConfig(mcp, fetchFn);
+    const config = mcp.config;
     for (const [key, value] of Object.entries(config)) {
       if (typeof value !== "object" || value === null || Array.isArray(value)) {
         throw new Error(
@@ -7676,7 +7617,7 @@ async function writeMcpToFilesystem(cwd, mcps, fetchFn) {
   }
   const resolvedServers = await resolveMcpPlaceholders(mcpServers, { cwd });
   const content = JSON.stringify(
-    { [MCP_SERVERS_JSON_KEY2]: resolvedServers },
+    { [MCP_SERVERS_JSON_KEY]: resolvedServers },
     null,
     2
   );
@@ -7688,6 +7629,9 @@ async function writeMcpToFilesystem(cwd, mcps, fetchFn) {
 // src/run-scenario/agents/claude-code/write-sub-agents.ts
 import { mkdir as mkdir4, writeFile as writeFile4 } from "fs/promises";
 import { join as join4 } from "path";
+import {
+  fetchGitHubFile as fetchGitHubFile2
+} from "@wix/evalforge-github-client";
 var AGENTS_DIR = ".claude/agents";
 function toAgentFilename(name, index, nameCount) {
   const base = (name || "").toLowerCase().replace(/\s+/g, "-").replace(/[^a-z0-9-]/g, "").replace(/^-+|-+$/g, "") || `sub-agent-${index}`;
@@ -7695,7 +7639,34 @@ function toAgentFilename(name, index, nameCount) {
   nameCount.set(base, count + 1);
   return count === 0 ? base : `${base}-${count + 1}`;
 }
-async function writeSubAgentsToFilesystem(cwd, subAgents, fetchFn) {
+async function resolveSubAgentContent(agent, fetchFn) {
+  if (agent.source) {
+    try {
+      const content = await fetchFn(agent.source, {
+        userAgent: "EvalForge-Evaluator"
+      });
+      console.log(
+        `[SubAgents] Fetched "${agent.name}" from ${agent.source.owner}/${agent.source.repo}/${agent.source.path}@${agent.source.ref}`
+      );
+      return content;
+    } catch (error) {
+      const message = error instanceof Error ? error.message : "Unknown error";
+      console.error(
+        `[SubAgents] "${agent.name}": GitHub fetch failed: ${message}`
+      );
+      throw new Error(
+        `Failed to fetch sub-agent "${agent.name}" from GitHub: ${message}`
+      );
+    }
+  }
+  if (!agent.subAgentMd) {
+    console.warn(
+      `[SubAgents] "${agent.name}" has empty inline content \u2013 the agent file will be blank`
+    );
+  }
+  return agent.subAgentMd;
+}
+async function writeSubAgentsToFilesystem(cwd, subAgents, fetchFn = fetchGitHubFile2) {
   if (subAgents.length === 0) return;
   const agentsDir = join4(cwd, AGENTS_DIR);
   await mkdir4(agentsDir, { recursive: true });
@@ -7703,7 +7674,7 @@ async function writeSubAgentsToFilesystem(cwd, subAgents, fetchFn) {
   for (const [i, agent] of subAgents.entries()) {
     const filename = toAgentFilename(agent.name, i, nameCount);
     const filePath = join4(agentsDir, `${filename}.md`);
-    const content = await resolveSubAgentMd(agent, fetchFn);
+    const content = await resolveSubAgentContent(agent, fetchFn);
     await writeFile4(filePath, content, "utf8");
   }
   console.log(`[SubAgents] Written to ${agentsDir}`);
@@ -7753,19 +7724,18 @@ function validateGenericDirectory(dir, cwd) {
   }
   return trimmed;
 }
-async function writeRulesToFilesystem(cwd, rules, fetchFn) {
+async function writeRulesToFilesystem(cwd, rules) {
   if (rules.length === 0) return;
   const nameCount = /* @__PURE__ */ new Map();
   let hasCursorRules = false;
   for (const [i, rule] of rules.entries()) {
-    const content = await resolveRuleText(rule, fetchFn);
     switch (rule.ruleType) {
       case "claude-md": {
-        await appendToFile(join5(cwd, "CLAUDE.md"), content);
+        await appendToFile(join5(cwd, "CLAUDE.md"), rule.content);
         break;
       }
       case "agents-md": {
-        await appendToFile(join5(cwd, "AGENTS.md"), content);
+        await appendToFile(join5(cwd, "AGENTS.md"), rule.content);
         break;
       }
       case "cursor-rule": {
@@ -7775,7 +7745,7 @@ async function writeRulesToFilesystem(cwd, rules, fetchFn) {
         }
         const filename = toRuleFilename(rule.name, i, nameCount);
         const filePath = join5(cwd, CURSOR_RULES_DIR, `${filename}.md`);
-        await writeFile5(filePath, content, "utf8");
+        await writeFile5(filePath, rule.content, "utf8");
         break;
       }
       case "generic": {
@@ -7786,7 +7756,7 @@ async function writeRulesToFilesystem(cwd, rules, fetchFn) {
         const dirPath = join5(cwd, directory);
         await mkdir5(dirPath, { recursive: true });
         const filename = toRuleFilename(rule.name, i, nameCount);
-        await writeFile5(join5(dirPath, `${filename}.md`), content, "utf8");
+        await writeFile5(join5(dirPath, `${filename}.md`), rule.content, "utf8");
         break;
       }
       default: {
@@ -9054,28 +9024,51 @@ function tryParseJson(text) {
 // src/run-scenario/agents/opencode/write-skills.ts
 import { mkdir as mkdir6 } from "fs/promises";
 import { join as join6 } from "path";
-async function writeSkillsToFilesystem2(cwd, skills, fetchFn) {
+import { fetchGitHubFolder as fetchGitHubFolder3 } from "@wix/evalforge-github-client";
+async function writeSkillsToFilesystem2(cwd, skills, fetchFn = fetchGitHubFolder3) {
   await Promise.all(
     skills.map((skill) => writeSkillToFilesystem2(cwd, skill, fetchFn))
   );
 }
 async function writeSkillToFilesystem2(cwd, skill, fetchFn) {
-  const skillDir = join6(cwd, ".opencode", "skills", skill.name);
+  const skillName = skill.name;
+  const skillDir = join6(cwd, ".opencode", "skills", skillName);
   await mkdir6(skillDir, { recursive: true });
-  try {
-    const files = await resolveSkillFiles(skill, fetchFn);
-    await writeFilesToDirectory(skillDir, files);
-  } catch (error) {
-    const message = error instanceof Error ? error.message : "Unknown error";
-    throw new Error(
-      `Failed to write skill ${skill.name} to filesystem: ${message}`
+  const version = skill.latestVersion;
+  if (version?.files && version.files.length > 0) {
+    await writeFilesToDirectory(skillDir, version.files);
+    console.log(
+      `[Skill] ${skillName}: wrote ${version.files.length} file(s) from snapshot`
     );
+  } else if (skill.source) {
+    try {
+      const files = await fetchFn(skill.source, {
+        userAgent: "EvalForge-Evaluator"
+      });
+      await writeFilesToDirectory(skillDir, files);
+      console.log(
+        `[Skill] ${skillName}: wrote ${files.length} file(s) from GitHub (live)`
+      );
+    } catch (error) {
+      const message = error instanceof Error ? error.message : "Unknown error";
+      console.error(
+        `[Skill] ${skillName}: GitHub fetch failed: ${message}, no files to fall back to`
+      );
+      throw new Error(
+        `Failed to write skill ${skillName} to filesystem: ${message}`
+      );
+    }
+  } else {
+    throw new Error(`Skill ${skillName} has no files and no source configured`);
   }
 }
 
 // src/run-scenario/agents/opencode/write-sub-agents.ts
 import { mkdir as mkdir7, writeFile as writeFile6 } from "fs/promises";
 import { join as join7 } from "path";
+import {
+  fetchGitHubFile as fetchGitHubFile3
+} from "@wix/evalforge-github-client";
 var AGENTS_DIR2 = ".opencode/agents";
 function toAgentFilename2(name, index, nameCount) {
   const base = (name || "").toLowerCase().replace(/\s+/g, "-").replace(/[^a-z0-9-]/g, "").replace(/^-+|-+$/g, "") || `sub-agent-${index}`;
@@ -9083,7 +9076,34 @@ function toAgentFilename2(name, index, nameCount) {
   nameCount.set(base, count + 1);
   return count === 0 ? base : `${base}-${count + 1}`;
 }
-async function writeSubAgentsToFilesystem2(cwd, subAgents, fetchFn) {
+async function resolveSubAgentContent2(agent, fetchFn) {
+  if (agent.source) {
+    try {
+      const content = await fetchFn(agent.source, {
+        userAgent: "EvalForge-Evaluator"
+      });
+      console.log(
+        `[SubAgents] Fetched "${agent.name}" from ${agent.source.owner}/${agent.source.repo}/${agent.source.path}@${agent.source.ref}`
+      );
+      return content;
+    } catch (error) {
+      const message = error instanceof Error ? error.message : "Unknown error";
+      console.error(
+        `[SubAgents] "${agent.name}": GitHub fetch failed: ${message}`
+      );
+      throw new Error(
+        `Failed to fetch sub-agent "${agent.name}" from GitHub: ${message}`
+      );
+    }
+  }
+  if (!agent.subAgentMd) {
+    console.warn(
+      `[SubAgents] "${agent.name}" has empty inline content \u2013 the agent file will be blank`
+    );
+  }
+  return agent.subAgentMd;
+}
+async function writeSubAgentsToFilesystem2(cwd, subAgents, fetchFn = fetchGitHubFile3) {
   if (subAgents.length === 0) return;
   const agentsDir = join7(cwd, AGENTS_DIR2);
   await mkdir7(agentsDir, { recursive: true });
@@ -9091,7 +9111,7 @@ async function writeSubAgentsToFilesystem2(cwd, subAgents, fetchFn) {
   for (const [i, agent] of subAgents.entries()) {
     const filename = toAgentFilename2(agent.name, i, nameCount);
     const filePath = join7(agentsDir, `${filename}.md`);
-    const content = await resolveSubAgentMd(agent, fetchFn);
+    const content = await resolveSubAgentContent2(agent, fetchFn);
     await writeFile6(filePath, content, "utf8");
   }
   console.log(`[SubAgents] Written to ${agentsDir}`);
@@ -9193,7 +9213,7 @@ async function buildOpenCodeEnv(options) {
   if (options.mcps && options.mcps.length > 0) {
     const mcpServers = {};
     for (const mcpEntity of options.mcps) {
-      const entityConfig = await resolveMcpConfig(mcpEntity);
+      const entityConfig = mcpEntity.config;
       for (const [key, value] of Object.entries(entityConfig)) {
         if (typeof value !== "object" || value === null || Array.isArray(value)) {
           throw new Error(