@wix/evalforge-evaluator 0.59.0 → 0.61.0

package/build/index.js

@@ -160,6 +160,16 @@ function createApiClient(serverUrl, options = "") {
     getSkill(projectId2, id) {
       return fetchJson(`/projects/${projectId2}/skills/${id}`);
     },
+    getSkillVersion(projectId2, skillId, versionId) {
+      return fetchJson(
+        `/projects/${projectId2}/skills/${skillId}/versions/${versionId}`
+      );
+    },
+    getLatestSkillVersion(projectId2, skillId) {
+      return fetchJson(
+        `/projects/${projectId2}/skills/${skillId}/versions/latest`
+      );
+    },
     getAgent(projectId2, id) {
       return fetchJson(`/projects/${projectId2}/agents/${id}`);
     },
@@ -294,6 +304,29 @@ async function fetchEvaluationData(api, projectId2, evalRunId2) {
         skillsGroup.skillIds.map((id) => api.getSkill(projectId2, id))
       );
     }
+    if (evalRun.skillVersions && Object.keys(evalRun.skillVersions).length > 0) {
+      skills = await Promise.all(
+        skills.map(async (skill) => {
+          const versionId = evalRun.skillVersions?.[skill.id];
+          if (versionId) {
+            const version = await api.getSkillVersion(
+              projectId2,
+              skill.id,
+              versionId
+            );
+            return { ...skill, latestVersion: version };
+          }
+          return skill;
+        })
+      );
+    }
+    skills = skills.map((skill) => {
+      const hasPinnedVersion = evalRun.skillVersions?.[skill.id];
+      if (!hasPinnedVersion && skill.source) {
+        return { ...skill, latestVersion: void 0 };
+      }
+      return skill;
+    });
   }
   let mcps = [];
   if (evalRun.mcpIds && evalRun.mcpIds.length > 0) {
@@ -1106,10 +1139,10 @@ var Minipass = class extends import_node_events.EventEmitter {
    * Return a void Promise that resolves once the stream ends.
    */
   async promise() {
-    return new Promise((resolve, reject) => {
+    return new Promise((resolve2, reject) => {
       this.on(DESTROYED, () => reject(new Error("stream destroyed")));
       this.on("error", (er) => reject(er));
-      this.on("end", () => resolve());
+      this.on("end", () => resolve2());
     });
   }
   /**
@@ -1133,7 +1166,7 @@ var Minipass = class extends import_node_events.EventEmitter {
         return Promise.resolve({ done: false, value: res });
       if (this[EOF])
         return stop();
-      let resolve;
+      let resolve2;
       let reject;
       const onerr = (er) => {
         this.off("data", ondata);
@@ -1147,19 +1180,19 @@ var Minipass = class extends import_node_events.EventEmitter {
         this.off("end", onend);
         this.off(DESTROYED, ondestroy);
         this.pause();
-        resolve({ value, done: !!this[EOF] });
+        resolve2({ value, done: !!this[EOF] });
       };
       const onend = () => {
         this.off("error", onerr);
         this.off("data", ondata);
         this.off(DESTROYED, ondestroy);
         stop();
-        resolve({ done: true, value: void 0 });
+        resolve2({ done: true, value: void 0 });
       };
       const ondestroy = () => onerr(new Error("stream destroyed"));
       return new Promise((res2, rej) => {
         reject = rej;
-        resolve = res2;
+        resolve2 = res2;
         this.once(DESTROYED, ondestroy);
         this.once("error", onerr);
         this.once("end", onend);
@@ -3287,9 +3320,9 @@ var listFile = (opt, _files) => {
   const parse4 = new Parser(opt);
   const readSize = opt.maxReadSize || 16 * 1024 * 1024;
   const file = opt.file;
-  const p = new Promise((resolve, reject) => {
+  const p = new Promise((resolve2, reject) => {
     parse4.on("error", reject);
-    parse4.on("end", resolve);
+    parse4.on("end", resolve2);
     import_node_fs.default.stat(file, (er, stat) => {
       if (er) {
         reject(er);
@@ -5930,9 +5963,9 @@ var extractFile = (opt, _) => {
   const u = new Unpack(opt);
   const readSize = opt.maxReadSize || 16 * 1024 * 1024;
   const file = opt.file;
-  const p = new Promise((resolve, reject) => {
+  const p = new Promise((resolve2, reject) => {
     u.on("error", reject);
-    u.on("close", resolve);
+    u.on("close", resolve2);
     import_node_fs5.default.stat(file, (er, stat) => {
       if (er) {
         reject(er);
@@ -6066,7 +6099,7 @@ var replaceAsync = (opt, files) => {
     };
     import_node_fs6.default.read(fd, headBuf, 0, 512, position, onread);
   };
-  const promise = new Promise((resolve, reject) => {
+  const promise = new Promise((resolve2, reject) => {
     p.on("error", reject);
     let flag = "r+";
     const onopen = (er, fd) => {
@@ -6091,7 +6124,7 @@ var replaceAsync = (opt, files) => {
           });
           p.pipe(stream);
           stream.on("error", reject);
-          stream.on("close", resolve);
+          stream.on("close", resolve2);
           addFilesAsync2(p, files);
         });
       });
@@ -6357,33 +6390,96 @@ var import_evalforge_types4 = require("@wix/evalforge-types");
 
 // src/run-scenario/agents/claude-code/execute.ts
 var import_evalforge_types3 = require("@wix/evalforge-types");
-var import_crypto = require("crypto");
-var import_promises5 = require("fs/promises");
-var import_path7 = require("path");
 
-// src/run-scenario/agents/claude-code/write-mcp.ts
+// src/run-scenario/agents/claude-code/write-skills.ts
 var import_promises3 = require("fs/promises");
 var import_path5 = require("path");
+var import_evalforge_github_client = require("@wix/evalforge-github-client");
+async function writeSkillsToFilesystem(cwd, skills, fetchFn = import_evalforge_github_client.fetchSkillFolderRaw) {
+  await Promise.all(
+    skills.map((skill) => writeSkillToFilesystem(cwd, skill, fetchFn))
+  );
+}
+async function writeSkillToFilesystem(cwd, skill, fetchFn = import_evalforge_github_client.fetchSkillFolderRaw) {
+  const skillName = skill.name;
+  const skillDir = (0, import_path5.join)(cwd, ".claude", "skills", skillName);
+  await (0, import_promises3.mkdir)(skillDir, { recursive: true });
+  const version = skill.latestVersion;
+  if (version?.files && version.files.length > 0) {
+    await writeSkillFiles(skillDir, version.files);
+    console.log(
+      `[Skill] ${skillName}: wrote ${version.files.length} file(s) from snapshot`
+    );
+  } else if (skill.source) {
+    try {
+      const files = await fetchFn(skill.source, {
+        userAgent: "EvalForge-Evaluator"
+      });
+      await writeSkillFiles(skillDir, files);
+      console.log(
+        `[Skill] ${skillName}: wrote ${files.length} file(s) from GitHub (live)`
+      );
+    } catch (error) {
+      const message = error instanceof Error ? error.message : "Unknown error";
+      console.error(
+        `[Skill] ${skillName}: GitHub fetch failed: ${message}, no files to fall back to`
+      );
+      throw new Error(
+        `Failed to write skill ${skillName} to filesystem: ${message}`
+      );
+    }
+  } else {
+    throw new Error(`Skill ${skillName} has no files and no source configured`);
+  }
+}
+async function writeSkillFiles(skillDir, files) {
+  const resolvedBase = (0, import_path5.resolve)(skillDir);
+  for (const file of files) {
+    const filePath = (0, import_path5.resolve)(skillDir, file.path);
+    if (!filePath.startsWith(resolvedBase + import_path5.sep) && filePath !== resolvedBase) {
+      throw new Error(
+        `Path traversal detected in skill file: "${file.path}" resolves outside skill directory`
+      );
+    }
+    await (0, import_promises3.mkdir)((0, import_path5.dirname)(filePath), { recursive: true });
+    await (0, import_promises3.writeFile)(filePath, file.content, "utf-8");
+  }
+}
+
+// src/run-scenario/agents/claude-code/execute.ts
+var import_crypto = require("crypto");
+
+// src/run-scenario/agents/claude-code/write-mcp.ts
+var import_promises4 = require("fs/promises");
+var import_path6 = require("path");
 var import_evalforge_types2 = require("@wix/evalforge-types");
 async function writeMcpToFilesystem(cwd, mcps) {
   if (mcps.length === 0) return;
   const mcpServers = {};
   for (const mcp of mcps) {
-    mcpServers[mcp.name] = mcp.config;
+    const config = mcp.config;
+    for (const [key, value] of Object.entries(config)) {
+      if (typeof value !== "object" || value === null || Array.isArray(value)) {
+        throw new Error(
+          `MCP "${mcp.name}" has invalid config: value for key "${key}" must be an object (got ${typeof value}). Config must use keyed format, e.g. { "server-name": { "command": "npx", ... } }`
+        );
+      }
+      mcpServers[key] = value;
+    }
   }
   const content = JSON.stringify(
     { [import_evalforge_types2.MCP_SERVERS_JSON_KEY]: mcpServers },
     null,
     2
   );
-  const filePath = (0, import_path5.join)(cwd, ".mcp.json");
-  await (0, import_promises3.writeFile)(filePath, content, "utf8");
+  const filePath = (0, import_path6.join)(cwd, ".mcp.json");
+  await (0, import_promises4.writeFile)(filePath, content, "utf8");
   console.log(`[MCP] Written to ${filePath}`);
 }
 
 // src/run-scenario/agents/claude-code/write-sub-agents.ts
-var import_promises4 = require("fs/promises");
-var import_path6 = require("path");
+var import_promises5 = require("fs/promises");
+var import_path7 = require("path");
 var AGENTS_DIR = ".claude/agents";
 function toAgentFilename(name2, index, nameCount) {
   const base = (name2 || "").toLowerCase().replace(/\s+/g, "-").replace(/[^a-z0-9-]/g, "").replace(/^-+|-+$/g, "") || `sub-agent-${index}`;
@@ -6393,13 +6489,13 @@ function toAgentFilename(name2, index, nameCount) {
 }
 async function writeSubAgentsToFilesystem(cwd, subAgents) {
   if (subAgents.length === 0) return;
-  const agentsDir = (0, import_path6.join)(cwd, AGENTS_DIR);
-  await (0, import_promises4.mkdir)(agentsDir, { recursive: true });
+  const agentsDir = (0, import_path7.join)(cwd, AGENTS_DIR);
+  await (0, import_promises5.mkdir)(agentsDir, { recursive: true });
   const nameCount = /* @__PURE__ */ new Map();
   for (const [i, agent] of subAgents.entries()) {
     const filename = toAgentFilename(agent.name, i, nameCount);
-    const filePath = (0, import_path6.join)(agentsDir, `${filename}.md`);
-    await (0, import_promises4.writeFile)(filePath, agent.subAgentMd, "utf8");
+    const filePath = (0, import_path7.join)(agentsDir, `${filename}.md`);
+    await (0, import_promises5.writeFile)(filePath, agent.subAgentMd, "utf8");
   }
   console.log(`[SubAgents] Written to ${agentsDir}`);
 }
@@ -6725,9 +6821,7 @@ async function executeWithClaudeCode(skills, scenario, options) {
     })
   );
   let messageCount = 0;
-  const canUseTool = async () => {
-    return { behavior: "allow" };
-  };
+  const canUseTool = async (_toolName, input) => ({ behavior: "allow", updatedInput: input });
   const baseAllowedTools = [
     "Skill",
     "Read",
@@ -7180,16 +7274,6 @@ Stack: ${errorStack.split("\n").slice(0, 5).join("\n")}` : "")
     llmTrace
   };
 }
-async function writeSkillsToFilesystem(cwd, skills) {
-  for (const skill of skills) {
-    const skillName = skill.name;
-    const skillDir = (0, import_path7.join)(cwd, ".claude", "skills", skillName);
-    await (0, import_promises5.mkdir)(skillDir, { recursive: true });
-    const skillPath = (0, import_path7.join)(skillDir, "SKILL.md");
-    await (0, import_promises5.writeFile)(skillPath, skill.skillMd, "utf-8");
-    console.log(`[Skill] Written to ${skillPath}`);
-  }
-}
 function buildSdkEnvironment(options) {
   const env = { ...process.env };
   const placeholderApiKey = "sk-ant-api03-placeholder-auth-handled-by-gateway-000000000000000000000000";
@@ -8187,7 +8271,7 @@ async function runAgentWithContext(config, evalRunId2, scenario, evalData, workD
     mcps: evalData.mcps.length > 0 ? evalData.mcps : void 0,
     subAgents: evalData.subAgents.length > 0 ? evalData.subAgents : void 0
   };
-  const result = await adapter.execute(executionContext);
+  const { outputText, durationMs, llmTrace } = await adapter.execute(executionContext);
   const completedAt = (/* @__PURE__ */ new Date()).toISOString();
   const afterSnapshot = workDir ? snapshotDirectory(workDir) : {};
   const fileDiffs = diffSnapshots(beforeSnapshot, afterSnapshot);
@@ -8199,13 +8283,13 @@ async function runAgentWithContext(config, evalRunId2, scenario, evalData, workD
     scenarioId: scenario.id,
     scenarioName: scenario.name,
     modelConfig: agent?.modelConfig,
-    duration: result.durationMs,
-    outputText: result.outputText,
+    duration: durationMs,
+    outputText,
     fileDiffs: fileDiffs.length > 0 ? fileDiffs : void 0,
     templateFiles: templateFiles && templateFiles.length > 0 ? templateFiles : void 0,
     startedAt,
     completedAt,
-    llmTrace: result.llmTrace
+    llmTrace
   };
 }