@wix/evalforge-evaluator 0.60.0 → 0.61.0

package/build/index.mjs

@@ -137,6 +137,16 @@ function createApiClient(serverUrl, options = "") {
     getSkill(projectId2, id) {
       return fetchJson(`/projects/${projectId2}/skills/${id}`);
     },
+    getSkillVersion(projectId2, skillId, versionId) {
+      return fetchJson(
+        `/projects/${projectId2}/skills/${skillId}/versions/${versionId}`
+      );
+    },
+    getLatestSkillVersion(projectId2, skillId) {
+      return fetchJson(
+        `/projects/${projectId2}/skills/${skillId}/versions/latest`
+      );
+    },
     getAgent(projectId2, id) {
       return fetchJson(`/projects/${projectId2}/agents/${id}`);
     },
@@ -274,6 +284,29 @@ async function fetchEvaluationData(api, projectId2, evalRunId2) {
         skillsGroup.skillIds.map((id) => api.getSkill(projectId2, id))
       );
     }
+    if (evalRun.skillVersions && Object.keys(evalRun.skillVersions).length > 0) {
+      skills = await Promise.all(
+        skills.map(async (skill) => {
+          const versionId = evalRun.skillVersions?.[skill.id];
+          if (versionId) {
+            const version = await api.getSkillVersion(
+              projectId2,
+              skill.id,
+              versionId
+            );
+            return { ...skill, latestVersion: version };
+          }
+          return skill;
+        })
+      );
+    }
+    skills = skills.map((skill) => {
+      const hasPinnedVersion = evalRun.skillVersions?.[skill.id];
+      if (!hasPinnedVersion && skill.source) {
+        return { ...skill, latestVersion: void 0 };
+      }
+      return skill;
+    });
   }
   let mcps = [];
   if (evalRun.mcpIds && evalRun.mcpIds.length > 0) {
@@ -1088,10 +1121,10 @@ var Minipass = class extends EventEmitter {
    * Return a void Promise that resolves once the stream ends.
    */
   async promise() {
-    return new Promise((resolve, reject) => {
+    return new Promise((resolve2, reject) => {
       this.on(DESTROYED, () => reject(new Error("stream destroyed")));
       this.on("error", (er) => reject(er));
-      this.on("end", () => resolve());
+      this.on("end", () => resolve2());
     });
   }
   /**
@@ -1115,7 +1148,7 @@ var Minipass = class extends EventEmitter {
         return Promise.resolve({ done: false, value: res });
       if (this[EOF])
         return stop();
-      let resolve;
+      let resolve2;
       let reject;
       const onerr = (er) => {
         this.off("data", ondata);
@@ -1129,19 +1162,19 @@ var Minipass = class extends EventEmitter {
         this.off("end", onend);
         this.off(DESTROYED, ondestroy);
         this.pause();
-        resolve({ value, done: !!this[EOF] });
+        resolve2({ value, done: !!this[EOF] });
       };
       const onend = () => {
         this.off("error", onerr);
         this.off("data", ondata);
         this.off(DESTROYED, ondestroy);
         stop();
-        resolve({ done: true, value: void 0 });
+        resolve2({ done: true, value: void 0 });
       };
       const ondestroy = () => onerr(new Error("stream destroyed"));
       return new Promise((res2, rej) => {
         reject = rej;
-        resolve = res2;
+        resolve2 = res2;
         this.once(DESTROYED, ondestroy);
         this.once("error", onerr);
         this.once("end", onend);
@@ -3269,9 +3302,9 @@ var listFile = (opt, _files) => {
   const parse4 = new Parser(opt);
   const readSize = opt.maxReadSize || 16 * 1024 * 1024;
   const file = opt.file;
-  const p = new Promise((resolve, reject) => {
+  const p = new Promise((resolve2, reject) => {
     parse4.on("error", reject);
-    parse4.on("end", resolve);
+    parse4.on("end", resolve2);
     fs2.stat(file, (er, stat) => {
       if (er) {
         reject(er);
@@ -5912,9 +5945,9 @@ var extractFile = (opt, _) => {
   const u = new Unpack(opt);
   const readSize = opt.maxReadSize || 16 * 1024 * 1024;
   const file = opt.file;
-  const p = new Promise((resolve, reject) => {
+  const p = new Promise((resolve2, reject) => {
     u.on("error", reject);
-    u.on("close", resolve);
+    u.on("close", resolve2);
     fs9.stat(file, (er, stat) => {
       if (er) {
         reject(er);
@@ -6048,7 +6081,7 @@ var replaceAsync = (opt, files) => {
     };
     fs10.read(fd, headBuf, 0, 512, position, onread);
   };
-  const promise = new Promise((resolve, reject) => {
+  const promise = new Promise((resolve2, reject) => {
     p.on("error", reject);
     let flag = "r+";
     const onopen = (er, fd) => {
@@ -6073,7 +6106,7 @@ var replaceAsync = (opt, files) => {
           });
           p.pipe(stream);
           stream.on("error", reject);
-          stream.on("close", resolve);
+          stream.on("close", resolve2);
           addFilesAsync2(p, files);
         });
       });
@@ -6344,13 +6377,68 @@ import {
   TRACE_EVENT_PREFIX,
   AVAILABLE_MODELS
 } from "@wix/evalforge-types";
+
+// src/run-scenario/agents/claude-code/write-skills.ts
+import { mkdir as mkdir2, writeFile } from "fs/promises";
+import { dirname as dirname2, join as join2, resolve, sep } from "path";
+import { fetchSkillFolderRaw } from "@wix/evalforge-github-client";
+async function writeSkillsToFilesystem(cwd, skills, fetchFn = fetchSkillFolderRaw) {
+  await Promise.all(
+    skills.map((skill) => writeSkillToFilesystem(cwd, skill, fetchFn))
+  );
+}
+async function writeSkillToFilesystem(cwd, skill, fetchFn = fetchSkillFolderRaw) {
+  const skillName = skill.name;
+  const skillDir = join2(cwd, ".claude", "skills", skillName);
+  await mkdir2(skillDir, { recursive: true });
+  const version = skill.latestVersion;
+  if (version?.files && version.files.length > 0) {
+    await writeSkillFiles(skillDir, version.files);
+    console.log(
+      `[Skill] ${skillName}: wrote ${version.files.length} file(s) from snapshot`
+    );
+  } else if (skill.source) {
+    try {
+      const files = await fetchFn(skill.source, {
+        userAgent: "EvalForge-Evaluator"
+      });
+      await writeSkillFiles(skillDir, files);
+      console.log(
+        `[Skill] ${skillName}: wrote ${files.length} file(s) from GitHub (live)`
+      );
+    } catch (error) {
+      const message = error instanceof Error ? error.message : "Unknown error";
+      console.error(
+        `[Skill] ${skillName}: GitHub fetch failed: ${message}, no files to fall back to`
+      );
+      throw new Error(
+        `Failed to write skill ${skillName} to filesystem: ${message}`
+      );
+    }
+  } else {
+    throw new Error(`Skill ${skillName} has no files and no source configured`);
+  }
+}
+async function writeSkillFiles(skillDir, files) {
+  const resolvedBase = resolve(skillDir);
+  for (const file of files) {
+    const filePath = resolve(skillDir, file.path);
+    if (!filePath.startsWith(resolvedBase + sep) && filePath !== resolvedBase) {
+      throw new Error(
+        `Path traversal detected in skill file: "${file.path}" resolves outside skill directory`
+      );
+    }
+    await mkdir2(dirname2(filePath), { recursive: true });
+    await writeFile(filePath, file.content, "utf-8");
+  }
+}
+
+// src/run-scenario/agents/claude-code/execute.ts
 import { randomUUID } from "crypto";
-import { mkdir as mkdir3, writeFile as writeFile3 } from "fs/promises";
-import { join as join4 } from "path";
 
 // src/run-scenario/agents/claude-code/write-mcp.ts
-import { writeFile } from "fs/promises";
-import { join as join2 } from "path";
+import { writeFile as writeFile2 } from "fs/promises";
+import { join as join3 } from "path";
 import { MCP_SERVERS_JSON_KEY } from "@wix/evalforge-types";
 async function writeMcpToFilesystem(cwd, mcps) {
   if (mcps.length === 0) return;
@@ -6371,14 +6459,14 @@ async function writeMcpToFilesystem(cwd, mcps) {
     null,
     2
   );
-  const filePath = join2(cwd, ".mcp.json");
-  await writeFile(filePath, content, "utf8");
+  const filePath = join3(cwd, ".mcp.json");
+  await writeFile2(filePath, content, "utf8");
   console.log(`[MCP] Written to ${filePath}`);
 }
 
 // src/run-scenario/agents/claude-code/write-sub-agents.ts
-import { mkdir as mkdir2, writeFile as writeFile2 } from "fs/promises";
-import { join as join3 } from "path";
+import { mkdir as mkdir3, writeFile as writeFile3 } from "fs/promises";
+import { join as join4 } from "path";
 var AGENTS_DIR = ".claude/agents";
 function toAgentFilename(name2, index, nameCount) {
   const base = (name2 || "").toLowerCase().replace(/\s+/g, "-").replace(/[^a-z0-9-]/g, "").replace(/^-+|-+$/g, "") || `sub-agent-${index}`;
@@ -6388,13 +6476,13 @@ function toAgentFilename(name2, index, nameCount) {
 }
 async function writeSubAgentsToFilesystem(cwd, subAgents) {
   if (subAgents.length === 0) return;
-  const agentsDir = join3(cwd, AGENTS_DIR);
-  await mkdir2(agentsDir, { recursive: true });
+  const agentsDir = join4(cwd, AGENTS_DIR);
+  await mkdir3(agentsDir, { recursive: true });
   const nameCount = /* @__PURE__ */ new Map();
   for (const [i, agent] of subAgents.entries()) {
     const filename = toAgentFilename(agent.name, i, nameCount);
-    const filePath = join3(agentsDir, `${filename}.md`);
-    await writeFile2(filePath, agent.subAgentMd, "utf8");
+    const filePath = join4(agentsDir, `${filename}.md`);
+    await writeFile3(filePath, agent.subAgentMd, "utf8");
   }
   console.log(`[SubAgents] Written to ${agentsDir}`);
 }
@@ -7173,16 +7261,6 @@ Stack: ${errorStack.split("\n").slice(0, 5).join("\n")}` : "")
     llmTrace
   };
 }
-async function writeSkillsToFilesystem(cwd, skills) {
-  for (const skill of skills) {
-    const skillName = skill.name;
-    const skillDir = join4(cwd, ".claude", "skills", skillName);
-    await mkdir3(skillDir, { recursive: true });
-    const skillPath = join4(skillDir, "SKILL.md");
-    await writeFile3(skillPath, skill.skillMd, "utf-8");
-    console.log(`[Skill] Written to ${skillPath}`);
-  }
-}
 function buildSdkEnvironment(options) {
   const env = { ...process.env };
   const placeholderApiKey = "sk-ant-api03-placeholder-auth-handled-by-gateway-000000000000000000000000";
@@ -8180,7 +8258,7 @@ async function runAgentWithContext(config, evalRunId2, scenario, evalData, workD
     mcps: evalData.mcps.length > 0 ? evalData.mcps : void 0,
     subAgents: evalData.subAgents.length > 0 ? evalData.subAgents : void 0
   };
-  const result = await adapter.execute(executionContext);
+  const { outputText, durationMs, llmTrace } = await adapter.execute(executionContext);
   const completedAt = (/* @__PURE__ */ new Date()).toISOString();
   const afterSnapshot = workDir ? snapshotDirectory(workDir) : {};
   const fileDiffs = diffSnapshots(beforeSnapshot, afterSnapshot);
@@ -8192,13 +8270,13 @@ async function runAgentWithContext(config, evalRunId2, scenario, evalData, workD
     scenarioId: scenario.id,
     scenarioName: scenario.name,
     modelConfig: agent?.modelConfig,
-    duration: result.durationMs,
-    outputText: result.outputText,
+    duration: durationMs,
+    outputText,
     fileDiffs: fileDiffs.length > 0 ? fileDiffs : void 0,
     templateFiles: templateFiles && templateFiles.length > 0 ? templateFiles : void 0,
     startedAt,
     completedAt,
-    llmTrace: result.llmTrace
+    llmTrace
   };
 }