@wix/evalforge-evaluator 0.163.0 → 0.165.0

package/build/index.mjs

@@ -423,9 +423,13 @@ import {
 
 // src/run-scenario/environment.ts
 import { mkdirSync, existsSync, rmSync, readFileSync, writeFileSync } from "fs";
+import { mkdir as mkdir2, writeFile as writeFile2 } from "fs/promises";
 import { tmpdir } from "os";
-import path from "path";
-import { fetchGitHubFolder } from "@wix/evalforge-github-client";
+import path, { sep as sep2 } from "path";
+import {
+  fetchGitHubFolder,
+  fetchGitHubFile
+} from "@wix/evalforge-github-client";
 
 // src/run-scenario/utils/write-files.ts
 import { mkdir, writeFile } from "fs/promises";
@@ -446,16 +450,34 @@ async function writeFilesToDirectory(targetDir, files) {
 
 // src/run-scenario/environment.ts
 async function fetchAndWriteTemplateFiles(template, workDir) {
-  if (!template.source) {
+  let sourceFiles = [];
+  if (template.source) {
+    sourceFiles = await fetchGitHubFolder(template.source, {
+      userAgent: "EvalForge-Evaluator"
+    });
+  } else if (template.sourceFiles?.length) {
+    sourceFiles = template.sourceFiles;
+  } else {
     console.warn(
       `Template "${template.name}" has no source configured, creating empty directory`
     );
-    return;
   }
-  const files = await fetchGitHubFolder(template.source, {
-    userAgent: "EvalForge-Evaluator"
-  });
-  await writeFilesToDirectory(workDir, files);
+  await writeFilesToDirectory(workDir, sourceFiles);
+  await Promise.all(
+    (template.extraFiles ?? []).map(async (ef) => {
+      const content = ef.gitSource ? await fetchGitHubFile(ef.gitSource, {
+        userAgent: "EvalForge-Evaluator"
+      }) : ef.content ?? "";
+      const dest = path.resolve(workDir, ef.path);
+      if (!dest.startsWith(workDir + sep2)) {
+        throw new Error(
+          `Extra file path escapes working directory: "${ef.path}"`
+        );
+      }
+      await mkdir2(path.dirname(dest), { recursive: true });
+      await writeFile2(dest, content, "utf8");
+    })
+  );
 }
 function writeWixEnvFile(workDir) {
   const configPath = path.join(workDir, "wix.config.json");
@@ -618,7 +640,7 @@ import {
 } from "@wix/evalforge-types";
 
 // src/run-scenario/agents/claude-code/write-skills.ts
-import { mkdir as mkdir2 } from "fs/promises";
+import { mkdir as mkdir3 } from "fs/promises";
 import { join } from "path";
 import { fetchGitHubFolder as fetchGitHubFolder2 } from "@wix/evalforge-github-client";
 async function writeSkillsToFilesystem(cwd, skills, fetchFn = fetchGitHubFolder2) {
@@ -629,7 +651,7 @@ async function writeSkillsToFilesystem(cwd, skills, fetchFn = fetchGitHubFolder2
 async function writeSkillToFilesystem(cwd, skill, fetchFn = fetchGitHubFolder2) {
   const skillName = skill.name;
   const skillDir = join(cwd, ".claude", "skills", skillName);
-  await mkdir2(skillDir, { recursive: true });
+  await mkdir3(skillDir, { recursive: true });
   const version = skill.latestVersion;
   if (version?.files && version.files.length > 0) {
     await writeFilesToDirectory(skillDir, version.files);
@@ -671,7 +693,7 @@ function resolveTimeoutMs(maxTurns, maxDurationMs) {
 import { randomUUID } from "crypto";
 
 // src/run-scenario/agents/claude-code/write-mcp.ts
-import { writeFile as writeFile2 } from "fs/promises";
+import { writeFile as writeFile3 } from "fs/promises";
 import { join as join3 } from "path";
 import { MCP_SERVERS_JSON_KEY } from "@wix/evalforge-types";
 
@@ -739,15 +761,15 @@ async function writeMcpToFilesystem(cwd, mcps) {
     2
   );
   const filePath = join3(cwd, ".mcp.json");
-  await writeFile2(filePath, content, "utf8");
+  await writeFile3(filePath, content, "utf8");
   console.log(`[MCP] Written to ${filePath}`);
 }
 
 // src/run-scenario/agents/claude-code/write-sub-agents.ts
-import { mkdir as mkdir3, writeFile as writeFile3 } from "fs/promises";
+import { mkdir as mkdir4, writeFile as writeFile4 } from "fs/promises";
 import { join as join4 } from "path";
 import {
-  fetchGitHubFile
+  fetchGitHubFile as fetchGitHubFile2
 } from "@wix/evalforge-github-client";
 var AGENTS_DIR = ".claude/agents";
 function toAgentFilename(name, index, nameCount) {
@@ -783,23 +805,23 @@ async function resolveSubAgentContent(agent, fetchFn) {
   }
   return agent.subAgentMd;
 }
-async function writeSubAgentsToFilesystem(cwd, subAgents, fetchFn = fetchGitHubFile) {
+async function writeSubAgentsToFilesystem(cwd, subAgents, fetchFn = fetchGitHubFile2) {
   if (subAgents.length === 0) return;
   const agentsDir = join4(cwd, AGENTS_DIR);
-  await mkdir3(agentsDir, { recursive: true });
+  await mkdir4(agentsDir, { recursive: true });
   const nameCount = /* @__PURE__ */ new Map();
   for (const [i, agent] of subAgents.entries()) {
     const filename = toAgentFilename(agent.name, i, nameCount);
     const filePath = join4(agentsDir, `${filename}.md`);
     const content = await resolveSubAgentContent(agent, fetchFn);
-    await writeFile3(filePath, content, "utf8");
+    await writeFile4(filePath, content, "utf8");
   }
   console.log(`[SubAgents] Written to ${agentsDir}`);
 }
 
 // src/run-scenario/agents/claude-code/write-rules.ts
-import { mkdir as mkdir4, writeFile as writeFile4, readFile as readFile2 } from "fs/promises";
-import { join as join5, resolve as resolve2, sep as sep2 } from "path";
+import { mkdir as mkdir5, writeFile as writeFile5, readFile as readFile2 } from "fs/promises";
+import { join as join5, resolve as resolve2, sep as sep3 } from "path";
 var CURSOR_RULES_DIR = ".cursor/rules";
 function toRuleFilename(name, index, nameCount) {
   const base = (name || "").toLowerCase().replace(/\s+/g, "-").replace(/[^a-z0-9-]/g, "").replace(/^-+|-+$/g, "") || `rule-${index}`;
@@ -816,7 +838,7 @@ async function appendToFile(filePath, content) {
   const merged = existing ? `${existing.trimEnd()}
 
 ${content}` : content;
-  await writeFile4(filePath, merged, "utf8");
+  await writeFile5(filePath, merged, "utf8");
 }
 function validateGenericDirectory(dir, cwd) {
   const trimmed = dir.trim();
@@ -832,9 +854,9 @@ function validateGenericDirectory(dir, cwd) {
       `Generic rule directory may not contain "..", got: "${dir}"`
     );
   }
-  const normalizedCwd = cwd.endsWith(sep2) ? cwd.slice(0, -1) : cwd;
+  const normalizedCwd = cwd.endsWith(sep3) ? cwd.slice(0, -1) : cwd;
   const resolved = resolve2(normalizedCwd, trimmed);
-  if (!resolved.startsWith(normalizedCwd + sep2)) {
+  if (!resolved.startsWith(normalizedCwd + sep3)) {
     throw new Error(
       `Generic rule directory escapes the working directory: "${dir}"`
     );
@@ -857,12 +879,12 @@ async function writeRulesToFilesystem(cwd, rules) {
       }
       case "cursor-rule": {
         if (!hasCursorRules) {
-          await mkdir4(join5(cwd, CURSOR_RULES_DIR), { recursive: true });
+          await mkdir5(join5(cwd, CURSOR_RULES_DIR), { recursive: true });
           hasCursorRules = true;
         }
         const filename = toRuleFilename(rule.name, i, nameCount);
         const filePath = join5(cwd, CURSOR_RULES_DIR, `${filename}.md`);
-        await writeFile4(filePath, rule.content, "utf8");
+        await writeFile5(filePath, rule.content, "utf8");
         break;
       }
       case "generic": {
@@ -871,9 +893,9 @@ async function writeRulesToFilesystem(cwd, rules) {
           cwd
         );
         const dirPath = join5(cwd, directory);
-        await mkdir4(dirPath, { recursive: true });
+        await mkdir5(dirPath, { recursive: true });
         const filename = toRuleFilename(rule.name, i, nameCount);
-        await writeFile4(join5(dirPath, `${filename}.md`), rule.content, "utf8");
+        await writeFile5(join5(dirPath, `${filename}.md`), rule.content, "utf8");
         break;
       }
       default: {
@@ -1182,10 +1204,10 @@ function createTraceEventFromAnyMessage(message, context, stepNumber, isComplete
   };
 }
 async function prepareClaudeCodeEnvironment(cwd, skills, options) {
-  const { mkdir: mkdirAsync, writeFile: writeFile7 } = await import("fs/promises");
+  const { mkdir: mkdirAsync, writeFile: writeFile8 } = await import("fs/promises");
   const claudeDir = `${cwd}/.claude`;
   await mkdirAsync(claudeDir, { recursive: true });
-  await writeFile7(`${claudeDir}/settings.json`, "{}", {
+  await writeFile8(`${claudeDir}/settings.json`, "{}", {
     flag: "wx"
   }).catch(() => {
   });
@@ -2186,7 +2208,7 @@ function tryParseJson(text) {
 }
 
 // src/run-scenario/agents/opencode/write-skills.ts
-import { mkdir as mkdir5 } from "fs/promises";
+import { mkdir as mkdir6 } from "fs/promises";
 import { join as join6 } from "path";
 import { fetchGitHubFolder as fetchGitHubFolder3 } from "@wix/evalforge-github-client";
 async function writeSkillsToFilesystem2(cwd, skills, fetchFn = fetchGitHubFolder3) {
@@ -2197,7 +2219,7 @@ async function writeSkillsToFilesystem2(cwd, skills, fetchFn = fetchGitHubFolder
 async function writeSkillToFilesystem2(cwd, skill, fetchFn) {
   const skillName = skill.name;
   const skillDir = join6(cwd, ".opencode", "skills", skillName);
-  await mkdir5(skillDir, { recursive: true });
+  await mkdir6(skillDir, { recursive: true });
   const version = skill.latestVersion;
   if (version?.files && version.files.length > 0) {
     await writeFilesToDirectory(skillDir, version.files);
@@ -2228,10 +2250,10 @@ async function writeSkillToFilesystem2(cwd, skill, fetchFn) {
 }
 
 // src/run-scenario/agents/opencode/write-sub-agents.ts
-import { mkdir as mkdir6, writeFile as writeFile5 } from "fs/promises";
+import { mkdir as mkdir7, writeFile as writeFile6 } from "fs/promises";
 import { join as join7 } from "path";
 import {
-  fetchGitHubFile as fetchGitHubFile2
+  fetchGitHubFile as fetchGitHubFile3
 } from "@wix/evalforge-github-client";
 var AGENTS_DIR2 = ".opencode/agents";
 function toAgentFilename2(name, index, nameCount) {
@@ -2267,16 +2289,16 @@ async function resolveSubAgentContent2(agent, fetchFn) {
   }
   return agent.subAgentMd;
 }
-async function writeSubAgentsToFilesystem2(cwd, subAgents, fetchFn = fetchGitHubFile2) {
+async function writeSubAgentsToFilesystem2(cwd, subAgents, fetchFn = fetchGitHubFile3) {
   if (subAgents.length === 0) return;
   const agentsDir = join7(cwd, AGENTS_DIR2);
-  await mkdir6(agentsDir, { recursive: true });
+  await mkdir7(agentsDir, { recursive: true });
   const nameCount = /* @__PURE__ */ new Map();
   for (const [i, agent] of subAgents.entries()) {
     const filename = toAgentFilename2(agent.name, i, nameCount);
     const filePath = join7(agentsDir, `${filename}.md`);
     const content = await resolveSubAgentContent2(agent, fetchFn);
-    await writeFile5(filePath, content, "utf8");
+    await writeFile6(filePath, content, "utf8");
   }
   console.log(`[SubAgents] Written to ${agentsDir}`);
 }
@@ -2742,7 +2764,7 @@ function buildConversation2(timestampedEvents) {
 }
 
 // src/run-scenario/agents/opencode/execute.ts
-import { writeFile as writeFile6, mkdir as mkdir7 } from "fs/promises";
+import { writeFile as writeFile7, mkdir as mkdir8 } from "fs/promises";
 import { join as join8 } from "path";
 var KILL_GRACE_PERIOD_MS = 5e3;
 var IDLE_TIMEOUT_MS = 12e4;
@@ -2769,7 +2791,7 @@ function extractToolAction(toolName, args) {
 }
 async function writePromptImages(cwd, images) {
   const imagesDir = join8(cwd, "prompt-images");
-  await mkdir7(imagesDir, { recursive: true });
+  await mkdir8(imagesDir, { recursive: true });
   const filePaths = [];
   for (let i = 0; i < images.length; i++) {
     const img = images[i];
@@ -2777,7 +2799,7 @@ async function writePromptImages(cwd, images) {
     const filename = `image-${i}.${ext}`;
     const filepath = join8(imagesDir, filename);
     const buffer = Buffer.from(img.base64, "base64");
-    await writeFile6(filepath, buffer);
+    await writeFile7(filepath, buffer);
     filePaths.push(filepath);
   }
   return filePaths;
@@ -2869,8 +2891,8 @@ async function prepareOpenCodeEnvironment(cwd, skills, options) {
 }
 async function writeSystemPromptRule(cwd, systemPrompt) {
   const rulesDir = join8(cwd, ".opencode", "rules");
-  await mkdir7(rulesDir, { recursive: true });
-  await writeFile6(
+  await mkdir8(rulesDir, { recursive: true });
+  await writeFile7(
     join8(rulesDir, "evalforge-system-prompt.md"),
     systemPrompt,
     "utf-8"