@wix/evalforge-evaluator 0.164.0 → 0.165.0

package/build/index.js

@@ -433,6 +433,7 @@ var import_eval_assertions = require("@wix/eval-assertions");
 
 // src/run-scenario/environment.ts
 var import_fs = require("fs");
+var import_promises2 = require("fs/promises");
 var import_os = require("os");
 var import_path2 = __toESM(require("path"));
 var import_evalforge_github_client = require("@wix/evalforge-github-client");
@@ -456,16 +457,34 @@ async function writeFilesToDirectory(targetDir, files) {
 
 // src/run-scenario/environment.ts
 async function fetchAndWriteTemplateFiles(template, workDir) {
-  if (!template.source) {
+  let sourceFiles = [];
+  if (template.source) {
+    sourceFiles = await (0, import_evalforge_github_client.fetchGitHubFolder)(template.source, {
+      userAgent: "EvalForge-Evaluator"
+    });
+  } else if (template.sourceFiles?.length) {
+    sourceFiles = template.sourceFiles;
+  } else {
     console.warn(
       `Template "${template.name}" has no source configured, creating empty directory`
     );
-    return;
   }
-  const files = await (0, import_evalforge_github_client.fetchGitHubFolder)(template.source, {
-    userAgent: "EvalForge-Evaluator"
-  });
-  await writeFilesToDirectory(workDir, files);
+  await writeFilesToDirectory(workDir, sourceFiles);
+  await Promise.all(
+    (template.extraFiles ?? []).map(async (ef) => {
+      const content = ef.gitSource ? await (0, import_evalforge_github_client.fetchGitHubFile)(ef.gitSource, {
+        userAgent: "EvalForge-Evaluator"
+      }) : ef.content ?? "";
+      const dest = import_path2.default.resolve(workDir, ef.path);
+      if (!dest.startsWith(workDir + import_path2.sep)) {
+        throw new Error(
+          `Extra file path escapes working directory: "${ef.path}"`
+        );
+      }
+      await (0, import_promises2.mkdir)(import_path2.default.dirname(dest), { recursive: true });
+      await (0, import_promises2.writeFile)(dest, content, "utf8");
+    })
+  );
 }
 function writeWixEnvFile(workDir) {
   const configPath = import_path2.default.join(workDir, "wix.config.json");
@@ -623,7 +642,7 @@ var import_evalforge_types5 = require("@wix/evalforge-types");
 var import_evalforge_types4 = require("@wix/evalforge-types");
 
 // src/run-scenario/agents/claude-code/write-skills.ts
-var import_promises2 = require("fs/promises");
+var import_promises3 = require("fs/promises");
 var import_path3 = require("path");
 var import_evalforge_github_client2 = require("@wix/evalforge-github-client");
 async function writeSkillsToFilesystem(cwd, skills, fetchFn = import_evalforge_github_client2.fetchGitHubFolder) {
@@ -634,7 +653,7 @@ async function writeSkillsToFilesystem(cwd, skills, fetchFn = import_evalforge_g
 async function writeSkillToFilesystem(cwd, skill, fetchFn = import_evalforge_github_client2.fetchGitHubFolder) {
   const skillName = skill.name;
   const skillDir = (0, import_path3.join)(cwd, ".claude", "skills", skillName);
-  await (0, import_promises2.mkdir)(skillDir, { recursive: true });
+  await (0, import_promises3.mkdir)(skillDir, { recursive: true });
   const version = skill.latestVersion;
   if (version?.files && version.files.length > 0) {
     await writeFilesToDirectory(skillDir, version.files);
@@ -676,18 +695,18 @@ function resolveTimeoutMs(maxTurns, maxDurationMs) {
 var import_crypto = require("crypto");
 
 // src/run-scenario/agents/claude-code/write-mcp.ts
-var import_promises4 = require("fs/promises");
+var import_promises5 = require("fs/promises");
 var import_path5 = require("path");
 var import_evalforge_types2 = require("@wix/evalforge-types");
 
 // src/run-scenario/agents/shared/resolve-mcp-placeholders.ts
-var import_promises3 = require("fs/promises");
+var import_promises4 = require("fs/promises");
 var import_path4 = require("path");
 var import_os2 = require("os");
 var WIX_AUTH_FILE = (0, import_path4.join)((0, import_os2.homedir)(), ".wix", "auth", "api-key.json");
 async function loadWixAuthPlaceholders(authFilePath = WIX_AUTH_FILE) {
   try {
-    const content = await (0, import_promises3.readFile)(authFilePath, "utf-8");
+    const content = await (0, import_promises4.readFile)(authFilePath, "utf-8");
     const auth = JSON.parse(content);
     if (!auth.token || !auth.userInfo?.userId) {
       return {};
@@ -744,12 +763,12 @@ async function writeMcpToFilesystem(cwd, mcps) {
     2
   );
   const filePath = (0, import_path5.join)(cwd, ".mcp.json");
-  await (0, import_promises4.writeFile)(filePath, content, "utf8");
+  await (0, import_promises5.writeFile)(filePath, content, "utf8");
   console.log(`[MCP] Written to ${filePath}`);
 }
 
 // src/run-scenario/agents/claude-code/write-sub-agents.ts
-var import_promises5 = require("fs/promises");
+var import_promises6 = require("fs/promises");
 var import_path6 = require("path");
 var import_evalforge_github_client3 = require("@wix/evalforge-github-client");
 var AGENTS_DIR = ".claude/agents";
@@ -789,19 +808,19 @@ async function resolveSubAgentContent(agent, fetchFn) {
 async function writeSubAgentsToFilesystem(cwd, subAgents, fetchFn = import_evalforge_github_client3.fetchGitHubFile) {
   if (subAgents.length === 0) return;
   const agentsDir = (0, import_path6.join)(cwd, AGENTS_DIR);
-  await (0, import_promises5.mkdir)(agentsDir, { recursive: true });
+  await (0, import_promises6.mkdir)(agentsDir, { recursive: true });
   const nameCount = /* @__PURE__ */ new Map();
   for (const [i, agent] of subAgents.entries()) {
     const filename = toAgentFilename(agent.name, i, nameCount);
     const filePath = (0, import_path6.join)(agentsDir, `${filename}.md`);
     const content = await resolveSubAgentContent(agent, fetchFn);
-    await (0, import_promises5.writeFile)(filePath, content, "utf8");
+    await (0, import_promises6.writeFile)(filePath, content, "utf8");
   }
   console.log(`[SubAgents] Written to ${agentsDir}`);
 }
 
 // src/run-scenario/agents/claude-code/write-rules.ts
-var import_promises6 = require("fs/promises");
+var import_promises7 = require("fs/promises");
 var import_path7 = require("path");
 var CURSOR_RULES_DIR = ".cursor/rules";
 function toRuleFilename(name, index, nameCount) {
@@ -813,13 +832,13 @@ function toRuleFilename(name, index, nameCount) {
 async function appendToFile(filePath, content) {
   let existing = "";
   try {
-    existing = await (0, import_promises6.readFile)(filePath, "utf8");
+    existing = await (0, import_promises7.readFile)(filePath, "utf8");
   } catch {
   }
   const merged = existing ? `${existing.trimEnd()}
 
 ${content}` : content;
-  await (0, import_promises6.writeFile)(filePath, merged, "utf8");
+  await (0, import_promises7.writeFile)(filePath, merged, "utf8");
 }
 function validateGenericDirectory(dir, cwd) {
   const trimmed = dir.trim();
@@ -860,12 +879,12 @@ async function writeRulesToFilesystem(cwd, rules) {
       }
       case "cursor-rule": {
         if (!hasCursorRules) {
-          await (0, import_promises6.mkdir)((0, import_path7.join)(cwd, CURSOR_RULES_DIR), { recursive: true });
+          await (0, import_promises7.mkdir)((0, import_path7.join)(cwd, CURSOR_RULES_DIR), { recursive: true });
           hasCursorRules = true;
         }
         const filename = toRuleFilename(rule.name, i, nameCount);
         const filePath = (0, import_path7.join)(cwd, CURSOR_RULES_DIR, `${filename}.md`);
-        await (0, import_promises6.writeFile)(filePath, rule.content, "utf8");
+        await (0, import_promises7.writeFile)(filePath, rule.content, "utf8");
         break;
       }
       case "generic": {
@@ -874,9 +893,9 @@ async function writeRulesToFilesystem(cwd, rules) {
           cwd
         );
         const dirPath = (0, import_path7.join)(cwd, directory);
-        await (0, import_promises6.mkdir)(dirPath, { recursive: true });
+        await (0, import_promises7.mkdir)(dirPath, { recursive: true });
         const filename = toRuleFilename(rule.name, i, nameCount);
-        await (0, import_promises6.writeFile)((0, import_path7.join)(dirPath, `${filename}.md`), rule.content, "utf8");
+        await (0, import_promises7.writeFile)((0, import_path7.join)(dirPath, `${filename}.md`), rule.content, "utf8");
         break;
       }
       default: {
@@ -1185,10 +1204,10 @@ function createTraceEventFromAnyMessage(message, context, stepNumber, isComplete
   };
 }
 async function prepareClaudeCodeEnvironment(cwd, skills, options) {
-  const { mkdir: mkdirAsync, writeFile: writeFile7 } = await import("fs/promises");
+  const { mkdir: mkdirAsync, writeFile: writeFile8 } = await import("fs/promises");
   const claudeDir = `${cwd}/.claude`;
   await mkdirAsync(claudeDir, { recursive: true });
-  await writeFile7(`${claudeDir}/settings.json`, "{}", {
+  await writeFile8(`${claudeDir}/settings.json`, "{}", {
     flag: "wx"
   }).catch(() => {
   });
@@ -2186,7 +2205,7 @@ function tryParseJson(text) {
 }
 
 // src/run-scenario/agents/opencode/write-skills.ts
-var import_promises7 = require("fs/promises");
+var import_promises8 = require("fs/promises");
 var import_path8 = require("path");
 var import_evalforge_github_client4 = require("@wix/evalforge-github-client");
 async function writeSkillsToFilesystem2(cwd, skills, fetchFn = import_evalforge_github_client4.fetchGitHubFolder) {
@@ -2197,7 +2216,7 @@ async function writeSkillsToFilesystem2(cwd, skills, fetchFn = import_evalforge_
 async function writeSkillToFilesystem2(cwd, skill, fetchFn) {
   const skillName = skill.name;
   const skillDir = (0, import_path8.join)(cwd, ".opencode", "skills", skillName);
-  await (0, import_promises7.mkdir)(skillDir, { recursive: true });
+  await (0, import_promises8.mkdir)(skillDir, { recursive: true });
   const version = skill.latestVersion;
   if (version?.files && version.files.length > 0) {
     await writeFilesToDirectory(skillDir, version.files);
@@ -2228,7 +2247,7 @@ async function writeSkillToFilesystem2(cwd, skill, fetchFn) {
 }
 
 // src/run-scenario/agents/opencode/write-sub-agents.ts
-var import_promises8 = require("fs/promises");
+var import_promises9 = require("fs/promises");
 var import_path9 = require("path");
 var import_evalforge_github_client5 = require("@wix/evalforge-github-client");
 var AGENTS_DIR2 = ".opencode/agents";
@@ -2268,13 +2287,13 @@ async function resolveSubAgentContent2(agent, fetchFn) {
 async function writeSubAgentsToFilesystem2(cwd, subAgents, fetchFn = import_evalforge_github_client5.fetchGitHubFile) {
   if (subAgents.length === 0) return;
   const agentsDir = (0, import_path9.join)(cwd, AGENTS_DIR2);
-  await (0, import_promises8.mkdir)(agentsDir, { recursive: true });
+  await (0, import_promises9.mkdir)(agentsDir, { recursive: true });
   const nameCount = /* @__PURE__ */ new Map();
   for (const [i, agent] of subAgents.entries()) {
     const filename = toAgentFilename2(agent.name, i, nameCount);
     const filePath = (0, import_path9.join)(agentsDir, `${filename}.md`);
     const content = await resolveSubAgentContent2(agent, fetchFn);
-    await (0, import_promises8.writeFile)(filePath, content, "utf8");
+    await (0, import_promises9.writeFile)(filePath, content, "utf8");
   }
   console.log(`[SubAgents] Written to ${agentsDir}`);
 }
@@ -2735,7 +2754,7 @@ function buildConversation2(timestampedEvents) {
 }
 
 // src/run-scenario/agents/opencode/execute.ts
-var import_promises9 = require("fs/promises");
+var import_promises10 = require("fs/promises");
 var import_path10 = require("path");
 var KILL_GRACE_PERIOD_MS = 5e3;
 var IDLE_TIMEOUT_MS = 12e4;
@@ -2762,7 +2781,7 @@ function extractToolAction(toolName, args) {
 }
 async function writePromptImages(cwd, images) {
   const imagesDir = (0, import_path10.join)(cwd, "prompt-images");
-  await (0, import_promises9.mkdir)(imagesDir, { recursive: true });
+  await (0, import_promises10.mkdir)(imagesDir, { recursive: true });
   const filePaths = [];
   for (let i = 0; i < images.length; i++) {
     const img = images[i];
@@ -2770,7 +2789,7 @@ async function writePromptImages(cwd, images) {
     const filename = `image-${i}.${ext}`;
     const filepath = (0, import_path10.join)(imagesDir, filename);
     const buffer = Buffer.from(img.base64, "base64");
-    await (0, import_promises9.writeFile)(filepath, buffer);
+    await (0, import_promises10.writeFile)(filepath, buffer);
     filePaths.push(filepath);
   }
   return filePaths;
@@ -2862,8 +2881,8 @@ async function prepareOpenCodeEnvironment(cwd, skills, options) {
 }
 async function writeSystemPromptRule(cwd, systemPrompt) {
   const rulesDir = (0, import_path10.join)(cwd, ".opencode", "rules");
-  await (0, import_promises9.mkdir)(rulesDir, { recursive: true });
-  await (0, import_promises9.writeFile)(
+  await (0, import_promises10.mkdir)(rulesDir, { recursive: true });
+  await (0, import_promises10.writeFile)(
     (0, import_path10.join)(rulesDir, "evalforge-system-prompt.md"),
     systemPrompt,
     "utf-8"