@wix/evalforge-evaluator 0.170.0 → 0.171.0

package/build/index.mjs

@@ -67,8 +67,8 @@ function createApiClient(serverUrl, options = "") {
     }
     return headers;
   }
-  async function fetchJson(path2) {
-    const url = `${serverUrl}${apiPrefix}${pathPrefix}${path2}`;
+  async function fetchJson(path3) {
+    const url = `${serverUrl}${apiPrefix}${pathPrefix}${path3}`;
     console.error(`[API] GET ${url}`);
     const headers = buildHeaders();
     const response = await fetch(url, {
@@ -82,8 +82,8 @@ function createApiClient(serverUrl, options = "") {
     }
     return response.json();
   }
-  async function postJson(path2, body) {
-    const url = `${serverUrl}${apiPrefix}${pathPrefix}${path2}`;
+  async function postJson(path3, body) {
+    const url = `${serverUrl}${apiPrefix}${pathPrefix}${path3}`;
     console.error(`[API] POST ${url}`);
     const response = await fetch(url, {
       method: "POST",
@@ -97,8 +97,8 @@ function createApiClient(serverUrl, options = "") {
       );
     }
   }
-  async function deleteRequest(path2) {
-    const url = `${serverUrl}${apiPrefix}${pathPrefix}${path2}`;
+  async function deleteRequest(path3) {
+    const url = `${serverUrl}${apiPrefix}${pathPrefix}${path3}`;
     console.error(`[API] DELETE ${url}`);
     const headers = buildHeaders();
     const response = await fetch(url, {
@@ -112,8 +112,8 @@ function createApiClient(serverUrl, options = "") {
       );
     }
   }
-  async function putJson(path2, body) {
-    const url = `${serverUrl}${apiPrefix}${pathPrefix}${path2}`;
+  async function putJson(path3, body) {
+    const url = `${serverUrl}${apiPrefix}${pathPrefix}${path3}`;
     console.error(`[API] PUT ${url}`);
     const response = await fetch(url, {
       method: "PUT",
@@ -415,17 +415,19 @@ async function fetchEvaluationData(api, projectId2, evalRunId2) {
 // src/run-scenario/index.ts
 import {
   AssertionResultStatus,
-  DEFAULT_JUDGE_MODEL
+  DEFAULT_JUDGE_MODEL,
+  LiveTraceEventType as LiveTraceEventType4,
+  formatTraceEventLine
 } from "@wix/evalforge-types";
 import {
   evaluateAssertions as evaluateAssertionsBase
 } from "@wix/eval-assertions";
 
 // src/run-scenario/environment.ts
-import { mkdirSync, existsSync, rmSync, readFileSync, writeFileSync } from "fs";
+import { mkdirSync as mkdirSync2, existsSync as existsSync2, rmSync, readFileSync as readFileSync2, writeFileSync as writeFileSync2 } from "fs";
 import { mkdir as mkdir2, writeFile as writeFile2 } from "fs/promises";
 import { tmpdir } from "os";
-import path, { sep as sep2 } from "path";
+import path2, { sep as sep2 } from "path";
 import {
   fetchGitHubFolder,
   fetchGitHubFile
@@ -448,6 +450,94 @@ async function writeFilesToDirectory(targetDir, files) {
   }
 }
 
+// src/run-scenario/install-dependencies.ts
+import {
+  mkdirSync,
+  existsSync,
+  readFileSync,
+  writeFileSync,
+  copyFileSync,
+  cpSync,
+  renameSync
+} from "fs";
+import { createHash } from "crypto";
+import path from "path";
+import { execFileSync } from "child_process";
+function detectPackageManager(workDir) {
+  if (existsSync(path.join(workDir, "pnpm-lock.yaml"))) {
+    return { cmd: "pnpm", args: ["install", "--frozen-lockfile"], cacheSourceFile: "pnpm-lock.yaml" };
+  }
+  if (existsSync(path.join(workDir, "package-lock.json"))) {
+    return { cmd: "npm", args: ["ci"], cacheSourceFile: "package-lock.json" };
+  }
+  if (existsSync(path.join(workDir, "yarn.lock"))) {
+    return { cmd: "yarn", args: ["install", "--frozen-lockfile"], cacheSourceFile: "yarn.lock" };
+  }
+  writeFileSync(path.join(workDir, "yarn.lock"), "", "utf-8");
+  return { cmd: "yarn", args: ["install"], cacheSourceFile: "package.json" };
+}
+function cloneDirectory(src, dest) {
+  if (process.platform === "darwin") {
+    execFileSync("cp", ["-rc", src, dest]);
+  } else {
+    cpSync(src, dest, { recursive: true });
+  }
+}
+function installWithCache(workDir, exec, cacheBase, pm) {
+  const sourceContent = readFileSync(path.join(workDir, pm.cacheSourceFile), "utf-8");
+  const cacheKey = createHash("sha256").update(sourceContent).digest("hex").slice(0, 16);
+  const cachedNodeModules = path.join(cacheBase, cacheKey, "node_modules");
+  const targetNodeModules = path.join(workDir, "node_modules");
+  const cacheDir = path.dirname(cachedNodeModules);
+  const cachedYarnLock = path.join(cacheDir, "yarn.lock");
+  if (existsSync(cachedNodeModules)) {
+    console.log(`[environment] Restoring node_modules from cache (key: ${cacheKey})`);
+    if (!existsSync(targetNodeModules)) {
+      cloneDirectory(cachedNodeModules, targetNodeModules);
+    }
+    if (existsSync(cachedYarnLock)) {
+      copyFileSync(cachedYarnLock, path.join(workDir, "yarn.lock"));
+    }
+    return;
+  }
+  console.log(`[environment] Running ${pm.cmd} ${pm.args.join(" ")} in ${workDir} (cache key: ${cacheKey})`);
+  try {
+    exec(pm.cmd, pm.args, { cwd: workDir, stdio: "inherit", timeout: 18e4 });
+  } catch (err) {
+    console.error("[environment] Dependency installation failed:", err instanceof Error ? err.message : String(err));
+    return;
+  }
+  console.log("[environment] Dependency installation complete \u2014 saving to cache");
+  try {
+    mkdirSync(cacheDir, { recursive: true });
+    const yarnLockPath = path.join(workDir, "yarn.lock");
+    if (existsSync(yarnLockPath)) {
+      copyFileSync(yarnLockPath, cachedYarnLock);
+    }
+    renameSync(targetNodeModules, cachedNodeModules);
+    cloneDirectory(cachedNodeModules, targetNodeModules);
+  } catch (err) {
+    console.error("[environment] Failed to save to cache (installation still succeeded):", err instanceof Error ? err.message : String(err));
+  }
+}
+async function installDependencies(workDir, exec = execFileSync, cacheBase) {
+  if (!existsSync(path.join(workDir, "package.json"))) {
+    return;
+  }
+  const pm = detectPackageManager(workDir);
+  if (cacheBase) {
+    installWithCache(workDir, exec, cacheBase, pm);
+    return;
+  }
+  console.log(`[environment] Running ${pm.cmd} ${pm.args.join(" ")} in ${workDir}`);
+  try {
+    exec(pm.cmd, pm.args, { cwd: workDir, stdio: "inherit", timeout: 18e4 });
+    console.log("[environment] Dependency installation complete");
+  } catch (err) {
+    console.error("[environment] Dependency installation failed:", err instanceof Error ? err.message : String(err));
+  }
+}
+
 // src/run-scenario/environment.ts
 async function fetchAndWriteTemplateFiles(template, workDir) {
   let sourceFiles = [];
@@ -468,27 +558,27 @@ async function fetchAndWriteTemplateFiles(template, workDir) {
       const content = ef.gitSource ? await fetchGitHubFile(ef.gitSource, {
         userAgent: "EvalForge-Evaluator"
       }) : ef.content ?? "";
-      const dest = path.resolve(workDir, ef.path);
+      const dest = path2.resolve(workDir, ef.path);
       if (!dest.startsWith(workDir + sep2)) {
         throw new Error(
           `Extra file path escapes working directory: "${ef.path}"`
         );
       }
-      await mkdir2(path.dirname(dest), { recursive: true });
+      await mkdir2(path2.dirname(dest), { recursive: true });
       await writeFile2(dest, content, "utf8");
     })
   );
 }
 function writeWixEnvFile(workDir) {
-  const configPath = path.join(workDir, "wix.config.json");
-  if (!existsSync(configPath)) {
+  const configPath = path2.join(workDir, "wix.config.json");
+  if (!existsSync2(configPath)) {
     return;
   }
   try {
-    const config = JSON.parse(readFileSync(configPath, "utf-8"));
+    const config = JSON.parse(readFileSync2(configPath, "utf-8"));
     if (config.appId) {
-      writeFileSync(
-        path.join(workDir, ".env"),
+      writeFileSync2(
+        path2.join(workDir, ".env"),
         `WIX_CLIENT_ID=${config.appId}
 `,
         "utf-8"
@@ -500,28 +590,30 @@ function writeWixEnvFile(workDir) {
   }
 }
 async function prepareWorkingDirectory(config, evalRunId2, targetId, scenarioId, template) {
-  const baseDir = config.evaluationsDir ?? path.join(tmpdir(), "evalforge-evaluations");
+  const baseDir = config.evaluationsDir ?? path2.join(tmpdir(), "evalforge-evaluations");
+  const nodeModulesCacheDir = path2.join(baseDir, "_node_modules_cache");
   if (template) {
     if (!config.evaluationsDir) {
       console.warn(
         "Template specified but EVALUATIONS_DIR not set, using temp directory"
       );
     }
-    const workDir2 = path.join(baseDir, `${evalRunId2}_${targetId}`);
-    if (existsSync(workDir2)) {
+    const workDir2 = path2.join(baseDir, `${evalRunId2}_${targetId}`);
+    if (existsSync2(workDir2)) {
       rmSync(workDir2, { recursive: true });
     }
-    mkdirSync(workDir2, { recursive: true });
+    mkdirSync2(workDir2, { recursive: true });
     await fetchAndWriteTemplateFiles(template, workDir2);
     console.log(`Template files written to ${workDir2}`);
     writeWixEnvFile(workDir2);
+    await installDependencies(workDir2, void 0, nodeModulesCacheDir);
     return workDir2;
   }
-  const workDir = path.join(baseDir, `${evalRunId2}_${targetId}_${scenarioId}`);
-  if (existsSync(workDir)) {
+  const workDir = path2.join(baseDir, `${evalRunId2}_${targetId}_${scenarioId}`);
+  if (existsSync2(workDir)) {
     rmSync(workDir, { recursive: true });
   }
-  mkdirSync(workDir, { recursive: true });
+  mkdirSync2(workDir, { recursive: true });
   console.log(`Empty working directory created at ${workDir}`);
   return workDir;
 }
@@ -1066,8 +1158,8 @@ function extractToolActionDescription(toolName, toolArgs) {
         }
       }
       if (toolName === "LS" || toolName === "ls" || toolName === "ListFiles") {
-        const path2 = args.path || args.directory || ".";
-        return `Listing: ${String(path2).slice(0, 50)}`;
+        const path3 = args.path || args.directory || ".";
+        return `Listing: ${String(path3).slice(0, 50)}`;
       }
       if ((toolName === "Read" || toolName === "read" || toolName === "View") && (args.file_path || args.path || args.target_file)) {
         const filePath = String(
@@ -4209,7 +4301,7 @@ var simpleAgentAdapter = new SimpleAgentAdapter();
 defaultRegistry.register(simpleAgentAdapter);
 
 // src/run-scenario/file-diff.ts
-import { readdirSync, readFileSync as readFileSync2, statSync, existsSync as existsSync2 } from "fs";
+import { readdirSync, readFileSync as readFileSync3, statSync, existsSync as existsSync3 } from "fs";
 import { join as join10, relative } from "path";
 
 // ../../node_modules/diff/lib/index.mjs
@@ -4314,11 +4406,11 @@ Diff.prototype = {
       }
     }
   },
-  addToPath: function addToPath(path2, added, removed, oldPosInc, options) {
-    var last = path2.lastComponent;
+  addToPath: function addToPath(path3, added, removed, oldPosInc, options) {
+    var last = path3.lastComponent;
     if (last && !options.oneChangePerToken && last.added === added && last.removed === removed) {
       return {
-        oldPos: path2.oldPos + oldPosInc,
+        oldPos: path3.oldPos + oldPosInc,
         lastComponent: {
           count: last.count + 1,
           added,
@@ -4328,7 +4420,7 @@ Diff.prototype = {
       };
     } else {
       return {
-        oldPos: path2.oldPos + oldPosInc,
+        oldPos: path3.oldPos + oldPosInc,
         lastComponent: {
           count: 1,
           added,
@@ -4768,9 +4860,9 @@ arrayDiff.join = arrayDiff.removeEmpty = function(value) {
 // src/run-scenario/file-diff.ts
 function deriveInfrastructurePaths(prePrep, postPrep) {
   const infraPaths = /* @__PURE__ */ new Set();
-  for (const path2 of Object.keys(postPrep)) {
-    if (prePrep[path2] === void 0 || prePrep[path2] !== postPrep[path2]) {
-      infraPaths.add(path2);
+  for (const path3 of Object.keys(postPrep)) {
+    if (prePrep[path3] === void 0 || prePrep[path3] !== postPrep[path3]) {
+      infraPaths.add(path3);
     }
   }
   return infraPaths;
@@ -4830,7 +4922,7 @@ function isBinaryFile(filename) {
 function snapshotDirectory(dir, baseDir) {
   const snapshot = {};
   const base = baseDir || dir;
-  if (!existsSync2(dir)) {
+  if (!existsSync3(dir)) {
     return snapshot;
   }
   const entries = readdirSync(dir, { withFileTypes: true });
@@ -4852,7 +4944,7 @@ function snapshotDirectory(dir, baseDir) {
         if (stats.size > MAX_FILE_SIZE) {
           continue;
         }
-        const content = readFileSync2(fullPath, "utf-8");
+        const content = readFileSync3(fullPath, "utf-8");
         snapshot[relativePath] = content;
       } catch {
         continue;
@@ -4881,19 +4973,19 @@ function generateDiffLines(before, after) {
 function diffSnapshots(before, after, infrastructurePaths) {
   const diffs = [];
   const allPaths = /* @__PURE__ */ new Set([...Object.keys(before), ...Object.keys(after)]);
-  for (const path2 of allPaths) {
-    const beforeContent = before[path2] ?? "";
-    const afterContent = after[path2] ?? "";
-    if (before[path2] !== void 0 && beforeContent === afterContent) {
+  for (const path3 of allPaths) {
+    const beforeContent = before[path3] ?? "";
+    const afterContent = after[path3] ?? "";
+    if (before[path3] !== void 0 && beforeContent === afterContent) {
       continue;
     }
     const diffLines2 = generateDiffLines(beforeContent, afterContent);
     diffs.push({
-      path: path2,
+      path: path3,
       expected: beforeContent,
       actual: afterContent,
       diffLines: diffLines2,
-      ...infrastructurePaths?.has(path2) && { isInfrastructure: true }
+      ...infrastructurePaths?.has(path3) && { isInfrastructure: true }
     });
   }
   const deletedPaths = [...allPaths].filter((p) => after[p] === void 0);
@@ -4920,9 +5012,9 @@ function diffSnapshots(before, after, infrastructurePaths) {
 function extractTemplateFiles(before, after, infrastructurePaths) {
   const files = [];
   const allPaths = /* @__PURE__ */ new Set([...Object.keys(before), ...Object.keys(after)]);
-  for (const path2 of allPaths) {
-    const beforeContent = before[path2];
-    const afterContent = after[path2];
+  for (const path3 of allPaths) {
+    const beforeContent = before[path3];
+    const afterContent = after[path3];
     if (afterContent === void 0) {
       continue;
     }
@@ -4935,10 +5027,10 @@ function extractTemplateFiles(before, after, infrastructurePaths) {
       status = "unchanged";
     }
     files.push({
-      path: path2,
+      path: path3,
       content: afterContent,
       status,
-      ...infrastructurePaths?.has(path2) && { isInfrastructure: true }
+      ...infrastructurePaths?.has(path3) && { isInfrastructure: true }
     });
   }
   files.sort((a, b) => a.path.localeCompare(b.path));
@@ -5043,6 +5135,24 @@ async function runAgentWithContext(config, evalRunId2, scenario, evalData, workD
 // src/run-scenario/index.ts
 async function runScenario(config, evalRunId2, scenario, evalData, template, resolvedAssertions) {
   const targetId = evalData.evalRun.presetId ?? evalData.agent?.id ?? evalData.evalRun.id;
+  const targetName = evalData.presetName ?? evalData.agent?.name ?? "";
+  if (template) {
+    console.log(
+      formatTraceEventLine({
+        evalRunId: evalRunId2,
+        scenarioId: scenario.id,
+        scenarioName: scenario.name,
+        targetId,
+        targetName,
+        stepNumber: 0,
+        type: LiveTraceEventType4.PROGRESS,
+        outputPreview: "Setting up environment (installing dependencies)...",
+        elapsedMs: 0,
+        timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+        isComplete: false
+      })
+    );
+  }
   const workDir = await prepareWorkingDirectory(
     config,
     evalRunId2,