@wix/evalforge-evaluator 0.190.0 → 0.192.0

package/build/index.mjs

@@ -7393,7 +7393,38 @@ async function writeFilesToDirectory(targetDir, files) {
 import { mkdirSync, existsSync, readFileSync, copyFileSync, cpSync } from "fs";
 import { createHash } from "crypto";
 import path from "path";
-import { execFileSync } from "child_process";
+import { spawn, execFileSync } from "child_process";
+var INSTALL_TIMEOUT_MS = 9e4;
+var HEARTBEAT_INTERVAL_MS = 5e3;
+function defaultExec(cmd, args, opts) {
+  return new Promise((resolve3, reject) => {
+    const child = spawn(cmd, args, {
+      cwd: opts.cwd,
+      env: opts.env,
+      stdio: "inherit"
+    });
+    const timer = setTimeout(() => {
+      child.kill("SIGKILL");
+      reject(
+        new Error(
+          `${cmd} ${args.join(" ")} timed out after ${opts.timeoutMs}ms`
+        )
+      );
+    }, opts.timeoutMs);
+    child.on("error", (err) => {
+      clearTimeout(timer);
+      reject(err);
+    });
+    child.on("close", (code) => {
+      clearTimeout(timer);
+      if (code === 0) {
+        resolve3();
+      } else {
+        reject(new Error(`${cmd} exited with code ${code}`));
+      }
+    });
+  });
+}
 function detectPackageManager(workDir) {
   if (existsSync(path.join(workDir, "pnpm-lock.yaml"))) {
     return {
@@ -7431,7 +7462,34 @@ function cloneDirectory(src, dest) {
     cpSync(src, dest, { recursive: true });
   }
 }
-function installWithCache(workDir, exec, cacheBase, pm) {
+async function runInstall(exec, pm, workDir, onProgress) {
+  onProgress(`[diag] npm install starting: ${pm.cmd} ${pm.args.join(" ")}`);
+  const startedAt = Date.now();
+  const heartbeat = setInterval(() => {
+    onProgress(
+      `[diag] npm install still running: ${Math.round(
+        (Date.now() - startedAt) / 1e3
+      )}s elapsed`
+    );
+  }, HEARTBEAT_INTERVAL_MS);
+  try {
+    await exec(pm.cmd, pm.args, {
+      cwd: workDir,
+      timeoutMs: INSTALL_TIMEOUT_MS,
+      env: { ...process.env, NODE_ENV: "development" }
+    });
+    onProgress(`[diag] npm install finished in ${Date.now() - startedAt}ms`);
+    return true;
+  } catch (err) {
+    onProgress(
+      `[diag] npm install FAILED after ${Date.now() - startedAt}ms: ${err instanceof Error ? err.message : String(err)}`
+    );
+    return false;
+  } finally {
+    clearInterval(heartbeat);
+  }
+}
+async function installWithCache(workDir, exec, cacheBase, pm, onProgress) {
   const sourceContent = readFileSync(
     path.join(workDir, pm.cacheSourceFile),
     "utf-8"
@@ -7445,29 +7503,19 @@ function installWithCache(workDir, exec, cacheBase, pm) {
     console.log(
       `[environment] Restoring node_modules from cache (key: ${cacheKey})`
     );
+    onProgress(`[diag] node_modules cache HIT (key: ${cacheKey}) \u2014 restoring`);
     if (!existsSync(targetNodeModules)) {
       cloneDirectory(cachedNodeModules, targetNodeModules);
     }
     if (existsSync(cachedYarnLock)) {
       copyFileSync(cachedYarnLock, path.join(workDir, "yarn.lock"));
     }
+    onProgress("[diag] node_modules cache restore complete");
     return;
   }
-  console.log(
-    `[environment] Running ${pm.cmd} ${pm.args.join(" ")} in ${workDir} (cache key: ${cacheKey})`
-  );
-  try {
-    exec(pm.cmd, pm.args, {
-      cwd: workDir,
-      stdio: "inherit",
-      timeout: 18e4,
-      env: { ...process.env, NODE_ENV: "development" }
-    });
-  } catch (err) {
-    console.error(
-      "[environment] Dependency installation failed:",
-      err instanceof Error ? err.message : String(err)
-    );
+  onProgress(`[diag] node_modules cache MISS (key: ${cacheKey})`);
+  const ok = await runInstall(exec, pm, workDir, onProgress);
+  if (!ok) {
     return;
   }
   console.log(
@@ -7491,31 +7539,15 @@ async function installDependencies(workDir, onProgress, options = {}) {
   if (!existsSync(path.join(workDir, "package.json"))) {
     return;
   }
-  const exec = options.exec ?? execFileSync;
+  const exec = options.exec ?? defaultExec;
   const cacheBase = options.cacheBase;
   onProgress("Installing dependencies...");
   const pm = detectPackageManager(workDir);
   if (cacheBase) {
-    installWithCache(workDir, exec, cacheBase, pm);
+    await installWithCache(workDir, exec, cacheBase, pm, onProgress);
     return;
   }
-  console.log(
-    `[environment] Running ${pm.cmd} ${pm.args.join(" ")} in ${workDir}`
-  );
-  try {
-    exec(pm.cmd, pm.args, {
-      cwd: workDir,
-      stdio: "inherit",
-      timeout: 18e4,
-      env: { ...process.env, NODE_ENV: "development" }
-    });
-    console.log("[environment] Dependency installation complete");
-  } catch (err) {
-    console.error(
-      "[environment] Dependency installation failed:",
-      err instanceof Error ? err.message : String(err)
-    );
-  }
+  await runInstall(exec, pm, workDir, onProgress);
 }
 
 // src/run-scenario/environment.ts
@@ -7591,12 +7623,39 @@ async function fetchAndWriteTemplateFiles(template, workDir, onProgress) {
       `Template "${template.name}" has no source configured, creating empty directory`
     );
   }
+  onProgress(`[diag] writing ${sourceFiles.length} source file(s) to workDir`);
   await writeFilesToDirectory(workDir, sourceFiles);
+  onProgress(`[diag] ${sourceFiles.length} source file(s) written`);
+  const extraFiles = template.extraFiles ?? [];
+  onProgress(`[diag] resolving ${extraFiles.length} extra file(s)`);
   await Promise.all(
-    (template.extraFiles ?? []).map(async (ef) => {
-      const content = ef.gitSource ? await fetchGitHubFile(ef.gitSource, {
-        userAgent: "EvalForge-Evaluator"
-      }) : ef.content ?? "";
+    extraFiles.map(async (ef) => {
+      let content;
+      if (ef.gitSource) {
+        const { owner, repo, path: gitPath, ref } = ef.gitSource;
+        const startedAt = Date.now();
+        onProgress(
+          `[diag] extra fetch start: ${ef.path} <- ${owner}/${repo}/${gitPath ?? ""}@${ref ?? "default"}`
+        );
+        try {
+          content = await withTimeout(
+            fetchGitHubFile(ef.gitSource, { userAgent: "EvalForge-Evaluator" }),
+            3e4,
+            `extra file fetch (${ef.path})`
+          );
+          onProgress(
+            `[diag] extra fetch done: ${ef.path} (${content.length} bytes) in ${Date.now() - startedAt}ms`
+          );
+        } catch (err) {
+          onProgress(
+            `[diag] extra fetch FAILED: ${ef.path} after ${Date.now() - startedAt}ms: ${err instanceof Error ? err.message : String(err)}`
+          );
+          throw err;
+        }
+      } else {
+        content = ef.content ?? "";
+        onProgress(`[diag] extra inline: ${ef.path} (${content.length} bytes)`);
+      }
       const dest = path2.resolve(workDir, ef.path);
       if (!dest.startsWith(workDir + sep2)) {
         throw new Error(
@@ -7607,6 +7666,7 @@ async function fetchAndWriteTemplateFiles(template, workDir, onProgress) {
       await writeFile2(dest, content, "utf8");
     })
   );
+  onProgress("[diag] all extra files written");
 }
 function writeWixEnvFile(workDir) {
   const configPath = path2.join(workDir, "wix.config.json");
@@ -7646,10 +7706,13 @@ async function prepareWorkingDirectory(config, evalRunId2, targetId, scenarioId,
     onProgress("Fetching template files...");
     await fetchAndWriteTemplateFiles(template, workDir2, onProgress);
     console.log(`Template files written to ${workDir2}`);
+    onProgress("[diag] writing wix env file");
     writeWixEnvFile(workDir2);
+    onProgress("[diag] entering installDependencies");
     await installDependencies(workDir2, onProgress, {
       cacheBase: nodeModulesCacheDir
     });
+    onProgress("[diag] installDependencies returned");
     onProgress("Environment ready");
     return workDir2;
   }
@@ -8463,7 +8526,7 @@ async function executeWithClaudeCode(skills, scenario, options) {
   const SDK_TIMEOUT_MS = resolveTimeoutMs(maxTurns, options.maxDurationMs);
   let timeoutHandle;
   let timedOut = false;
-  const HEARTBEAT_INTERVAL_MS = 1e4;
+  const HEARTBEAT_INTERVAL_MS2 = 1e4;
   let heartbeatHandle;
   const executionStartTime = Date.now();
   try {
@@ -8515,7 +8578,7 @@ async function executeWithClaudeCode(skills, scenario, options) {
           isComplete: false
         };
         emitTraceEvent(progressEvent, traceContext.pushEvent);
-      }, HEARTBEAT_INTERVAL_MS);
+      }, HEARTBEAT_INTERVAL_MS2);
     }
     const sdkPromise = (async () => {
       const hasImages = scenario.triggerPromptImages && scenario.triggerPromptImages.length > 0;
@@ -9281,7 +9344,7 @@ defaultRegistry.register(claudeCodeAdapter);
 import { AgentRunCommand as AgentRunCommand2, OpenCodeConfigSchema as OpenCodeConfigSchema2 } from "@wix/evalforge-types";
 
 // src/run-scenario/agents/opencode/execute.ts
-import { spawn } from "child_process";
+import { spawn as spawn2 } from "child_process";
 import {
   DEFAULT_EVALUATOR_SYSTEM_PROMPT as DEFAULT_EVALUATOR_SYSTEM_PROMPT2,
   LiveTraceEventType as LiveTraceEventType2
@@ -10088,7 +10151,7 @@ function spawnOpenCodeProcess(opts) {
     };
     let child;
     try {
-      child = spawn("opencode", args, {
+      child = spawn2("opencode", args, {
         cwd,
         env,
         stdio: ["ignore", "pipe", "pipe"],
@@ -12132,11 +12195,15 @@ async function runScenario(config, evalRunId2, scenario, evalData, template, res
   );
   let provisionedSite;
   if (apiClient && projectId2 && scenario.siteSetup && scenario.siteSetup.mode !== "none") {
+    emitSetupProgress(
+      `[diag] provisioning scenario site (mode: ${scenario.siteSetup.mode})...`
+    );
     provisionedSite = await apiClient.provisionScenarioSite(
       projectId2,
       evalRunId2,
       scenario.id
     );
+    emitSetupProgress("[diag] scenario site provisioned");
   }
   const failedStep = provisionedSite?.bootstrapResult?.steps.find((s) => !s.ok);
   if (failedStep) {
@@ -12169,6 +12236,7 @@ Site ID: ${provisionedSite.id}` : scenario.triggerPrompt;
       emitSetupProgress,
       { template }
     );
+    emitSetupProgress("[diag] starting agent run");
     const partialResult = await runAgentWithContext(
       config,
       evalRunId2,