@wix/evalforge-evaluator 0.187.0 → 0.188.0

package/README.md

@@ -50,6 +50,10 @@ Backend calls go through the evalforge ambassador packages (gRPC via
 user-editable fields, not the system state transitions
 (`status`/`completedAt`/`jobError`/`jobStatus`) the evaluator writes.
 
+## Live trace during environment setup
+
+For templated runs, the evaluator emits `PROGRESS` trace events during environment setup — "Setting up environment", "Fetching template files", "Installing dependencies", "Environment ready" — via the shared `emitTraceEvent` helper. Because `emitTraceEvent` writes to stdout (captured by the backend for local runs) and also calls the `pushEvent` callback (used for remote jobs via `tracePushUrl`), these events appear in the live trace in both local and remote runs. Without them, the trace panel stays blank during the often multi-minute setup phase before the agent starts.
+
 ## Scripts
 
 ```bash

package/build/index.js

@@ -7462,10 +7462,13 @@ function installWithCache(workDir, exec, cacheBase, pm) {
     );
   }
 }
-async function installDependencies(workDir, exec = import_child_process.execFileSync, cacheBase) {
+async function installDependencies(workDir, onProgress, options = {}) {
   if (!(0, import_fs.existsSync)(import_path2.default.join(workDir, "package.json"))) {
     return;
   }
+  const exec = options.exec ?? import_child_process.execFileSync;
+  const cacheBase = options.cacheBase;
+  onProgress("Installing dependencies...");
   const pm = detectPackageManager(workDir);
   if (cacheBase) {
     installWithCache(workDir, exec, cacheBase, pm);
@@ -7541,7 +7544,8 @@ function writeWixEnvFile(workDir) {
     console.warn("[environment] Failed to read wix.config.json");
   }
 }
-async function prepareWorkingDirectory(config, evalRunId2, targetId, scenarioId, template) {
+async function prepareWorkingDirectory(config, evalRunId2, targetId, scenarioId, onProgress, options = {}) {
+  const template = options.template;
   const baseDir = config.evaluationsDir ?? import_path3.default.join((0, import_os.tmpdir)(), "evalforge-evaluations");
   const nodeModulesCacheDir = import_path3.default.join(baseDir, "_node_modules_cache");
   if (template) {
@@ -7555,10 +7559,14 @@ async function prepareWorkingDirectory(config, evalRunId2, targetId, scenarioId,
       (0, import_fs2.rmSync)(workDir2, { recursive: true });
     }
     (0, import_fs2.mkdirSync)(workDir2, { recursive: true });
+    onProgress("Fetching template files...");
     await fetchAndWriteTemplateFiles(template, workDir2);
     console.log(`Template files written to ${workDir2}`);
     writeWixEnvFile(workDir2);
-    await installDependencies(workDir2, void 0, nodeModulesCacheDir);
+    await installDependencies(workDir2, onProgress, {
+      cacheBase: nodeModulesCacheDir
+    });
+    onProgress("Environment ready");
     return workDir2;
   }
   const workDir = import_path3.default.join(baseDir, `${evalRunId2}_${targetId}_${scenarioId}`);
@@ -7570,6 +7578,13 @@ async function prepareWorkingDirectory(config, evalRunId2, targetId, scenarioId,
   return workDir;
 }
 
+// src/run-scenario/agents/shared/trace-emit.ts
+var import_evalforge_types2 = require("@wix/evalforge-types");
+function emitTraceEvent(event, pushEvent) {
+  console.log(`${import_evalforge_types2.TRACE_EVENT_PREFIX}${JSON.stringify(event)}`);
+  pushEvent?.(event);
+}
+
 // src/run-scenario/run-agent-with-context.ts
 var import_crypto5 = require("crypto");
 
@@ -7734,7 +7749,7 @@ var import_crypto2 = require("crypto");
 // src/run-scenario/agents/claude-code/write-mcp.ts
 var import_promises5 = require("fs/promises");
 var import_path6 = require("path");
-var import_evalforge_types2 = require("@wix/evalforge-types");
+var import_evalforge_types3 = require("@wix/evalforge-types");
 
 // src/run-scenario/agents/shared/resolve-mcp-placeholders.ts
 var import_promises4 = require("fs/promises");
@@ -7795,7 +7810,7 @@ async function writeMcpToFilesystem(cwd, mcps) {
   }
   const resolvedServers = await resolveMcpPlaceholders(mcpServers, { cwd });
   const content = JSON.stringify(
-    { [import_evalforge_types2.MCP_SERVERS_JSON_KEY]: resolvedServers },
+    { [import_evalforge_types3.MCP_SERVERS_JSON_KEY]: resolvedServers },
     null,
     2
   );
@@ -8021,13 +8036,6 @@ function buildConversation(timestampedMessages) {
   return messages;
 }
 
-// src/run-scenario/agents/shared/trace-emit.ts
-var import_evalforge_types3 = require("@wix/evalforge-types");
-function emitTraceEvent(event, pushEvent) {
-  console.log(`${import_evalforge_types3.TRACE_EVENT_PREFIX}${JSON.stringify(event)}`);
-  pushEvent?.(event);
-}
-
 // src/run-scenario/agents/claude-code/execute.ts
 var DEFAULT_MODEL = import_evalforge_types4.ClaudeModel.CLAUDE_4_5_SONNET_1_0;
 async function* buildPromptStream(triggerPrompt, images) {
@@ -11992,6 +12000,22 @@ async function runAgentWithContext(config, evalRunId2, scenario, evalData, workD
 async function runScenario(config, evalRunId2, scenario, evalData, template, resolvedAssertions, pushEvent, apiClient, projectId2) {
   const targetId = evalData.evalRun.presetId ?? evalData.agent?.id ?? evalData.evalRun.id;
   const targetName = evalData.presetName ?? evalData.agent?.name ?? "";
+  const emitSetupProgress = (outputPreview) => emitTraceEvent(
+    {
+      evalRunId: evalRunId2,
+      scenarioId: scenario.id,
+      scenarioName: scenario.name,
+      targetId,
+      targetName,
+      stepNumber: 0,
+      type: import_evalforge_types13.LiveTraceEventType.PROGRESS,
+      outputPreview,
+      elapsedMs: 0,
+      timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+      isComplete: false
+    },
+    pushEvent
+  );
   let provisionedSite;
   if (apiClient && projectId2 && scenario.siteSetup && scenario.siteSetup.mode !== "none") {
     provisionedSite = await apiClient.provisionScenarioSite(
@@ -12022,29 +12046,14 @@ async function runScenario(config, evalRunId2, scenario, evalData, template, res
 
 Site ID: ${provisionedSite.id}` : scenario.triggerPrompt;
   try {
-    if (template) {
-      console.log(
-        (0, import_evalforge_types13.formatTraceEventLine)({
-          evalRunId: evalRunId2,
-          scenarioId: scenario.id,
-          scenarioName: scenario.name,
-          targetId,
-          targetName,
-          stepNumber: 0,
-          type: import_evalforge_types13.LiveTraceEventType.PROGRESS,
-          outputPreview: "Setting up environment (installing dependencies)...",
-          elapsedMs: 0,
-          timestamp: (/* @__PURE__ */ new Date()).toISOString(),
-          isComplete: false
-        })
-      );
-    }
+    emitSetupProgress("Setting up environment...");
     const workDir = await prepareWorkingDirectory(
       config,
       evalRunId2,
       targetId,
       scenario.id,
-      template
+      emitSetupProgress,
+      { template }
     );
     const partialResult = await runAgentWithContext(
       config,