@wix/evalforge-evaluator 0.198.0 → 0.200.0

package/build/index.js

@@ -7424,6 +7424,46 @@ function defaultExec(cmd, args, opts) {
     });
   });
 }
+function createLoggingInstallExec(onProgress) {
+  return (cmd, args, opts) => new Promise((resolve3, reject) => {
+    const child = (0, import_child_process.spawn)(cmd, args, {
+      cwd: opts.cwd,
+      env: opts.env,
+      stdio: ["ignore", "pipe", "pipe"]
+    });
+    let output = "";
+    const collect = (chunk) => {
+      output += chunk.toString("utf8");
+    };
+    child.stdout?.on("data", collect);
+    child.stderr?.on("data", collect);
+    const timer = setTimeout(() => {
+      child.kill("SIGKILL");
+      reject(
+        new Error(
+          `${cmd} ${args.join(" ")} timed out after ${opts.timeoutMs}ms`
+        )
+      );
+    }, opts.timeoutMs);
+    child.on("error", (err) => {
+      clearTimeout(timer);
+      reject(err);
+    });
+    child.on("close", (code) => {
+      clearTimeout(timer);
+      const trimmed = output.trimEnd();
+      if (trimmed.length > 0) {
+        onProgress(`[install:output]
+${trimmed}`);
+      }
+      if (code === 0) {
+        resolve3();
+      } else {
+        reject(new Error(`${cmd} exited with code ${code}`));
+      }
+    });
+  });
+}
 function detectPackageManager(workDir) {
   if ((0, import_fs.existsSync)(import_path2.default.join(workDir, "pnpm-lock.yaml"))) {
     return {
@@ -7712,7 +7752,8 @@ async function prepareWorkingDirectory(config, evalRunId2, targetId, scenarioId,
     writeWixEnvFile(workDir2);
     onProgress("[diag] entering installDependencies");
     await installDependencies(workDir2, onProgress, {
-      cacheBase: nodeModulesCacheDir
+      cacheBase: nodeModulesCacheDir,
+      exec: createLoggingInstallExec(onProgress)
     });
     onProgress("[diag] installDependencies returned");
     onProgress("Environment ready");
@@ -10980,22 +11021,38 @@ async function executeWithAiSdk(context) {
       ...topLevelExtras,
       model,
       abortSignal: abortController.signal,
-      system: systemPrompt,
-      ...hasImages ? {
-        messages: [
+      // Anthropic prompt caching: the Vercel AI SDK can't attach cache_control
+      // to the top-level `system` string, so the system prompt is sent as a
+      // `system` message carrying an ephemeral cache breakpoint (Anthropic only).
+      // Anthropic renders tools -> system -> messages, so a breakpoint on the
+      // system block caches the tool definitions too — high value for MCP
+      // scenarios that share a large tool set + system prompt across runs.
+      messages: [
+        ...systemPrompt ? [
           {
-            role: "user",
-            content: [
-              { type: "text", text: scenario.triggerPrompt },
-              ...triggerPromptImages.map((img) => ({
-                type: "image",
-                image: img.base64,
-                mediaType: img.mediaType
-              }))
-            ]
+            role: "system",
+            content: systemPrompt,
+            ...isAnthropic ? {
+              providerOptions: {
+                anthropic: {
+                  cacheControl: { type: "ephemeral" }
+                }
+              }
+            } : {}
           }
-        ]
-      } : { prompt: scenario.triggerPrompt },
+        ] : [],
+        {
+          role: "user",
+          content: hasImages ? [
+            { type: "text", text: scenario.triggerPrompt },
+            ...triggerPromptImages.map((img) => ({
+              type: "image",
+              image: img.base64,
+              mediaType: img.mediaType
+            }))
+          ] : scenario.triggerPrompt
+        }
+      ],
       temperature: supportsThinking ? void 0 : cfg.temperature,
       topP: supportsThinking ? void 0 : cfg.topP,
       frequencyPenalty: cfg.frequencyPenalty,