@wix/evalforge-evaluator 0.139.0 → 0.141.0

package/build/index.js

@@ -287,13 +287,16 @@ function resolveSystemAssertion(assertionId, params) {
         }
       };
       break;
-    case "build_passed":
+    case "build_passed": {
+      const rawCmd = params?.command;
+      const command = typeof rawCmd === "string" && (0, import_evalforge_types.isAllowedBuildCommandString)(rawCmd) ? rawCmd.trim() : void 0;
       baseAssertion = {
         type: "build_passed",
-        command: params?.command ?? void 0,
+        ...command !== void 0 && { command },
         expectedExitCode: params?.expectedExitCode ?? void 0
       };
       break;
+    }
     case "time_limit":
       baseAssertion = {
         type: "time_limit",
@@ -972,6 +975,26 @@ function emitTraceEvent(event, tracePushUrl, routeHeader, authToken) {
 
 // src/run-scenario/agents/claude-code/execute.ts
 var DEFAULT_MODEL = import_evalforge_types4.ClaudeModel.CLAUDE_4_5_SONNET_1_0;
+async function* buildPromptStream(triggerPrompt, images) {
+  yield {
+    type: "user",
+    message: {
+      role: "user",
+      content: [
+        { type: "text", text: triggerPrompt },
+        ...images.map((img) => ({
+          type: "image",
+          source: {
+            type: "base64",
+            data: img.base64,
+            media_type: img.mediaType
+          }
+        }))
+      ]
+    },
+    parent_tool_use_id: null
+  };
+}
 function extractToolActionDescription(toolName, toolArgs) {
   if (!toolName) {
     return "Using tool...";
@@ -1347,8 +1370,13 @@ async function executeWithClaudeCode(skills, scenario, options) {
       }, HEARTBEAT_INTERVAL_MS);
     }
     const sdkPromise = (async () => {
+      const hasImages = scenario.triggerPromptImages && scenario.triggerPromptImages.length > 0;
+      const prompt = hasImages ? buildPromptStream(
+        scenario.triggerPrompt,
+        scenario.triggerPromptImages
+      ) : scenario.triggerPrompt;
       for await (const message of query({
-        prompt: scenario.triggerPrompt,
+        prompt,
         options: queryOptions
       })) {
         messageCount++;
@@ -2651,6 +2679,21 @@ function extractToolAction(toolName, args) {
   }
   return `Using ${toolName}...`;
 }
+async function writePromptImages(cwd, images) {
+  const imagesDir = (0, import_path10.join)(cwd, "prompt-images");
+  await (0, import_promises9.mkdir)(imagesDir, { recursive: true });
+  const filePaths = [];
+  for (let i = 0; i < images.length; i++) {
+    const img = images[i];
+    const ext = img.mediaType.split("/")[1] || "png";
+    const filename = `image-${i}.${ext}`;
+    const filepath = (0, import_path10.join)(imagesDir, filename);
+    const buffer = Buffer.from(img.base64, "base64");
+    await (0, import_promises9.writeFile)(filepath, buffer);
+    filePaths.push(filepath);
+  }
+  return filePaths;
+}
 function createTraceEventFromNdjson(evt, context, stepNumber, isComplete) {
   const base = {
     evalRunId: context.evalRunId,
@@ -3080,6 +3123,17 @@ async function executeWithOpenCode(skills, scenario, options) {
     "--dir",
     options.cwd
   ];
+  const hasImages = scenario.triggerPromptImages && scenario.triggerPromptImages.length > 0;
+  const imageFileArgs = [];
+  if (hasImages) {
+    const imagePaths = await writePromptImages(
+      options.cwd,
+      scenario.triggerPromptImages
+    );
+    for (const imgPath of imagePaths) {
+      imageFileArgs.push("-f", imgPath);
+    }
+  }
   const accumulatedEvents = [];
   let traceStepNumber = 0;
   let lastAttemptResult;
@@ -3114,7 +3168,7 @@ async function executeWithOpenCode(skills, scenario, options) {
         );
       }
     }
-    const args = [...baseArgs, prompt];
+    const args = [...baseArgs, ...imageFileArgs, prompt];
     console.log(
       `[executeWithOpenCode] Spawning attempt ${attempt}: opencode`,
       args.slice(0, 5)
@@ -3597,10 +3651,26 @@ async function executeWithAiSdk(context) {
       }
     };
     const stepTimestamps = [];
-    const result = await (0, import_ai.generateText)({
+    const { triggerPromptImages } = context;
+    const hasImages = triggerPromptImages && triggerPromptImages.length > 0;
+    const generateTextParams = {
       model,
       system: systemPrompt,
-      prompt: scenario.triggerPrompt,
+      ...hasImages ? {
+        messages: [
+          {
+            role: "user",
+            content: [
+              { type: "text", text: scenario.triggerPrompt },
+              ...triggerPromptImages.map((img) => ({
+                type: "image",
+                image: img.base64,
+                mediaType: img.mediaType
+              }))
+            ]
+          }
+        ]
+      } : { prompt: scenario.triggerPrompt },
       temperature: supportsThinking ? void 0 : modelConfig.temperature,
       maxOutputTokens: modelConfig.maxTokens,
       tools: mcpTools,
@@ -3633,7 +3703,8 @@ async function executeWithAiSdk(context) {
           );
         }
       }
-    });
+    };
+    const result = await (0, import_ai.generateText)(generateTextParams);
     const durationMs = Date.now() - startTime;
     const usage = {
       inputTokens: result.usage.inputTokens ?? 0,
@@ -4577,7 +4648,8 @@ async function runAgentWithContext(config, evalRunId2, scenario, evalData, workD
     mcps: evalData.mcps.length > 0 ? evalData.mcps : void 0,
     subAgents: evalData.subAgents.length > 0 ? evalData.subAgents : void 0,
     rules: evalData.rules?.length > 0 ? evalData.rules : void 0,
-    systemPrompt: agent?.systemPrompt
+    systemPrompt: agent?.systemPrompt,
+    triggerPromptImages: scenario.triggerPromptImages && scenario.triggerPromptImages.length > 0 ? scenario.triggerPromptImages : void 0
   };
   const hasPrepare = !!adapter.prepareEnvironment;
   const prePrepSnapshot = hasPrepare && workDir ? snapshotDirectory(workDir) : {};