@mindstudio-ai/remy 0.1.179 → 0.1.180

package/dist/headless.js

@@ -3813,6 +3813,121 @@ function resolveModel(surfaceId, models, fallback) {
 
 // src/subagents/browserAutomation/index.ts
 var log6 = createLogger("browser-automation");
+async function runBrowserAutomation(task, context) {
+  const release = await acquireBrowserLock();
+  try {
+    const result = await runSubAgent({
+      system: getBrowserAutomationPrompt(),
+      task,
+      tools: BROWSER_TOOLS,
+      externalTools: BROWSER_EXTERNAL_TOOLS,
+      executeTool: async (name, _input, _toolCallId, onLog) => {
+        if (name === "setupBrowser") {
+          try {
+            const result2 = await sidecarRequest(
+              "/setup-browser",
+              {
+                auth: _input.auth,
+                path: _input.path
+              },
+              { timeout: 15e3 }
+            );
+            return JSON.stringify(result2);
+          } catch (err) {
+            return `Error setting up browser: ${err.message}`;
+          }
+        }
+        if (name === "screenshotFullPage") {
+          try {
+            return await captureAndAnalyzeScreenshot({
+              path: _input.path,
+              onLog,
+              model: resolveModel(
+                "imageAnalysis",
+                context.models,
+                context.model
+              )
+            });
+          } catch (err) {
+            return `Error taking screenshot: ${err.message}`;
+          }
+        }
+        return `Error: unknown local tool "${name}"`;
+      },
+      apiConfig: context.apiConfig,
+      model: resolveModel("browserAutomation", context.models, context.model),
+      subAgentId: "browserAutomation",
+      signal: context.signal,
+      parentToolId: context.toolCallId,
+      requestId: context.requestId,
+      onEvent: context.onEvent,
+      resolveExternalTool: async (id, name, input) => {
+        if (!context.resolveExternalTool) {
+          return "Error: no external tool resolver";
+        }
+        const result2 = await context.resolveExternalTool(id, name, input);
+        if (name === "browserCommand") {
+          try {
+            const parsed = JSON.parse(result2);
+            const screenshotSteps = (parsed.steps || []).filter(
+              (s) => s.command === "screenshotViewport" && s.result?.url
+            );
+            if (screenshotSteps.length > 0) {
+              const visionOverride = {
+                model: resolveModel(
+                  "imageAnalysis",
+                  context.models,
+                  context.model
+                )
+              };
+              const batchInput = screenshotSteps.map((s) => ({
+                stepType: "analyzeImage",
+                step: {
+                  imageUrl: s.result.url,
+                  prompt: buildScreenshotAnalysisPrompt({
+                    styleMap: s.result.styleMap
+                  }),
+                  visionModelOverride: visionOverride
+                }
+              }));
+              const batchResult = await runMindstudioCli(
+                ["batch", JSON.stringify(batchInput)],
+                { timeout: 2e5, caller: "browserAutomation" }
+              );
+              try {
+                const analyses = JSON.parse(batchResult);
+                let ai = 0;
+                for (const step of parsed.steps) {
+                  if (step.command === "screenshotViewport" && step.result?.url && ai < analyses.length) {
+                    step.result.analysis = analyses[ai]?.output?.analysis || analyses[ai]?.output || "";
+                    ai++;
+                  }
+                }
+              } catch {
+                log6.debug("Failed to parse batch analysis result", {
+                  batchResult
+                });
+              }
+              return JSON.stringify(parsed);
+            }
+          } catch {
+          }
+        }
+        return result2;
+      },
+      toolRegistry: context.toolRegistry,
+      captureArtifacts: ["screenshotFullPage"]
+    });
+    context.subAgentMessages?.set(context.toolCallId, result.messages);
+    const ss = result.artifacts?.screenshotFullPage;
+    return {
+      text: result.text,
+      ...ss?.url ? { screenshot: { url: ss.url, styleMap: ss.styleMap } } : {}
+    };
+  } finally {
+    release();
+  }
+}
 var browserAutomationTool = {
   clearable: true,
   definition: {
@@ -3833,121 +3948,13 @@ var browserAutomationTool = {
     if (!context) {
       return "Error: browser automation requires execution context (only available in headless mode)";
     }
-    const release = await acquireBrowserLock();
-    try {
-      const result = await runSubAgent({
-        system: getBrowserAutomationPrompt(),
-        task: input.task,
-        tools: BROWSER_TOOLS,
-        externalTools: BROWSER_EXTERNAL_TOOLS,
-        executeTool: async (name, _input, _toolCallId, onLog) => {
-          if (name === "setupBrowser") {
-            try {
-              const result2 = await sidecarRequest(
-                "/setup-browser",
-                {
-                  auth: _input.auth,
-                  path: _input.path
-                },
-                { timeout: 15e3 }
-              );
-              return JSON.stringify(result2);
-            } catch (err) {
-              return `Error setting up browser: ${err.message}`;
-            }
-          }
-          if (name === "screenshotFullPage") {
-            try {
-              return await captureAndAnalyzeScreenshot({
-                path: _input.path,
-                onLog,
-                model: resolveModel(
-                  "imageAnalysis",
-                  context.models,
-                  context.model
-                )
-              });
-            } catch (err) {
-              return `Error taking screenshot: ${err.message}`;
-            }
-          }
-          return `Error: unknown local tool "${name}"`;
-        },
-        apiConfig: context.apiConfig,
-        model: resolveModel("browserAutomation", context.models, context.model),
-        subAgentId: "browserAutomation",
-        signal: context.signal,
-        parentToolId: context.toolCallId,
-        requestId: context.requestId,
-        onEvent: context.onEvent,
-        resolveExternalTool: async (id, name, input2) => {
-          if (!context.resolveExternalTool) {
-            return "Error: no external tool resolver";
-          }
-          const result2 = await context.resolveExternalTool(id, name, input2);
-          if (name === "browserCommand") {
-            try {
-              const parsed = JSON.parse(result2);
-              const screenshotSteps = (parsed.steps || []).filter(
-                (s) => s.command === "screenshotViewport" && s.result?.url
-              );
-              if (screenshotSteps.length > 0) {
-                const visionOverride = {
-                  model: resolveModel(
-                    "imageAnalysis",
-                    context.models,
-                    context.model
-                  )
-                };
-                const batchInput = screenshotSteps.map((s) => ({
-                  stepType: "analyzeImage",
-                  step: {
-                    imageUrl: s.result.url,
-                    prompt: buildScreenshotAnalysisPrompt({
-                      styleMap: s.result.styleMap
-                    }),
-                    visionModelOverride: visionOverride
-                  }
-                }));
-                const batchResult = await runMindstudioCli(
-                  ["batch", JSON.stringify(batchInput)],
-                  { timeout: 2e5, caller: "browserAutomation" }
-                );
-                try {
-                  const analyses = JSON.parse(batchResult);
-                  let ai = 0;
-                  for (const step of parsed.steps) {
-                    if (step.command === "screenshotViewport" && step.result?.url && ai < analyses.length) {
-                      step.result.analysis = analyses[ai]?.output?.analysis || analyses[ai]?.output || "";
-                      ai++;
-                    }
-                  }
-                } catch {
-                  log6.debug("Failed to parse batch analysis result", {
-                    batchResult
-                  });
-                }
-                return JSON.stringify(parsed);
-              }
-            } catch {
-            }
-          }
-          return result2;
-        },
-        toolRegistry: context.toolRegistry,
-        captureArtifacts: ["screenshotFullPage"]
-      });
-      context.subAgentMessages?.set(context.toolCallId, result.messages);
-      const ss = result.artifacts?.screenshotFullPage;
-      if (ss?.url) {
-        return `${result.text}
+    const result = await runBrowserAutomation(input.task, context);
+    if (result.screenshot) {
+      return `${result.text}
 
-![Final state](${ss.url})`;
-      }
-      return result.text;
-    } finally {
-      release();
+![Final state](${result.screenshot.url})`;
     }
+    return result.text;
   }
 };
 
@@ -3991,23 +3998,14 @@ var screenshotTool = {
       }
       if (input.instructions && context) {
         const task = input.path ? `Navigate to "${input.path}", then: ${input.instructions}. After completing these steps, take a full-page screenshot.` : `${input.instructions}. After completing these steps, take a full-page screenshot.`;
-        const result = await browserAutomationTool.execute({ task }, context);
-        const resultStr = result;
-        let url;
-        let styleMap;
-        try {
-          const parsed = JSON.parse(resultStr);
-          url = parsed.screenshotUrl;
-          styleMap = parsed.styleMap;
-        } catch {
-        }
-        if (!url) {
-          return `Error: browser navigation completed but no screenshot URL was returned. Agent output: ${resultStr}`;
+        const result = await runBrowserAutomation(task, context);
+        if (!result.screenshot) {
+          return result.text;
         }
         return await streamScreenshotAnalysis({
-          url,
+          url: result.screenshot.url,
           prompt: input.prompt,
-          styleMap,
+          styleMap: result.screenshot.styleMap,
           onLog: context?.onLog,
           model: resolveModel("imageAnalysis", context?.models, context?.model)
         });
@@ -4339,23 +4337,14 @@ async function execute5(input, onLog, context) {
   if (input.instructions && context) {
     try {
       const task = input.path ? `Navigate to "${input.path}", then: ${input.instructions}. After completing these steps, take a full-page screenshot.` : `${input.instructions}. After completing these steps, take a full-page screenshot.`;
-      const result = await browserAutomationTool.execute({ task }, context);
-      const resultStr = result;
-      let url;
-      let styleMap;
-      try {
-        const parsed = JSON.parse(resultStr);
-        url = parsed.screenshotUrl;
-        styleMap = parsed.styleMap;
-      } catch {
-      }
-      if (!url) {
-        return `Error: browser navigation completed but no screenshot URL was returned. Agent output: ${resultStr}`;
+      const result = await runBrowserAutomation(task, context);
+      if (!result.screenshot) {
+        return result.text;
       }
       return await streamScreenshotAnalysis({
-        url,
+        url: result.screenshot.url,
         prompt: input.prompt,
-        styleMap,
+        styleMap: result.screenshot.styleMap,
         onLog,
         model: resolveModel("imageAnalysis", context?.models, context?.model)
       });

package/dist/index.js

@@ -4176,6 +4176,121 @@ var init_prompt2 = __esm({
 });
 
 // src/subagents/browserAutomation/index.ts
+async function runBrowserAutomation(task, context) {
+  const release = await acquireBrowserLock();
+  try {
+    const result = await runSubAgent({
+      system: getBrowserAutomationPrompt(),
+      task,
+      tools: BROWSER_TOOLS,
+      externalTools: BROWSER_EXTERNAL_TOOLS,
+      executeTool: async (name, _input, _toolCallId, onLog) => {
+        if (name === "setupBrowser") {
+          try {
+            const result2 = await sidecarRequest(
+              "/setup-browser",
+              {
+                auth: _input.auth,
+                path: _input.path
+              },
+              { timeout: 15e3 }
+            );
+            return JSON.stringify(result2);
+          } catch (err) {
+            return `Error setting up browser: ${err.message}`;
+          }
+        }
+        if (name === "screenshotFullPage") {
+          try {
+            return await captureAndAnalyzeScreenshot({
+              path: _input.path,
+              onLog,
+              model: resolveModel(
+                "imageAnalysis",
+                context.models,
+                context.model
+              )
+            });
+          } catch (err) {
+            return `Error taking screenshot: ${err.message}`;
+          }
+        }
+        return `Error: unknown local tool "${name}"`;
+      },
+      apiConfig: context.apiConfig,
+      model: resolveModel("browserAutomation", context.models, context.model),
+      subAgentId: "browserAutomation",
+      signal: context.signal,
+      parentToolId: context.toolCallId,
+      requestId: context.requestId,
+      onEvent: context.onEvent,
+      resolveExternalTool: async (id, name, input) => {
+        if (!context.resolveExternalTool) {
+          return "Error: no external tool resolver";
+        }
+        const result2 = await context.resolveExternalTool(id, name, input);
+        if (name === "browserCommand") {
+          try {
+            const parsed = JSON.parse(result2);
+            const screenshotSteps = (parsed.steps || []).filter(
+              (s) => s.command === "screenshotViewport" && s.result?.url
+            );
+            if (screenshotSteps.length > 0) {
+              const visionOverride = {
+                model: resolveModel(
+                  "imageAnalysis",
+                  context.models,
+                  context.model
+                )
+              };
+              const batchInput = screenshotSteps.map((s) => ({
+                stepType: "analyzeImage",
+                step: {
+                  imageUrl: s.result.url,
+                  prompt: buildScreenshotAnalysisPrompt({
+                    styleMap: s.result.styleMap
+                  }),
+                  visionModelOverride: visionOverride
+                }
+              }));
+              const batchResult = await runMindstudioCli(
+                ["batch", JSON.stringify(batchInput)],
+                { timeout: 2e5, caller: "browserAutomation" }
+              );
+              try {
+                const analyses = JSON.parse(batchResult);
+                let ai = 0;
+                for (const step of parsed.steps) {
+                  if (step.command === "screenshotViewport" && step.result?.url && ai < analyses.length) {
+                    step.result.analysis = analyses[ai]?.output?.analysis || analyses[ai]?.output || "";
+                    ai++;
+                  }
+                }
+              } catch {
+                log6.debug("Failed to parse batch analysis result", {
+                  batchResult
+                });
+              }
+              return JSON.stringify(parsed);
+            }
+          } catch {
+          }
+        }
+        return result2;
+      },
+      toolRegistry: context.toolRegistry,
+      captureArtifacts: ["screenshotFullPage"]
+    });
+    context.subAgentMessages?.set(context.toolCallId, result.messages);
+    const ss = result.artifacts?.screenshotFullPage;
+    return {
+      text: result.text,
+      ...ss?.url ? { screenshot: { url: ss.url, styleMap: ss.styleMap } } : {}
+    };
+  } finally {
+    release();
+  }
+}
 var log6, browserAutomationTool;
 var init_browserAutomation = __esm({
   "src/subagents/browserAutomation/index.ts"() {
@@ -4210,121 +4325,13 @@ var init_browserAutomation = __esm({
         if (!context) {
           return "Error: browser automation requires execution context (only available in headless mode)";
         }
-        const release = await acquireBrowserLock();
-        try {
-          const result = await runSubAgent({
-            system: getBrowserAutomationPrompt(),
-            task: input.task,
-            tools: BROWSER_TOOLS,
-            externalTools: BROWSER_EXTERNAL_TOOLS,
-            executeTool: async (name, _input, _toolCallId, onLog) => {
-              if (name === "setupBrowser") {
-                try {
-                  const result2 = await sidecarRequest(
-                    "/setup-browser",
-                    {
-                      auth: _input.auth,
-                      path: _input.path
-                    },
-                    { timeout: 15e3 }
-                  );
-                  return JSON.stringify(result2);
-                } catch (err) {
-                  return `Error setting up browser: ${err.message}`;
-                }
-              }
-              if (name === "screenshotFullPage") {
-                try {
-                  return await captureAndAnalyzeScreenshot({
-                    path: _input.path,
-                    onLog,
-                    model: resolveModel(
-                      "imageAnalysis",
-                      context.models,
-                      context.model
-                    )
-                  });
-                } catch (err) {
-                  return `Error taking screenshot: ${err.message}`;
-                }
-              }
-              return `Error: unknown local tool "${name}"`;
-            },
-            apiConfig: context.apiConfig,
-            model: resolveModel("browserAutomation", context.models, context.model),
-            subAgentId: "browserAutomation",
-            signal: context.signal,
-            parentToolId: context.toolCallId,
-            requestId: context.requestId,
-            onEvent: context.onEvent,
-            resolveExternalTool: async (id, name, input2) => {
-              if (!context.resolveExternalTool) {
-                return "Error: no external tool resolver";
-              }
-              const result2 = await context.resolveExternalTool(id, name, input2);
-              if (name === "browserCommand") {
-                try {
-                  const parsed = JSON.parse(result2);
-                  const screenshotSteps = (parsed.steps || []).filter(
-                    (s) => s.command === "screenshotViewport" && s.result?.url
-                  );
-                  if (screenshotSteps.length > 0) {
-                    const visionOverride = {
-                      model: resolveModel(
-                        "imageAnalysis",
-                        context.models,
-                        context.model
-                      )
-                    };
-                    const batchInput = screenshotSteps.map((s) => ({
-                      stepType: "analyzeImage",
-                      step: {
-                        imageUrl: s.result.url,
-                        prompt: buildScreenshotAnalysisPrompt({
-                          styleMap: s.result.styleMap
-                        }),
-                        visionModelOverride: visionOverride
-                      }
-                    }));
-                    const batchResult = await runMindstudioCli(
-                      ["batch", JSON.stringify(batchInput)],
-                      { timeout: 2e5, caller: "browserAutomation" }
-                    );
-                    try {
-                      const analyses = JSON.parse(batchResult);
-                      let ai = 0;
-                      for (const step of parsed.steps) {
-                        if (step.command === "screenshotViewport" && step.result?.url && ai < analyses.length) {
-                          step.result.analysis = analyses[ai]?.output?.analysis || analyses[ai]?.output || "";
-                          ai++;
-                        }
-                      }
-                    } catch {
-                      log6.debug("Failed to parse batch analysis result", {
-                        batchResult
-                      });
-                    }
-                    return JSON.stringify(parsed);
-                  }
-                } catch {
-                }
-              }
-              return result2;
-            },
-            toolRegistry: context.toolRegistry,
-            captureArtifacts: ["screenshotFullPage"]
-          });
-          context.subAgentMessages?.set(context.toolCallId, result.messages);
-          const ss = result.artifacts?.screenshotFullPage;
-          if (ss?.url) {
-            return `${result.text}
+        const result = await runBrowserAutomation(input.task, context);
+        if (result.screenshot) {
+          return `${result.text}
 
-![Final state](${ss.url})`;
-          }
-          return result.text;
-        } finally {
-          release();
+![Final state](${result.screenshot.url})`;
         }
+        return result.text;
       }
     };
   }
@@ -4378,23 +4385,14 @@ var init_screenshot2 = __esm({
           }
           if (input.instructions && context) {
             const task = input.path ? `Navigate to "${input.path}", then: ${input.instructions}. After completing these steps, take a full-page screenshot.` : `${input.instructions}. After completing these steps, take a full-page screenshot.`;
-            const result = await browserAutomationTool.execute({ task }, context);
-            const resultStr = result;
-            let url;
-            let styleMap;
-            try {
-              const parsed = JSON.parse(resultStr);
-              url = parsed.screenshotUrl;
-              styleMap = parsed.styleMap;
-            } catch {
-            }
-            if (!url) {
-              return `Error: browser navigation completed but no screenshot URL was returned. Agent output: ${resultStr}`;
+            const result = await runBrowserAutomation(task, context);
+            if (!result.screenshot) {
+              return result.text;
             }
             return await streamScreenshotAnalysis({
-              url,
+              url: result.screenshot.url,
               prompt: input.prompt,
-              styleMap,
+              styleMap: result.screenshot.styleMap,
               onLog: context?.onLog,
               model: resolveModel("imageAnalysis", context?.models, context?.model)
             });
@@ -4744,23 +4742,14 @@ async function execute5(input, onLog, context) {
   if (input.instructions && context) {
     try {
       const task = input.path ? `Navigate to "${input.path}", then: ${input.instructions}. After completing these steps, take a full-page screenshot.` : `${input.instructions}. After completing these steps, take a full-page screenshot.`;
-      const result = await browserAutomationTool.execute({ task }, context);
-      const resultStr = result;
-      let url;
-      let styleMap;
-      try {
-        const parsed = JSON.parse(resultStr);
-        url = parsed.screenshotUrl;
-        styleMap = parsed.styleMap;
-      } catch {
-      }
-      if (!url) {
-        return `Error: browser navigation completed but no screenshot URL was returned. Agent output: ${resultStr}`;
+      const result = await runBrowserAutomation(task, context);
+      if (!result.screenshot) {
+        return result.text;
       }
       return await streamScreenshotAnalysis({
-        url,
+        url: result.screenshot.url,
         prompt: input.prompt,
-        styleMap,
+        styleMap: result.screenshot.styleMap,
         onLog,
         model: resolveModel("imageAnalysis", context?.models, context?.model)
       });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mindstudio-ai/remy",
-  "version": "0.1.179",
+  "version": "0.1.180",
   "description": "MindStudio coding agent",
   "repository": {
     "type": "git",