open-agents-ai 0.184.79 → 0.184.81

package/dist/index.js

@@ -16206,7 +16206,7 @@ var init_desktop_click = __esm({
     DesktopClickTool = class {
       workingDir;
       name = "desktop_click";
-      description = "Click on a UI element identified by natural language description. Takes a screenshot, uses vision to find the described element, then clicks at that location using xdotool (Linux) or cliclick (macOS). Example: desktop_click({ target: 'the Save button' })";
+      description = "Click on a UI element identified by natural language description. Takes a screenshot, uses Moondream vision to find the element's coordinates, then clicks at that location. Supports single click, double click, and right click.\n  Single click: desktop_click({ target: 'the Save button' })\n  Double click: desktop_click({ target: 'document.pdf', click_type: 'double' })\n  Right click:  desktop_click({ target: 'file icon', button: 'right' })\nChain multiple calls for complex interactions (e.g. right-click \u2192 select menu item).";
       /** Active Ollama model name — used as vision fallback if it has vision capability */
       _activeModel = "";
       /** Whether the active model has vision capability */
@@ -16303,20 +16303,51 @@ var init_desktop_click = __esm({
               const envModel = process.env["OLLAMA_VISION_MODEL"];
               const ollamaModel = envModel || (this._activeModelHasVision && this._activeModel ? this._activeModel : "moondream");
               const imageBase64 = readFileSync15(screenshotPath).toString("base64");
-              const res = await fetch(`${ollamaHost}/api/generate`, {
+              let res = await fetch(`${ollamaHost}/api/generate`, {
                 method: "POST",
                 headers: { "Content-Type": "application/json" },
-                body: JSON.stringify({ model: ollamaModel, prompt: `Point to ${target}`, images: [imageBase64], stream: false }),
+                body: JSON.stringify({ model: ollamaModel, prompt: `Locate the "${target}" in this image and give its x,y position as normalized coordinates.`, images: [imageBase64], stream: false }),
                 signal: AbortSignal.timeout(6e4)
               });
+              if (!res.ok && ollamaModel === "moondream") {
+                const errText = await res.text().catch(() => "");
+                if (res.status === 404 || /not found|does not exist/i.test(errText)) {
+                  try {
+                    const { execSync: es } = await import("node:child_process");
+                    es("ollama pull moondream", { timeout: 3e5, stdio: "pipe" });
+                    res = await fetch(`${ollamaHost}/api/generate`, {
+                      method: "POST",
+                      headers: { "Content-Type": "application/json" },
+                      body: JSON.stringify({ model: ollamaModel, prompt: `Locate the "${target}" in this image and give its x,y position as normalized coordinates.`, images: [imageBase64], stream: false }),
+                      signal: AbortSignal.timeout(6e4)
+                    });
+                  } catch {
+                  }
+                }
+              }
               if (res.ok) {
                 const data = await res.json();
                 const response = data.response || "";
-                const pointMatches = [...response.matchAll(/<point\s+x="([\d.]+)"\s+y="([\d.]+)"\s*\/?>/g)];
-                if (pointMatches.length > 0) {
-                  points = pointMatches.map((m) => ({ x: parseFloat(m[1]), y: parseFloat(m[2]) }));
+                const xmlMatches = [...response.matchAll(/<point\s+x="([\d.]+)"\s+y="([\d.]+)"\s*\/?>/g)];
+                if (xmlMatches.length > 0) {
+                  points = xmlMatches.map((m) => ({ x: parseFloat(m[1]), y: parseFloat(m[2]) }));
                   visionWorked = true;
                 }
+                if (!visionWorked) {
+                  const arrayMatch = response.match(/\[\s*([\d.]+)\s*,\s*([\d.]+)(?:\s*,\s*([\d.]+)\s*,\s*([\d.]+))?\s*\]/);
+                  if (arrayMatch) {
+                    const x1 = parseFloat(arrayMatch[1]);
+                    const y1 = parseFloat(arrayMatch[2]);
+                    if (arrayMatch[3] && arrayMatch[4]) {
+                      const x2 = parseFloat(arrayMatch[3]);
+                      const y2 = parseFloat(arrayMatch[4]);
+                      points = [{ x: (x1 + x2) / 2, y: (y1 + y2) / 2 }];
+                    } else {
+                      points = [{ x: x1, y: y1 }];
+                    }
+                    visionWorked = true;
+                  }
+                }
               }
             } catch {
             }
@@ -16480,12 +16511,28 @@ ${caption}`);
               const ollamaModel = envModel || (this._activeModelHasVision && this._activeModel ? this._activeModel : "moondream");
               const imageBase64 = imageBuffer.toString("base64");
               const ollamaPrompt = question || "Describe what you see on this desktop screenshot in detail. Include visible applications, windows, text, and UI elements.";
-              const res = await fetch(`${ollamaHost}/api/generate`, {
+              let res = await fetch(`${ollamaHost}/api/generate`, {
                 method: "POST",
                 headers: { "Content-Type": "application/json" },
                 body: JSON.stringify({ model: ollamaModel, prompt: ollamaPrompt, images: [imageBase64], stream: false }),
                 signal: AbortSignal.timeout(6e4)
               });
+              if (!res.ok && ollamaModel === "moondream") {
+                const errText = await res.text().catch(() => "");
+                if (res.status === 404 || /not found|does not exist/i.test(errText)) {
+                  try {
+                    const { execSync: es } = await import("node:child_process");
+                    es("ollama pull moondream", { timeout: 3e5, stdio: "pipe" });
+                    res = await fetch(`${ollamaHost}/api/generate`, {
+                      method: "POST",
+                      headers: { "Content-Type": "application/json" },
+                      body: JSON.stringify({ model: ollamaModel, prompt: ollamaPrompt, images: [imageBase64], stream: false }),
+                      signal: AbortSignal.timeout(6e4)
+                    });
+                  } catch {
+                  }
+                }
+              }
               if (res.ok) {
                 const data = await res.json();
                 if (data.response) {
@@ -64433,26 +64480,28 @@ ${lines.join("\n")}
     dynamicContext += `
 
 <vision-capabilities>
-You have vision capabilities available. You can analyze images using these tools:
-
-1. **image_read** \u2014 Read/view an image file directly. Use this for screenshots, photos,
-   diagrams, or any image the user provides. Pass the file path.
-   Example: image_read(image="/path/to/image.png")
-
-2. **vision** \u2014 Analyze images with Moondream vision model. Supports:
-   - caption: Describe what's in an image
-   - query: Ask a question about an image (visual QA)
-   - detect: Find all instances of an object (bounding boxes)
-   - point: Find the center location of an object (for click targeting)
-   Example: vision(image="/tmp/screenshot.png", action="caption")
-   Example: vision(image="photo.jpg", action="query", prompt="What color is the car?")
-
-3. **screenshot** \u2014 Capture the current screen (desktop automation).
-   Returns a screenshot path you can then analyze with vision or image_read.
-
-When the user asks you to look at, describe, or analyze an image or camera feed,
-use these tools proactively. Do not say you cannot see images \u2014 you can.
-For video streams or camera feeds, capture frames and analyze them sequentially.
+You have vision capabilities available.
+
+IMAGE ANALYSIS:
+  image_read(image="path")  \u2014 View/read an image file directly
+  vision(image="path", action="caption")  \u2014 Describe image contents
+  vision(image="path", action="query", prompt="question")  \u2014 Visual QA
+  vision(image="path", action="detect", prompt="object")  \u2014 Find objects (bounding boxes)
+  vision(image="path", action="point", prompt="element")  \u2014 Find element center (for clicking)
+
+DESKTOP INTERACTION PIPELINE:
+  1. desktop_describe()  \u2014 Screenshot + describe what's on screen
+  2. desktop_click({ target: "element" })  \u2014 Find + click an element
+  3. desktop_click({ target: "file.pdf", click_type: "double" })  \u2014 Double click to open
+  4. desktop_click({ target: "icon", button: "right" })  \u2014 Right click for context menu
+  Chain calls iteratively: describe \u2192 click \u2192 wait \u2192 describe \u2192 click next item
+
+RULES:
+  - Do NOT say you cannot see images \u2014 you can.
+  - Use desktop_describe first to survey the screen, then desktop_click to interact.
+  - After clicking, call desktop_describe again to verify the result.
+  - For file operations: double-click to open, right-click for context menus.
+  - Use delay_ms parameter if UI needs time to transition between clicks.
 </vision-capabilities>`;
   }
   let localFirstOverride = false;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "open-agents-ai",
-  "version": "0.184.79",
+  "version": "0.184.81",
   "description": "AI coding agent powered by open-source models (Ollama/vLLM) — interactive TUI with agentic tool-calling loop",
   "type": "module",
   "main": "./dist/index.js",