open-agents-ai 0.184.80 → 0.184.81

package/dist/index.js

@@ -16206,7 +16206,7 @@ var init_desktop_click = __esm({
     DesktopClickTool = class {
       workingDir;
       name = "desktop_click";
-      description = "Click on a UI element identified by natural language description. Takes a screenshot, uses vision to find the described element, then clicks at that location using xdotool (Linux) or cliclick (macOS). Example: desktop_click({ target: 'the Save button' })";
+      description = "Click on a UI element identified by natural language description. Takes a screenshot, uses Moondream vision to find the element's coordinates, then clicks at that location. Supports single click, double click, and right click.\n  Single click: desktop_click({ target: 'the Save button' })\n  Double click: desktop_click({ target: 'document.pdf', click_type: 'double' })\n  Right click:  desktop_click({ target: 'file icon', button: 'right' })\nChain multiple calls for complex interactions (e.g. right-click \u2192 select menu item).";
       /** Active Ollama model name — used as vision fallback if it has vision capability */
       _activeModel = "";
       /** Whether the active model has vision capability */
@@ -64480,26 +64480,28 @@ ${lines.join("\n")}
     dynamicContext += `
 
 <vision-capabilities>
-You have vision capabilities available. You can analyze images using these tools:
-
-1. **image_read** \u2014 Read/view an image file directly. Use this for screenshots, photos,
-   diagrams, or any image the user provides. Pass the file path.
-   Example: image_read(image="/path/to/image.png")
-
-2. **vision** \u2014 Analyze images with Moondream vision model. Supports:
-   - caption: Describe what's in an image
-   - query: Ask a question about an image (visual QA)
-   - detect: Find all instances of an object (bounding boxes)
-   - point: Find the center location of an object (for click targeting)
-   Example: vision(image="/tmp/screenshot.png", action="caption")
-   Example: vision(image="photo.jpg", action="query", prompt="What color is the car?")
-
-3. **screenshot** \u2014 Capture the current screen (desktop automation).
-   Returns a screenshot path you can then analyze with vision or image_read.
-
-When the user asks you to look at, describe, or analyze an image or camera feed,
-use these tools proactively. Do not say you cannot see images \u2014 you can.
-For video streams or camera feeds, capture frames and analyze them sequentially.
+You have vision capabilities available.
+
+IMAGE ANALYSIS:
+  image_read(image="path")  \u2014 View/read an image file directly
+  vision(image="path", action="caption")  \u2014 Describe image contents
+  vision(image="path", action="query", prompt="question")  \u2014 Visual QA
+  vision(image="path", action="detect", prompt="object")  \u2014 Find objects (bounding boxes)
+  vision(image="path", action="point", prompt="element")  \u2014 Find element center (for clicking)
+
+DESKTOP INTERACTION PIPELINE:
+  1. desktop_describe()  \u2014 Screenshot + describe what's on screen
+  2. desktop_click({ target: "element" })  \u2014 Find + click an element
+  3. desktop_click({ target: "file.pdf", click_type: "double" })  \u2014 Double click to open
+  4. desktop_click({ target: "icon", button: "right" })  \u2014 Right click for context menu
+  Chain calls iteratively: describe \u2192 click \u2192 wait \u2192 describe \u2192 click next item
+
+RULES:
+  - Do NOT say you cannot see images \u2014 you can.
+  - Use desktop_describe first to survey the screen, then desktop_click to interact.
+  - After clicking, call desktop_describe again to verify the result.
+  - For file operations: double-click to open, right-click for context menus.
+  - Use delay_ms parameter if UI needs time to transition between clicks.
 </vision-capabilities>`;
   }
   let localFirstOverride = false;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "open-agents-ai",
-  "version": "0.184.80",
+  "version": "0.184.81",
   "description": "AI coding agent powered by open-source models (Ollama/vLLM) — interactive TUI with agentic tool-calling loop",
   "type": "module",
   "main": "./dist/index.js",