open-agents-ai 0.184.74 → 0.184.76

package/dist/index.js

@@ -15944,10 +15944,10 @@ Coordinates are normalized (0-1). Multiply by image width/height for pixel value
             ollamaPrompt = prompt;
             break;
           case "detect":
-            ollamaPrompt = `Detect all instances of "${prompt}" in this image. For each, describe its location.`;
+            ollamaPrompt = `Locate all instances of "${prompt}" in this image. For each, give bounding box coordinates as [x1, y1, x2, y2].`;
             break;
           case "point":
-            ollamaPrompt = `Point to ${prompt}`;
+            ollamaPrompt = `Locate the "${prompt}" in this image and give its x,y position as normalized coordinates.`;
             break;
           default:
             return null;
@@ -15982,19 +15982,48 @@ Coordinates are normalized (0-1). Multiply by image width/height for pixel value
           if (!response)
             return null;
           if (action === "point") {
-            const matches = [...response.matchAll(/<point\s+x="([\d.]+)"\s+y="([\d.]+)"\s*\/?>/g)];
-            if (matches.length > 0) {
-              const formatted = matches.map((m, i) => `  ${i + 1}. (${parseFloat(m[1]).toFixed(4)}, ${parseFloat(m[2]).toFixed(4)}) \u2014 normalized 0-1`).join("\n");
+            const xmlMatches = [...response.matchAll(/<point\s+x="([\d.]+)"\s+y="([\d.]+)"\s*\/?>/g)];
+            if (xmlMatches.length > 0) {
+              const formatted = xmlMatches.map((m, i) => `  ${i + 1}. (${parseFloat(m[1]).toFixed(4)}, ${parseFloat(m[2]).toFixed(4)}) \u2014 normalized 0-1`).join("\n");
               return {
                 success: true,
-                output: `Found ${matches.length} "${prompt}" location(s) in ${filename} (via Ollama):
+                output: `Found ${xmlMatches.length} "${prompt}" location(s) in ${filename} (via Ollama):
 ${formatted}
 
 Coordinates are normalized (0-1). Multiply by image width/height for pixel values.`,
                 durationMs: performance.now() - start
               };
             }
-            return { success: true, output: `Could not extract coordinates for "${prompt}" from ${filename}. Model response: ${response}`, durationMs: performance.now() - start };
+            const arrayMatch = response.match(/\[\s*([\d.]+)\s*,\s*([\d.]+)(?:\s*,\s*([\d.]+)\s*,\s*([\d.]+))?\s*\]/);
+            if (arrayMatch) {
+              const x1 = parseFloat(arrayMatch[1]);
+              const y1 = parseFloat(arrayMatch[2]);
+              if (arrayMatch[3] && arrayMatch[4]) {
+                const x2 = parseFloat(arrayMatch[3]);
+                const y2 = parseFloat(arrayMatch[4]);
+                const cx = ((x1 + x2) / 2).toFixed(4);
+                const cy = ((y1 + y2) / 2).toFixed(4);
+                return {
+                  success: true,
+                  output: `Found "${prompt}" in ${filename} (via Ollama):
+  Center: (${cx}, ${cy}) \u2014 normalized 0-1
+  Bounding box: [${x1}, ${y1}, ${x2}, ${y2}]
+
+Coordinates are normalized (0-1). Multiply by image width/height for pixel values.`,
+                  durationMs: performance.now() - start
+                };
+              }
+              return {
+                success: true,
+                output: `Found "${prompt}" in ${filename} (via Ollama):
+  Position: (${x1.toFixed(4)}, ${y1.toFixed(4)}) \u2014 normalized 0-1
+
+Coordinates are normalized (0-1). Multiply by image width/height for pixel values.`,
+                durationMs: performance.now() - start
+              };
+            }
+            return { success: true, output: `Location of "${prompt}" in ${filename} (via Ollama):
+${response}`, durationMs: performance.now() - start };
           }
           if (action === "caption") {
             return { success: true, output: `Caption (${length}) for ${filename} (via Ollama):
@@ -51415,13 +51444,18 @@ async function handleUpdate(subcommand, ctx) {
   const { execPath, argv } = process;
   try {
     const { execFileSync } = await import("node:child_process");
+    if (process.stdout.isTTY) {
+      process.stdout.write("\x1B[?1002l\x1B[?1003l\x1B[?1006l\x1B[?25h\x1B[?1049l\x1B[0m");
+    }
     process.env.__OA_RESUMED = resumeFlag;
     execFileSync(execPath, argv.slice(1), {
       stdio: "inherit",
       env: { ...process.env, __OA_RESUMED: resumeFlag }
     });
+    process.exit(0);
   } catch {
-    renderWarning("Restart oa manually to use the new version.");
+    process.stderr.write("\x1B[0m\nRestart oa manually to use the new version.\n");
+    process.exit(1);
   }
 }
 async function switchModel(query, ctx, local = false) {
@@ -64391,6 +64425,32 @@ ${lines.join("\n")}
   if (flowEnabled) {
     dynamicContext += "\n\n" + FLOWSTATE_PROMPT;
   }
+  if (modelCaps?.vision) {
+    dynamicContext += `
+
+<vision-capabilities>
+You have vision capabilities available. You can analyze images using these tools:
+
+1. **image_read** \u2014 Read/view an image file directly. Use this for screenshots, photos,
+   diagrams, or any image the user provides. Pass the file path.
+   Example: image_read(image="/path/to/image.png")
+
+2. **vision** \u2014 Analyze images with Moondream vision model. Supports:
+   - caption: Describe what's in an image
+   - query: Ask a question about an image (visual QA)
+   - detect: Find all instances of an object (bounding boxes)
+   - point: Find the center location of an object (for click targeting)
+   Example: vision(image="/tmp/screenshot.png", action="caption")
+   Example: vision(image="photo.jpg", action="query", prompt="What color is the car?")
+
+3. **screenshot** \u2014 Capture the current screen (desktop automation).
+   Returns a screenshot path you can then analyze with vision or image_read.
+
+When the user asks you to look at, describe, or analyze an image or camera feed,
+use these tools proactively. Do not say you cannot see images \u2014 you can.
+For video streams or camera feeds, capture frames and analyze them sequentially.
+</vision-capabilities>`;
+  }
   let localFirstOverride = false;
   if (config.backendType === "nexus") {
     try {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "open-agents-ai",
-  "version": "0.184.74",
+  "version": "0.184.76",
   "description": "AI coding agent powered by open-source models (Ollama/vLLM) — interactive TUI with agentic tool-calling loop",
   "type": "module",
   "main": "./dist/index.js",