open-agents-ai 0.184.73 → 0.184.75

package/dist/index.js

@@ -15839,10 +15839,34 @@ var init_vision = __esm({
           const ollamaResult = await this.tryOllamaVision(buffer, filename, action, prompt, length, start);
           if (ollamaResult)
             return ollamaResult;
+          try {
+            const { execSync: execSync34 } = await import("node:child_process");
+            try {
+              execSync34("pip3 install --user moondream 2>/dev/null || pip install --user moondream 2>/dev/null", {
+                timeout: 12e4,
+                stdio: "pipe"
+              });
+              moondreamError = null;
+              moondreamClient = null;
+              const retryClient = await getMoondreamClient().catch(() => null);
+              if (retryClient) {
+                return await this.runMoondream(retryClient, buffer, filename, action, prompt, length, start);
+              }
+            } catch {
+            }
+            try {
+              execSync34("ollama pull moondream", { timeout: 3e5, stdio: "pipe" });
+              const retryOllama = await this.tryOllamaVision(buffer, filename, action, prompt, length, start);
+              if (retryOllama)
+                return retryOllama;
+            } catch {
+            }
+          } catch {
+          }
           return {
             success: false,
             output: "",
-            error: "No vision backend available.\nTo enable vision, either:\n  1. ollama pull moondream  \u2014 uses Ollama (easiest)\n  2. pip install moondream-station  \u2014 dedicated server\n  3. Set MOONDREAM_API_KEY for cloud inference",
+            error: "No vision backend available (auto-install attempted but failed).\nManual options:\n  1. ollama pull moondream  \u2014 uses Ollama (easiest)\n  2. pip install moondream-station  \u2014 dedicated server\n  3. Set MOONDREAM_API_KEY for cloud inference",
             durationMs: performance.now() - start
           };
         } catch (error) {
@@ -15929,12 +15953,28 @@ Coordinates are normalized (0-1). Multiply by image width/height for pixel value
             return null;
         }
         try {
-          const res = await fetch(`${ollamaHost}/api/generate`, {
+          let res = await fetch(`${ollamaHost}/api/generate`, {
             method: "POST",
             headers: { "Content-Type": "application/json" },
             body: JSON.stringify({ model, prompt: ollamaPrompt, images: [imageBase64], stream: false }),
             signal: AbortSignal.timeout(6e4)
           });
+          if (!res.ok && model === "moondream") {
+            const errText = await res.text().catch(() => "");
+            if (res.status === 404 || /not found|does not exist/i.test(errText)) {
+              try {
+                const { execSync: execSync34 } = await import("node:child_process");
+                execSync34("ollama pull moondream", { timeout: 3e5, stdio: "pipe" });
+                res = await fetch(`${ollamaHost}/api/generate`, {
+                  method: "POST",
+                  headers: { "Content-Type": "application/json" },
+                  body: JSON.stringify({ model, prompt: ollamaPrompt, images: [imageBase64], stream: false }),
+                  signal: AbortSignal.timeout(6e4)
+                });
+              } catch {
+              }
+            }
+          }
           if (!res.ok)
             return null;
           const data = await res.json();
@@ -64351,6 +64391,32 @@ ${lines.join("\n")}
   if (flowEnabled) {
     dynamicContext += "\n\n" + FLOWSTATE_PROMPT;
   }
+  if (modelCaps?.vision) {
+    dynamicContext += `
+
+<vision-capabilities>
+You have vision capabilities available. You can analyze images using these tools:
+
+1. **image_read** \u2014 Read/view an image file directly. Use this for screenshots, photos,
+   diagrams, or any image the user provides. Pass the file path.
+   Example: image_read(image="/path/to/image.png")
+
+2. **vision** \u2014 Analyze images with Moondream vision model. Supports:
+   - caption: Describe what's in an image
+   - query: Ask a question about an image (visual QA)
+   - detect: Find all instances of an object (bounding boxes)
+   - point: Find the center location of an object (for click targeting)
+   Example: vision(image="/tmp/screenshot.png", action="caption")
+   Example: vision(image="photo.jpg", action="query", prompt="What color is the car?")
+
+3. **screenshot** \u2014 Capture the current screen (desktop automation).
+   Returns a screenshot path you can then analyze with vision or image_read.
+
+When the user asks you to look at, describe, or analyze an image or camera feed,
+use these tools proactively. Do not say you cannot see images \u2014 you can.
+For video streams or camera feeds, capture frames and analyze them sequentially.
+</vision-capabilities>`;
+  }
   let localFirstOverride = false;
   if (config.backendType === "nexus") {
     try {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "open-agents-ai",
-  "version": "0.184.73",
+  "version": "0.184.75",
   "description": "AI coding agent powered by open-source models (Ollama/vLLM) — interactive TUI with agentic tool-calling loop",
   "type": "module",
   "main": "./dist/index.js",