open-agents-ai 0.23.2 → 0.24.0

package/dist/index.js

@@ -6628,111 +6628,158 @@ var init_vision = __esm({
         if (!rawPath) {
           return { success: false, output: "", error: "image path is required", durationMs: 0 };
         }
+        if ((action === "query" || action === "detect" || action === "point") && !prompt) {
+          return {
+            success: false,
+            output: "",
+            error: `prompt is required for ${action} action`,
+            durationMs: performance.now() - start
+          };
+        }
         try {
           const { buffer, fullPath } = loadImageBuffer(this.workingDir, rawPath);
-          const client = await getMoondreamClient();
-          switch (action) {
-            case "caption": {
-              const result = await client.caption({ image: buffer, length });
-              const caption = typeof result.caption === "string" ? result.caption : "(streaming not supported)";
-              return {
-                success: true,
-                output: `Caption (${length}) for ${basename4(fullPath)}:
-${caption}`,
-                durationMs: performance.now() - start
-              };
-            }
-            case "query": {
-              if (!prompt) {
-                return {
-                  success: false,
-                  output: "",
-                  error: "prompt is required for query action (the question to ask about the image)",
-                  durationMs: performance.now() - start
-                };
-              }
-              const result = await client.query({ image: buffer, question: prompt });
-              const answer = typeof result.answer === "string" ? result.answer : "(streaming not supported)";
-              return {
-                success: true,
-                output: `Q: ${prompt}
+          const filename = basename4(fullPath);
+          let client = null;
+          try {
+            client = await getMoondreamClient();
+          } catch {
+          }
+          if (client) {
+            return await this.runMoondream(client, buffer, filename, action, prompt, length, start);
+          }
+          const ollamaResult = await this.tryOllamaVision(buffer, filename, action, prompt, length, start);
+          if (ollamaResult)
+            return ollamaResult;
+          return {
+            success: false,
+            output: "",
+            error: "No vision backend available.\nTo enable vision, either:\n  1. ollama pull moondream  \u2014 uses Ollama (easiest)\n  2. pip install moondream-station  \u2014 dedicated server\n  3. Set MOONDREAM_API_KEY for cloud inference",
+            durationMs: performance.now() - start
+          };
+        } catch (error) {
+          return {
+            success: false,
+            output: "",
+            error: error instanceof Error ? error.message : String(error),
+            durationMs: performance.now() - start
+          };
+        }
+      }
+      async runMoondream(client, buffer, filename, action, prompt, length, start) {
+        switch (action) {
+          case "caption": {
+            const result = await client.caption({ image: buffer, length });
+            const caption = typeof result.caption === "string" ? result.caption : "(streaming not supported)";
+            return { success: true, output: `Caption (${length}) for ${filename}:
+${caption}`, durationMs: performance.now() - start };
+          }
+          case "query": {
+            const result = await client.query({ image: buffer, question: prompt });
+            const answer = typeof result.answer === "string" ? result.answer : "(streaming not supported)";
+            return { success: true, output: `Q: ${prompt}
 A: ${answer}
 
-Image: ${basename4(fullPath)}`,
-                durationMs: performance.now() - start
-              };
+Image: ${filename}`, durationMs: performance.now() - start };
+          }
+          case "detect": {
+            const result = await client.detect({ image: buffer, object: prompt });
+            const objects = result.objects ?? [];
+            if (objects.length === 0) {
+              return { success: true, output: `No "${prompt}" detected in ${filename}`, durationMs: performance.now() - start };
             }
-            case "detect": {
-              if (!prompt) {
-                return {
-                  success: false,
-                  output: "",
-                  error: "prompt is required for detect action (the object to find)",
-                  durationMs: performance.now() - start
-                };
-              }
-              const result = await client.detect({ image: buffer, object: prompt });
-              const objects = result.objects ?? [];
-              if (objects.length === 0) {
-                return {
-                  success: true,
-                  output: `No "${prompt}" detected in ${basename4(fullPath)}`,
-                  durationMs: performance.now() - start
-                };
-              }
-              const formatted = objects.map((obj, i) => `  ${i + 1}. bbox: [${obj.x_min.toFixed(3)}, ${obj.y_min.toFixed(3)}, ${obj.x_max.toFixed(3)}, ${obj.y_max.toFixed(3)}] (normalized 0-1)`).join("\n");
-              return {
-                success: true,
-                output: `Detected ${objects.length} "${prompt}" in ${basename4(fullPath)}:
+            const formatted = objects.map((obj, i) => `  ${i + 1}. bbox: [${obj.x_min.toFixed(3)}, ${obj.y_min.toFixed(3)}, ${obj.x_max.toFixed(3)}, ${obj.y_max.toFixed(3)}] (normalized 0-1)`).join("\n");
+            return {
+              success: true,
+              output: `Detected ${objects.length} "${prompt}" in ${filename}:
 ${formatted}
 
 Coordinates are normalized (0-1). Multiply by image width/height for pixel values.`,
-                durationMs: performance.now() - start
-              };
+              durationMs: performance.now() - start
+            };
+          }
+          case "point": {
+            const result = await client.point({ image: buffer, object: prompt });
+            const points = result.points ?? [];
+            if (points.length === 0) {
+              return { success: true, output: `No "${prompt}" found in ${filename}`, durationMs: performance.now() - start };
             }
-            case "point": {
-              if (!prompt) {
-                return {
-                  success: false,
-                  output: "",
-                  error: "prompt is required for point action (the object to locate)",
-                  durationMs: performance.now() - start
-                };
-              }
-              const result = await client.point({ image: buffer, object: prompt });
-              const points = result.points ?? [];
-              if (points.length === 0) {
-                return {
-                  success: true,
-                  output: `No "${prompt}" found in ${basename4(fullPath)}`,
-                  durationMs: performance.now() - start
-                };
-              }
-              const formatted = points.map((pt, i) => `  ${i + 1}. (${pt.x.toFixed(4)}, ${pt.y.toFixed(4)}) \u2014 normalized 0-1`).join("\n");
+            const formatted = points.map((pt, i) => `  ${i + 1}. (${pt.x.toFixed(4)}, ${pt.y.toFixed(4)}) \u2014 normalized 0-1`).join("\n");
+            return {
+              success: true,
+              output: `Found ${points.length} "${prompt}" location(s) in ${filename}:
+${formatted}
+
+Coordinates are normalized (0-1). Multiply by image width/height for pixel values.`,
+              durationMs: performance.now() - start
+            };
+          }
+          default:
+            return { success: false, output: "", error: `Unknown action: ${action}. Use: caption, query, detect, point`, durationMs: performance.now() - start };
+        }
+      }
+      async tryOllamaVision(buffer, filename, action, prompt, length, start) {
+        const ollamaHost = process.env["OLLAMA_HOST"] || "http://localhost:11434";
+        const model = process.env["OLLAMA_VISION_MODEL"] || "moondream";
+        const imageBase64 = buffer.toString("base64");
+        let ollamaPrompt;
+        switch (action) {
+          case "caption":
+            ollamaPrompt = length === "short" ? "Briefly describe this image." : length === "long" ? "Describe this image in detail." : "Describe this image.";
+            break;
+          case "query":
+            ollamaPrompt = prompt;
+            break;
+          case "detect":
+            ollamaPrompt = `Detect all instances of "${prompt}" in this image. For each, describe its location.`;
+            break;
+          case "point":
+            ollamaPrompt = `Point to ${prompt}`;
+            break;
+          default:
+            return null;
+        }
+        try {
+          const res = await fetch(`${ollamaHost}/api/generate`, {
+            method: "POST",
+            headers: { "Content-Type": "application/json" },
+            body: JSON.stringify({ model, prompt: ollamaPrompt, images: [imageBase64], stream: false }),
+            signal: AbortSignal.timeout(6e4)
+          });
+          if (!res.ok)
+            return null;
+          const data = await res.json();
+          const response = data.response || "";
+          if (!response)
+            return null;
+          if (action === "point") {
+            const matches = [...response.matchAll(/<point\s+x="([\d.]+)"\s+y="([\d.]+)"\s*\/?>/g)];
+            if (matches.length > 0) {
+              const formatted = matches.map((m, i) => `  ${i + 1}. (${parseFloat(m[1]).toFixed(4)}, ${parseFloat(m[2]).toFixed(4)}) \u2014 normalized 0-1`).join("\n");
               return {
                 success: true,
-                output: `Found ${points.length} "${prompt}" location(s) in ${basename4(fullPath)}:
+                output: `Found ${matches.length} "${prompt}" location(s) in ${filename} (via Ollama):
 ${formatted}
 
 Coordinates are normalized (0-1). Multiply by image width/height for pixel values.`,
                 durationMs: performance.now() - start
               };
             }
-            default:
-              return {
-                success: false,
-                output: "",
-                error: `Unknown action: ${action}. Use: caption, query, detect, point`,
-                durationMs: performance.now() - start
-              };
+            return { success: true, output: `Could not extract coordinates for "${prompt}" from ${filename}. Model response: ${response}`, durationMs: performance.now() - start };
           }
-        } catch (error) {
-          return {
-            success: false,
-            output: "",
-            error: error instanceof Error ? error.message : String(error),
-            durationMs: performance.now() - start
-          };
+          if (action === "caption") {
+            return { success: true, output: `Caption (${length}) for ${filename} (via Ollama):
+${response}`, durationMs: performance.now() - start };
+          }
+          if (action === "query") {
+            return { success: true, output: `Q: ${prompt}
+A: ${response}
+
+Image: ${filename} (via Ollama)`, durationMs: performance.now() - start };
+          }
+          return { success: true, output: `Detection results for "${prompt}" in ${filename} (via Ollama):
+${response}`, durationMs: performance.now() - start };
+        } catch {
+          return null;
         }
       }
     };
@@ -6981,18 +7028,44 @@ var init_desktop_click = __esm({
             visionWorked = true;
           } catch {
           }
+          if (!visionWorked) {
+            try {
+              const ollamaHost = process.env["OLLAMA_HOST"] || "http://localhost:11434";
+              const ollamaModel = process.env["OLLAMA_VISION_MODEL"] || "moondream";
+              const imageBase64 = readFileSync11(screenshotPath).toString("base64");
+              const res = await fetch(`${ollamaHost}/api/generate`, {
+                method: "POST",
+                headers: { "Content-Type": "application/json" },
+                body: JSON.stringify({ model: ollamaModel, prompt: `Point to ${target}`, images: [imageBase64], stream: false }),
+                signal: AbortSignal.timeout(6e4)
+              });
+              if (res.ok) {
+                const data = await res.json();
+                const response = data.response || "";
+                const pointMatches = [...response.matchAll(/<point\s+x="([\d.]+)"\s+y="([\d.]+)"\s*\/?>/g)];
+                if (pointMatches.length > 0) {
+                  points = pointMatches.map((m) => ({ x: parseFloat(m[1]), y: parseFloat(m[2]) }));
+                  visionWorked = true;
+                }
+              }
+            } catch {
+            }
+          }
           if (!visionWorked) {
             const hints = [
-              `(Moondream vision not available \u2014 cannot locate "${target}" on screen)`,
+              `(No vision backend available \u2014 cannot locate "${target}" on screen)`,
               `Screenshot saved: ${screenshotPath}`,
               `Screen: ${dims.width}x${dims.height}`,
               "",
+              "To enable vision-guided clicking, either:",
+              "  ollama pull moondream  \u2014 then Ollama handles point detection",
+              "  pip install moondream-station  \u2014 dedicated Moondream server",
+              "",
               "Use shell commands to interact with the desktop instead:",
               "  xdotool search --name 'pattern' \u2014 find windows by title",
               "  xdotool key 'ctrl+s' \u2014 send keyboard shortcuts",
               "  xdotool mousemove X Y click 1 \u2014 click at known coordinates",
-              "  wmctrl -a 'window title' \u2014 activate a window by name",
-              "  xdg-open <url> \u2014 open a URL in the default browser"
+              "  wmctrl -a 'window title' \u2014 activate a window by name"
             ];
             return {
               success: true,
@@ -7120,26 +7193,61 @@ ${caption}`);
           } catch {
           }
           if (!visionWorked) {
-            let ocrText = "";
             try {
-              ocrText = execSync11(`tesseract ${JSON.stringify(screenshotPath)} stdout 2>/dev/null`, {
-                encoding: "utf8",
-                timeout: 15e3
-              }).trim();
+              const ollamaHost = process.env["OLLAMA_HOST"] || "http://localhost:11434";
+              const ollamaModel = process.env["OLLAMA_VISION_MODEL"] || "moondream";
+              const imageBase64 = imageBuffer.toString("base64");
+              const ollamaPrompt = question || "Describe what you see on this desktop screenshot in detail. Include visible applications, windows, text, and UI elements.";
+              const res = await fetch(`${ollamaHost}/api/generate`, {
+                method: "POST",
+                headers: { "Content-Type": "application/json" },
+                body: JSON.stringify({ model: ollamaModel, prompt: ollamaPrompt, images: [imageBase64], stream: false }),
+                signal: AbortSignal.timeout(6e4)
+              });
+              if (res.ok) {
+                const data = await res.json();
+                if (data.response) {
+                  if (question) {
+                    parts.push(`Q: ${question}`);
+                    parts.push(`A: ${data.response}`);
+                  } else {
+                    parts.push(`Desktop description (via Ollama):
+${data.response}`);
+                  }
+                  visionWorked = true;
+                }
+              }
             } catch {
             }
+          }
+          if (!visionWorked) {
+            let ocrText = "";
+            const tess = ensureCommand("tesseract");
+            if (tess.available) {
+              try {
+                ocrText = execSync11(`tesseract ${JSON.stringify(screenshotPath)} stdout 2>/dev/null`, {
+                  encoding: "utf8",
+                  timeout: 15e3
+                }).trim();
+              } catch {
+              }
+            }
             if (ocrText) {
-              parts.push("(Moondream vision not available \u2014 using OCR text extraction)");
+              parts.push("(Vision models not available \u2014 using OCR text extraction)");
               parts.push(`
 Visible text on screen:
 ${ocrText}`);
             } else {
-              parts.push("(Moondream vision not available, OCR failed)");
+              parts.push("(No vision backend available)");
               parts.push("Screenshot captured but cannot describe contents.");
+              parts.push("To enable desktop vision, either:");
+              parts.push("  ollama pull moondream  \u2014 then Ollama handles vision");
+              parts.push("  pip install moondream-station  \u2014 dedicated Moondream server");
+              parts.push("  sudo apt install tesseract-ocr  \u2014 basic OCR text extraction");
+              parts.push("");
               parts.push("Use shell commands to check window state instead:");
               parts.push("  xdotool getactivewindow getwindowname  \u2014 get active window title");
               parts.push("  wmctrl -l  \u2014 list all open windows");
-              parts.push("  xdg-open <url>  \u2014 open a URL in the default browser");
             }
           }
           if (dims) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "open-agents-ai",
-  "version": "0.23.2",
+  "version": "0.24.0",
   "description": "AI coding agent powered by open-source models (Ollama/vLLM) — interactive TUI with agentic tool-calling loop",
   "type": "module",
   "main": "./dist/index.js",