open-agents-ai 0.23.1 → 0.24.0

package/dist/index.js

@@ -6628,111 +6628,158 @@ var init_vision = __esm({
         if (!rawPath) {
           return { success: false, output: "", error: "image path is required", durationMs: 0 };
         }
+        if ((action === "query" || action === "detect" || action === "point") && !prompt) {
+          return {
+            success: false,
+            output: "",
+            error: `prompt is required for ${action} action`,
+            durationMs: performance.now() - start
+          };
+        }
         try {
           const { buffer, fullPath } = loadImageBuffer(this.workingDir, rawPath);
-          const client = await getMoondreamClient();
-          switch (action) {
-            case "caption": {
-              const result = await client.caption({ image: buffer, length });
-              const caption = typeof result.caption === "string" ? result.caption : "(streaming not supported)";
-              return {
-                success: true,
-                output: `Caption (${length}) for ${basename4(fullPath)}:
-${caption}`,
-                durationMs: performance.now() - start
-              };
-            }
-            case "query": {
-              if (!prompt) {
-                return {
-                  success: false,
-                  output: "",
-                  error: "prompt is required for query action (the question to ask about the image)",
-                  durationMs: performance.now() - start
-                };
-              }
-              const result = await client.query({ image: buffer, question: prompt });
-              const answer = typeof result.answer === "string" ? result.answer : "(streaming not supported)";
-              return {
-                success: true,
-                output: `Q: ${prompt}
+          const filename = basename4(fullPath);
+          let client = null;
+          try {
+            client = await getMoondreamClient();
+          } catch {
+          }
+          if (client) {
+            return await this.runMoondream(client, buffer, filename, action, prompt, length, start);
+          }
+          const ollamaResult = await this.tryOllamaVision(buffer, filename, action, prompt, length, start);
+          if (ollamaResult)
+            return ollamaResult;
+          return {
+            success: false,
+            output: "",
+            error: "No vision backend available.\nTo enable vision, either:\n  1. ollama pull moondream  \u2014 uses Ollama (easiest)\n  2. pip install moondream-station  \u2014 dedicated server\n  3. Set MOONDREAM_API_KEY for cloud inference",
+            durationMs: performance.now() - start
+          };
+        } catch (error) {
+          return {
+            success: false,
+            output: "",
+            error: error instanceof Error ? error.message : String(error),
+            durationMs: performance.now() - start
+          };
+        }
+      }
+      async runMoondream(client, buffer, filename, action, prompt, length, start) {
+        switch (action) {
+          case "caption": {
+            const result = await client.caption({ image: buffer, length });
+            const caption = typeof result.caption === "string" ? result.caption : "(streaming not supported)";
+            return { success: true, output: `Caption (${length}) for ${filename}:
+${caption}`, durationMs: performance.now() - start };
+          }
+          case "query": {
+            const result = await client.query({ image: buffer, question: prompt });
+            const answer = typeof result.answer === "string" ? result.answer : "(streaming not supported)";
+            return { success: true, output: `Q: ${prompt}
 A: ${answer}
 
-Image: ${basename4(fullPath)}`,
-                durationMs: performance.now() - start
-              };
+Image: ${filename}`, durationMs: performance.now() - start };
+          }
+          case "detect": {
+            const result = await client.detect({ image: buffer, object: prompt });
+            const objects = result.objects ?? [];
+            if (objects.length === 0) {
+              return { success: true, output: `No "${prompt}" detected in ${filename}`, durationMs: performance.now() - start };
             }
-            case "detect": {
-              if (!prompt) {
-                return {
-                  success: false,
-                  output: "",
-                  error: "prompt is required for detect action (the object to find)",
-                  durationMs: performance.now() - start
-                };
-              }
-              const result = await client.detect({ image: buffer, object: prompt });
-              const objects = result.objects ?? [];
-              if (objects.length === 0) {
-                return {
-                  success: true,
-                  output: `No "${prompt}" detected in ${basename4(fullPath)}`,
-                  durationMs: performance.now() - start
-                };
-              }
-              const formatted = objects.map((obj, i) => `  ${i + 1}. bbox: [${obj.x_min.toFixed(3)}, ${obj.y_min.toFixed(3)}, ${obj.x_max.toFixed(3)}, ${obj.y_max.toFixed(3)}] (normalized 0-1)`).join("\n");
-              return {
-                success: true,
-                output: `Detected ${objects.length} "${prompt}" in ${basename4(fullPath)}:
+            const formatted = objects.map((obj, i) => `  ${i + 1}. bbox: [${obj.x_min.toFixed(3)}, ${obj.y_min.toFixed(3)}, ${obj.x_max.toFixed(3)}, ${obj.y_max.toFixed(3)}] (normalized 0-1)`).join("\n");
+            return {
+              success: true,
+              output: `Detected ${objects.length} "${prompt}" in ${filename}:
 ${formatted}
 
 Coordinates are normalized (0-1). Multiply by image width/height for pixel values.`,
-                durationMs: performance.now() - start
-              };
+              durationMs: performance.now() - start
+            };
+          }
+          case "point": {
+            const result = await client.point({ image: buffer, object: prompt });
+            const points = result.points ?? [];
+            if (points.length === 0) {
+              return { success: true, output: `No "${prompt}" found in ${filename}`, durationMs: performance.now() - start };
             }
-            case "point": {
-              if (!prompt) {
-                return {
-                  success: false,
-                  output: "",
-                  error: "prompt is required for point action (the object to locate)",
-                  durationMs: performance.now() - start
-                };
-              }
-              const result = await client.point({ image: buffer, object: prompt });
-              const points = result.points ?? [];
-              if (points.length === 0) {
-                return {
-                  success: true,
-                  output: `No "${prompt}" found in ${basename4(fullPath)}`,
-                  durationMs: performance.now() - start
-                };
-              }
-              const formatted = points.map((pt, i) => `  ${i + 1}. (${pt.x.toFixed(4)}, ${pt.y.toFixed(4)}) \u2014 normalized 0-1`).join("\n");
+            const formatted = points.map((pt, i) => `  ${i + 1}. (${pt.x.toFixed(4)}, ${pt.y.toFixed(4)}) \u2014 normalized 0-1`).join("\n");
+            return {
+              success: true,
+              output: `Found ${points.length} "${prompt}" location(s) in ${filename}:
+${formatted}
+
+Coordinates are normalized (0-1). Multiply by image width/height for pixel values.`,
+              durationMs: performance.now() - start
+            };
+          }
+          default:
+            return { success: false, output: "", error: `Unknown action: ${action}. Use: caption, query, detect, point`, durationMs: performance.now() - start };
+        }
+      }
+      async tryOllamaVision(buffer, filename, action, prompt, length, start) {
+        const ollamaHost = process.env["OLLAMA_HOST"] || "http://localhost:11434";
+        const model = process.env["OLLAMA_VISION_MODEL"] || "moondream";
+        const imageBase64 = buffer.toString("base64");
+        let ollamaPrompt;
+        switch (action) {
+          case "caption":
+            ollamaPrompt = length === "short" ? "Briefly describe this image." : length === "long" ? "Describe this image in detail." : "Describe this image.";
+            break;
+          case "query":
+            ollamaPrompt = prompt;
+            break;
+          case "detect":
+            ollamaPrompt = `Detect all instances of "${prompt}" in this image. For each, describe its location.`;
+            break;
+          case "point":
+            ollamaPrompt = `Point to ${prompt}`;
+            break;
+          default:
+            return null;
+        }
+        try {
+          const res = await fetch(`${ollamaHost}/api/generate`, {
+            method: "POST",
+            headers: { "Content-Type": "application/json" },
+            body: JSON.stringify({ model, prompt: ollamaPrompt, images: [imageBase64], stream: false }),
+            signal: AbortSignal.timeout(6e4)
+          });
+          if (!res.ok)
+            return null;
+          const data = await res.json();
+          const response = data.response || "";
+          if (!response)
+            return null;
+          if (action === "point") {
+            const matches = [...response.matchAll(/<point\s+x="([\d.]+)"\s+y="([\d.]+)"\s*\/?>/g)];
+            if (matches.length > 0) {
+              const formatted = matches.map((m, i) => `  ${i + 1}. (${parseFloat(m[1]).toFixed(4)}, ${parseFloat(m[2]).toFixed(4)}) \u2014 normalized 0-1`).join("\n");
               return {
                 success: true,
-                output: `Found ${points.length} "${prompt}" location(s) in ${basename4(fullPath)}:
+                output: `Found ${matches.length} "${prompt}" location(s) in ${filename} (via Ollama):
 ${formatted}
 
 Coordinates are normalized (0-1). Multiply by image width/height for pixel values.`,
                 durationMs: performance.now() - start
               };
             }
-            default:
-              return {
-                success: false,
-                output: "",
-                error: `Unknown action: ${action}. Use: caption, query, detect, point`,
-                durationMs: performance.now() - start
-              };
+            return { success: true, output: `Could not extract coordinates for "${prompt}" from ${filename}. Model response: ${response}`, durationMs: performance.now() - start };
           }
-        } catch (error) {
-          return {
-            success: false,
-            output: "",
-            error: error instanceof Error ? error.message : String(error),
-            durationMs: performance.now() - start
-          };
+          if (action === "caption") {
+            return { success: true, output: `Caption (${length}) for ${filename} (via Ollama):
+${response}`, durationMs: performance.now() - start };
+          }
+          if (action === "query") {
+            return { success: true, output: `Q: ${prompt}
+A: ${response}
+
+Image: ${filename} (via Ollama)`, durationMs: performance.now() - start };
+          }
+          return { success: true, output: `Detection results for "${prompt}" in ${filename} (via Ollama):
+${response}`, durationMs: performance.now() - start };
+        } catch {
+          return null;
         }
       }
     };
@@ -6981,18 +7028,44 @@ var init_desktop_click = __esm({
             visionWorked = true;
           } catch {
           }
+          if (!visionWorked) {
+            try {
+              const ollamaHost = process.env["OLLAMA_HOST"] || "http://localhost:11434";
+              const ollamaModel = process.env["OLLAMA_VISION_MODEL"] || "moondream";
+              const imageBase64 = readFileSync11(screenshotPath).toString("base64");
+              const res = await fetch(`${ollamaHost}/api/generate`, {
+                method: "POST",
+                headers: { "Content-Type": "application/json" },
+                body: JSON.stringify({ model: ollamaModel, prompt: `Point to ${target}`, images: [imageBase64], stream: false }),
+                signal: AbortSignal.timeout(6e4)
+              });
+              if (res.ok) {
+                const data = await res.json();
+                const response = data.response || "";
+                const pointMatches = [...response.matchAll(/<point\s+x="([\d.]+)"\s+y="([\d.]+)"\s*\/?>/g)];
+                if (pointMatches.length > 0) {
+                  points = pointMatches.map((m) => ({ x: parseFloat(m[1]), y: parseFloat(m[2]) }));
+                  visionWorked = true;
+                }
+              }
+            } catch {
+            }
+          }
           if (!visionWorked) {
             const hints = [
-              `(Moondream vision not available \u2014 cannot locate "${target}" on screen)`,
+              `(No vision backend available \u2014 cannot locate "${target}" on screen)`,
               `Screenshot saved: ${screenshotPath}`,
               `Screen: ${dims.width}x${dims.height}`,
               "",
+              "To enable vision-guided clicking, either:",
+              "  ollama pull moondream  \u2014 then Ollama handles point detection",
+              "  pip install moondream-station  \u2014 dedicated Moondream server",
+              "",
               "Use shell commands to interact with the desktop instead:",
               "  xdotool search --name 'pattern' \u2014 find windows by title",
               "  xdotool key 'ctrl+s' \u2014 send keyboard shortcuts",
               "  xdotool mousemove X Y click 1 \u2014 click at known coordinates",
-              "  wmctrl -a 'window title' \u2014 activate a window by name",
-              "  xdg-open <url> \u2014 open a URL in the default browser"
+              "  wmctrl -a 'window title' \u2014 activate a window by name"
             ];
             return {
               success: true,
@@ -7120,26 +7193,61 @@ ${caption}`);
           } catch {
           }
           if (!visionWorked) {
-            let ocrText = "";
             try {
-              ocrText = execSync11(`tesseract ${JSON.stringify(screenshotPath)} stdout 2>/dev/null`, {
-                encoding: "utf8",
-                timeout: 15e3
-              }).trim();
+              const ollamaHost = process.env["OLLAMA_HOST"] || "http://localhost:11434";
+              const ollamaModel = process.env["OLLAMA_VISION_MODEL"] || "moondream";
+              const imageBase64 = imageBuffer.toString("base64");
+              const ollamaPrompt = question || "Describe what you see on this desktop screenshot in detail. Include visible applications, windows, text, and UI elements.";
+              const res = await fetch(`${ollamaHost}/api/generate`, {
+                method: "POST",
+                headers: { "Content-Type": "application/json" },
+                body: JSON.stringify({ model: ollamaModel, prompt: ollamaPrompt, images: [imageBase64], stream: false }),
+                signal: AbortSignal.timeout(6e4)
+              });
+              if (res.ok) {
+                const data = await res.json();
+                if (data.response) {
+                  if (question) {
+                    parts.push(`Q: ${question}`);
+                    parts.push(`A: ${data.response}`);
+                  } else {
+                    parts.push(`Desktop description (via Ollama):
+${data.response}`);
+                  }
+                  visionWorked = true;
+                }
+              }
             } catch {
             }
+          }
+          if (!visionWorked) {
+            let ocrText = "";
+            const tess = ensureCommand("tesseract");
+            if (tess.available) {
+              try {
+                ocrText = execSync11(`tesseract ${JSON.stringify(screenshotPath)} stdout 2>/dev/null`, {
+                  encoding: "utf8",
+                  timeout: 15e3
+                }).trim();
+              } catch {
+              }
+            }
             if (ocrText) {
-              parts.push("(Moondream vision not available \u2014 using OCR text extraction)");
+              parts.push("(Vision models not available \u2014 using OCR text extraction)");
               parts.push(`
 Visible text on screen:
 ${ocrText}`);
             } else {
-              parts.push("(Moondream vision not available, OCR failed)");
+              parts.push("(No vision backend available)");
               parts.push("Screenshot captured but cannot describe contents.");
+              parts.push("To enable desktop vision, either:");
+              parts.push("  ollama pull moondream  \u2014 then Ollama handles vision");
+              parts.push("  pip install moondream-station  \u2014 dedicated Moondream server");
+              parts.push("  sudo apt install tesseract-ocr  \u2014 basic OCR text extraction");
+              parts.push("");
               parts.push("Use shell commands to check window state instead:");
               parts.push("  xdotool getactivewindow getwindowname  \u2014 get active window title");
               parts.push("  wmctrl -l  \u2014 list all open windows");
-              parts.push("  xdg-open <url>  \u2014 open a URL in the default browser");
             }
           }
           if (dims) {

package/dist/launcher.cjs

@@ -1,29 +1,32 @@
 #!/usr/bin/env node
 // Node version gate — uses only ES5 syntax so it parses on any Node version.
-// On old Node: offers to install Node 20 via nvm with y/n prompts.
+// On old Node: walks through nvm install, then Node 20, then reinstall.
 var nodeVersion = parseInt(process.versions.node, 10);
 
 if (nodeVersion < 18) {
   var os = require("os");
   var path = require("path");
+  var fs = require("fs");
   var childProcess = require("child_process");
   var readline = require("readline");
 
   var platform = os.platform();
   var arch = os.arch();
 
-  console.log(
-    "\n  open-agents requires Node.js >= 18 (you have " + process.version + ")" +
-    "\n  Platform: " + platform + "/" + arch + "\n"
-  );
+  console.log("");
+  console.log("  ┌─────────────────────────────────────────────────┐");
+  console.log("  │         open-agents — Node.js Upgrade           │");
+  console.log("  └─────────────────────────────────────────────────┘");
+  console.log("");
+  console.log("  Your Node.js:  " + process.version);
+  console.log("  Required:      >= 18.0.0");
+  console.log("  Platform:      " + platform + "/" + arch);
+  console.log("");
 
-  // Check if nvm is already installed
+  // Detect if nvm is already installed
   var nvmDir = process.env.NVM_DIR || path.join(os.homedir(), ".nvm");
   var hasNvm = false;
-  try {
-    require("fs").statSync(path.join(nvmDir, "nvm.sh"));
-    hasNvm = true;
-  } catch (e) {}
+  try { fs.statSync(path.join(nvmDir, "nvm.sh")); hasNvm = true; } catch (e) {}
 
   var rl = readline.createInterface({ input: process.stdin, output: process.stdout });
 
@@ -33,10 +36,10 @@ if (nodeVersion < 18) {
     });
   }
 
-  function run(cmd, opts) {
+  function run(cmd) {
     console.log("  $ " + cmd);
     try {
-      childProcess.execSync(cmd, Object.assign({ stdio: "inherit" }, opts || {}));
+      childProcess.execSync(cmd, { stdio: "inherit" });
       return true;
     } catch (e) {
       console.error("  Command failed: " + (e.message || e));
@@ -44,70 +47,123 @@ if (nodeVersion < 18) {
     }
   }
 
-  function installNode() {
-    if (!hasNvm) {
-      console.log("\n  nvm not found. Installing nvm first...\n");
+  function bail(msg) {
+    console.log(msg);
+    rl.close();
+    process.exit(1);
+  }
+
+  function doInstallNvm(next) {
+    if (hasNvm) {
+      console.log("  nvm found at " + nvmDir);
+      console.log("");
+      next();
+      return;
+    }
+    ask("  nvm is not installed. Install nvm now? [Y/n] ", function(a) {
+      if (a === "n" || a === "no") {
+        bail(
+          "\n  Install nvm manually:\n" +
+          "    curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash\n" +
+          "    source ~/.bashrc\n" +
+          "    nvm install 20\n" +
+          "    npm i -g open-agents-ai\n"
+        );
+        return;
+      }
+      console.log("");
       var ok = run("curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash");
       if (!ok) {
-        console.error("\n  nvm install failed. Install manually:");
-        console.error("    https://github.com/nvm-sh/nvm#installing-and-updating\n");
-        rl.close();
-        process.exit(1);
+        bail("\n  nvm install failed. See https://github.com/nvm-sh/nvm#installing-and-updating\n");
+        return;
       }
-      // Re-source nvm for this session
       nvmDir = process.env.NVM_DIR || path.join(os.homedir(), ".nvm");
-    } else {
-      console.log("\n  nvm found at " + nvmDir + "\n");
-    }
-
-    // nvm is a shell function, so we run through bash
-    var shell = process.env.SHELL || "/bin/bash";
-    var nvmCmd = 'export NVM_DIR="' + nvmDir + '" && ' +
-      '[ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" && ' +
-      "nvm install 20 && nvm use 20 && nvm alias default 20";
+      hasNvm = true;
+      console.log("");
+      next();
+    });
+  }
 
-    console.log("  Installing Node.js 20...\n");
-    var ok = run(shell + ' -c \'' + nvmCmd + "'");
-    if (!ok) {
-      console.error("\n  Node 20 install failed.\n");
-      rl.close();
-      process.exit(1);
-    }
+  function doInstallNode(next) {
+    ask("  Install Node.js 20 via nvm? [Y/n] ", function(a) {
+      if (a === "n" || a === "no") {
+        bail(
+          "\n  Install manually:\n" +
+          "    nvm install 20 && nvm alias default 20\n" +
+          "    npm i -g open-agents-ai\n"
+        );
+        return;
+      }
+      console.log("");
+      var shell = process.env.SHELL || "/bin/bash";
+      var cmd = 'export NVM_DIR="' + nvmDir + '" && ' +
+        '[ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" && ' +
+        "nvm install 20 && nvm use 20 && nvm alias default 20";
+      var ok = run(shell + " -c '" + cmd + "'");
+      if (!ok) {
+        bail("\n  Node 20 install failed.\n");
+        return;
+      }
+      console.log("");
+      next();
+    });
+  }
 
-    // Now reinstall open-agents under the new Node
-    console.log("\n  Node 20 installed. Reinstalling open-agents...\n");
-    var reinstallCmd = 'export NVM_DIR="' + nvmDir + '" && ' +
-      '[ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" && ' +
-      "npm i -g open-agents-ai";
-    var ok2 = run(shell + ' -c \'' + reinstallCmd + "'");
-    if (!ok2) {
-      console.error("\n  Reinstall failed. After opening a new terminal, run:");
-      console.error("    npm i -g open-agents-ai\n");
+  function doReinstall() {
+    ask("  Reinstall open-agents under Node 20? [Y/n] ", function(a) {
+      if (a === "n" || a === "no") {
+        console.log(
+          "\n  Open a new terminal, then run:\n" +
+          "    npm i -g open-agents-ai\n"
+        );
+        rl.close();
+        process.exit(0);
+        return;
+      }
+      console.log("");
+      var shell = process.env.SHELL || "/bin/bash";
+      var cmd = 'export NVM_DIR="' + nvmDir + '" && ' +
+        '[ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" && ' +
+        "nvm use 20 && npm i -g open-agents-ai";
+      var ok = run(shell + " -c '" + cmd + "'");
+      if (!ok) {
+        console.log(
+          "\n  Reinstall failed. Open a new terminal, then run:\n" +
+          "    npm i -g open-agents-ai\n"
+        );
+        rl.close();
+        process.exit(1);
+        return;
+      }
+      console.log("");
+      console.log("  ┌─────────────────────────────────────────────────┐");
+      console.log("  │  Done! Open a new terminal, then run:  oa       │");
+      console.log("  └─────────────────────────────────────────────────┘");
+      console.log("");
       rl.close();
-      process.exit(1);
-    }
-
-    console.log(
-      "\n  Done! Open a new terminal (or run: source ~/.bashrc) then run:" +
-      "\n    oa\n"
-    );
-    rl.close();
-    process.exit(0);
+      process.exit(0);
+    });
   }
 
-  ask("  Install Node.js 20 now? [Y/n] ", function(answer) {
-    if (answer === "n" || answer === "no") {
-      console.log(
+  // Walk through the steps
+  ask("  Upgrade to Node.js 20 now? [Y/n] ", function(a) {
+    if (a === "n" || a === "no") {
+      bail(
         "\n  To install manually:\n" +
         "    curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash\n" +
         "    source ~/.bashrc\n" +
         "    nvm install 20\n" +
+        "    nvm alias default 20\n" +
         "    npm i -g open-agents-ai\n"
       );
-      rl.close();
-      process.exit(1);
+      return;
     }
-    installNode();
+    console.log("");
+    doInstallNvm(function() {
+      doInstallNode(function() {
+        doReinstall();
+      });
+    });
   });
 } else {
   // Node >= 18 — load the ESM bundle

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "open-agents-ai",
-  "version": "0.23.1",
+  "version": "0.24.0",
   "description": "AI coding agent powered by open-source models (Ollama/vLLM) — interactive TUI with agentic tool-calling loop",
   "type": "module",
   "main": "./dist/index.js",