open-agents-ai 0.23.0 → 0.23.1

package/dist/index.js

@@ -6961,20 +6961,45 @@ var init_desktop_click = __esm({
               durationMs: performance.now() - start
             };
           }
-          const { vl } = await import("moondream");
-          const apiKey = process.env["MOONDREAM_API_KEY"];
-          const endpoint = process.env["MOONDREAM_ENDPOINT"];
-          let client;
-          if (apiKey) {
-            client = new vl({ apiKey });
-          } else if (endpoint) {
-            client = new vl({ endpoint });
-          } else {
-            client = new vl({ endpoint: "http://localhost:2020/v1" });
+          let points = [];
+          let visionWorked = false;
+          try {
+            const { vl } = await import("moondream");
+            const apiKey = process.env["MOONDREAM_API_KEY"];
+            const endpoint = process.env["MOONDREAM_ENDPOINT"];
+            let client;
+            if (apiKey) {
+              client = new vl({ apiKey });
+            } else if (endpoint) {
+              client = new vl({ endpoint });
+            } else {
+              client = new vl({ endpoint: "http://localhost:2020/v1" });
+            }
+            const imageBuffer = readFileSync11(screenshotPath);
+            const pointResult = await client.point({ image: imageBuffer, object: target });
+            points = pointResult.points ?? [];
+            visionWorked = true;
+          } catch {
+          }
+          if (!visionWorked) {
+            const hints = [
+              `(Moondream vision not available \u2014 cannot locate "${target}" on screen)`,
+              `Screenshot saved: ${screenshotPath}`,
+              `Screen: ${dims.width}x${dims.height}`,
+              "",
+              "Use shell commands to interact with the desktop instead:",
+              "  xdotool search --name 'pattern' \u2014 find windows by title",
+              "  xdotool key 'ctrl+s' \u2014 send keyboard shortcuts",
+              "  xdotool mousemove X Y click 1 \u2014 click at known coordinates",
+              "  wmctrl -a 'window title' \u2014 activate a window by name",
+              "  xdg-open <url> \u2014 open a URL in the default browser"
+            ];
+            return {
+              success: true,
+              output: hints.join("\n"),
+              durationMs: performance.now() - start
+            };
           }
-          const imageBuffer = readFileSync11(screenshotPath);
-          const pointResult = await client.point({ image: imageBuffer, object: target });
-          const points = pointResult.points ?? [];
           if (points.length === 0) {
             return {
               success: false,
@@ -7066,37 +7091,62 @@ Screenshot: ${screenshotPath}`,
           captureScreenshot(screenshotPath);
           const dims = getImageDimensions2(screenshotPath);
           const imageBuffer = readFileSync11(screenshotPath);
-          const { vl } = await import("moondream");
-          const apiKey = process.env["MOONDREAM_API_KEY"];
-          const endpoint = process.env["MOONDREAM_ENDPOINT"];
-          let client;
-          if (apiKey) {
-            client = new vl({ apiKey });
-          } else if (endpoint) {
-            client = new vl({ endpoint });
-          } else {
-            client = new vl({ endpoint: "http://localhost:2020/v1" });
-          }
           const parts = [];
-          if (question) {
-            const result = await client.query({ image: imageBuffer, question });
-            const answer = typeof result.answer === "string" ? result.answer : "(streaming not supported)";
-            parts.push(`Q: ${question}`);
-            parts.push(`A: ${answer}`);
-          } else {
-            const result = await client.caption({ image: imageBuffer, length });
-            const caption = typeof result.caption === "string" ? result.caption : "(streaming not supported)";
-            parts.push(`Desktop description:
+          let visionWorked = false;
+          try {
+            const { vl } = await import("moondream");
+            const apiKey = process.env["MOONDREAM_API_KEY"];
+            const endpoint = process.env["MOONDREAM_ENDPOINT"];
+            let client;
+            if (apiKey) {
+              client = new vl({ apiKey });
+            } else if (endpoint) {
+              client = new vl({ endpoint });
+            } else {
+              client = new vl({ endpoint: "http://localhost:2020/v1" });
+            }
+            if (question) {
+              const result = await client.query({ image: imageBuffer, question });
+              const answer = typeof result.answer === "string" ? result.answer : "(streaming not supported)";
+              parts.push(`Q: ${question}`);
+              parts.push(`A: ${answer}`);
+            } else {
+              const result = await client.caption({ image: imageBuffer, length });
+              const caption = typeof result.caption === "string" ? result.caption : "(streaming not supported)";
+              parts.push(`Desktop description:
 ${caption}`);
+            }
+            visionWorked = true;
+          } catch {
+          }
+          if (!visionWorked) {
+            let ocrText = "";
+            try {
+              ocrText = execSync11(`tesseract ${JSON.stringify(screenshotPath)} stdout 2>/dev/null`, {
+                encoding: "utf8",
+                timeout: 15e3
+              }).trim();
+            } catch {
+            }
+            if (ocrText) {
+              parts.push("(Moondream vision not available \u2014 using OCR text extraction)");
+              parts.push(`
+Visible text on screen:
+${ocrText}`);
+            } else {
+              parts.push("(Moondream vision not available, OCR failed)");
+              parts.push("Screenshot captured but cannot describe contents.");
+              parts.push("Use shell commands to check window state instead:");
+              parts.push("  xdotool getactivewindow getwindowname  \u2014 get active window title");
+              parts.push("  wmctrl -l  \u2014 list all open windows");
+              parts.push("  xdg-open <url>  \u2014 open a URL in the default browser");
+            }
           }
           if (dims) {
             parts.push(`
 Screen: ${dims.width}x${dims.height}`);
           }
           parts.push(`Screenshot: ${screenshotPath}`);
-          const base64 = imageBuffer.toString("base64");
-          parts.push(`
-[IMAGE_BASE64:image/png:${base64}]`);
           return {
             success: true,
             output: parts.join("\n"),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "open-agents-ai",
-  "version": "0.23.0",
+  "version": "0.23.1",
   "description": "AI coding agent powered by open-source models (Ollama/vLLM) — interactive TUI with agentic tool-calling loop",
   "type": "module",
   "main": "./dist/index.js",