@pulso/companion 0.3.3 → 0.4.1

package/dist/index.js

@@ -27,7 +27,7 @@ if (!TOKEN) {
 var ACCESS_LEVEL = process.env.PULSO_ACCESS ?? process.argv.find((_, i, a) => a[i - 1] === "--access") ?? "sandboxed";
 var WAKE_WORD_ENABLED = process.argv.includes("--wake-word");
 var PICOVOICE_ACCESS_KEY = process.env.PICOVOICE_ACCESS_KEY ?? process.argv.find((_, i, a) => a[i - 1] === "--picovoice-key") ?? "";
-var WS_URL = API_URL.replace("https://", "wss://").replace("http://", "ws://") + "/ws/browser?token=" + TOKEN;
+var WS_URL = API_URL.replace("https://", "wss://").replace("http://", "ws://") + "/ws/companion?token=" + TOKEN;
 var HOME = homedir();
 var RECONNECT_DELAY = 5e3;
 var SAFE_DIRS = ["Documents", "Desktop", "Downloads", "Projects", "Projetos"];
@@ -478,13 +478,68 @@ async function handleCommand(command, params) {
       case "sys_open_app": {
         const app = params.app;
         if (!app) return { success: false, error: "Missing app name" };
-        await runShell(`open -a "${app.replace(/"/g, "")}"`);
-        return { success: true, data: { opened: app } };
+        const sanitizedApp = app.replace(/"/g, "");
+        try {
+          await runShell(`open -a "${sanitizedApp}"`);
+        } catch (e) {
+          return { success: false, error: `Failed to open "${sanitizedApp}": ${e.message}`, errorCode: "APP_NOT_FOUND" };
+        }
+        let launched = false;
+        for (let i = 0; i < 10; i++) {
+          await new Promise((r) => setTimeout(r, 500));
+          try {
+            const running = await runAppleScript(
+              `tell application "System Events" to (name of processes) contains "${sanitizedApp}"`
+            );
+            if (running.trim() === "true") {
+              launched = true;
+              break;
+            }
+          } catch {
+          }
+        }
+        let windowInfo;
+        if (launched) {
+          try {
+            const winCheck = await runAppleScript(`
+              tell application "System Events"
+                tell process "${sanitizedApp}"
+                  if (count of windows) > 0 then
+                    set w to window 1
+                    return (name of w) & "|" & (position of w as string) & "|" & (size of w as string)
+                  else
+                    return "no-window"
+                  end if
+                end tell
+              end tell`);
+            windowInfo = winCheck.trim();
+          } catch {
+          }
+        }
+        return {
+          success: true,
+          data: {
+            opened: sanitizedApp,
+            launched,
+            hasWindow: windowInfo ? windowInfo !== "no-window" : false,
+            windowInfo: windowInfo && windowInfo !== "no-window" ? windowInfo : void 0,
+            note: launched ? `"${sanitizedApp}" is running${windowInfo && windowInfo !== "no-window" ? " with a visible window" : " (may still be loading)"}.` : `"${sanitizedApp}" was requested to open but process not detected yet. It may still be launching \u2014 take a screenshot to verify.`
+          }
+        };
       }
       case "sys_open_url": {
         const url = params.url;
         if (!url) return { success: false, error: "Missing URL" };
-        await runShell(`open "${url.replace(/"/g, "")}"`);
+        const sanitizedUrl = url.replace(/"/g, '\\"');
+        try {
+          await runAppleScript(`
+            tell application "Google Chrome"
+              activate
+              tell front window to make new tab with properties {URL:"${sanitizedUrl}"}
+            end tell`);
+        } catch {
+          await runShell(`open "${url.replace(/"/g, "")}"`);
+        }
         return { success: true, data: { opened: url } };
       }
       case "sys_speak": {
@@ -685,29 +740,33 @@ async function handleCommand(command, params) {
         return { success: true, data: { path, written: content.length } };
       }
       case "sys_screenshot": {
+        const display = params.display || 0;
         const ts = Date.now();
         const pngPath = `/tmp/pulso-ss-${ts}.png`;
         const jpgPath = `/tmp/pulso-ss-${ts}.jpg`;
         try {
-          await runShell(`screencapture -C -x -D1 ${pngPath}`, 15e3);
+          const displayFlag = display > 0 ? `-D${display}` : "";
+          await runShell(`screencapture -C -x ${displayFlag} ${pngPath}`, 15e3);
         } catch (ssErr) {
           const msg = ssErr.message || "";
           if (msg.includes("could not create image") || msg.includes("display")) {
             return {
               success: false,
-              error: "Screen Recording permission required. Go to System Settings \u2192 Privacy & Security \u2192 Screen Recording \u2192 enable your terminal app (Terminal, iTerm, etc). Then restart the companion."
+              error: "Screen Recording permission required. Go to System Settings \u2192 Privacy & Security \u2192 Screen Recording \u2192 enable your terminal app (Terminal, iTerm, etc). Then restart the companion.",
+              errorCode: "PERMISSION_DENIED"
             };
           }
-          return { success: false, error: `Screenshot failed: ${msg}` };
+          return { success: false, error: `Screenshot failed: ${msg}`, errorCode: "SCREENSHOT_FAILED" };
         }
         if (!existsSync(pngPath))
           return {
             success: false,
-            error: "Screenshot failed \u2014 Screen Recording permission needed. System Settings \u2192 Privacy & Security \u2192 Screen Recording \u2192 enable your terminal app, then restart companion."
+            error: "Screenshot failed \u2014 Screen Recording permission needed.",
+            errorCode: "PERMISSION_DENIED"
           };
         try {
           await runShell(
-            `sips --resampleWidth 1280 --setProperty format jpeg --setProperty formatOptions 60 ${pngPath} --out ${jpgPath}`,
+            `sips --resampleWidth 1600 --setProperty format jpeg --setProperty formatOptions 75 ${pngPath} --out ${jpgPath}`,
             1e4
           );
         } catch {
@@ -721,7 +780,8 @@ async function handleCommand(command, params) {
             data: {
               image: `data:image/png;base64,${buf2.toString("base64")}`,
               format: "png",
-              note: "Full screen screenshot"
+              display: display || "all",
+              note: "Full screen screenshot (PNG fallback)"
             }
           };
         }
@@ -735,11 +795,18 @@ async function handleCommand(command, params) {
           unlinkSync(jpgPath);
         } catch {
         }
-        let screenSize = "unknown";
+        let displayInfo = { width: 0, height: 0, displays: 1 };
         try {
-          screenSize = await runShell(
-            `system_profiler SPDisplaysDataType 2>/dev/null | grep Resolution | head -1 | sed 's/.*: //'`
-          );
+          const diSwift = `
+import Cocoa
+let screens = NSScreen.screens
+let main = screens.first!
+let w = Int(main.frame.width)
+let h = Int(main.frame.height)
+print("\\(w),\\(h),\\(screens.count)")`;
+          const diResult = await runSwift(diSwift);
+          const [dw, dh, dc] = diResult.trim().split(",").map(Number);
+          displayInfo = { width: dw || 0, height: dh || 0, displays: dc || 1 };
         } catch {
         }
         return {
@@ -747,12 +814,110 @@ async function handleCommand(command, params) {
           data: {
             image: `data:image/jpeg;base64,${base64}`,
             format: "jpeg",
-            resolution: screenSize.trim(),
-            imageWidth: 1280,
-            note: "Screenshot captured. Coordinates in this image are scaled \u2014 multiply x by (screen_width/1280) for actual clicks."
+            display: display || "all",
+            screenWidth: displayInfo.width,
+            screenHeight: displayInfo.height,
+            totalDisplays: displayInfo.displays,
+            imageWidth: 1600,
+            note: display > 0 ? `Screenshot of display ${display}. Coordinates: multiply x by (${displayInfo.width}/1600) and y by (${displayInfo.height}/(1600*${displayInfo.height}/${displayInfo.width})) for actual clicks.` : `Screenshot of all ${displayInfo.displays} display(s) stitched horizontally. Use sys_list_displays to get individual display bounds for coordinate mapping.`
+          }
+        };
+      }
+      case "sys_screenshot_region": {
+        const rx = params.x;
+        const ry = params.y;
+        const rw = params.width;
+        const rh = params.height;
+        if (rx == null || ry == null || rw == null || rh == null)
+          return { success: false, error: "Missing x, y, width, or height" };
+        const ts2 = Date.now();
+        const regPath = `/tmp/pulso-ss-region-${ts2}.png`;
+        const regJpg = `/tmp/pulso-ss-region-${ts2}.jpg`;
+        try {
+          await runShell(`screencapture -x -R${rx},${ry},${rw},${rh} ${regPath}`, 15e3);
+        } catch (e) {
+          return { success: false, error: `Region screenshot failed: ${e.message}`, errorCode: "SCREENSHOT_FAILED" };
+        }
+        if (!existsSync(regPath))
+          return { success: false, error: "Region screenshot failed", errorCode: "SCREENSHOT_FAILED" };
+        try {
+          await runShell(`sips --setProperty format jpeg --setProperty formatOptions 85 ${regPath} --out ${regJpg}`, 1e4);
+        } catch {
+          const rb = readFileSync(regPath);
+          try {
+            unlinkSync(regPath);
+          } catch {
+          }
+          return { success: true, data: { image: `data:image/png;base64,${rb.toString("base64")}`, format: "png", region: { x: rx, y: ry, width: rw, height: rh } } };
+        }
+        const rb2 = readFileSync(regJpg);
+        try {
+          unlinkSync(regPath);
+        } catch {
+        }
+        try {
+          unlinkSync(regJpg);
+        } catch {
+        }
+        return {
+          success: true,
+          data: {
+            image: `data:image/jpeg;base64,${rb2.toString("base64")}`,
+            format: "jpeg",
+            region: { x: rx, y: ry, width: rw, height: rh },
+            note: "Region screenshot at actual resolution (no scaling). Coordinates are absolute screen coordinates."
           }
         };
       }
+      case "sys_list_displays": {
+        try {
+          const swift = `
+import Cocoa
+let screens = NSScreen.screens
+var result = ""
+for (i, screen) in screens.enumerated() {
+    let f = screen.frame
+    let vf = screen.visibleFrame
+    let isMain = (screen == NSScreen.main)
+    let scale = screen.backingScaleFactor
+    let name = screen.localizedName
+    result += "\\(i+1)|\\(name)|\\(Int(f.origin.x)),\\(Int(f.origin.y))|\\(Int(f.width)),\\(Int(f.height))|\\(Int(vf.origin.x)),\\(Int(vf.origin.y))|\\(Int(vf.width)),\\(Int(vf.height))|\\(scale)|\\(isMain)\\n"
+}
+print(result)`;
+          const raw = await runSwift(swift, 15e3);
+          const displays = raw.trim().split("\n").filter(Boolean).map((line) => {
+            const [index, name, origin, size, visOrigin, visSize, scale, isMain] = line.split("|");
+            const [ox, oy] = (origin || "0,0").split(",").map(Number);
+            const [sw2, sh2] = (size || "0,0").split(",").map(Number);
+            const [vox, voy] = (visOrigin || "0,0").split(",").map(Number);
+            const [vsw, vsh] = (visSize || "0,0").split(",").map(Number);
+            return {
+              display: parseInt(index) || 0,
+              name: name?.trim() || "Unknown",
+              x: ox,
+              y: oy,
+              width: sw2,
+              height: sh2,
+              visibleX: vox,
+              visibleY: voy,
+              visibleWidth: vsw,
+              visibleHeight: vsh,
+              scale: parseFloat(scale) || 1,
+              isMain: isMain?.trim() === "true"
+            };
+          });
+          return {
+            success: true,
+            data: {
+              displays,
+              count: displays.length,
+              note: "Display coordinates use macOS coordinate system (origin bottom-left). For screencapture: use display number (1-based). For mouse clicks: windows on display 2 have x >= display1.width."
+            }
+          };
+        } catch (e) {
+          return { success: false, error: `Failed to list displays: ${e.message}` };
+        }
+      }
       // ── Computer-Use: Mouse & Keyboard ────────────────────
       case "sys_mouse_click": {
         const x = Number(params.x);
@@ -1038,7 +1203,7 @@ print("\\(x),\\(y)")`;
               tell application "Safari"
                 activate
                 if (count of windows) = 0 then make new document
-                set URL of front document to "${url.replace(/"/g, '\\"')}"
+                tell front window to set current tab to (make new tab with properties {URL:"${url.replace(/"/g, '\\"')}"})
               end tell`);
           } else {
             await runAppleScript(`
@@ -1046,10 +1211,8 @@ print("\\(x),\\(y)")`;
                 activate
                 if (count of windows) = 0 then
                   make new window
-                  set URL of active tab of front window to "${url.replace(/"/g, '\\"')}"
-                else
-                  set URL of active tab of front window to "${url.replace(/"/g, '\\"')}"
                 end if
+                tell front window to make new tab with properties {URL:"${url.replace(/"/g, '\\"')}"}
               end tell`);
           }
           return { success: true, data: { navigated: url, browser } };
@@ -1732,6 +1895,19 @@ end tell`);
       }
       // ── Window Management ───────────────────────────────────
       case "sys_window_list": {
+        let displayBounds = [];
+        try {
+          const dbSwift = `
+import Cocoa
+let screens = NSScreen.screens
+for s in screens { print("\\(Int(s.frame.origin.x)),\\(Int(s.frame.width))") }`;
+          const dbRaw = await runSwift(dbSwift, 5e3);
+          displayBounds = dbRaw.trim().split("\n").filter(Boolean).map((l) => {
+            const [bx, bw] = l.split(",").map(Number);
+            return { x: bx || 0, width: bw || 0 };
+          });
+        } catch {
+        }
         const raw4 = await runAppleScript(`
           tell application "System Events"
             set output to ""
@@ -1744,15 +1920,42 @@ end tell`);
             return output
           end tell`);
         const windows = raw4.split("\n").filter(Boolean).map((line) => {
-          const [app, title, pos, sz] = line.split(" | ");
+          const [appW, title, pos, sz] = line.split(" | ");
+          const posMatch = (pos || "").match(/(\d+),\s*(\d+)/);
+          const wx = posMatch ? parseInt(posMatch[1]) : 0;
+          const wy = posMatch ? parseInt(posMatch[2]) : 0;
+          const szMatch = (sz || "").match(/(\d+),\s*(\d+)/);
+          const ww = szMatch ? parseInt(szMatch[1]) : 0;
+          const wh = szMatch ? parseInt(szMatch[2]) : 0;
+          let displayIndex = 1;
+          if (displayBounds.length > 1) {
+            for (let di = 0; di < displayBounds.length; di++) {
+              const db = displayBounds[di];
+              if (wx >= db.x && wx < db.x + db.width) {
+                displayIndex = di + 1;
+                break;
+              }
+            }
+          }
           return {
-            app: app?.trim(),
+            app: appW?.trim(),
             title: title?.trim(),
-            position: pos?.trim(),
-            size: sz?.trim()
+            x: wx,
+            y: wy,
+            width: ww,
+            height: wh,
+            display: displayIndex
           };
         });
-        return { success: true, data: { windows, count: windows.length } };
+        return {
+          success: true,
+          data: {
+            windows,
+            count: windows.length,
+            displays: displayBounds.length || 1,
+            note: "Window positions are in global coordinates. 'display' indicates which monitor the window is on (1=primary, 2=secondary, etc)."
+          }
+        };
       }
       case "sys_window_focus": {
         const appName = params.app;
@@ -2926,14 +3129,14 @@ var CAPABILITY_PROBES = [
     name: "screenshot",
     test: async () => {
       try {
-        await runShell("screencapture -x -D1 /tmp/pulso-probe-ss.png", 5e3);
+        await runShell("screencapture -x /tmp/pulso-probe-ss.png", 5e3);
         unlinkSync("/tmp/pulso-probe-ss.png");
         return true;
       } catch {
         return false;
       }
     },
-    tools: ["sys_screenshot"]
+    tools: ["sys_screenshot", "sys_screenshot_region", "sys_list_displays"]
   },
   {
     name: "chrome_js",
@@ -3100,7 +3303,7 @@ function connect() {
       ws.send(JSON.stringify({
         type: "extension_ready",
         platform: "macos",
-        version: "0.3.2",
+        version: "0.4.0",
         accessLevel: ACCESS_LEVEL,
         capabilities: cap.available,
         unavailable: cap.unavailable,
@@ -3358,7 +3561,7 @@ function writeString(view, offset, str) {
 }
 console.log("");
 console.log("  \u2554\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2557");
-console.log("  \u2551    \u{1FAC0} Pulso Mac Companion v0.3.0     \u2551");
+console.log("  \u2551    \u{1FAC0} Pulso Mac Companion v0.4.0     \u2551");
 console.log("  \u255A\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u255D");
 console.log("");
 setupPermissions().then(() => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pulso/companion",
-  "version": "0.3.3",
+  "version": "0.4.1",
   "type": "module",
   "description": "Pulso Companion — gives your AI agent real control over your computer",
   "bin": {