@pulso/companion 0.3.2 → 0.4.0

package/dist/index.js

@@ -27,7 +27,7 @@ if (!TOKEN) {
 var ACCESS_LEVEL = process.env.PULSO_ACCESS ?? process.argv.find((_, i, a) => a[i - 1] === "--access") ?? "sandboxed";
 var WAKE_WORD_ENABLED = process.argv.includes("--wake-word");
 var PICOVOICE_ACCESS_KEY = process.env.PICOVOICE_ACCESS_KEY ?? process.argv.find((_, i, a) => a[i - 1] === "--picovoice-key") ?? "";
-var WS_URL = API_URL.replace("https://", "wss://").replace("http://", "ws://") + "/ws/browser?token=" + TOKEN;
+var WS_URL = API_URL.replace("https://", "wss://").replace("http://", "ws://") + "/ws/companion?token=" + TOKEN;
 var HOME = homedir();
 var RECONNECT_DELAY = 5e3;
 var SAFE_DIRS = ["Documents", "Desktop", "Downloads", "Projects", "Projetos"];
@@ -478,8 +478,54 @@ async function handleCommand(command, params) {
       case "sys_open_app": {
         const app = params.app;
         if (!app) return { success: false, error: "Missing app name" };
-        await runShell(`open -a "${app.replace(/"/g, "")}"`);
-        return { success: true, data: { opened: app } };
+        const sanitizedApp = app.replace(/"/g, "");
+        try {
+          await runShell(`open -a "${sanitizedApp}"`);
+        } catch (e) {
+          return { success: false, error: `Failed to open "${sanitizedApp}": ${e.message}`, errorCode: "APP_NOT_FOUND" };
+        }
+        let launched = false;
+        for (let i = 0; i < 10; i++) {
+          await new Promise((r) => setTimeout(r, 500));
+          try {
+            const running = await runAppleScript(
+              `tell application "System Events" to (name of processes) contains "${sanitizedApp}"`
+            );
+            if (running.trim() === "true") {
+              launched = true;
+              break;
+            }
+          } catch {
+          }
+        }
+        let windowInfo;
+        if (launched) {
+          try {
+            const winCheck = await runAppleScript(`
+              tell application "System Events"
+                tell process "${sanitizedApp}"
+                  if (count of windows) > 0 then
+                    set w to window 1
+                    return (name of w) & "|" & (position of w as string) & "|" & (size of w as string)
+                  else
+                    return "no-window"
+                  end if
+                end tell
+              end tell`);
+            windowInfo = winCheck.trim();
+          } catch {
+          }
+        }
+        return {
+          success: true,
+          data: {
+            opened: sanitizedApp,
+            launched,
+            hasWindow: windowInfo ? windowInfo !== "no-window" : false,
+            windowInfo: windowInfo && windowInfo !== "no-window" ? windowInfo : void 0,
+            note: launched ? `"${sanitizedApp}" is running${windowInfo && windowInfo !== "no-window" ? " with a visible window" : " (may still be loading)"}.` : `"${sanitizedApp}" was requested to open but process not detected yet. It may still be launching \u2014 take a screenshot to verify.`
+          }
+        };
       }
       case "sys_open_url": {
         const url = params.url;
@@ -685,29 +731,33 @@ async function handleCommand(command, params) {
         return { success: true, data: { path, written: content.length } };
       }
       case "sys_screenshot": {
+        const display = params.display || 0;
         const ts = Date.now();
         const pngPath = `/tmp/pulso-ss-${ts}.png`;
         const jpgPath = `/tmp/pulso-ss-${ts}.jpg`;
         try {
-          await runShell(`screencapture -C -x -D1 ${pngPath}`, 15e3);
+          const displayFlag = display > 0 ? `-D${display}` : "";
+          await runShell(`screencapture -C -x ${displayFlag} ${pngPath}`, 15e3);
         } catch (ssErr) {
           const msg = ssErr.message || "";
           if (msg.includes("could not create image") || msg.includes("display")) {
             return {
               success: false,
-              error: "Screen Recording permission required. Go to System Settings \u2192 Privacy & Security \u2192 Screen Recording \u2192 enable your terminal app (Terminal, iTerm, etc). Then restart the companion."
+              error: "Screen Recording permission required. Go to System Settings \u2192 Privacy & Security \u2192 Screen Recording \u2192 enable your terminal app (Terminal, iTerm, etc). Then restart the companion.",
+              errorCode: "PERMISSION_DENIED"
             };
           }
-          return { success: false, error: `Screenshot failed: ${msg}` };
+          return { success: false, error: `Screenshot failed: ${msg}`, errorCode: "SCREENSHOT_FAILED" };
         }
         if (!existsSync(pngPath))
           return {
             success: false,
-            error: "Screenshot failed \u2014 Screen Recording permission needed. System Settings \u2192 Privacy & Security \u2192 Screen Recording \u2192 enable your terminal app, then restart companion."
+            error: "Screenshot failed \u2014 Screen Recording permission needed.",
+            errorCode: "PERMISSION_DENIED"
           };
         try {
           await runShell(
-            `sips --resampleWidth 1280 --setProperty format jpeg --setProperty formatOptions 60 ${pngPath} --out ${jpgPath}`,
+            `sips --resampleWidth 1600 --setProperty format jpeg --setProperty formatOptions 75 ${pngPath} --out ${jpgPath}`,
             1e4
           );
         } catch {
@@ -721,7 +771,8 @@ async function handleCommand(command, params) {
             data: {
               image: `data:image/png;base64,${buf2.toString("base64")}`,
               format: "png",
-              note: "Full screen screenshot"
+              display: display || "all",
+              note: "Full screen screenshot (PNG fallback)"
             }
           };
         }
@@ -735,11 +786,18 @@ async function handleCommand(command, params) {
           unlinkSync(jpgPath);
         } catch {
         }
-        let screenSize = "unknown";
+        let displayInfo = { width: 0, height: 0, displays: 1 };
         try {
-          screenSize = await runShell(
-            `system_profiler SPDisplaysDataType 2>/dev/null | grep Resolution | head -1 | sed 's/.*: //'`
-          );
+          const diSwift = `
+import Cocoa
+let screens = NSScreen.screens
+let main = screens.first!
+let w = Int(main.frame.width)
+let h = Int(main.frame.height)
+print("\\(w),\\(h),\\(screens.count)")`;
+          const diResult = await runSwift(diSwift);
+          const [dw, dh, dc] = diResult.trim().split(",").map(Number);
+          displayInfo = { width: dw || 0, height: dh || 0, displays: dc || 1 };
         } catch {
         }
         return {
@@ -747,12 +805,110 @@ async function handleCommand(command, params) {
           data: {
             image: `data:image/jpeg;base64,${base64}`,
             format: "jpeg",
-            resolution: screenSize.trim(),
-            imageWidth: 1280,
-            note: "Screenshot captured. Coordinates in this image are scaled \u2014 multiply x by (screen_width/1280) for actual clicks."
+            display: display || "all",
+            screenWidth: displayInfo.width,
+            screenHeight: displayInfo.height,
+            totalDisplays: displayInfo.displays,
+            imageWidth: 1600,
+            note: display > 0 ? `Screenshot of display ${display}. Coordinates: multiply x by (${displayInfo.width}/1600) and y by (${displayInfo.height}/(1600*${displayInfo.height}/${displayInfo.width})) for actual clicks.` : `Screenshot of all ${displayInfo.displays} display(s) stitched horizontally. Use sys_list_displays to get individual display bounds for coordinate mapping.`
+          }
+        };
+      }
+      case "sys_screenshot_region": {
+        const rx = params.x;
+        const ry = params.y;
+        const rw = params.width;
+        const rh = params.height;
+        if (rx == null || ry == null || rw == null || rh == null)
+          return { success: false, error: "Missing x, y, width, or height" };
+        const ts2 = Date.now();
+        const regPath = `/tmp/pulso-ss-region-${ts2}.png`;
+        const regJpg = `/tmp/pulso-ss-region-${ts2}.jpg`;
+        try {
+          await runShell(`screencapture -x -R${rx},${ry},${rw},${rh} ${regPath}`, 15e3);
+        } catch (e) {
+          return { success: false, error: `Region screenshot failed: ${e.message}`, errorCode: "SCREENSHOT_FAILED" };
+        }
+        if (!existsSync(regPath))
+          return { success: false, error: "Region screenshot failed", errorCode: "SCREENSHOT_FAILED" };
+        try {
+          await runShell(`sips --setProperty format jpeg --setProperty formatOptions 85 ${regPath} --out ${regJpg}`, 1e4);
+        } catch {
+          const rb = readFileSync(regPath);
+          try {
+            unlinkSync(regPath);
+          } catch {
+          }
+          return { success: true, data: { image: `data:image/png;base64,${rb.toString("base64")}`, format: "png", region: { x: rx, y: ry, width: rw, height: rh } } };
+        }
+        const rb2 = readFileSync(regJpg);
+        try {
+          unlinkSync(regPath);
+        } catch {
+        }
+        try {
+          unlinkSync(regJpg);
+        } catch {
+        }
+        return {
+          success: true,
+          data: {
+            image: `data:image/jpeg;base64,${rb2.toString("base64")}`,
+            format: "jpeg",
+            region: { x: rx, y: ry, width: rw, height: rh },
+            note: "Region screenshot at actual resolution (no scaling). Coordinates are absolute screen coordinates."
           }
         };
       }
+      case "sys_list_displays": {
+        try {
+          const swift = `
+import Cocoa
+let screens = NSScreen.screens
+var result = ""
+for (i, screen) in screens.enumerated() {
+    let f = screen.frame
+    let vf = screen.visibleFrame
+    let isMain = (screen == NSScreen.main)
+    let scale = screen.backingScaleFactor
+    let name = screen.localizedName
+    result += "\\(i+1)|\\(name)|\\(Int(f.origin.x)),\\(Int(f.origin.y))|\\(Int(f.width)),\\(Int(f.height))|\\(Int(vf.origin.x)),\\(Int(vf.origin.y))|\\(Int(vf.width)),\\(Int(vf.height))|\\(scale)|\\(isMain)\\n"
+}
+print(result)`;
+          const raw = await runSwift(swift, 15e3);
+          const displays = raw.trim().split("\n").filter(Boolean).map((line) => {
+            const [index, name, origin, size, visOrigin, visSize, scale, isMain] = line.split("|");
+            const [ox, oy] = (origin || "0,0").split(",").map(Number);
+            const [sw2, sh2] = (size || "0,0").split(",").map(Number);
+            const [vox, voy] = (visOrigin || "0,0").split(",").map(Number);
+            const [vsw, vsh] = (visSize || "0,0").split(",").map(Number);
+            return {
+              display: parseInt(index) || 0,
+              name: name?.trim() || "Unknown",
+              x: ox,
+              y: oy,
+              width: sw2,
+              height: sh2,
+              visibleX: vox,
+              visibleY: voy,
+              visibleWidth: vsw,
+              visibleHeight: vsh,
+              scale: parseFloat(scale) || 1,
+              isMain: isMain?.trim() === "true"
+            };
+          });
+          return {
+            success: true,
+            data: {
+              displays,
+              count: displays.length,
+              note: "Display coordinates use macOS coordinate system (origin bottom-left). For screencapture: use display number (1-based). For mouse clicks: windows on display 2 have x >= display1.width."
+            }
+          };
+        } catch (e) {
+          return { success: false, error: `Failed to list displays: ${e.message}` };
+        }
+      }
       // ── Computer-Use: Mouse & Keyboard ────────────────────
       case "sys_mouse_click": {
         const x = Number(params.x);
@@ -1732,6 +1888,19 @@ end tell`);
       }
       // ── Window Management ───────────────────────────────────
       case "sys_window_list": {
+        let displayBounds = [];
+        try {
+          const dbSwift = `
+import Cocoa
+let screens = NSScreen.screens
+for s in screens { print("\\(Int(s.frame.origin.x)),\\(Int(s.frame.width))") }`;
+          const dbRaw = await runSwift(dbSwift, 5e3);
+          displayBounds = dbRaw.trim().split("\n").filter(Boolean).map((l) => {
+            const [bx, bw] = l.split(",").map(Number);
+            return { x: bx || 0, width: bw || 0 };
+          });
+        } catch {
+        }
         const raw4 = await runAppleScript(`
           tell application "System Events"
             set output to ""
@@ -1744,15 +1913,42 @@ end tell`);
             return output
           end tell`);
         const windows = raw4.split("\n").filter(Boolean).map((line) => {
-          const [app, title, pos, sz] = line.split(" | ");
+          const [appW, title, pos, sz] = line.split(" | ");
+          const posMatch = (pos || "").match(/(\d+),\s*(\d+)/);
+          const wx = posMatch ? parseInt(posMatch[1]) : 0;
+          const wy = posMatch ? parseInt(posMatch[2]) : 0;
+          const szMatch = (sz || "").match(/(\d+),\s*(\d+)/);
+          const ww = szMatch ? parseInt(szMatch[1]) : 0;
+          const wh = szMatch ? parseInt(szMatch[2]) : 0;
+          let displayIndex = 1;
+          if (displayBounds.length > 1) {
+            for (let di = 0; di < displayBounds.length; di++) {
+              const db = displayBounds[di];
+              if (wx >= db.x && wx < db.x + db.width) {
+                displayIndex = di + 1;
+                break;
+              }
+            }
+          }
           return {
-            app: app?.trim(),
+            app: appW?.trim(),
             title: title?.trim(),
-            position: pos?.trim(),
-            size: sz?.trim()
+            x: wx,
+            y: wy,
+            width: ww,
+            height: wh,
+            display: displayIndex
           };
         });
-        return { success: true, data: { windows, count: windows.length } };
+        return {
+          success: true,
+          data: {
+            windows,
+            count: windows.length,
+            displays: displayBounds.length || 1,
+            note: "Window positions are in global coordinates. 'display' indicates which monitor the window is on (1=primary, 2=secondary, etc)."
+          }
+        };
       }
       case "sys_window_focus": {
         const appName = params.app;
@@ -2885,6 +3081,184 @@ var reconnectTimer = null;
 var heartbeatTimer = null;
 var HEARTBEAT_INTERVAL = 3e4;
 var reconnectAttempts = 0;
+var CAPABILITY_PROBES = [
+  {
+    name: "calendar",
+    test: async () => {
+      try {
+        await runAppleScript('tell application "Calendar" to name of calendars');
+        return true;
+      } catch {
+        return false;
+      }
+    },
+    tools: ["sys_calendar_list", "sys_calendar_create"]
+  },
+  {
+    name: "contacts",
+    test: async () => {
+      try {
+        await runAppleScript('tell application "Contacts" to count of people');
+        return true;
+      } catch {
+        return false;
+      }
+    },
+    tools: ["sys_contacts_search"]
+  },
+  {
+    name: "reminders",
+    test: async () => {
+      try {
+        await runAppleScript('tell application "Reminders" to name of lists');
+        return true;
+      } catch {
+        return false;
+      }
+    },
+    tools: ["sys_reminder_create", "sys_reminder_list"]
+  },
+  {
+    name: "screenshot",
+    test: async () => {
+      try {
+        await runShell("screencapture -x /tmp/pulso-probe-ss.png", 5e3);
+        unlinkSync("/tmp/pulso-probe-ss.png");
+        return true;
+      } catch {
+        return false;
+      }
+    },
+    tools: ["sys_screenshot", "sys_screenshot_region", "sys_list_displays"]
+  },
+  {
+    name: "chrome_js",
+    test: async () => {
+      try {
+        await runShell("pgrep -x 'Google Chrome' >/dev/null 2>&1");
+        return true;
+      } catch {
+        return false;
+      }
+    },
+    tools: ["sys_browser_execute_js"]
+  },
+  {
+    name: "safari_js",
+    test: async () => {
+      try {
+        await runAppleScript('tell application "Safari" to name of front window');
+        return true;
+      } catch {
+        return false;
+      }
+    },
+    tools: ["sys_browser_execute_js"]
+  },
+  {
+    name: "spotify",
+    test: async () => {
+      try {
+        await runShell("pgrep -x Spotify >/dev/null 2>&1 || ls /Applications/Spotify.app >/dev/null 2>&1");
+        return true;
+      } catch {
+        return false;
+      }
+    },
+    tools: ["sys_spotify_play", "sys_spotify_pause", "sys_spotify_current", "sys_spotify_next", "sys_spotify_previous", "sys_spotify_search"]
+  },
+  {
+    name: "tts",
+    test: async () => {
+      try {
+        await runShell("which say >/dev/null 2>&1");
+        return true;
+      } catch {
+        return false;
+      }
+    },
+    tools: ["sys_tts"]
+  },
+  {
+    name: "shortcuts",
+    test: async () => {
+      try {
+        await runShell("which shortcuts >/dev/null 2>&1");
+        return true;
+      } catch {
+        return false;
+      }
+    },
+    tools: ["sys_shortcuts_run", "sys_shortcuts_list"]
+  }
+];
+var verifiedCapabilities = {
+  available: [],
+  unavailable: [],
+  tools: []
+};
+async function probeCapabilities() {
+  console.log("\u{1F50D} Probing system capabilities...");
+  const available = [];
+  const unavailable = [];
+  const tools = /* @__PURE__ */ new Set();
+  const alwaysAvailable = [
+    "sys_open_app",
+    "sys_open_url",
+    "sys_clipboard_read",
+    "sys_clipboard_write",
+    "sys_notification",
+    "sys_shell",
+    "sys_file_read",
+    "sys_file_write",
+    "sys_file_list",
+    "sys_file_search",
+    "sys_volume",
+    "sys_window_list",
+    "sys_window_focus",
+    "sys_system_info",
+    "sys_wifi_info",
+    "sys_disk_info",
+    "sys_process_list",
+    "sys_process_kill",
+    "sys_dark_mode",
+    "sys_spotlight_search",
+    "sys_settings",
+    "sys_screen_lock",
+    "sys_trash",
+    "sys_power",
+    "sys_focus_mode",
+    "sys_audio_devices",
+    "sys_bluetooth",
+    "sys_browser_open_tab",
+    "sys_browser_close_tab",
+    "sys_browser_tabs",
+    "sys_browser_read_page",
+    "sys_browser_navigate"
+  ];
+  for (const t of alwaysAvailable) tools.add(t);
+  const results = await Promise.allSettled(
+    CAPABILITY_PROBES.map(async (probe) => {
+      const ok = await probe.test();
+      return { name: probe.name, ok, tools: probe.tools };
+    })
+  );
+  for (const r of results) {
+    if (r.status === "fulfilled") {
+      if (r.value.ok) {
+        available.push(r.value.name);
+        for (const t of r.value.tools) tools.add(t);
+      } else {
+        unavailable.push(r.value.name);
+      }
+    }
+  }
+  const cap = { available, unavailable, tools: Array.from(tools) };
+  console.log(`   \u2705 Available: ${available.join(", ") || "none"}`);
+  if (unavailable.length) console.log(`   \u26A0\uFE0F  Unavailable: ${unavailable.join(", ")}`);
+  console.log(`   \u{1F4E6} ${cap.tools.length} tools verified`);
+  return cap;
+}
 function connect() {
   console.log("\u{1F50C} Connecting to Pulso...");
   console.log(`   ${WS_URL.replace(/token=.*/, "token=***")}`);
@@ -2917,7 +3291,19 @@ function connect() {
       `   Access: ${ACCESS_LEVEL === "full" ? "\u{1F513} Full (unrestricted)" : "\u{1F512} Sandboxed (safe dirs only)"}`
     );
     console.log("   Waiting for commands from Pulso agent...");
-    ws.send(JSON.stringify({ type: "extension_ready" }));
+    probeCapabilities().then((cap) => {
+      verifiedCapabilities = cap;
+      ws.send(JSON.stringify({
+        type: "extension_ready",
+        platform: "macos",
+        version: "0.4.0",
+        accessLevel: ACCESS_LEVEL,
+        capabilities: cap.available,
+        unavailable: cap.unavailable,
+        tools: cap.tools,
+        totalTools: cap.tools.length
+      }));
+    });
     if (heartbeatTimer) clearInterval(heartbeatTimer);
     heartbeatTimer = setInterval(() => {
       if (ws && ws.readyState === WebSocket.OPEN) {
@@ -3168,7 +3554,7 @@ function writeString(view, offset, str) {
 }
 console.log("");
 console.log("  \u2554\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2557");
-console.log("  \u2551    \u{1FAC0} Pulso Mac Companion v0.3.0     \u2551");
+console.log("  \u2551    \u{1FAC0} Pulso Mac Companion v0.4.0     \u2551");
 console.log("  \u255A\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u255D");
 console.log("");
 setupPermissions().then(() => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pulso/companion",
-  "version": "0.3.2",
+  "version": "0.4.0",
   "type": "module",
   "description": "Pulso Companion — gives your AI agent real control over your computer",
   "bin": {