@pulso/companion 0.1.4 → 0.1.5

package/dist/index.js

@@ -45,6 +45,16 @@ function runShell(cmd, timeout = 1e4) {
     });
   });
 }
+function runSwift(code, timeout = 1e4) {
+  return new Promise((resolve2, reject) => {
+    const child = exec(`swift -`, { timeout }, (err, stdout, stderr) => {
+      if (err) reject(new Error(stderr || err.message));
+      else resolve2(stdout.trim());
+    });
+    child.stdin?.write(code);
+    child.stdin?.end();
+  });
+}
 var spotifyToken = null;
 var spotifyTokenExpiry = 0;
 async function getSpotifyToken() {
@@ -226,20 +236,219 @@ async function handleCommand(command, params) {
         return { success: true, data: { path, written: content.length } };
       }
       case "sys_screenshot": {
-        const tmpPath = `/tmp/pulso-screenshot-${Date.now()}.png`;
-        await runShell(`screencapture -x ${tmpPath}`, 15e3);
-        if (!existsSync(tmpPath)) return { success: false, error: "Screenshot failed" };
-        const buf = readFileSync(tmpPath);
+        const ts = Date.now();
+        const pngPath = `/tmp/pulso-ss-${ts}.png`;
+        const jpgPath = `/tmp/pulso-ss-${ts}.jpg`;
+        await runShell(`screencapture -x ${pngPath}`, 15e3);
+        if (!existsSync(pngPath)) return { success: false, error: "Screenshot failed" };
+        try {
+          await runShell(`sips --resampleWidth 1280 --setProperty format jpeg --setProperty formatOptions 60 ${pngPath} --out ${jpgPath}`, 1e4);
+        } catch {
+          const buf2 = readFileSync(pngPath);
+          exec(`rm -f ${pngPath}`);
+          return { success: true, data: { image: `data:image/png;base64,${buf2.toString("base64")}`, format: "png", note: "Full screen screenshot" } };
+        }
+        const buf = readFileSync(jpgPath);
         const base64 = buf.toString("base64");
-        exec(`rm -f ${tmpPath}`);
+        exec(`rm -f ${pngPath} ${jpgPath}`);
+        let screenSize = "unknown";
+        try {
+          screenSize = await runShell(`system_profiler SPDisplaysDataType 2>/dev/null | grep Resolution | head -1 | sed 's/.*: //'`);
+        } catch {
+        }
         return {
           success: true,
           data: {
-            image: `data:image/png;base64,${base64}`,
-            note: "Full screen screenshot captured"
+            image: `data:image/jpeg;base64,${base64}`,
+            format: "jpeg",
+            resolution: screenSize.trim(),
+            imageWidth: 1280,
+            note: "Screenshot captured. Coordinates in this image are scaled \u2014 multiply x by (screen_width/1280) for actual clicks."
           }
         };
       }
+      // ── Computer-Use: Mouse & Keyboard ────────────────────
+      case "sys_mouse_click": {
+        const x = Number(params.x);
+        const y = Number(params.y);
+        const button = params.button || "left";
+        if (isNaN(x) || isNaN(y)) return { success: false, error: "Missing x, y coordinates" };
+        const mouseType = button === "right" ? "rightMouseDown" : "leftMouseDown";
+        const mouseTypeUp = button === "right" ? "rightMouseUp" : "leftMouseUp";
+        const mouseButton = button === "right" ? ".right" : ".left";
+        const swift = `
+import Cocoa
+let p = CGPoint(x: ${x}, y: ${y})
+let d = CGEvent(mouseEventSource: nil, mouseType: .${mouseType}, mouseCursorPosition: p, mouseButton: ${mouseButton})!
+d.post(tap: .cghidEventTap)
+usleep(50000)
+let u = CGEvent(mouseEventSource: nil, mouseType: .${mouseTypeUp}, mouseCursorPosition: p, mouseButton: ${mouseButton})!
+u.post(tap: .cghidEventTap)
+print("clicked")`;
+        await runSwift(swift);
+        return { success: true, data: { clicked: { x, y }, button } };
+      }
+      case "sys_mouse_double_click": {
+        const x = Number(params.x);
+        const y = Number(params.y);
+        if (isNaN(x) || isNaN(y)) return { success: false, error: "Missing x, y coordinates" };
+        const swift = `
+import Cocoa
+let p = CGPoint(x: ${x}, y: ${y})
+let d1 = CGEvent(mouseEventSource: nil, mouseType: .leftMouseDown, mouseCursorPosition: p, mouseButton: .left)!
+d1.setIntegerValueField(.mouseEventClickState, value: 1)
+d1.post(tap: .cghidEventTap)
+usleep(30000)
+let u1 = CGEvent(mouseEventSource: nil, mouseType: .leftMouseUp, mouseCursorPosition: p, mouseButton: .left)!
+u1.setIntegerValueField(.mouseEventClickState, value: 1)
+u1.post(tap: .cghidEventTap)
+usleep(80000)
+let d2 = CGEvent(mouseEventSource: nil, mouseType: .leftMouseDown, mouseCursorPosition: p, mouseButton: .left)!
+d2.setIntegerValueField(.mouseEventClickState, value: 2)
+d2.post(tap: .cghidEventTap)
+usleep(30000)
+let u2 = CGEvent(mouseEventSource: nil, mouseType: .leftMouseUp, mouseCursorPosition: p, mouseButton: .left)!
+u2.setIntegerValueField(.mouseEventClickState, value: 2)
+u2.post(tap: .cghidEventTap)
+print("double-clicked")`;
+        await runSwift(swift);
+        return { success: true, data: { doubleClicked: { x, y } } };
+      }
+      case "sys_mouse_scroll": {
+        const x = Number(params.x) || 0;
+        const y = Number(params.y) || 0;
+        const scrollY = Number(params.scrollY) || 0;
+        const scrollX = Number(params.scrollX) || 0;
+        if (!scrollY && !scrollX) return { success: false, error: "Missing scrollY or scrollX" };
+        const swift = `
+import Cocoa
+let p = CGPoint(x: ${x}, y: ${y})
+let move = CGEvent(mouseEventSource: nil, mouseType: .mouseMoved, mouseCursorPosition: p, mouseButton: .left)!
+move.post(tap: .cghidEventTap)
+usleep(50000)
+let scroll = CGEvent(scrollWheelEvent2Source: nil, units: .pixel, wheelCount: 2, wheel1: Int32(${-scrollY}), wheel2: Int32(${-scrollX}))!
+scroll.post(tap: .cghidEventTap)
+print("scrolled")`;
+        await runSwift(swift);
+        return { success: true, data: { scrolled: { x, y, scrollY, scrollX } } };
+      }
+      case "sys_keyboard_type": {
+        const text = params.text;
+        if (!text) return { success: false, error: "Missing text" };
+        await runAppleScript(`tell application "System Events" to keystroke "${text.replace(/\\/g, "\\\\").replace(/"/g, '\\"')}"`);
+        return { success: true, data: { typed: text.slice(0, 100) } };
+      }
+      case "sys_key_press": {
+        const key = params.key;
+        if (!key) return { success: false, error: "Missing key" };
+        const rawMods = params.modifiers;
+        const modifiers = Array.isArray(rawMods) ? rawMods : typeof rawMods === "string" ? rawMods.split(",").map((m) => m.trim()).filter(Boolean) : [];
+        const modMap = {
+          cmd: "command down",
+          command: "command down",
+          shift: "shift down",
+          alt: "option down",
+          option: "option down",
+          ctrl: "control down",
+          control: "control down"
+        };
+        const modStr = modifiers.map((m) => modMap[m.toLowerCase()] || "").filter(Boolean);
+        const keyCodeMap = {
+          enter: 36,
+          return: 36,
+          tab: 48,
+          escape: 27,
+          esc: 27,
+          delete: 51,
+          backspace: 51,
+          space: 49,
+          up: 126,
+          down: 125,
+          left: 123,
+          right: 124,
+          f1: 122,
+          f2: 120,
+          f3: 99,
+          f4: 118,
+          f5: 96,
+          f6: 97,
+          f7: 98,
+          f8: 100,
+          f9: 101,
+          f10: 109,
+          f11: 103,
+          f12: 111,
+          home: 115,
+          end: 119,
+          pageup: 116,
+          pagedown: 121
+        };
+        const keyCode = keyCodeMap[key.toLowerCase()];
+        if (keyCode !== void 0) {
+          const using = modStr.length > 0 ? ` using {${modStr.join(", ")}}` : "";
+          await runAppleScript(`tell application "System Events" to key code ${keyCode}${using}`);
+        } else if (key.length === 1) {
+          const using = modStr.length > 0 ? ` using {${modStr.join(", ")}}` : "";
+          await runAppleScript(`tell application "System Events" to keystroke "${key}"${using}`);
+        } else {
+          return { success: false, error: `Unknown key: ${key}. Use single characters or: enter, tab, escape, delete, space, up, down, left, right, f1-f12, home, end, pageup, pagedown` };
+        }
+        return { success: true, data: { pressed: key, modifiers } };
+      }
+      case "sys_mouse_move": {
+        const x = Number(params.x);
+        const y = Number(params.y);
+        if (isNaN(x) || isNaN(y)) return { success: false, error: "Missing x, y coordinates" };
+        const swift = `
+import Cocoa
+let p = CGPoint(x: ${x}, y: ${y})
+let m = CGEvent(mouseEventSource: nil, mouseType: .mouseMoved, mouseCursorPosition: p, mouseButton: .left)!
+m.post(tap: .cghidEventTap)
+print("moved")`;
+        await runSwift(swift);
+        return { success: true, data: { movedTo: { x, y } } };
+      }
+      case "sys_drag": {
+        const fromX = Number(params.fromX);
+        const fromY = Number(params.fromY);
+        const toX = Number(params.toX);
+        const toY = Number(params.toY);
+        if ([fromX, fromY, toX, toY].some(isNaN)) return { success: false, error: "Missing fromX, fromY, toX, toY" };
+        const swift = `
+import Cocoa
+let from = CGPoint(x: ${fromX}, y: ${fromY})
+let to = CGPoint(x: ${toX}, y: ${toY})
+let d = CGEvent(mouseEventSource: nil, mouseType: .leftMouseDown, mouseCursorPosition: from, mouseButton: .left)!
+d.post(tap: .cghidEventTap)
+usleep(100000)
+let steps = 10
+for i in 1...steps {
+    let frac = Double(i) / Double(steps)
+    let p = CGPoint(x: from.x + (to.x - from.x) * frac, y: from.y + (to.y - from.y) * frac)
+    let drag = CGEvent(mouseEventSource: nil, mouseType: .leftMouseDragged, mouseCursorPosition: p, mouseButton: .left)!
+    drag.post(tap: .cghidEventTap)
+    usleep(20000)
+}
+let u = CGEvent(mouseEventSource: nil, mouseType: .leftMouseUp, mouseCursorPosition: to, mouseButton: .left)!
+u.post(tap: .cghidEventTap)
+print("dragged")`;
+        await runSwift(swift);
+        return { success: true, data: { dragged: { from: { x: fromX, y: fromY }, to: { x: toX, y: toY } } } };
+      }
+      case "sys_get_cursor_position": {
+        const swift = `
+import Cocoa
+let loc = NSEvent.mouseLocation
+let screens = NSScreen.screens
+let mainHeight = screens.first?.frame.height ?? 0
+// Convert from bottom-left to top-left coordinate system
+let x = Int(loc.x)
+let y = Int(mainHeight - loc.y)
+print("\\(x),\\(y)")`;
+        const pos = await runSwift(swift);
+        const [cx, cy] = pos.trim().split(",").map(Number);
+        return { success: true, data: { x: cx, y: cy } };
+      }
       case "sys_run_shortcut": {
         const name = params.name;
         const input = params.input;
@@ -334,7 +543,7 @@ function scheduleReconnect() {
 }
 console.log("");
 console.log("  \u2554\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2557");
-console.log("  \u2551    \u{1FAC0} Pulso Mac Companion v0.1.4     \u2551");
+console.log("  \u2551    \u{1FAC0} Pulso Mac Companion v0.1.5     \u2551");
 console.log("  \u255A\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u255D");
 console.log("");
 connect();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pulso/companion",
-  "version": "0.1.4",
+  "version": "0.1.5",
   "type": "module",
   "description": "Pulso Companion — gives your AI agent real control over your computer",
   "bin": {