@mseep/clawdcursor 1.5.5

package/dist/platform/native-desktop.js

@@ -0,0 +1,936 @@
+"use strict";
+/**
+ * Native Desktop Control — direct OS-level input
+ * using @nut-tree-fork/nut-js for mouse/keyboard and screen capture.
+ *
+ * No network connection needed — controls the local desktop directly.
+ *
+ * - captureScreen() returns full-resolution frames
+ * - captureForLLM() returns resized frames (1280px wide) with scaling metadata
+ * - Coordinate scaling handled transparently
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.NativeDesktop = void 0;
+const os_1 = __importDefault(require("os"));
+const events_1 = require("events");
+const sharp_1 = __importDefault(require("sharp"));
+const nut_js_1 = require("@nut-tree-fork/nut-js");
+const keys_1 = require("./keys");
+const native_helper_1 = require("./native-helper");
+const fs = __importStar(require("fs"));
+// On macOS, Command key = Key.LeftCmd. On other platforms, Super = Key.LeftSuper.
+const SUPER_KEY = os_1.default.platform() === 'darwin' ? nut_js_1.Key.LeftCmd : nut_js_1.Key.LeftSuper;
+const IS_MAC = os_1.default.platform() === 'darwin';
+/** Safely resolve a nut-js Key enum value from multiple candidate names */
+function resolveNutKey(...candidates) {
+    for (const name of candidates) {
+        const value = nut_js_1.Key[name];
+        if (value !== undefined)
+            return value;
+    }
+    throw new Error(`Unable to resolve nut-js key from candidates: ${candidates.join(', ')}`);
+}
+// nut-js Key enum mapping from canonical key names (see keys.ts for normalization)
+const KEY_MAP = {
+    'Return': nut_js_1.Key.Enter,
+    'Tab': nut_js_1.Key.Tab,
+    'Escape': nut_js_1.Key.Escape,
+    'Backspace': nut_js_1.Key.Backspace,
+    'Delete': nut_js_1.Key.Delete,
+    'Home': nut_js_1.Key.Home,
+    'End': nut_js_1.Key.End,
+    'PageUp': nut_js_1.Key.PageUp,
+    'PageDown': nut_js_1.Key.PageDown,
+    'Left': nut_js_1.Key.Left,
+    'Up': nut_js_1.Key.Up,
+    'Right': nut_js_1.Key.Right,
+    'Down': nut_js_1.Key.Down,
+    'F1': nut_js_1.Key.F1, 'F2': nut_js_1.Key.F2, 'F3': nut_js_1.Key.F3, 'F4': nut_js_1.Key.F4,
+    'F5': nut_js_1.Key.F5, 'F6': nut_js_1.Key.F6, 'F7': nut_js_1.Key.F7, 'F8': nut_js_1.Key.F8,
+    'F9': nut_js_1.Key.F9, 'F10': nut_js_1.Key.F10, 'F11': nut_js_1.Key.F11, 'F12': nut_js_1.Key.F12,
+    'Shift': nut_js_1.Key.LeftShift,
+    'Control': nut_js_1.Key.LeftControl,
+    'Alt': nut_js_1.Key.LeftAlt,
+    'Super': SUPER_KEY,
+    'Space': nut_js_1.Key.Space,
+    // Symbol keys for combos like ctrl+plus / ctrl+minus
+    '=': resolveNutKey('Equal', 'Equals'),
+    '+': resolveNutKey('Equal', 'Equals', 'Add', 'NumAdd'),
+    '-': resolveNutKey('Minus', 'Subtract', 'NumSubtract'),
+    '_': resolveNutKey('Minus', 'Subtract', 'NumSubtract'),
+};
+/** LLM screenshot target width — smaller = faster API calls + fewer tokens */
+// Higher resolution = better tool/icon identification. 1280 is Anthropic's recommended max.
+// At 2560 screen: 1280 → scale 2x (was 1024 → 2.5x). Icons go from ~12px to ~20px.
+const LLM_TARGET_WIDTH = 1280;
+class NativeDesktop extends events_1.EventEmitter {
+    config;
+    screenWidth = 0;
+    screenHeight = 0;
+    connected = false;
+    monitors = [];
+    helper = IS_MAC ? (0, native_helper_1.getNativeHelper)() : null;
+    /** Scale factor: LLM coordinates × scaleFactor = real screen coordinates */
+    scaleFactor = 1;
+    /**
+     * DPI ratio: physical pixels / logical (mouse) pixels.
+     * OCR coordinates (physical) / dpiRatio = mouse coordinates (logical).
+     * Detected at connect() time via System.Windows.Forms.
+     */
+    dpiRatio = 1;
+    constructor(config) {
+        super();
+        this.config = config;
+    }
+    /**
+     * Enumerate all connected monitors with their positions and sizes.
+     * Returns best-effort results — falls back to primary only on errors.
+     */
+    async getMonitors() {
+        if (this.monitors.length > 0)
+            return this.monitors;
+        try {
+            if (process.platform === 'win32') {
+                const { exec } = await Promise.resolve().then(() => __importStar(require('child_process')));
+                const { promisify } = await Promise.resolve().then(() => __importStar(require('util')));
+                const execAsync = promisify(exec);
+                const ps = `Add-Type -AssemblyName System.Windows.Forms; [System.Windows.Forms.Screen]::AllScreens | ForEach-Object { "$($_.Bounds.X),$($_.Bounds.Y),$($_.Bounds.Width),$($_.Bounds.Height),$($_.Primary),$($_.DeviceName)" }`;
+                const { stdout } = await execAsync(`powershell -NoProfile -Command "${ps}"`);
+                const lines = stdout.trim().split('\n').filter(Boolean);
+                this.monitors = lines.map((line, i) => {
+                    const [x, y, w, h, primary, name] = line.trim().split(',');
+                    return {
+                        index: i,
+                        x: parseInt(x), y: parseInt(y),
+                        width: parseInt(w), height: parseInt(h),
+                        primary: primary.trim().toLowerCase() === 'true',
+                        name: name?.trim() || `Monitor ${i + 1}`,
+                    };
+                });
+            }
+            else if (process.platform === 'darwin') {
+                const { exec } = await Promise.resolve().then(() => __importStar(require('child_process')));
+                const { promisify } = await Promise.resolve().then(() => __importStar(require('util')));
+                const execAsync = promisify(exec);
+                // system_profiler gives display info; for bounds we use osascript
+                const { stdout } = await execAsync(`osascript -e 'tell application "System Events" to get bounds of every desktop'`).catch(() => ({ stdout: '' }));
+                if (stdout.trim()) {
+                    // fallback: just return primary
+                    this.monitors = [{ index: 0, x: 0, y: 0, width: this.screenWidth, height: this.screenHeight, primary: true, name: 'Primary' }];
+                }
+                else {
+                    this.monitors = [{ index: 0, x: 0, y: 0, width: this.screenWidth, height: this.screenHeight, primary: true, name: 'Primary' }];
+                }
+            }
+            else {
+                // Linux: use xrandr
+                const { exec } = await Promise.resolve().then(() => __importStar(require('child_process')));
+                const { promisify } = await Promise.resolve().then(() => __importStar(require('util')));
+                const execAsync = promisify(exec);
+                const { stdout } = await execAsync('xrandr --query 2>/dev/null').catch(() => ({ stdout: '' }));
+                const re = /(\S+) connected(?: primary)? (\d+)x(\d+)\+(\d+)\+(\d+)/g;
+                let m;
+                let i = 0;
+                const results = [];
+                while ((m = re.exec(stdout)) !== null) {
+                    results.push({ index: i++, x: parseInt(m[4]), y: parseInt(m[5]), width: parseInt(m[2]), height: parseInt(m[3]), primary: stdout.includes(m[1] + ' connected primary'), name: m[1] });
+                }
+                this.monitors = results.length > 0 ? results : [{ index: 0, x: 0, y: 0, width: this.screenWidth, height: this.screenHeight, primary: true, name: 'Primary' }];
+            }
+        }
+        catch {
+            this.monitors = [{ index: 0, x: 0, y: 0, width: this.screenWidth, height: this.screenHeight, primary: true, name: 'Primary' }];
+        }
+        return this.monitors;
+    }
+    /**
+     * Capture a specific monitor by index.
+     * Falls back to primary grab if region capture fails.
+     */
+    async captureMonitor(monitorIndex = 0) {
+        const monitors = await this.getMonitors();
+        const mon = monitors[monitorIndex] ?? monitors.find(m => m.primary) ?? monitors[0];
+        if (!mon)
+            return this.captureForLLM();
+        try {
+            const { Region } = await Promise.resolve().then(() => __importStar(require('@nut-tree-fork/nut-js')));
+            const region = new Region(mon.x, mon.y, mon.width, mon.height);
+            const img = await nut_js_1.screen.grabRegion(region);
+            const scaleFactor = mon.width > LLM_TARGET_WIDTH ? mon.width / LLM_TARGET_WIDTH : 1;
+            const llmW = Math.round(mon.width / scaleFactor);
+            const llmH = Math.round(mon.height / scaleFactor);
+            const processed = await (0, sharp_1.default)(img.data, { raw: { width: img.width, height: img.height, channels: 4 } })
+                .resize(llmW, llmH)
+                .png()
+                .toBuffer();
+            // Release the raw RGBA buffer immediately after processing
+            img.data = null;
+            return { width: mon.width, height: mon.height, buffer: processed, timestamp: Date.now(), format: 'png', scaleFactor, llmWidth: llmW, llmHeight: llmH };
+        }
+        catch {
+            // Fallback: full primary grab
+            return this.captureForLLM();
+        }
+    }
+    /**
+     * "Connect" to the native desktop — detects screen size and configures nut-js.
+     * No actual network connection; just initializes the local screen interface.
+     */
+    async connect() {
+        try {
+            if (IS_MAC) {
+                // Use the standalone screenshot-helper binary — avoids ReplayKit CPU spin
+                // bug and runs in an isolated subprocess for clean TCC permission scoping.
+                try {
+                    const result = await (0, native_helper_1.captureScreenViaHelper)();
+                    this.screenWidth = result.width;
+                    this.screenHeight = result.height;
+                    this.scaleFactor = this.screenWidth > LLM_TARGET_WIDTH ? this.screenWidth / LLM_TARGET_WIDTH : 1;
+                    // Clean up the temp file from connect probe
+                    try {
+                        fs.unlinkSync(result.path);
+                    }
+                    catch { /* ignore */ }
+                    this.connected = true;
+                    console.log(`🐾 Native desktop connected (macOS screenshot-helper)`);
+                    console.log(`   Screen: ${this.screenWidth}x${this.screenHeight}`);
+                    console.log(`   LLM scale factor: ${this.scaleFactor.toFixed(2)}x`);
+                    return;
+                }
+                catch (err) {
+                    console.warn(`⚠️  macOS screenshot-helper failed, falling back to nut-js: ${err?.message || err}`);
+                }
+            }
+            // Configure nut-js for speed
+            nut_js_1.mouse.config.mouseSpeed = 2000; // Fast mouse movement
+            nut_js_1.mouse.config.autoDelayMs = 0; // No auto-delay between actions
+            nut_js_1.keyboard.config.autoDelayMs = 0; // No auto-delay between keystrokes
+            // Grab a screenshot to determine screen dimensions
+            const img = await nut_js_1.screen.grab();
+            this.screenWidth = img.width;
+            this.screenHeight = img.height;
+            // Calculate scale factor
+            if (this.screenWidth > LLM_TARGET_WIDTH) {
+                this.scaleFactor = this.screenWidth / LLM_TARGET_WIDTH;
+            }
+            else {
+                this.scaleFactor = 1;
+            }
+            // Detect DPI ratio (physical / logical) for OCR coordinate conversion.
+            // On Windows, System.Windows.Forms.Screen returns logical (DPI-scaled) dimensions,
+            // while screen.grab() returns physical pixels. Mouse API uses logical coords.
+            if (process.platform === 'win32') {
+                try {
+                    const { execFileSync } = await Promise.resolve().then(() => __importStar(require('child_process')));
+                    const result = execFileSync('powershell.exe', [
+                        '-NoProfile', '-Command',
+                        "Add-Type -AssemblyName System.Windows.Forms; $s=[System.Windows.Forms.Screen]::PrimaryScreen.Bounds; \"$($s.Width),$($s.Height)\"",
+                    ], { timeout: 10000, encoding: 'utf-8' }).trim();
+                    const [logicalW] = result.split(',').map(Number);
+                    if (logicalW > 0 && logicalW < this.screenWidth) {
+                        this.dpiRatio = this.screenWidth / logicalW;
+                    }
+                }
+                catch { /* non-fatal — dpiRatio stays 1 */ }
+            }
+            else if (process.platform === 'darwin') {
+                try {
+                    const { execFileSync } = await Promise.resolve().then(() => __importStar(require('child_process')));
+                    // NSScreen reports in logical (point) dimensions — compare with physical pixels from screen.grab()
+                    // TODO: Multi-monitor support — currently uses mainScreen only. For multi-monitor,
+                    // enumerate NSScreen.screens and sum widths to get the full virtual canvas size.
+                    const result = execFileSync('osascript', ['-e',
+                        'use framework "AppKit"\nreturn (current application\'s NSScreen\'s mainScreen\'s frame()\'s size\'s width) as integer',
+                    ], { timeout: 5000, encoding: 'utf-8' }).trim();
+                    const logicalW = parseInt(result);
+                    if (logicalW > 0 && logicalW < this.screenWidth) {
+                        this.dpiRatio = this.screenWidth / logicalW;
+                    }
+                }
+                catch { /* non-fatal — dpiRatio stays 1 */ }
+            }
+            else if (process.platform === 'linux') {
+                try {
+                    // Check common DE scale environment variables first
+                    const gdkScale = parseInt(process.env.GDK_SCALE || '1');
+                    const qtScale = parseFloat(process.env.QT_SCALE_FACTOR || '1');
+                    const envScale = Math.max(gdkScale, qtScale);
+                    if (envScale > 1) {
+                        this.dpiRatio = envScale;
+                    }
+                    else {
+                        const { execFileSync } = await Promise.resolve().then(() => __importStar(require('child_process')));
+                        const output = execFileSync('xrandr', ['--query'], { timeout: 5000, encoding: 'utf-8' });
+                        const match = output.match(/primary\s+(\d+)x(\d+)/);
+                        if (match) {
+                            const logicalW = parseInt(match[1]);
+                            if (logicalW > 0 && logicalW < this.screenWidth) {
+                                this.dpiRatio = this.screenWidth / logicalW;
+                            }
+                        }
+                    }
+                }
+                catch { /* non-fatal — dpiRatio stays 1 */ }
+            }
+            this.connected = true;
+            console.log(`🐾 Native desktop connected`);
+            console.log(`   Screen: ${this.screenWidth}x${this.screenHeight}`);
+            console.log(`   LLM scale factor: ${this.scaleFactor.toFixed(2)}x`);
+            if (this.dpiRatio > 1) {
+                console.log(`   DPI ratio: ${this.dpiRatio.toFixed(2)}x (physical/logical)`);
+            }
+        }
+        catch (err) {
+            console.error('Native desktop init error:', err?.message);
+            this.connected = false;
+            throw err;
+        }
+    }
+    /**
+     * Capture a full-resolution screenshot.
+     */
+    async captureScreen() {
+        if (!this.connected) {
+            throw new Error('Not connected to native desktop');
+        }
+        if (IS_MAC) {
+            try {
+                const result = await (0, native_helper_1.captureScreenViaHelper)();
+                const buffer = fs.readFileSync(result.path);
+                try {
+                    fs.unlinkSync(result.path);
+                }
+                catch { /* cleanup */ }
+                this.screenWidth = result.width;
+                this.screenHeight = result.height;
+                return {
+                    width: result.width,
+                    height: result.height,
+                    buffer,
+                    timestamp: Date.now(),
+                    format: 'png',
+                };
+            }
+            catch (err) {
+                console.warn(`⚠️  macOS screenshot-helper failed, falling back to nut-js: ${err?.message || err}`);
+            }
+        }
+        const img = await nut_js_1.screen.grab();
+        // Update screen dimensions in case of resolution change
+        this.screenWidth = img.width;
+        this.screenHeight = img.height;
+        const processed = await this.processFrame(img.data, img.width, img.height, this.screenWidth, this.screenHeight);
+        // Release the raw RGBA buffer immediately after processing
+        img.data = null;
+        return {
+            width: this.screenWidth,
+            height: this.screenHeight,
+            buffer: processed,
+            timestamp: Date.now(),
+            format: this.config.capture.format,
+        };
+    }
+    /**
+     * Capture a RESIZED screenshot optimized for LLM vision.
+     * - Resized to 1280px wide (or less if screen is smaller)
+     * - Much smaller payload = fewer tokens = faster API calls
+     * - Returns scaleFactor so coordinates in AI response can be mapped back
+     */
+    async captureForLLM() {
+        if (!this.connected) {
+            throw new Error('Not connected to native desktop');
+        }
+        if (IS_MAC) {
+            try {
+                const frame = await this.captureScreen();
+                this.screenWidth = frame.width;
+                this.screenHeight = frame.height;
+                this.scaleFactor = this.screenWidth > LLM_TARGET_WIDTH ? this.screenWidth / LLM_TARGET_WIDTH : 1;
+                const llmWidth = Math.min(this.screenWidth, LLM_TARGET_WIDTH);
+                const llmHeight = Math.round(this.screenHeight / this.scaleFactor);
+                const pipeline = (0, sharp_1.default)(frame.buffer).resize(llmWidth, llmHeight);
+                const processed = this.config.capture.format === 'jpeg'
+                    ? await pipeline.jpeg({ quality: this.config.capture.quality }).toBuffer()
+                    : await pipeline.png().toBuffer();
+                return {
+                    width: this.screenWidth,
+                    height: this.screenHeight,
+                    buffer: processed,
+                    timestamp: Date.now(),
+                    format: this.config.capture.format,
+                    scaleFactor: this.scaleFactor,
+                    llmWidth,
+                    llmHeight,
+                };
+            }
+            catch (err) {
+                console.warn(`⚠️  macOS screenshot-helper LLM capture failed, falling back to nut-js: ${err?.message || err}`);
+            }
+        }
+        const img = await nut_js_1.screen.grab();
+        // Update screen dimensions
+        this.screenWidth = img.width;
+        this.screenHeight = img.height;
+        // Recalculate scale factor in case resolution changed
+        if (this.screenWidth > LLM_TARGET_WIDTH) {
+            this.scaleFactor = this.screenWidth / LLM_TARGET_WIDTH;
+        }
+        else {
+            this.scaleFactor = 1;
+        }
+        const llmWidth = Math.min(this.screenWidth, LLM_TARGET_WIDTH);
+        const llmHeight = Math.round(this.screenHeight / this.scaleFactor);
+        const processed = await this.processFrame(img.data, img.width, img.height, llmWidth, llmHeight);
+        // Release the raw RGBA buffer immediately after processing
+        img.data = null;
+        return {
+            width: this.screenWidth, // real screen width
+            height: this.screenHeight, // real screen height
+            buffer: processed,
+            timestamp: Date.now(),
+            format: this.config.capture.format,
+            scaleFactor: this.scaleFactor,
+            llmWidth,
+            llmHeight,
+        };
+    }
+    /**
+     * Capture a CROPPED region of the screen, resized for LLM.
+     * Coordinates are in REAL screen pixels.
+     * Returns the cropped image at higher effective resolution (more detail per pixel).
+     * @future — not yet used; intended for focused region analysis
+     */
+    async captureRegionForLLM(x, y, w, h) {
+        if (!this.connected)
+            throw new Error('Not connected');
+        if (IS_MAC && this.helper) {
+            const full = await this.captureScreen();
+            const rx = Math.max(0, Math.min(x, full.width - 1));
+            const ry = Math.max(0, Math.min(y, full.height - 1));
+            const rw = Math.min(w, full.width - rx);
+            const rh = Math.min(h, full.height - ry);
+            const cropScale = rw > LLM_TARGET_WIDTH ? rw / LLM_TARGET_WIDTH : 1;
+            const llmWidth = Math.min(rw, LLM_TARGET_WIDTH);
+            const llmHeight = Math.round(rh / cropScale);
+            const { format, quality } = this.config.capture;
+            let pipeline = (0, sharp_1.default)(full.buffer).extract({ left: rx, top: ry, width: rw, height: rh });
+            if (llmWidth < rw) {
+                pipeline = pipeline.resize(llmWidth, llmHeight, { fit: 'fill', kernel: 'lanczos3' });
+            }
+            const buffer = format === 'jpeg' ? await pipeline.jpeg({ quality }).toBuffer() : await pipeline.png().toBuffer();
+            return { width: rw, height: rh, buffer, timestamp: Date.now(), format, scaleFactor: cropScale, llmWidth, llmHeight, regionX: rx, regionY: ry };
+        }
+        const img = await nut_js_1.screen.grab();
+        // Clamp to screen bounds
+        const rx = Math.max(0, Math.min(x, img.width - 1));
+        const ry = Math.max(0, Math.min(y, img.height - 1));
+        const rw = Math.min(w, img.width - rx);
+        const rh = Math.min(h, img.height - ry);
+        // Scale crop to LLM-sized output (max 1280px wide)
+        const cropScale = rw > LLM_TARGET_WIDTH ? rw / LLM_TARGET_WIDTH : 1;
+        const llmWidth = Math.min(rw, LLM_TARGET_WIDTH);
+        const llmHeight = Math.round(rh / cropScale);
+        const { format, quality } = this.config.capture;
+        let pipeline = (0, sharp_1.default)(img.data, {
+            raw: { width: img.width, height: img.height, channels: 4 },
+        }).extract({ left: rx, top: ry, width: rw, height: rh });
+        if (llmWidth < rw) {
+            pipeline = pipeline.resize(llmWidth, llmHeight, { fit: 'fill', kernel: 'lanczos3' });
+        }
+        const buffer = format === 'jpeg'
+            ? await pipeline.jpeg({ quality }).toBuffer()
+            : await pipeline.png().toBuffer();
+        // Release the raw RGBA buffer immediately after processing
+        img.data = null;
+        return {
+            width: rw,
+            height: rh,
+            buffer,
+            timestamp: Date.now(),
+            format,
+            scaleFactor: cropScale,
+            llmWidth,
+            llmHeight,
+            regionX: rx,
+            regionY: ry,
+        };
+    }
+    /**
+     * Get the scaling factor (LLM pixels → real screen pixels)
+     */
+    getScaleFactor() {
+        return this.scaleFactor;
+    }
+    /**
+     * Get the DPI ratio (physical pixels / logical mouse pixels).
+     * Returns 1 on non-HiDPI screens or non-Windows platforms.
+     */
+    getDpiRatio() {
+        return this.dpiRatio;
+    }
+    /**
+     * Convert physical pixel coordinates (from OCR/screenshot) to mouse coordinates.
+     * On Windows with DPI scaling, nut-js mouse API uses logical (DPI-scaled) coords,
+     * while screen.grab() returns physical pixels. This method bridges the gap.
+     */
+    physicalToMouse(x, y) {
+        if (this.dpiRatio <= 1)
+            return { x, y };
+        return {
+            x: Math.round(x / this.dpiRatio),
+            y: Math.round(y / this.dpiRatio),
+        };
+    }
+    /**
+     * Process a raw RGBA buffer into the configured output format.
+     * nut-js screen.grab() returns RGBA data directly — no BGRA swap needed.
+     */
+    async processFrame(rawData, srcWidth, srcHeight, targetWidth, targetHeight) {
+        const { format, quality } = this.config.capture;
+        let pipeline = (0, sharp_1.default)(rawData, {
+            raw: {
+                width: srcWidth,
+                height: srcHeight,
+                channels: 4,
+            },
+        });
+        // Resize if target is smaller than source
+        if (targetWidth < srcWidth || targetHeight < srcHeight) {
+            pipeline = pipeline.resize(targetWidth, targetHeight, {
+                fit: 'fill',
+                kernel: 'lanczos3',
+            });
+        }
+        if (format === 'jpeg') {
+            return pipeline.jpeg({ quality }).toBuffer();
+        }
+        return pipeline.png().toBuffer();
+    }
+    // --- Input Methods ---
+    async mouseClick(x, y, button = 1) {
+        if (!this.connected)
+            throw new Error('Not connected');
+        // On macOS: skip the Swift helper (CGEvent blocked by TCC), use nut-js directly.
+        // nut-js mouse events ARE delivered on macOS (unlike CGEvent from child processes).
+        await nut_js_1.mouse.setPosition(new nut_js_1.Point(x, y));
+        await this.delay(50);
+        const btn = this.mapButton(button);
+        await nut_js_1.mouse.click(btn);
+    }
+    async mouseDoubleClick(x, y) {
+        if (!this.connected)
+            throw new Error('Not connected');
+        await nut_js_1.mouse.setPosition(new nut_js_1.Point(x, y));
+        await this.delay(50);
+        await nut_js_1.mouse.doubleClick(nut_js_1.Button.LEFT);
+        console.log(`   🖱️  Double-click at (${x}, ${y})`);
+    }
+    async mouseRightClick(x, y) {
+        if (!this.connected)
+            throw new Error('Not connected');
+        await nut_js_1.mouse.setPosition(new nut_js_1.Point(x, y));
+        await this.delay(50);
+        await nut_js_1.mouse.rightClick();
+    }
+    async mouseMove(x, y) {
+        if (!this.connected)
+            throw new Error('Not connected');
+        await nut_js_1.mouse.setPosition(new nut_js_1.Point(x, y));
+    }
+    async mouseScroll(x, y, delta) {
+        if (!this.connected)
+            throw new Error('Not connected');
+        await nut_js_1.mouse.setPosition(new nut_js_1.Point(x, y));
+        await this.delay(30);
+        const steps = Math.abs(Math.round(delta));
+        for (let i = 0; i < steps; i++) {
+            if (delta > 0) {
+                await nut_js_1.mouse.scrollDown(3);
+            }
+            else {
+                await nut_js_1.mouse.scrollUp(3);
+            }
+            await this.delay(30);
+        }
+        console.log(`   🖱️  Scroll at (${x}, ${y}) delta=${delta}`);
+    }
+    async typeText(text) {
+        if (!this.connected)
+            throw new Error('Not connected');
+        if (IS_MAC) {
+            // Use System Events for typing — same reason as keyPress:
+            // CGEvent from helper subprocess is silently blocked by TCC.
+            const { execFile } = await Promise.resolve().then(() => __importStar(require('child_process')));
+            const { promisify } = await Promise.resolve().then(() => __importStar(require('util')));
+            const execFileAsync = promisify(execFile);
+            const escaped = text.replace(/\\/g, '\\\\').replace(/"/g, '\\"');
+            try {
+                await execFileAsync('osascript', ['-e',
+                    `tell application "System Events" to keystroke "${escaped}"`
+                ], { timeout: 10000 });
+            }
+            catch (err) {
+                console.warn(`   ⌨️  macOS type failed: ${err.message?.substring(0, 100)}`);
+                // Fallback: try helper anyway
+                if (this.helper) {
+                    await this.helper.type(text);
+                }
+            }
+            return;
+        }
+        await nut_js_1.keyboard.type(text);
+        console.log(`   ⌨️  Typed: "${text.substring(0, 60)}${text.length > 60 ? '...' : ''}"`);
+    }
+    async keyPress(keyCombo) {
+        if (!this.connected)
+            throw new Error('Not connected');
+        if (IS_MAC) {
+            // macOS: use System Events via osascript for reliable keystroke delivery.
+            // CGEvent.post() from a spawned helper is silently blocked by TCC on macOS 15+
+            // because the helper inherits the parent's TCC context but CGEvent posting
+            // requires the calling process itself to be trusted for input.
+            // System Events is the proven, reliable method for keyboard automation on macOS.
+            await this.macKeyPress(keyCombo);
+            return;
+        }
+        // Special case: literal "+" character (can't split on "+" since it IS the separator)
+        if (keyCombo === '+') {
+            await nut_js_1.keyboard.type('+');
+            await this.delay(30);
+            console.log(`   ⌨️  Key press: +`);
+            return;
+        }
+        const parts = keyCombo.split('+').map(k => k.trim()).filter(k => k.length > 0);
+        const keys = parts.map(k => this.mapKey(k));
+        // If the only key is a TYPE_CHAR (single printable char like *, +, ., etc.),
+        // use keyboard.type() which handles shift combos automatically
+        if (keys.length === 1 && keys[0] === 'TYPE_CHAR') {
+            await nut_js_1.keyboard.type(parts[0]);
+            await this.delay(30);
+        }
+        else if (keys.length === 1) {
+            await nut_js_1.keyboard.pressKey(keys[0]);
+            await this.delay(30);
+            await nut_js_1.keyboard.releaseKey(keys[0]);
+        }
+        else {
+            // Press all modifier keys down, then the final key, then release in reverse
+            for (const key of keys) {
+                if (key === 'TYPE_CHAR') {
+                    await nut_js_1.keyboard.type(parts[keys.indexOf(key)]);
+                }
+                else {
+                    await nut_js_1.keyboard.pressKey(key);
+                }
+                await this.delay(30);
+            }
+            for (const key of [...keys].reverse()) {
+                if (key !== 'TYPE_CHAR') {
+                    await nut_js_1.keyboard.releaseKey(key);
+                }
+                await this.delay(30);
+            }
+        }
+        console.log(`   ⌨️  Key press: ${keyCombo}`);
+    }
+    /**
+     * macOS keyboard input via System Events (osascript).
+     * Reliable because System Events has its own TCC grant for input,
+     * unlike CGEvent.post() from a spawned child process.
+     */
+    async macKeyPress(keyCombo) {
+        const { execFile } = await Promise.resolve().then(() => __importStar(require('child_process')));
+        const { promisify } = await Promise.resolve().then(() => __importStar(require('util')));
+        const execFileAsync = promisify(execFile);
+        if (keyCombo === '+') {
+            await execFileAsync('osascript', ['-e', 'tell application "System Events" to keystroke "+"']);
+            return;
+        }
+        const parts = keyCombo.split('+').map(k => k.trim()).filter(Boolean);
+        const key = parts[parts.length - 1] || keyCombo;
+        if (!key) {
+            throw new Error(`macKeyPress: empty key after parsing combo "${keyCombo}"`);
+        }
+        const mods = parts.slice(0, -1).map(k => k.toLowerCase());
+        // Map modifier names to System Events syntax.
+        // `mod` is the platform-aware "primary" modifier that resolves to Cmd
+        // on macOS — without this branch macKeyPress would silently type the
+        // bare letter (e.g. `mod+s` becomes a literal `s` keystroke).
+        const modUsing = [];
+        for (const m of mods) {
+            if (m === 'cmd' || m === 'command' || m === 'super' || m === 'mod')
+                modUsing.push('command down');
+            else if (m === 'shift')
+                modUsing.push('shift down');
+            else if (m === 'alt' || m === 'option')
+                modUsing.push('option down');
+            else if (m === 'ctrl' || m === 'control')
+                modUsing.push('control down');
+        }
+        // Map special key names to System Events key code actions
+        const specialKeys = {
+            'return': 'key code 36', 'enter': 'key code 36',
+            'tab': 'key code 48',
+            'escape': 'key code 53', 'esc': 'key code 53',
+            'delete': 'key code 51', 'backspace': 'key code 51',
+            'space': 'key code 49',
+            'left': 'key code 123', 'right': 'key code 124',
+            'down': 'key code 125', 'up': 'key code 126',
+            'f1': 'key code 122', 'f2': 'key code 120', 'f3': 'key code 99',
+            'f4': 'key code 118', 'f5': 'key code 96', 'f6': 'key code 97',
+            'f7': 'key code 98', 'f8': 'key code 100', 'f9': 'key code 101',
+            'f10': 'key code 109', 'f11': 'key code 103', 'f12': 'key code 111',
+            'pageup': 'key code 116', 'pagedown': 'key code 121',
+            'home': 'key code 115', 'end': 'key code 119',
+            'forwarddelete': 'key code 117',
+        };
+        let script;
+        const special = specialKeys[key.toLowerCase()];
+        if (special) {
+            // Special key — use key code
+            if (modUsing.length > 0) {
+                script = `tell application "System Events" to ${special} using {${modUsing.join(', ')}}`;
+            }
+            else {
+                script = `tell application "System Events" to ${special}`;
+            }
+        }
+        else if (key.length === 1) {
+            // Single character — use keystroke
+            const escaped = key.replace(/\\/g, '\\\\').replace(/"/g, '\\"');
+            if (modUsing.length > 0) {
+                script = `tell application "System Events" to keystroke "${escaped}" using {${modUsing.join(', ')}}`;
+            }
+            else {
+                script = `tell application "System Events" to keystroke "${escaped}"`;
+            }
+        }
+        else {
+            // Unknown key — try keystroke as-is
+            const escaped = key.replace(/\\/g, '\\\\').replace(/"/g, '\\"');
+            script = `tell application "System Events" to keystroke "${escaped}"`;
+        }
+        try {
+            await execFileAsync('osascript', ['-e', script], { timeout: 5000 });
+        }
+        catch (err) {
+            console.warn(`   ⌨️  macOS key press failed: ${err.message?.substring(0, 100)}`);
+        }
+    }
+    async executeMouseAction(action) {
+        switch (action.kind) {
+            case 'click':
+                await this.mouseClick(action.x, action.y);
+                break;
+            case 'double_click':
+                await this.mouseDoubleClick(action.x, action.y);
+                break;
+            case 'right_click':
+                await this.mouseRightClick(action.x, action.y);
+                break;
+            case 'move':
+                await this.mouseMove(action.x, action.y);
+                break;
+            case 'scroll':
+                await this.mouseScroll(action.x, action.y, action.scrollDelta || 3);
+                break;
+            case 'drag':
+                await this.mouseDrag(action.x, action.y, action.endX || action.x, action.endY || action.y);
+                break;
+        }
+    }
+    async executeKeyboardAction(action) {
+        switch (action.kind) {
+            case 'type':
+                if (action.text)
+                    await this.typeText(action.text);
+                break;
+            case 'key_press':
+                if (action.key)
+                    await this.keyPress(action.key);
+                break;
+        }
+    }
+    isConnected() {
+        return this.connected;
+    }
+    // ─── Low-level key control (for Computer Use API hold_key) ────────
+    async keyDown(keyCombo) {
+        if (!this.connected)
+            throw new Error('Not connected');
+        console.log(`   ⌨️  Key down: ${keyCombo}`);
+        if (keyCombo === '+') {
+            await nut_js_1.keyboard.type('+');
+            return;
+        }
+        const parts = keyCombo.split('+').map(k => k.trim()).filter(k => k.length > 0);
+        for (const k of parts) {
+            const key = this.mapKey(k);
+            if (key === 'TYPE_CHAR') {
+                await nut_js_1.keyboard.type(k);
+            }
+            else {
+                await nut_js_1.keyboard.pressKey(key);
+            }
+            await this.delay(20);
+        }
+    }
+    async keyUp(keyCombo) {
+        if (!this.connected)
+            throw new Error('Not connected');
+        console.log(`   ⌨️  Key up: ${keyCombo}`);
+        if (keyCombo === '+')
+            return; // type() already released
+        const parts = keyCombo.split('+').map(k => k.trim()).filter(k => k.length > 0);
+        for (const k of [...parts].reverse()) {
+            const key = this.mapKey(k);
+            if (key !== 'TYPE_CHAR') {
+                await nut_js_1.keyboard.releaseKey(key);
+            }
+            await this.delay(20);
+        }
+    }
+    // ─── Low-level pointer control (for Computer Use API) ────────────
+    async mouseDown(x, y, button = 1) {
+        if (!this.connected)
+            throw new Error('Not connected');
+        console.log(`   🖱️  Mouse down at (${x}, ${y})`);
+        await nut_js_1.mouse.setPosition(new nut_js_1.Point(x, y));
+        const btn = this.mapButton(button);
+        await nut_js_1.mouse.pressButton(btn);
+    }
+    async mouseUp(x, y, button = 1) {
+        if (!this.connected)
+            throw new Error('Not connected');
+        console.log(`   🖱️  Mouse up at (${x}, ${y})`);
+        await nut_js_1.mouse.setPosition(new nut_js_1.Point(x, y));
+        const btn = this.mapButton(button);
+        await nut_js_1.mouse.releaseButton(btn);
+    }
+    async mouseDrag(sx, sy, ex, ey) {
+        if (!this.connected)
+            throw new Error('Not connected');
+        // nut-js drag works on all platforms including macOS
+        console.log(`   🖱️  Drag (${sx},${sy}) → (${ex},${ey})`);
+        await nut_js_1.mouse.setPosition(new nut_js_1.Point(sx, sy));
+        await this.delay(50);
+        await nut_js_1.mouse.pressButton(nut_js_1.Button.LEFT);
+        await this.delay(100);
+        // Interpolate intermediate points for smoother drag
+        const steps = Math.max(5, Math.floor(Math.hypot(ex - sx, ey - sy) / 20));
+        for (let i = 1; i <= steps; i++) {
+            const t = i / steps;
+            const ix = Math.round(sx + (ex - sx) * t);
+            const iy = Math.round(sy + (ey - sy) * t);
+            await nut_js_1.mouse.setPosition(new nut_js_1.Point(ix, iy));
+            await this.delay(15);
+        }
+        await nut_js_1.mouse.releaseButton(nut_js_1.Button.LEFT);
+    }
+    getScreenSize() {
+        return { width: this.screenWidth, height: this.screenHeight };
+    }
+    disconnect() {
+        this.connected = false;
+        this.screenWidth = 0;
+        this.screenHeight = 0;
+        this.emit('disconnected');
+        // Remove all listeners so this instance can be GCd after disconnect.
+        // Must come after emit so 'disconnected' handlers still fire.
+        this.removeAllListeners();
+        console.log('🐾 Native desktop disconnected');
+    }
+    // ─── Private helpers ──────────────────────────────────────────────
+    /**
+     * Map a button number to nut-js Button enum.
+     * 1=left, 2=middle, 4=right
+     */
+    mapButton(buttonId) {
+        switch (buttonId) {
+            case 1: return nut_js_1.Button.LEFT;
+            case 2: return nut_js_1.Button.MIDDLE;
+            case 4: return nut_js_1.Button.RIGHT;
+            default: return nut_js_1.Button.LEFT;
+        }
+    }
+    /**
+     * Map a string key name to nut-js Key enum value.
+     * Falls back to character-based lookup for single characters.
+     */
+    mapKey(keyName) {
+        // Normalize via canonical key names first
+        const normalized = (0, keys_1.normalizeKey)(keyName);
+        // Direct lookup in our map
+        const mapped = KEY_MAP[normalized];
+        if (mapped !== undefined)
+            return mapped;
+        // Single character — try to find matching Key enum entry
+        if (keyName.length === 1) {
+            const upper = keyName.toUpperCase();
+            // Letters A-Z
+            if (upper >= 'A' && upper <= 'Z') {
+                const keyEntry = nut_js_1.Key[upper];
+                if (keyEntry !== undefined)
+                    return keyEntry;
+            }
+            // Digits 0-9
+            if (upper >= '0' && upper <= '9') {
+                const numKey = `Num${upper}`;
+                const keyEntry = nut_js_1.Key[numKey];
+                if (keyEntry !== undefined)
+                    return keyEntry;
+            }
+            // Single printable character (symbols like *, +, -, ., etc.)
+            // Use keyboard.type() for these — it handles shift combos automatically
+            if (keyName.charCodeAt(0) >= 32 && keyName.charCodeAt(0) <= 126) {
+                return 'TYPE_CHAR';
+            }
+        }
+        // Last resort: try exact enum name match
+        const enumKey = keyName;
+        if (nut_js_1.Key[enumKey] !== undefined)
+            return nut_js_1.Key[enumKey];
+        throw new Error(`Unknown key: "${keyName}" — no mapping found in KEY_MAP or Key enum`);
+    }
+    delay(ms) {
+        return new Promise(resolve => setTimeout(resolve, ms));
+    }
+}
+exports.NativeDesktop = NativeDesktop;
+//# sourceMappingURL=native-desktop.js.map