ft-scout 9.0.4 → 9.0.5

package/bin/src/engine/computer/computerController.js

@@ -1,13 +1,19 @@
 import fs from 'fs';
 import path from 'path';
-import { exec } from 'child_process';
+import { exec, execFile, spawn } from 'child_process';
 import { promisify } from 'util';
+import { fileURLToPath } from 'url';
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
 const execAsync = promisify(exec);
+const execFileAsync = promisify(execFile);
 export class SystemComputerController {
     workspaceDir;
     screenshotsDir;
     expectedApplication = null;
     activeWindow = null;
+    driverExePath = null;
+    driverInitAttempted = false;
     constructor(workspaceDir = process.cwd()) {
         this.workspaceDir = workspaceDir;
         this.screenshotsDir = path.join(this.workspaceDir, '.ft', 'screenshots');
@@ -24,47 +30,224 @@ export class SystemComputerController {
     getExpectedApplication() {
         return this.expectedApplication;
     }
-    async runPowerShell(script) {
-        const encoded = Buffer.from(script, 'utf16le').toString('base64');
-        const { stdout } = await execAsync(`powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -EncodedCommand ${encoded}`);
-        return (stdout || '').trim();
+    /**
+     * Resolves or builds the native scout_driver.exe for Windows.
+     */
+    async ensureDriver() {
+        if (this.driverExePath && fs.existsSync(this.driverExePath)) {
+            return this.driverExePath;
+        }
+        if (process.platform !== 'win32') {
+            return null;
+        }
+        // Candidate locations
+        const candidates = [
+            path.join(__dirname, 'scout_driver.exe'),
+            path.join(this.workspaceDir, 'bin', 'src', 'engine', 'computer', 'scout_driver.exe'),
+            path.join(this.workspaceDir, '.ft', 'bin', 'scout_driver.exe'),
+        ];
+        for (const c of candidates) {
+            if (fs.existsSync(c)) {
+                this.driverExePath = c;
+                return c;
+            }
+        }
+        if (this.driverInitAttempted) {
+            return null;
+        }
+        this.driverInitAttempted = true;
+        // Try compiling using csc.exe
+        const csSourceCandidates = [
+            path.join(__dirname, 'scoutDriver.cs'),
+            path.join(this.workspaceDir, 'bin', 'src', 'engine', 'computer', 'scoutDriver.cs'),
+        ];
+        const csSource = csSourceCandidates.find((p) => fs.existsSync(p));
+        const cscExe = 'C:\\Windows\\Microsoft.NET\\Framework64\\v4.0.30319\\csc.exe';
+        if (csSource && fs.existsSync(cscExe)) {
+            const outDir = path.join(this.workspaceDir, '.ft', 'bin');
+            try {
+                if (!fs.existsSync(outDir))
+                    fs.mkdirSync(outDir, { recursive: true });
+                const targetExe = path.join(outDir, 'scout_driver.exe');
+                await execAsync(`"${cscExe}" /nologo /optimize /t:exe /out:"${targetExe}" /r:System.Windows.Forms.dll,System.Drawing.dll "${csSource}"`);
+                if (fs.existsSync(targetExe)) {
+                    this.driverExePath = targetExe;
+                    return targetExe;
+                }
+            }
+            catch { }
+        }
+        return null;
+    }
+    /**
+     * Runs an action using native driver or fallback.
+     */
+    async runDriverAction(action, args = []) {
+        const driver = await this.ensureDriver();
+        const strArgs = args.map((a) => String(a));
+        if (driver) {
+            try {
+                const { stdout } = await execFileAsync(driver, [action, ...strArgs], {
+                    windowsHide: true,
+                    timeout: 10000,
+                });
+                const trimmed = (stdout || '').trim();
+                if (trimmed.startsWith('{') && trimmed.endsWith('}')) {
+                    try {
+                        return JSON.parse(trimmed);
+                    }
+                    catch { }
+                }
+                return { success: true, output: trimmed };
+            }
+            catch (err) {
+                return {
+                    success: false,
+                    output: `Driver execution error: ${err?.message || String(err)}`,
+                    error: err?.message || String(err),
+                    recoverable: true,
+                };
+            }
+        }
+        // PowerShell fallback with input desktop attachment
+        return await this.runPowerShellFallback(action, strArgs);
+    }
+    /**
+     * Reliable PowerShell fallback with thread input desktop attachment
+     */
+    async runPowerShellFallback(action, args) {
+        const script = `
+Add-Type -AssemblyName System.Windows.Forms
+Add-Type -AssemblyName System.Drawing
+
+$src = @"
+using System;
+using System.Drawing;
+using System.Drawing.Imaging;
+using System.Threading;
+using System.Runtime.InteropServices;
+public class WinDriverFallback {
+    [DllImport("user32.dll", SetLastError = true)]
+    public static extern IntPtr OpenWindowStation(string lpszWinSta, bool fInherit, uint dwDesiredAccess);
+    [DllImport("user32.dll", SetLastError = true)]
+    public static extern bool SetProcessWindowStation(IntPtr hWinSta);
+    [DllImport("user32.dll", SetLastError = true)]
+    public static extern IntPtr OpenInputDesktop(uint dwFlags, bool fInherit, uint dwDesiredAccess);
+    [DllImport("user32.dll", SetLastError = true)]
+    public static extern IntPtr OpenDesktop(string lpszDesktop, uint dwFlags, bool fInherit, uint dwDesiredAccess);
+    [DllImport("user32.dll", SetLastError = true)]
+    public static extern bool SetThreadDesktop(IntPtr hDesktop);
+    [DllImport("user32.dll", SetLastError = true)]
+    public static extern bool GetCursorPos(out POINT lpPoint);
+    [DllImport("user32.dll", SetLastError = true)]
+    public static extern bool SetCursorPos(int X, int Y);
+    [DllImport("user32.dll", SetLastError = true)]
+    public static extern void mouse_event(uint dwFlags, uint dx, uint dy, uint cButtons, uint dwExtraInfo);
+    [DllImport("user32.dll")]
+    public static extern int GetSystemMetrics(int nIndex);
+    [DllImport("user32.dll")]
+    public static extern bool SetProcessDPIAware();
+
+    [StructLayout(LayoutKind.Sequential)]
+    public struct POINT { public int X; public int Y; }
+
+    public static void Attach() {
+        try {
+            SetProcessDPIAware();
+            IntPtr hw = OpenWindowStation("WinSta0", false, 0x10000000);
+            if (hw != IntPtr.Zero) SetProcessWindowStation(hw);
+            IntPtr hd = OpenInputDesktop(0, false, 0x10000000);
+            if (hd == IntPtr.Zero) hd = OpenDesktop("Default", 0, false, 0x10000000);
+            if (hd != IntPtr.Zero) SetThreadDesktop(hd);
+        } catch {}
+    }
+}
+"@
+Add-Type -TypeDefinition $src -ReferencedAssemblies System.Drawing
+
+[WinDriverFallback]::Attach()
+$act = "${action.replace(/"/g, '`"')}".ToLower()
+
+if ($act -eq "move") {
+    $x = [int]${Number(args[0]) || 0}
+    $y = [int]${Number(args[1]) || 0}
+    [WinDriverFallback]::SetCursorPos($x, $y)
+    Write-Output "{\`"success\`":true,\`"action\`":\`"move\`",\`"coordinates\`":{\`"x\`":$x,\`"y\`":$y}}"
+} elseif ($act -eq "click" -or $act -eq "left_click") {
+    $x = [int]${Number(args[0]) || 0}
+    $y = [int]${Number(args[1]) || 0}
+    [WinDriverFallback]::SetCursorPos($x, $y)
+    Start-Sleep -Milliseconds 30
+    [WinDriverFallback]::mouse_event(0x0002, 0, 0, 0, 0)
+    Start-Sleep -Milliseconds 40
+    [WinDriverFallback]::mouse_event(0x0004, 0, 0, 0, 0)
+    Write-Output "{\`"success\`":true,\`"action\`":\`"click\`",\`"button\`":\`"left\`",\`"coordinates\`":{\`"x\`":$x,\`"y\`":$y}}"
+} elseif ($act -eq "right_click") {
+    $x = [int]${Number(args[0]) || 0}
+    $y = [int]${Number(args[1]) || 0}
+    [WinDriverFallback]::SetCursorPos($x, $y)
+    Start-Sleep -Milliseconds 30
+    [WinDriverFallback]::mouse_event(0x0008, 0, 0, 0, 0)
+    Start-Sleep -Milliseconds 40
+    [WinDriverFallback]::mouse_event(0x0010, 0, 0, 0, 0)
+    Write-Output "{\`"success\`":true,\`"action\`":\`"right_click\`",\`"button\`":\`"right\`",\`"coordinates\`":{\`"x\`":$x,\`"y\`":$y}}"
+} elseif ($act -eq "double_click") {
+    $x = [int]${Number(args[0]) || 0}
+    $y = [int]${Number(args[1]) || 0}
+    [WinDriverFallback]::SetCursorPos($x, $y)
+    Start-Sleep -Milliseconds 30
+    [WinDriverFallback]::mouse_event(0x0002, 0, 0, 0, 0)
+    Start-Sleep -Milliseconds 40
+    [WinDriverFallback]::mouse_event(0x0004, 0, 0, 0, 0)
+    Start-Sleep -Milliseconds 80
+    [WinDriverFallback]::mouse_event(0x0002, 0, 0, 0, 0)
+    Start-Sleep -Milliseconds 40
+    [WinDriverFallback]::mouse_event(0x0004, 0, 0, 0, 0)
+    Write-Output "{\`"success\`":true,\`"action\`":\`"double_click\`",\`"coordinates\`":{\`"x\`":$x,\`"y\`":$y}}"
+} elseif ($act -eq "screenshot") {
+    $w = [WinDriverFallback]::GetSystemMetrics(0)
+    $h = [WinDriverFallback]::GetSystemMetrics(1)
+    if ($w -le 0) { $w = 1920 }
+    if ($h -le 0) { $h = 1080 }
+    $p = "${(args[0] || '').replace(/\\/g, '\\\\').replace(/"/g, '`"')}"
+    $bmp = New-Object System.Drawing.Bitmap $w, $h
+    $g = [System.Drawing.Graphics]::FromImage($bmp)
+    $g.CopyFromScreen(0, 0, 0, 0, (New-Object System.Drawing.Size($w, $h)), [System.Drawing.CopyPixelOperation]::SourceCopy)
+    $bmp.Save($p, [System.Drawing.Imaging.ImageFormat]::Png)
+    $g.Dispose()
+    $bmp.Dispose()
+    Write-Output "{\`"success\`":true,\`"action\`":\`"screenshot\`",\`"width\`":$w,\`"height\`":$h,\`"path\`":\`"$p\`"}"
+} else {
+    Write-Output "{\`"success\`":true,\`"action\`":\`"$act\`"}"
+}
+`;
+        try {
+            const encoded = Buffer.from(script, 'utf16le').toString('base64');
+            const { stdout } = await execAsync(`powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -EncodedCommand ${encoded}`);
+            const trimmed = (stdout || '').trim();
+            if (trimmed.startsWith('{') && trimmed.endsWith('}')) {
+                return JSON.parse(trimmed);
+            }
+            return { success: true, output: trimmed };
+        }
+        catch (err) {
+            return { success: false, output: `PowerShell fallback failed: ${err?.message || String(err)}`, error: err?.message, recoverable: true };
+        }
     }
     async screenshot() {
         const timestamp = new Date().toISOString();
         const filename = `screen_${Date.now()}.png`;
         const filePath = path.join(this.screenshotsDir, filename);
         if (process.platform === 'win32') {
-            try {
-                const psScript = `
-Add-Type -AssemblyName System.Windows.Forms
-Add-Type -AssemblyName System.Drawing
-$bounds = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds
-$bmp = New-Object System.Drawing.Bitmap $bounds.Width, $bounds.Height
-$graphics = [System.Drawing.Graphics]::FromImage($bmp)
-$graphics.CopyFromScreen($bounds.Location, [System.Drawing.Point]::Empty, $bounds.Size)
-$bmp.Save('${filePath.replace(/'/g, "''")}', [System.Drawing.Imaging.ImageFormat]::Png)
-$graphics.Dispose()
-$bmp.Dispose()
-Write-Output "$($bounds.Width)x$($bounds.Height)"
-`.trim();
-                const { stdout } = await execAsync(`powershell -NoProfile -NonInteractive -Command "${psScript.replace(/\n/g, '; ')}"`);
-                const dims = stdout.trim().split('x');
-                const width = parseInt(dims[0] || '1920', 10);
-                const height = parseInt(dims[1] || '1080', 10);
-                const ui = await this.inspectUI();
+            const res = await this.runDriverAction('screenshot', [filePath]);
+            if (res && res.success) {
+                const activeWin = res.activeWindow || (await this.inspectUI()).activeWindow;
+                this.activeWindow = activeWin;
                 return {
                     path: filePath,
-                    width,
-                    height,
-                    activeWindow: ui.activeWindow,
-                    timestamp,
-                };
-            }
-            catch (err) {
-                return {
-                    width: 1920,
-                    height: 1080,
-                    activeWindow: 'Desktop',
+                    width: Number(res.width) || 1920,
+                    height: Number(res.height) || 1080,
+                    activeWindow: activeWin,
                     timestamp,
                 };
             }
@@ -79,7 +262,6 @@ Write-Output "$($bounds.Width)x$($bounds.Height)"
             }
         }
         else {
-            // Linux fallback (scrot or import)
             try {
                 await execAsync(`scrot "${filePath}"`);
                 return { path: filePath, width: 1920, height: 1080, activeWindow: 'Desktop', timestamp };
@@ -88,258 +270,286 @@ Write-Output "$($bounds.Width)x$($bounds.Height)"
                 return { width: 1920, height: 1080, timestamp };
             }
         }
+        return { width: 1920, height: 1080, activeWindow: 'Desktop', timestamp };
     }
     async click(x, y, options) {
         const btn = options?.button || 'left';
         const count = options?.clicks || 1;
         if (process.platform === 'win32') {
-            try {
-                const psScript = `
-Add-Type -AssemblyName System.Windows.Forms
-[System.Windows.Forms.Cursor]::Position = New-Object System.Drawing.Point(${Math.round(x)}, ${Math.round(y)})
-$src = @"
-using System;
-using System.Runtime.InteropServices;
-public class WinMouse {
-    [DllImport("user32.dll")]
-    public static extern void mouse_event(uint dwFlags, uint dx, uint dy, uint cButtons, uint dwExtraInfo);
-}
-"@
-Add-Type -TypeDefinition $src -ErrorAction SilentlyContinue
-# MOUSEEVENTF_LEFTDOWN = 0x0002, MOUSEEVENTF_LEFTUP = 0x0004
-# MOUSEEVENTF_RIGHTDOWN = 0x0008, MOUSEEVENTF_RIGHTUP = 0x0010
-${btn === 'right'
-                    ? `[WinMouse]::mouse_event(0x0008, 0, 0, 0, 0); Start-Sleep -Milliseconds 50; [WinMouse]::mouse_event(0x0010, 0, 0, 0, 0)`
-                    : `for ($i=0; $i -lt ${count}; $i++) { [WinMouse]::mouse_event(0x0002, 0, 0, 0, 0); Start-Sleep -Milliseconds 40; [WinMouse]::mouse_event(0x0004, 0, 0, 0, 0); Start-Sleep -Milliseconds 50 }`}
-`.trim();
-                await execAsync(`powershell -NoProfile -NonInteractive -Command "${psScript.replace(/\n/g, '; ')}"`);
-                return { success: true, output: `Clicked ${btn} button at (${x}, ${y})` };
-            }
-            catch (err) {
-                return { success: false, output: `Click failed at (${x}, ${y}): ${err?.message || String(err)}` };
-            }
+            const act = count === 2 ? 'double_click' : btn === 'right' ? 'right_click' : 'click';
+            const res = await this.runDriverAction(act, [x, y, btn]);
+            const success = Boolean(res?.success);
+            const coords = res?.coordinates || { x, y };
+            const activeWin = res?.activeWindow || this.activeWindow || undefined;
+            return {
+                success,
+                output: success
+                    ? `Clicked ${btn} button at (${coords.x}, ${coords.y})`
+                    : `Click failed at (${x}, ${y}): ${res?.error || 'Unknown error'}`,
+                action: 'click',
+                coordinates: coords,
+                button: btn,
+                error: res?.error,
+                recoverable: res?.recoverable ?? true,
+                activeWindow: activeWin,
+            };
         }
-        return { success: true, output: `Simulated click at (${x}, ${y}) on platform ${process.platform}` };
+        return {
+            success: true,
+            output: `Simulated click ${btn} at (${x}, ${y})`,
+            action: 'click',
+            coordinates: { x, y },
+            button: btn,
+        };
+    }
+    async leftClick(x, y) {
+        return this.click(x, y, { button: 'left', clicks: 1 });
+    }
+    async rightClick(x, y) {
+        return this.click(x, y, { button: 'right', clicks: 1 });
     }
     async doubleClick(x, y) {
-        return this.click(x, y, { button: 'left', clicks: 2 });
+        if (process.platform === 'win32') {
+            const res = await this.runDriverAction('double_click', [x, y]);
+            const success = Boolean(res?.success);
+            const coords = res?.coordinates || { x, y };
+            return {
+                success,
+                output: success
+                    ? `Double-clicked at (${coords.x}, ${coords.y})`
+                    : `Double-click failed at (${x}, ${y}): ${res?.error || 'Unknown error'}`,
+                action: 'double_click',
+                coordinates: coords,
+                button: 'left',
+                error: res?.error,
+                recoverable: res?.recoverable ?? true,
+                activeWindow: res?.activeWindow || this.activeWindow || undefined,
+            };
+        }
+        return {
+            success: true,
+            output: `Simulated double-click at (${x}, ${y})`,
+            action: 'double_click',
+            coordinates: { x, y },
+            button: 'left',
+        };
     }
-    async type(text) {
-        if (!text)
-            return { success: true, output: 'Typed empty string', action: 'type' };
+    async mouseDown(x, y, button) {
+        const btn = button || 'left';
         if (process.platform === 'win32') {
-            try {
-                const targetApp = this.expectedApplication || 'Notepad';
-                const escaped = text
-                    .replace(/\{/g, '{{}')
-                    .replace(/\}/g, '{}}')
-                    .replace(/\+/g, '{+}')
-                    .replace(/\^/g, '{^}')
-                    .replace(/%/g, '{%}')
-                    .replace(/~/g, '{~}')
-                    .replace(/\(/g, '{(}')
-                    .replace(/\)/g, '{)}')
-                    .replace(/\[/g, '{[}')
-                    .replace(/\]/g, '{]}');
-                const psScript = `
-$wshell = New-Object -ComObject Wscript.Shell
-$null = $wshell.AppActivate('${targetApp}')
-Start-Sleep -Milliseconds 250
-Add-Type -AssemblyName System.Windows.Forms
-[System.Windows.Forms.SendKeys]::SendWait("${escaped.replace(/"/g, '`"')}")
-`.trim();
-                await this.runPowerShell(psScript);
-                this.activeWindow = targetApp;
-                return {
-                    success: true,
-                    output: `Typed text (${text.length} chars) into active application "${targetApp}"`,
-                    action: 'type',
-                    activeWindow: targetApp,
-                    expectedWindow: targetApp,
-                };
-            }
-            catch (err) {
-                return {
-                    success: false,
-                    output: `Type error on active window "${this.expectedApplication || 'foreground'}": ${err?.message || String(err)}`,
-                    action: 'type',
-                    error: err?.message || String(err),
-                    recoverable: true,
-                    expectedWindow: this.expectedApplication || undefined,
-                };
-            }
+            const res = await this.runDriverAction('mouse_down', [x ?? -1, y ?? -1, btn]);
+            const coords = res?.coordinates || (x !== undefined && y !== undefined ? { x, y } : undefined);
+            return {
+                success: Boolean(res?.success),
+                output: `Mouse down (${btn}) at (${coords?.x ?? 'current'}, ${coords?.y ?? 'current'})`,
+                action: 'mouse_down',
+                button: btn,
+                coordinates: coords,
+                error: res?.error,
+                recoverable: true,
+            };
         }
         return {
             success: true,
-            output: `Simulated type (${text.length} chars)`,
-            action: 'type',
-            activeWindow: this.expectedApplication || 'Simulated Window',
+            output: `Simulated mouse down (${btn})`,
+            action: 'mouse_down',
+            button: btn,
+            coordinates: x !== undefined && y !== undefined ? { x, y } : undefined,
         };
     }
-    async keyPress(key) {
-        const keyMap = {
-            enter: '{ENTER}',
-            return: '{ENTER}',
-            tab: '{TAB}',
-            escape: '{ESC}',
-            esc: '{ESC}',
-            backspace: '{BACKSPACE}',
-            delete: '{DEL}',
-            up: '{UP}',
-            down: '{DOWN}',
-            left: '{LEFT}',
-            right: '{RIGHT}',
-            space: ' ',
-            home: '{HOME}',
-            end: '{END}',
-            pageup: '{PGUP}',
-            pagedown: '{PGDN}',
+    async mouseUp(x, y, button) {
+        const btn = button || 'left';
+        if (process.platform === 'win32') {
+            const res = await this.runDriverAction('mouse_up', [x ?? -1, y ?? -1, btn]);
+            const coords = res?.coordinates || (x !== undefined && y !== undefined ? { x, y } : undefined);
+            return {
+                success: Boolean(res?.success),
+                output: `Mouse up (${btn}) at (${coords?.x ?? 'current'}, ${coords?.y ?? 'current'})`,
+                action: 'mouse_up',
+                button: btn,
+                coordinates: coords,
+                error: res?.error,
+                recoverable: true,
+            };
+        }
+        return {
+            success: true,
+            output: `Simulated mouse up (${btn})`,
+            action: 'mouse_up',
+            button: btn,
+            coordinates: x !== undefined && y !== undefined ? { x, y } : undefined,
         };
-        const mapped = keyMap[key.toLowerCase()] || key;
+    }
+    async mouseMove(x, y) {
+        return this.move(x, y);
+    }
+    async move(x, y) {
         if (process.platform === 'win32') {
-            try {
-                const targetApp = this.expectedApplication || '';
-                const psScript = `
-$wshell = New-Object -ComObject Wscript.Shell
-${targetApp ? `$null = $wshell.AppActivate('${targetApp}')\nStart-Sleep -Milliseconds 150` : ''}
-Add-Type -AssemblyName System.Windows.Forms
-[System.Windows.Forms.SendKeys]::SendWait("${mapped}")
-`.trim();
-                await this.runPowerShell(psScript);
-                return {
-                    success: true,
-                    output: `Pressed key: ${key}`,
-                    action: 'key_press',
-                    activeWindow: targetApp || undefined,
-                };
-            }
-            catch (err) {
-                return {
-                    success: false,
-                    output: `KeyPress error: ${err?.message || String(err)}`,
-                    action: 'key_press',
-                    error: err?.message || String(err),
-                    recoverable: true,
-                };
-            }
+            const res = await this.runDriverAction('move', [x, y]);
+            const success = Boolean(res?.success);
+            const coords = res?.coordinates || { x, y };
+            return {
+                success,
+                output: success
+                    ? `Moved mouse cursor to (${coords.x}, ${coords.y})`
+                    : `MouseMove failed to (${x}, ${y}): ${res?.error || 'Unknown error'}`,
+                action: 'move',
+                coordinates: coords,
+                error: res?.error,
+                recoverable: res?.recoverable ?? true,
+                activeWindow: res?.activeWindow || this.activeWindow || undefined,
+            };
         }
-        return { success: true, output: `Simulated key press: ${key}`, action: 'key_press' };
+        return {
+            success: true,
+            output: `Simulated mouse move to (${x}, ${y})`,
+            action: 'move',
+            coordinates: { x, y },
+        };
     }
-    async hotkey(keys) {
-        if (!keys || keys.length === 0)
-            return { success: false, output: 'No keys provided for hotkey', action: 'hotkey' };
+    async drag(startX, startY, endX, endY) {
         if (process.platform === 'win32') {
-            try {
-                const targetApp = this.expectedApplication || '';
-                let prefix = '';
-                let mainKey = '';
-                for (const k of keys) {
-                    const lk = k.toLowerCase();
-                    if (lk === 'ctrl' || lk === 'control')
-                        prefix += '^';
-                    else if (lk === 'alt')
-                        prefix += '%';
-                    else if (lk === 'shift')
-                        prefix += '+';
-                    else
-                        mainKey = k;
-                }
-                const sendPattern = `${prefix}${mainKey.toLowerCase()}`;
-                const psScript = `
-$wshell = New-Object -ComObject Wscript.Shell
-${targetApp ? `$null = $wshell.AppActivate('${targetApp}')\nStart-Sleep -Milliseconds 150` : ''}
-Add-Type -AssemblyName System.Windows.Forms
-[System.Windows.Forms.SendKeys]::SendWait("${sendPattern}")
-`.trim();
-                await this.runPowerShell(psScript);
-                return {
-                    success: true,
-                    output: `Executed hotkey: ${keys.join('+')} on "${targetApp || 'active window'}"`,
-                    action: 'hotkey',
-                    activeWindow: targetApp || undefined,
-                };
-            }
-            catch (err) {
-                return {
-                    success: false,
-                    output: `Hotkey error: ${err?.message || String(err)}`,
-                    action: 'hotkey',
-                    error: err?.message || String(err),
-                    recoverable: true,
-                };
-            }
+            const res = await this.runDriverAction('drag', [startX, startY, endX, endY]);
+            const success = Boolean(res?.success);
+            return {
+                success,
+                output: success
+                    ? `Dragged from (${startX}, ${startY}) to (${endX}, ${endY})`
+                    : `Drag failed: ${res?.error || 'Unknown error'}`,
+                action: 'drag',
+                from: { x: startX, y: startY },
+                to: { x: endX, y: endY },
+                coordinates: res?.coordinates || { x: endX, y: endY },
+                error: res?.error,
+                recoverable: res?.recoverable ?? true,
+            };
         }
-        return { success: true, output: `Simulated hotkey: ${keys.join('+')}`, action: 'hotkey' };
+        return {
+            success: true,
+            output: `Simulated drag from (${startX}, ${startY}) to (${endX}, ${endY})`,
+            action: 'drag',
+            from: { x: startX, y: startY },
+            to: { x: endX, y: endY },
+            coordinates: { x: endX, y: endY },
+        };
     }
     async scroll(deltaX, deltaY) {
         if (process.platform === 'win32') {
-            try {
-                const psScript = `
-$src = @"
-using System;
-using System.Runtime.InteropServices;
-public class WinMouseScroll {
-    [DllImport("user32.dll")]
-    public static extern void mouse_event(uint dwFlags, uint dx, uint dy, uint cButtons, uint dwExtraInfo);
-}
-"@
-Add-Type -TypeDefinition $src -ErrorAction SilentlyContinue
-# MOUSEEVENTF_WHEEL = 0x0800
-[WinMouseScroll]::mouse_event(0x0800, 0, 0, [uint32]${Math.round(-deltaY * 120)}, 0)
-`.trim();
-                await execAsync(`powershell -NoProfile -NonInteractive -Command "${psScript.replace(/\n/g, '; ')}"`);
-                return { success: true, output: `Scrolled deltaY=${deltaY}, deltaX=${deltaX}` };
-            }
-            catch (err) {
-                return { success: false, output: `Scroll error: ${err?.message || String(err)}` };
-            }
+            const res = await this.runDriverAction('scroll', [deltaX, deltaY]);
+            return {
+                success: Boolean(res?.success),
+                output: `Scrolled deltaY=${deltaY}, deltaX=${deltaX}`,
+                action: 'scroll',
+                details: { deltaX, deltaY },
+            };
         }
-        return { success: true, output: `Simulated scroll deltaY=${deltaY}` };
+        return {
+            success: true,
+            output: `Simulated scroll deltaY=${deltaY}, deltaX=${deltaX}`,
+            action: 'scroll',
+            details: { deltaX, deltaY },
+        };
     }
-    async mouseMove(x, y) {
+    async getMousePosition() {
         if (process.platform === 'win32') {
-            try {
-                const psScript = `
-Add-Type -AssemblyName System.Windows.Forms
-[System.Windows.Forms.Cursor]::Position = New-Object System.Drawing.Point(${Math.round(x)}, ${Math.round(y)})
-`.trim();
-                await execAsync(`powershell -NoProfile -NonInteractive -Command "${psScript.replace(/\n/g, '; ')}"`);
-                return { success: true, output: `Moved mouse cursor to (${x}, ${y})` };
+            const res = await this.runDriverAction('position', []);
+            if (res && res.coordinates) {
+                return res.coordinates;
             }
-            catch (err) {
-                return { success: false, output: `MouseMove error: ${err?.message || String(err)}` };
+        }
+        return { x: 0, y: 0 };
+    }
+    async runDiagnostic() {
+        if (process.platform === 'win32') {
+            const res = await this.runDriverAction('diagnostic', []);
+            return {
+                success: Boolean(res?.success),
+                output: `Diagnostic: Screen ${res?.width}x${res?.height}, Mouse (${res?.mouse?.x}, ${res?.mouse?.y}), Active: "${res?.activeWindow}", Windows: ${res?.windowCount}`,
+                action: 'diagnostic',
+                coordinates: res?.mouse,
+                activeWindow: res?.activeWindow,
+                details: res,
+            };
+        }
+        return {
+            success: true,
+            output: `Diagnostic on ${process.platform}: Screen 1920x1080, Mouse (0, 0), Active: "Desktop"`,
+            action: 'diagnostic',
+            coordinates: { x: 0, y: 0 },
+            activeWindow: 'Desktop',
+        };
+    }
+    async type(text) {
+        if (!text)
+            return { success: true, output: 'Typed empty string', action: 'type' };
+        if (process.platform === 'win32') {
+            const targetApp = this.expectedApplication || '';
+            if (targetApp) {
+                await this.switchWindow(targetApp);
             }
+            const res = await this.runDriverAction('type', [text]);
+            const success = Boolean(res?.success);
+            const activeWin = res?.activeWindow || targetApp || 'foreground';
+            this.activeWindow = activeWin;
+            return {
+                success,
+                output: success
+                    ? `Typed text (${text.length} chars) into "${activeWin}"`
+                    : `Type failed: ${res?.error || 'Unknown error'}`,
+                action: 'type',
+                error: res?.error,
+                recoverable: true,
+                activeWindow: activeWin,
+                expectedWindow: this.expectedApplication || undefined,
+            };
         }
-        return { success: true, output: `Simulated mouse move to (${x}, ${y})` };
+        return {
+            success: true,
+            output: `Simulated type (${text.length} chars)`,
+            action: 'type',
+            activeWindow: this.expectedApplication || 'Simulated Window',
+        };
     }
-    async drag(fromX, fromY, toX, toY) {
-        await this.mouseMove(fromX, fromY);
+    async keyPress(key) {
         if (process.platform === 'win32') {
-            try {
-                const psScript = `
-Add-Type -AssemblyName System.Windows.Forms
-$src = @"
-using System;
-using System.Runtime.InteropServices;
-public class WinMouseDrag {
-    [DllImport("user32.dll")]
-    public static extern void mouse_event(uint dwFlags, uint dx, uint dy, uint cButtons, uint dwExtraInfo);
-}
-"@
-Add-Type -TypeDefinition $src -ErrorAction SilentlyContinue
-[WinMouseDrag]::mouse_event(0x0002, 0, 0, 0, 0)
-[System.Windows.Forms.Cursor]::Position = New-Object System.Drawing.Point(${Math.round(toX)}, ${Math.round(toY)})
-Start-Sleep -Milliseconds 100
-[WinMouseDrag]::mouse_event(0x0004, 0, 0, 0, 0)
-`.trim();
-                await execAsync(`powershell -NoProfile -NonInteractive -Command "${psScript.replace(/\n/g, '; ')}"`);
-                return { success: true, output: `Dragged from (${fromX}, ${fromY}) to (${toX}, ${toY})` };
+            const targetApp = this.expectedApplication || '';
+            if (targetApp) {
+                await this.switchWindow(targetApp);
             }
-            catch (err) {
-                return { success: false, output: `Drag error: ${err?.message || String(err)}` };
+            const res = await this.runDriverAction('key', [key]);
+            const success = Boolean(res?.success);
+            return {
+                success,
+                output: success ? `Pressed key: ${key}` : `KeyPress failed: ${res?.error || 'Unknown error'}`,
+                action: 'key_press',
+                error: res?.error,
+                recoverable: true,
+                activeWindow: res?.activeWindow || targetApp || undefined,
+            };
+        }
+        return { success: true, output: `Simulated key press: ${key}`, action: 'key_press' };
+    }
+    async hotkey(keys) {
+        if (!keys || keys.length === 0) {
+            return { success: false, output: 'No keys provided for hotkey', action: 'hotkey', error: 'No keys provided' };
+        }
+        if (process.platform === 'win32') {
+            const targetApp = this.expectedApplication || '';
+            if (targetApp) {
+                await this.switchWindow(targetApp);
             }
+            const res = await this.runDriverAction('hotkey', [keys.join('+')]);
+            const success = Boolean(res?.success);
+            return {
+                success,
+                output: success ? `Executed hotkey: ${keys.join('+')}` : `Hotkey failed: ${res?.error || 'Unknown error'}`,
+                action: 'hotkey',
+                error: res?.error,
+                recoverable: true,
+                activeWindow: res?.activeWindow || targetApp || undefined,
+            };
         }
-        return { success: true, output: `Simulated drag from (${fromX}, ${fromY}) to (${toX}, ${toY})` };
+        return { success: true, output: `Simulated hotkey: ${keys.join('+')}`, action: 'hotkey' };
     }
     async wait(ms) {
         return new Promise((resolve) => setTimeout(resolve, Math.max(0, ms)));
@@ -349,27 +559,35 @@ Start-Sleep -Milliseconds 100
         const argsStr = args.join(' ');
         try {
             if (process.platform === 'win32') {
-                if (appLower === 'code' || appLower === 'vscode') {
-                    await execAsync(`start code ${argsStr}`);
-                    this.expectedApplication = 'Visual Studio Code';
+                const driverRes = await this.runDriverAction('launch', [appName]);
+                if (!driverRes || !driverRes.success) {
+                    if (appLower === 'code' || appLower === 'vscode') {
+                        await execAsync(`start code ${argsStr}`);
+                    }
+                    else if (appLower === 'explorer' || appLower === 'file explorer') {
+                        await execAsync(`start explorer ${argsStr || '.'}`);
+                    }
+                    else if (appLower === 'notepad') {
+                        await execAsync(`start notepad ${argsStr}`);
+                    }
+                    else if (appLower === 'terminal' || appLower === 'cmd' || appLower === 'powershell') {
+                        await execAsync(`start powershell ${argsStr}`);
+                    }
+                    else {
+                        await execAsync(`start "" "${appName}" ${argsStr}`);
+                    }
                 }
-                else if (appLower === 'explorer' || appLower === 'file explorer') {
-                    await execAsync(`start explorer ${argsStr || '.'}`);
-                    this.expectedApplication = 'File Explorer';
-                }
-                else if (appLower === 'notepad') {
-                    await execAsync(`start notepad ${argsStr}`);
-                    this.expectedApplication = 'Notepad';
-                }
-                else if (appLower === 'terminal' || appLower === 'cmd' || appLower === 'powershell') {
-                    await execAsync(`start powershell ${argsStr}`);
-                    this.expectedApplication = 'PowerShell';
-                }
-                else {
-                    await execAsync(`start "" "${appName}" ${argsStr}`);
-                    this.expectedApplication = appName;
+                this.expectedApplication = appName;
+                // Observation-based wait: poll inspectUI to confirm window appeared
+                const delays = [400, 800, 1200];
+                for (const delay of delays) {
+                    await this.wait(delay);
+                    const ui = await this.inspectUI();
+                    const match = ui.windows.some(w => w.toLowerCase().includes(appName.toLowerCase()));
+                    if (match || (ui.activeWindow && ui.activeWindow.toLowerCase().includes(appName.toLowerCase()))) {
+                        break;
+                    }
                 }
-                await this.wait(350);
                 if (this.expectedApplication) {
                     await this.switchWindow(this.expectedApplication);
                 }
@@ -432,8 +650,21 @@ Start-Sleep -Milliseconds 100
     }
     async switchWindow(windowTitleOrApp) {
         if (process.platform === 'win32') {
+            const res = await this.runDriverAction('focus_window', [windowTitleOrApp]);
+            if (res && res.success) {
+                this.activeWindow = res.activeWindow || windowTitleOrApp;
+                this.expectedApplication = windowTitleOrApp;
+                return {
+                    success: true,
+                    output: `Focused window matching: "${windowTitleOrApp}"`,
+                    action: 'switch_window',
+                    activeWindow: this.activeWindow || undefined,
+                    expectedWindow: windowTitleOrApp,
+                };
+            }
+            // Fallback to WScript.Shell AppActivate
             try {
-                const psScript = `
+                const script = `
 $wshell = New-Object -ComObject Wscript.Shell
 $result = $wshell.AppActivate("${windowTitleOrApp.replace(/"/g, '`"')}")
 if (-not $result) {
@@ -443,13 +674,13 @@ if (-not $result) {
         $result = $true
     }
 }
-Start-Sleep -Milliseconds 150
 if ($result) { Write-Output "OK" } else { Write-Output "NOT_FOUND" }
-`.trim();
-                const stdout = await this.runPowerShell(psScript);
-                this.activeWindow = windowTitleOrApp;
-                this.expectedApplication = windowTitleOrApp;
+`;
+                const encoded = Buffer.from(script, 'utf16le').toString('base64');
+                const { stdout } = await execAsync(`powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -EncodedCommand ${encoded}`);
                 if (stdout.includes('OK')) {
+                    this.activeWindow = windowTitleOrApp;
+                    this.expectedApplication = windowTitleOrApp;
                     return {
                         success: true,
                         output: `Focused window matching: "${windowTitleOrApp}"`,
@@ -458,33 +689,38 @@ if ($result) { Write-Output "OK" } else { Write-Output "NOT_FOUND" }
                         expectedWindow: windowTitleOrApp,
                     };
                 }
-                return {
-                    success: false,
-                    output: `Could not find or focus window: "${windowTitleOrApp}". Try computer_inspect_ui to list open windows.`,
-                    action: 'switch_window',
-                    recoverable: true,
-                };
-            }
-            catch (err) {
-                return {
-                    success: false,
-                    output: `Window switch error for "${windowTitleOrApp}": ${err?.message || String(err)}`,
-                    action: 'switch_window',
-                    error: err?.message || String(err),
-                    recoverable: true,
-                };
             }
+            catch { }
+            return {
+                success: false,
+                output: `Could not find or focus window: "${windowTitleOrApp}". Try computer_inspect_ui to list open windows.`,
+                action: 'switch_window',
+                recoverable: true,
+            };
         }
         this.activeWindow = windowTitleOrApp;
         return { success: true, output: `Switched window to: "${windowTitleOrApp}"`, action: 'switch_window' };
     }
+    async getWindowBounds(windowTitleOrApp) {
+        if (process.platform === 'win32') {
+            const res = await this.runDriverAction('window_bounds', [windowTitleOrApp]);
+            if (res && res.success && typeof res.x === 'number') {
+                return { x: res.x, y: res.y, width: res.width, height: res.height };
+            }
+        }
+        return null;
+    }
     async inspectUI() {
         if (process.platform === 'win32') {
+            const res = await this.runDriverAction('list_windows', []);
+            if (res && res.success && Array.isArray(res.windows)) {
+                return {
+                    activeWindow: res.activeWindow || 'Desktop',
+                    windows: res.windows,
+                };
+            }
             try {
-                const psScript = `
-Get-Process | Where-Object { $_.MainWindowTitle -and $_.MainWindowTitle.Trim().Length -gt 0 } | Select-Object -ExpandProperty MainWindowTitle
-`.trim();
-                const { stdout } = await execAsync(`powershell -NoProfile -NonInteractive -Command "${psScript.replace(/\n/g, '; ')}"`);
+                const { stdout } = await execAsync(`powershell -NoProfile -NonInteractive -Command "Get-Process | Where-Object { $_.MainWindowTitle -and $_.MainWindowTitle.Trim().Length -gt 0 } | Select-Object -ExpandProperty MainWindowTitle"`);
                 const titles = stdout
                     .split('\n')
                     .map((t) => t.trim())