ft-scout 9.0.3 → 9.0.5

package/bin/src/engine/computer/computerController.js

@@ -1,295 +1,741 @@
-import { openApp, executeInApp, takeoverControl, releaseTakeover, captureScreen, analyzeScreen, getScreenInfo, } from '../appControl.js';
-/**
- * OSComputerController: Production implementation driving host desktop OS
- * (Windows, macOS, Linux) through Scout's platform automation engine.
- */
-export class OSComputerController {
-    async screenshot() {
+import fs from 'fs';
+import path from 'path';
+import { exec, execFile, spawn } from 'child_process';
+import { promisify } from 'util';
+import { fileURLToPath } from 'url';
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const execAsync = promisify(exec);
+const execFileAsync = promisify(execFile);
+export class SystemComputerController {
+    workspaceDir;
+    screenshotsDir;
+    expectedApplication = null;
+    activeWindow = null;
+    driverExePath = null;
+    driverInitAttempted = false;
+    constructor(workspaceDir = process.cwd()) {
+        this.workspaceDir = workspaceDir;
+        this.screenshotsDir = path.join(this.workspaceDir, '.ft', 'screenshots');
         try {
-            const shotPath = await captureScreen();
-            return {
-                success: true,
-                output: `Captured screen to ${shotPath}`,
-                details: { path: shotPath },
-            };
-        }
-        catch (err) {
-            return {
-                success: false,
-                output: `Failed to capture screen: ${err.message}`,
-            };
+            if (!fs.existsSync(this.screenshotsDir)) {
+                fs.mkdirSync(this.screenshotsDir, { recursive: true });
+            }
         }
+        catch { }
     }
-    async click(x, y, button = 'left') {
-        const res = await executeInApp('desktop', 'click_app', { x, y, button });
-        return {
-            success: res.success,
-            output: res.output,
-            details: res.details,
-        };
-    }
-    async double_click(x, y) {
-        const res = await executeInApp('desktop', 'click_app', { x, y, double: true });
-        return {
-            success: res.success,
-            output: res.output,
-            details: res.details,
-        };
-    }
-    async type(text) {
-        const res = await executeInApp('desktop', 'type_text', { text });
-        return {
-            success: res.success,
-            output: res.output,
-            details: res.details,
-        };
-    }
-    async key_press(key) {
-        const res = await executeInApp('desktop', 'send_keys', { key });
-        return {
-            success: res.success,
-            output: res.output,
-            details: res.details,
-        };
-    }
-    async hotkey(keys) {
-        const keyCombo = Array.isArray(keys) ? keys.join('+') : keys;
-        const res = await executeInApp('desktop', 'key_combo', { keyCombo });
-        return {
-            success: res.success,
-            output: res.output,
-            details: res.details,
-        };
+    setExpectedApplication(app) {
+        this.expectedApplication = app;
     }
-    async scroll(direction, amount = 4) {
-        const res = await executeInApp('desktop', 'scroll', { direction, amount });
-        return {
-            success: res.success,
-            output: res.output,
-            details: res.details,
-        };
+    getExpectedApplication() {
+        return this.expectedApplication;
     }
-    async move_mouse(x, y) {
-        const res = await executeInApp('desktop', 'move_mouse', { x, y });
-        return {
-            success: res.success,
-            output: res.output,
-            details: res.details,
-        };
+    /**
+     * Resolves or builds the native scout_driver.exe for Windows.
+     */
+    async ensureDriver() {
+        if (this.driverExePath && fs.existsSync(this.driverExePath)) {
+            return this.driverExePath;
+        }
+        if (process.platform !== 'win32') {
+            return null;
+        }
+        // Candidate locations
+        const candidates = [
+            path.join(__dirname, 'scout_driver.exe'),
+            path.join(this.workspaceDir, 'bin', 'src', 'engine', 'computer', 'scout_driver.exe'),
+            path.join(this.workspaceDir, '.ft', 'bin', 'scout_driver.exe'),
+        ];
+        for (const c of candidates) {
+            if (fs.existsSync(c)) {
+                this.driverExePath = c;
+                return c;
+            }
+        }
+        if (this.driverInitAttempted) {
+            return null;
+        }
+        this.driverInitAttempted = true;
+        // Try compiling using csc.exe
+        const csSourceCandidates = [
+            path.join(__dirname, 'scoutDriver.cs'),
+            path.join(this.workspaceDir, 'bin', 'src', 'engine', 'computer', 'scoutDriver.cs'),
+        ];
+        const csSource = csSourceCandidates.find((p) => fs.existsSync(p));
+        const cscExe = 'C:\\Windows\\Microsoft.NET\\Framework64\\v4.0.30319\\csc.exe';
+        if (csSource && fs.existsSync(cscExe)) {
+            const outDir = path.join(this.workspaceDir, '.ft', 'bin');
+            try {
+                if (!fs.existsSync(outDir))
+                    fs.mkdirSync(outDir, { recursive: true });
+                const targetExe = path.join(outDir, 'scout_driver.exe');
+                await execAsync(`"${cscExe}" /nologo /optimize /t:exe /out:"${targetExe}" /r:System.Windows.Forms.dll,System.Drawing.dll "${csSource}"`);
+                if (fs.existsSync(targetExe)) {
+                    this.driverExePath = targetExe;
+                    return targetExe;
+                }
+            }
+            catch { }
+        }
+        return null;
+    }
+    /**
+     * Runs an action using native driver or fallback.
+     */
+    async runDriverAction(action, args = []) {
+        const driver = await this.ensureDriver();
+        const strArgs = args.map((a) => String(a));
+        if (driver) {
+            try {
+                const { stdout } = await execFileAsync(driver, [action, ...strArgs], {
+                    windowsHide: true,
+                    timeout: 10000,
+                });
+                const trimmed = (stdout || '').trim();
+                if (trimmed.startsWith('{') && trimmed.endsWith('}')) {
+                    try {
+                        return JSON.parse(trimmed);
+                    }
+                    catch { }
+                }
+                return { success: true, output: trimmed };
+            }
+            catch (err) {
+                return {
+                    success: false,
+                    output: `Driver execution error: ${err?.message || String(err)}`,
+                    error: err?.message || String(err),
+                    recoverable: true,
+                };
+            }
+        }
+        // PowerShell fallback with input desktop attachment
+        return await this.runPowerShellFallback(action, strArgs);
+    }
+    /**
+     * Reliable PowerShell fallback with thread input desktop attachment
+     */
+    async runPowerShellFallback(action, args) {
+        const script = `
+Add-Type -AssemblyName System.Windows.Forms
+Add-Type -AssemblyName System.Drawing
+
+$src = @"
+using System;
+using System.Drawing;
+using System.Drawing.Imaging;
+using System.Threading;
+using System.Runtime.InteropServices;
+public class WinDriverFallback {
+    [DllImport("user32.dll", SetLastError = true)]
+    public static extern IntPtr OpenWindowStation(string lpszWinSta, bool fInherit, uint dwDesiredAccess);
+    [DllImport("user32.dll", SetLastError = true)]
+    public static extern bool SetProcessWindowStation(IntPtr hWinSta);
+    [DllImport("user32.dll", SetLastError = true)]
+    public static extern IntPtr OpenInputDesktop(uint dwFlags, bool fInherit, uint dwDesiredAccess);
+    [DllImport("user32.dll", SetLastError = true)]
+    public static extern IntPtr OpenDesktop(string lpszDesktop, uint dwFlags, bool fInherit, uint dwDesiredAccess);
+    [DllImport("user32.dll", SetLastError = true)]
+    public static extern bool SetThreadDesktop(IntPtr hDesktop);
+    [DllImport("user32.dll", SetLastError = true)]
+    public static extern bool GetCursorPos(out POINT lpPoint);
+    [DllImport("user32.dll", SetLastError = true)]
+    public static extern bool SetCursorPos(int X, int Y);
+    [DllImport("user32.dll", SetLastError = true)]
+    public static extern void mouse_event(uint dwFlags, uint dx, uint dy, uint cButtons, uint dwExtraInfo);
+    [DllImport("user32.dll")]
+    public static extern int GetSystemMetrics(int nIndex);
+    [DllImport("user32.dll")]
+    public static extern bool SetProcessDPIAware();
+
+    [StructLayout(LayoutKind.Sequential)]
+    public struct POINT { public int X; public int Y; }
+
+    public static void Attach() {
+        try {
+            SetProcessDPIAware();
+            IntPtr hw = OpenWindowStation("WinSta0", false, 0x10000000);
+            if (hw != IntPtr.Zero) SetProcessWindowStation(hw);
+            IntPtr hd = OpenInputDesktop(0, false, 0x10000000);
+            if (hd == IntPtr.Zero) hd = OpenDesktop("Default", 0, false, 0x10000000);
+            if (hd != IntPtr.Zero) SetThreadDesktop(hd);
+        } catch {}
     }
-    async drag(start, end) {
-        const res = await executeInApp('desktop', 'drag', { startX: start.x, startY: start.y, endX: end.x, endY: end.y });
-        return {
-            success: res.success,
-            output: res.output,
-            details: res.details,
-        };
+}
+"@
+Add-Type -TypeDefinition $src -ReferencedAssemblies System.Drawing
+
+[WinDriverFallback]::Attach()
+$act = "${action.replace(/"/g, '`"')}".ToLower()
+
+if ($act -eq "move") {
+    $x = [int]${Number(args[0]) || 0}
+    $y = [int]${Number(args[1]) || 0}
+    [WinDriverFallback]::SetCursorPos($x, $y)
+    Write-Output "{\`"success\`":true,\`"action\`":\`"move\`",\`"coordinates\`":{\`"x\`":$x,\`"y\`":$y}}"
+} elseif ($act -eq "click" -or $act -eq "left_click") {
+    $x = [int]${Number(args[0]) || 0}
+    $y = [int]${Number(args[1]) || 0}
+    [WinDriverFallback]::SetCursorPos($x, $y)
+    Start-Sleep -Milliseconds 30
+    [WinDriverFallback]::mouse_event(0x0002, 0, 0, 0, 0)
+    Start-Sleep -Milliseconds 40
+    [WinDriverFallback]::mouse_event(0x0004, 0, 0, 0, 0)
+    Write-Output "{\`"success\`":true,\`"action\`":\`"click\`",\`"button\`":\`"left\`",\`"coordinates\`":{\`"x\`":$x,\`"y\`":$y}}"
+} elseif ($act -eq "right_click") {
+    $x = [int]${Number(args[0]) || 0}
+    $y = [int]${Number(args[1]) || 0}
+    [WinDriverFallback]::SetCursorPos($x, $y)
+    Start-Sleep -Milliseconds 30
+    [WinDriverFallback]::mouse_event(0x0008, 0, 0, 0, 0)
+    Start-Sleep -Milliseconds 40
+    [WinDriverFallback]::mouse_event(0x0010, 0, 0, 0, 0)
+    Write-Output "{\`"success\`":true,\`"action\`":\`"right_click\`",\`"button\`":\`"right\`",\`"coordinates\`":{\`"x\`":$x,\`"y\`":$y}}"
+} elseif ($act -eq "double_click") {
+    $x = [int]${Number(args[0]) || 0}
+    $y = [int]${Number(args[1]) || 0}
+    [WinDriverFallback]::SetCursorPos($x, $y)
+    Start-Sleep -Milliseconds 30
+    [WinDriverFallback]::mouse_event(0x0002, 0, 0, 0, 0)
+    Start-Sleep -Milliseconds 40
+    [WinDriverFallback]::mouse_event(0x0004, 0, 0, 0, 0)
+    Start-Sleep -Milliseconds 80
+    [WinDriverFallback]::mouse_event(0x0002, 0, 0, 0, 0)
+    Start-Sleep -Milliseconds 40
+    [WinDriverFallback]::mouse_event(0x0004, 0, 0, 0, 0)
+    Write-Output "{\`"success\`":true,\`"action\`":\`"double_click\`",\`"coordinates\`":{\`"x\`":$x,\`"y\`":$y}}"
+} elseif ($act -eq "screenshot") {
+    $w = [WinDriverFallback]::GetSystemMetrics(0)
+    $h = [WinDriverFallback]::GetSystemMetrics(1)
+    if ($w -le 0) { $w = 1920 }
+    if ($h -le 0) { $h = 1080 }
+    $p = "${(args[0] || '').replace(/\\/g, '\\\\').replace(/"/g, '`"')}"
+    $bmp = New-Object System.Drawing.Bitmap $w, $h
+    $g = [System.Drawing.Graphics]::FromImage($bmp)
+    $g.CopyFromScreen(0, 0, 0, 0, (New-Object System.Drawing.Size($w, $h)), [System.Drawing.CopyPixelOperation]::SourceCopy)
+    $bmp.Save($p, [System.Drawing.Imaging.ImageFormat]::Png)
+    $g.Dispose()
+    $bmp.Dispose()
+    Write-Output "{\`"success\`":true,\`"action\`":\`"screenshot\`",\`"width\`":$w,\`"height\`":$h,\`"path\`":\`"$p\`"}"
+} else {
+    Write-Output "{\`"success\`":true,\`"action\`":\`"$act\`"}"
+}
+`;
+        try {
+            const encoded = Buffer.from(script, 'utf16le').toString('base64');
+            const { stdout } = await execAsync(`powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -EncodedCommand ${encoded}`);
+            const trimmed = (stdout || '').trim();
+            if (trimmed.startsWith('{') && trimmed.endsWith('}')) {
+                return JSON.parse(trimmed);
+            }
+            return { success: true, output: trimmed };
+        }
+        catch (err) {
+            return { success: false, output: `PowerShell fallback failed: ${err?.message || String(err)}`, error: err?.message, recoverable: true };
+        }
     }
-    async wait(durationMs) {
-        await new Promise((r) => setTimeout(r, Math.min(10000, durationMs)));
+    async screenshot() {
+        const timestamp = new Date().toISOString();
+        const filename = `screen_${Date.now()}.png`;
+        const filePath = path.join(this.screenshotsDir, filename);
+        if (process.platform === 'win32') {
+            const res = await this.runDriverAction('screenshot', [filePath]);
+            if (res && res.success) {
+                const activeWin = res.activeWindow || (await this.inspectUI()).activeWindow;
+                this.activeWindow = activeWin;
+                return {
+                    path: filePath,
+                    width: Number(res.width) || 1920,
+                    height: Number(res.height) || 1080,
+                    activeWindow: activeWin,
+                    timestamp,
+                };
+            }
+        }
+        else if (process.platform === 'darwin') {
+            try {
+                await execAsync(`screencapture -x "${filePath}"`);
+                return { path: filePath, width: 1920, height: 1080, activeWindow: 'Desktop', timestamp };
+            }
+            catch {
+                return { width: 1920, height: 1080, timestamp };
+            }
+        }
+        else {
+            try {
+                await execAsync(`scrot "${filePath}"`);
+                return { path: filePath, width: 1920, height: 1080, activeWindow: 'Desktop', timestamp };
+            }
+            catch {
+                return { width: 1920, height: 1080, timestamp };
+            }
+        }
+        return { width: 1920, height: 1080, activeWindow: 'Desktop', timestamp };
+    }
+    async click(x, y, options) {
+        const btn = options?.button || 'left';
+        const count = options?.clicks || 1;
+        if (process.platform === 'win32') {
+            const act = count === 2 ? 'double_click' : btn === 'right' ? 'right_click' : 'click';
+            const res = await this.runDriverAction(act, [x, y, btn]);
+            const success = Boolean(res?.success);
+            const coords = res?.coordinates || { x, y };
+            const activeWin = res?.activeWindow || this.activeWindow || undefined;
+            return {
+                success,
+                output: success
+                    ? `Clicked ${btn} button at (${coords.x}, ${coords.y})`
+                    : `Click failed at (${x}, ${y}): ${res?.error || 'Unknown error'}`,
+                action: 'click',
+                coordinates: coords,
+                button: btn,
+                error: res?.error,
+                recoverable: res?.recoverable ?? true,
+                activeWindow: activeWin,
+            };
+        }
         return {
             success: true,
-            output: `Waited for ${durationMs}ms`,
-            details: { durationMs },
-        };
-    }
-    async open_application(name, target) {
-        const res = await openApp(name, target);
-        return {
-            success: res.success,
-            output: res.output,
-            details: res.details,
-        };
-    }
-    async close_application(name) {
-        const res = await executeInApp(name, 'close_app');
-        return {
-            success: res.success,
-            output: res.output,
-            details: res.details,
+            output: `Simulated click ${btn} at (${x}, ${y})`,
+            action: 'click',
+            coordinates: { x, y },
+            button: btn,
         };
     }
-    async switch_application(name) {
-        const res = await executeInApp('desktop', 'focus_window', { app: name });
-        return {
-            success: res.success,
-            output: res.output,
-            details: res.details,
-        };
+    async leftClick(x, y) {
+        return this.click(x, y, { button: 'left', clicks: 1 });
     }
-    async inspect_page() {
-        const res = await executeInApp('desktop', 'analyze_screen');
-        return {
-            success: res.success,
-            output: res.output,
-            details: res.details,
-        };
+    async rightClick(x, y) {
+        return this.click(x, y, { button: 'right', clicks: 1 });
     }
-    async get_active_window() {
-        const res = await getScreenInfo();
+    async doubleClick(x, y) {
+        if (process.platform === 'win32') {
+            const res = await this.runDriverAction('double_click', [x, y]);
+            const success = Boolean(res?.success);
+            const coords = res?.coordinates || { x, y };
+            return {
+                success,
+                output: success
+                    ? `Double-clicked at (${coords.x}, ${coords.y})`
+                    : `Double-click failed at (${x}, ${y}): ${res?.error || 'Unknown error'}`,
+                action: 'double_click',
+                coordinates: coords,
+                button: 'left',
+                error: res?.error,
+                recoverable: res?.recoverable ?? true,
+                activeWindow: res?.activeWindow || this.activeWindow || undefined,
+            };
+        }
         return {
-            success: res.success,
-            output: res.output,
-            details: res.details,
+            success: true,
+            output: `Simulated double-click at (${x}, ${y})`,
+            action: 'double_click',
+            coordinates: { x, y },
+            button: 'left',
         };
     }
-}
-/**
- * MockComputerController: Fast, deterministic in-memory computer controller
- * for automated unit tests, CI pipelines, and desktop simulations.
- */
-export class MockComputerController {
-    cursor = { x: 0, y: 0 };
-    activeApp = null;
-    openApps = new Set();
-    typedHistory = [];
-    actionLog = [];
-    async screenshot() {
-        this.actionLog.push({ action: 'screenshot', params: {} });
+    async mouseDown(x, y, button) {
+        const btn = button || 'left';
+        if (process.platform === 'win32') {
+            const res = await this.runDriverAction('mouse_down', [x ?? -1, y ?? -1, btn]);
+            const coords = res?.coordinates || (x !== undefined && y !== undefined ? { x, y } : undefined);
+            return {
+                success: Boolean(res?.success),
+                output: `Mouse down (${btn}) at (${coords?.x ?? 'current'}, ${coords?.y ?? 'current'})`,
+                action: 'mouse_down',
+                button: btn,
+                coordinates: coords,
+                error: res?.error,
+                recoverable: true,
+            };
+        }
         return {
             success: true,
-            output: 'Mock screenshot captured: 1920x1080 virtual desktop.',
-            details: { width: 1920, height: 1080, format: 'png' },
+            output: `Simulated mouse down (${btn})`,
+            action: 'mouse_down',
+            button: btn,
+            coordinates: x !== undefined && y !== undefined ? { x, y } : undefined,
         };
     }
-    async click(x, y, button) {
-        const btn = typeof button === 'object' ? (button.button || 'left') : (button || 'left');
-        this.cursor = { x, y };
-        this.actionLog.push({ action: 'click', params: { x, y, button: btn } });
+    async mouseUp(x, y, button) {
+        const btn = button || 'left';
+        if (process.platform === 'win32') {
+            const res = await this.runDriverAction('mouse_up', [x ?? -1, y ?? -1, btn]);
+            const coords = res?.coordinates || (x !== undefined && y !== undefined ? { x, y } : undefined);
+            return {
+                success: Boolean(res?.success),
+                output: `Mouse up (${btn}) at (${coords?.x ?? 'current'}, ${coords?.y ?? 'current'})`,
+                action: 'mouse_up',
+                button: btn,
+                coordinates: coords,
+                error: res?.error,
+                recoverable: true,
+            };
+        }
         return {
             success: true,
-            output: `Mock clicked ${btn} button at (${x}, ${y}).`,
-            details: { x, y, button: btn },
+            output: `Simulated mouse up (${btn})`,
+            action: 'mouse_up',
+            button: btn,
+            coordinates: x !== undefined && y !== undefined ? { x, y } : undefined,
         };
     }
-    async getWindowList() {
-        return Array.from(this.openApps).length > 0 ? Array.from(this.openApps) : ['Code', 'Chrome', 'Terminal'];
-    }
-    async typeText(text) {
-        return this.type(text);
-    }
-    async takeScreenshot() {
-        return {
-            format: 'base64',
-            width: 1920,
-            height: 1080,
-            data: 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==',
-        };
+    async mouseMove(x, y) {
+        return this.move(x, y);
     }
-    async double_click(x, y) {
-        this.cursor = { x, y };
-        this.actionLog.push({ action: 'double_click', params: { x, y } });
+    async move(x, y) {
+        if (process.platform === 'win32') {
+            const res = await this.runDriverAction('move', [x, y]);
+            const success = Boolean(res?.success);
+            const coords = res?.coordinates || { x, y };
+            return {
+                success,
+                output: success
+                    ? `Moved mouse cursor to (${coords.x}, ${coords.y})`
+                    : `MouseMove failed to (${x}, ${y}): ${res?.error || 'Unknown error'}`,
+                action: 'move',
+                coordinates: coords,
+                error: res?.error,
+                recoverable: res?.recoverable ?? true,
+                activeWindow: res?.activeWindow || this.activeWindow || undefined,
+            };
+        }
         return {
             success: true,
-            output: `Mock double-clicked at (${x}, ${y}).`,
-            details: { x, y },
+            output: `Simulated mouse move to (${x}, ${y})`,
+            action: 'move',
+            coordinates: { x, y },
         };
     }
-    async type(text) {
-        this.typedHistory.push(text);
-        this.actionLog.push({ action: 'type', params: { text } });
+    async drag(startX, startY, endX, endY) {
+        if (process.platform === 'win32') {
+            const res = await this.runDriverAction('drag', [startX, startY, endX, endY]);
+            const success = Boolean(res?.success);
+            return {
+                success,
+                output: success
+                    ? `Dragged from (${startX}, ${startY}) to (${endX}, ${endY})`
+                    : `Drag failed: ${res?.error || 'Unknown error'}`,
+                action: 'drag',
+                from: { x: startX, y: startY },
+                to: { x: endX, y: endY },
+                coordinates: res?.coordinates || { x: endX, y: endY },
+                error: res?.error,
+                recoverable: res?.recoverable ?? true,
+            };
+        }
         return {
             success: true,
-            output: `Mock typed "${text}" into active application ${this.activeApp || 'desktop'}.`,
-            details: { text, activeApp: this.activeApp },
+            output: `Simulated drag from (${startX}, ${startY}) to (${endX}, ${endY})`,
+            action: 'drag',
+            from: { x: startX, y: startY },
+            to: { x: endX, y: endY },
+            coordinates: { x: endX, y: endY },
         };
     }
-    async key_press(key) {
-        this.actionLog.push({ action: 'key_press', params: { key } });
+    async scroll(deltaX, deltaY) {
+        if (process.platform === 'win32') {
+            const res = await this.runDriverAction('scroll', [deltaX, deltaY]);
+            return {
+                success: Boolean(res?.success),
+                output: `Scrolled deltaY=${deltaY}, deltaX=${deltaX}`,
+                action: 'scroll',
+                details: { deltaX, deltaY },
+            };
+        }
         return {
             success: true,
-            output: `Mock pressed key "${key}".`,
-            details: { key },
+            output: `Simulated scroll deltaY=${deltaY}, deltaX=${deltaX}`,
+            action: 'scroll',
+            details: { deltaX, deltaY },
         };
     }
-    async hotkey(keys) {
-        const combo = Array.isArray(keys) ? keys.join('+') : keys;
-        this.actionLog.push({ action: 'hotkey', params: { combo } });
-        return {
-            success: true,
-            output: `Mock pressed hotkey combo "${combo}".`,
-            details: { combo },
-        };
+    async getMousePosition() {
+        if (process.platform === 'win32') {
+            const res = await this.runDriverAction('position', []);
+            if (res && res.coordinates) {
+                return res.coordinates;
+            }
+        }
+        return { x: 0, y: 0 };
     }
-    async scroll(direction, amount = 4) {
-        this.actionLog.push({ action: 'scroll', params: { direction, amount } });
+    async runDiagnostic() {
+        if (process.platform === 'win32') {
+            const res = await this.runDriverAction('diagnostic', []);
+            return {
+                success: Boolean(res?.success),
+                output: `Diagnostic: Screen ${res?.width}x${res?.height}, Mouse (${res?.mouse?.x}, ${res?.mouse?.y}), Active: "${res?.activeWindow}", Windows: ${res?.windowCount}`,
+                action: 'diagnostic',
+                coordinates: res?.mouse,
+                activeWindow: res?.activeWindow,
+                details: res,
+            };
+        }
         return {
             success: true,
-            output: `Mock scrolled ${direction} by ${amount} units.`,
-            details: { direction, amount },
+            output: `Diagnostic on ${process.platform}: Screen 1920x1080, Mouse (0, 0), Active: "Desktop"`,
+            action: 'diagnostic',
+            coordinates: { x: 0, y: 0 },
+            activeWindow: 'Desktop',
         };
     }
-    async move_mouse(x, y) {
-        this.cursor = { x, y };
-        this.actionLog.push({ action: 'move_mouse', params: { x, y } });
+    async type(text) {
+        if (!text)
+            return { success: true, output: 'Typed empty string', action: 'type' };
+        if (process.platform === 'win32') {
+            const targetApp = this.expectedApplication || '';
+            if (targetApp) {
+                await this.switchWindow(targetApp);
+            }
+            const res = await this.runDriverAction('type', [text]);
+            const success = Boolean(res?.success);
+            const activeWin = res?.activeWindow || targetApp || 'foreground';
+            this.activeWindow = activeWin;
+            return {
+                success,
+                output: success
+                    ? `Typed text (${text.length} chars) into "${activeWin}"`
+                    : `Type failed: ${res?.error || 'Unknown error'}`,
+                action: 'type',
+                error: res?.error,
+                recoverable: true,
+                activeWindow: activeWin,
+                expectedWindow: this.expectedApplication || undefined,
+            };
+        }
         return {
             success: true,
-            output: `Mock moved cursor to (${x}, ${y}).`,
-            details: { x, y },
+            output: `Simulated type (${text.length} chars)`,
+            action: 'type',
+            activeWindow: this.expectedApplication || 'Simulated Window',
         };
     }
-    async drag(start, end) {
-        this.cursor = { ...end };
-        this.actionLog.push({ action: 'drag', params: { start, end } });
-        return {
-            success: true,
-            output: `Mock dragged from (${start.x}, ${start.y}) to (${end.x}, ${end.y}).`,
-            details: { start, end },
-        };
+    async keyPress(key) {
+        if (process.platform === 'win32') {
+            const targetApp = this.expectedApplication || '';
+            if (targetApp) {
+                await this.switchWindow(targetApp);
+            }
+            const res = await this.runDriverAction('key', [key]);
+            const success = Boolean(res?.success);
+            return {
+                success,
+                output: success ? `Pressed key: ${key}` : `KeyPress failed: ${res?.error || 'Unknown error'}`,
+                action: 'key_press',
+                error: res?.error,
+                recoverable: true,
+                activeWindow: res?.activeWindow || targetApp || undefined,
+            };
+        }
+        return { success: true, output: `Simulated key press: ${key}`, action: 'key_press' };
     }
-    async wait(durationMs) {
-        this.actionLog.push({ action: 'wait', params: { durationMs } });
-        return {
-            success: true,
-            output: `Mock waited ${durationMs}ms.`,
-            details: { durationMs },
-        };
+    async hotkey(keys) {
+        if (!keys || keys.length === 0) {
+            return { success: false, output: 'No keys provided for hotkey', action: 'hotkey', error: 'No keys provided' };
+        }
+        if (process.platform === 'win32') {
+            const targetApp = this.expectedApplication || '';
+            if (targetApp) {
+                await this.switchWindow(targetApp);
+            }
+            const res = await this.runDriverAction('hotkey', [keys.join('+')]);
+            const success = Boolean(res?.success);
+            return {
+                success,
+                output: success ? `Executed hotkey: ${keys.join('+')}` : `Hotkey failed: ${res?.error || 'Unknown error'}`,
+                action: 'hotkey',
+                error: res?.error,
+                recoverable: true,
+                activeWindow: res?.activeWindow || targetApp || undefined,
+            };
+        }
+        return { success: true, output: `Simulated hotkey: ${keys.join('+')}`, action: 'hotkey' };
     }
-    async open_application(name, target) {
-        this.openApps.add(name);
-        this.activeApp = name;
-        this.actionLog.push({ action: 'open_application', params: { name, target } });
-        return {
-            success: true,
-            output: `Mock opened application "${name}" with target "${target || 'default'}".`,
-            details: { name, target, active: true },
-        };
+    async wait(ms) {
+        return new Promise((resolve) => setTimeout(resolve, Math.max(0, ms)));
     }
-    async close_application(name) {
-        this.openApps.delete(name);
-        if (this.activeApp === name)
-            this.activeApp = null;
-        this.actionLog.push({ action: 'close_application', params: { name } });
-        return {
-            success: true,
-            output: `Mock closed application "${name}".`,
-            details: { name },
-        };
+    async openApplication(appName, args = []) {
+        const appLower = appName.toLowerCase().trim();
+        const argsStr = args.join(' ');
+        try {
+            if (process.platform === 'win32') {
+                const driverRes = await this.runDriverAction('launch', [appName]);
+                if (!driverRes || !driverRes.success) {
+                    if (appLower === 'code' || appLower === 'vscode') {
+                        await execAsync(`start code ${argsStr}`);
+                    }
+                    else if (appLower === 'explorer' || appLower === 'file explorer') {
+                        await execAsync(`start explorer ${argsStr || '.'}`);
+                    }
+                    else if (appLower === 'notepad') {
+                        await execAsync(`start notepad ${argsStr}`);
+                    }
+                    else if (appLower === 'terminal' || appLower === 'cmd' || appLower === 'powershell') {
+                        await execAsync(`start powershell ${argsStr}`);
+                    }
+                    else {
+                        await execAsync(`start "" "${appName}" ${argsStr}`);
+                    }
+                }
+                this.expectedApplication = appName;
+                // Observation-based wait: poll inspectUI to confirm window appeared
+                const delays = [400, 800, 1200];
+                for (const delay of delays) {
+                    await this.wait(delay);
+                    const ui = await this.inspectUI();
+                    const match = ui.windows.some(w => w.toLowerCase().includes(appName.toLowerCase()));
+                    if (match || (ui.activeWindow && ui.activeWindow.toLowerCase().includes(appName.toLowerCase()))) {
+                        break;
+                    }
+                }
+                if (this.expectedApplication) {
+                    await this.switchWindow(this.expectedApplication);
+                }
+                return {
+                    success: true,
+                    output: `Launched and focused application: "${this.expectedApplication || appName}"`,
+                    action: 'open_app',
+                    activeWindow: this.expectedApplication || appName,
+                    expectedWindow: this.expectedApplication || appName,
+                };
+            }
+            else if (process.platform === 'darwin') {
+                await execAsync(`open -a "${appName}" ${argsStr}`);
+                this.expectedApplication = appName;
+            }
+            else {
+                await execAsync(`xdg-open "${appName}" ${argsStr} &`);
+                this.expectedApplication = appName;
+            }
+            return {
+                success: true,
+                output: `Launched application: "${appName}"`,
+                action: 'open_app',
+                activeWindow: appName,
+                expectedWindow: appName,
+            };
+        }
+        catch (err) {
+            return {
+                success: false,
+                output: `Failed to launch "${appName}": ${err?.message || String(err)}`,
+                action: 'open_app',
+                error: err?.message || String(err),
+                recoverable: true,
+            };
+        }
     }
-    async switch_application(name) {
-        this.openApps.add(name);
-        this.activeApp = name;
-        this.actionLog.push({ action: 'switch_application', params: { name } });
-        return {
-            success: true,
-            output: `Mock focused application "${name}".`,
-            details: { name },
-        };
+    async closeApplication(appName) {
+        try {
+            if (process.platform === 'win32') {
+                const cleanName = appName.replace(/\.exe$/i, '');
+                await execAsync(`powershell -NoProfile -Command "Stop-Process -Name '${cleanName}' -Force -ErrorAction SilentlyContinue"`);
+            }
+            else {
+                await execAsync(`pkill -f "${appName}"`);
+            }
+            if (this.expectedApplication && this.expectedApplication.toLowerCase().includes(appName.toLowerCase())) {
+                this.expectedApplication = null;
+            }
+            return { success: true, output: `Closed application: "${appName}"`, action: 'close_app' };
+        }
+        catch (err) {
+            return {
+                success: false,
+                output: `Failed to close "${appName}": ${err?.message || String(err)}`,
+                action: 'close_app',
+                error: err?.message || String(err),
+            };
+        }
     }
-    async inspect_page() {
-        this.actionLog.push({ action: 'inspect_page', params: {} });
-        return {
-            success: true,
-            output: `Mock visual inspection of active app "${this.activeApp || 'desktop'}": 3 windows, 12 controls detected.`,
-            details: { activeApp: this.activeApp, controlsCount: 12 },
-        };
+    async switchWindow(windowTitleOrApp) {
+        if (process.platform === 'win32') {
+            const res = await this.runDriverAction('focus_window', [windowTitleOrApp]);
+            if (res && res.success) {
+                this.activeWindow = res.activeWindow || windowTitleOrApp;
+                this.expectedApplication = windowTitleOrApp;
+                return {
+                    success: true,
+                    output: `Focused window matching: "${windowTitleOrApp}"`,
+                    action: 'switch_window',
+                    activeWindow: this.activeWindow || undefined,
+                    expectedWindow: windowTitleOrApp,
+                };
+            }
+            // Fallback to WScript.Shell AppActivate
+            try {
+                const script = `
+$wshell = New-Object -ComObject Wscript.Shell
+$result = $wshell.AppActivate("${windowTitleOrApp.replace(/"/g, '`"')}")
+if (-not $result) {
+    $p = Get-Process | Where-Object { $_.MainWindowTitle -like "*${windowTitleOrApp.replace(/"/g, '`"')}*" -or $_.ProcessName -like "*${windowTitleOrApp.replace(/"/g, '`"')}*" } | Select-Object -First 1
+    if ($p) {
+        $null = $wshell.AppActivate($p.Id)
+        $result = $true
     }
-    async get_active_window() {
-        return {
-            success: true,
-            output: `Active window: "${this.activeApp || 'Desktop'}"`,
-            details: { activeApp: this.activeApp },
-        };
+}
+if ($result) { Write-Output "OK" } else { Write-Output "NOT_FOUND" }
+`;
+                const encoded = Buffer.from(script, 'utf16le').toString('base64');
+                const { stdout } = await execAsync(`powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -EncodedCommand ${encoded}`);
+                if (stdout.includes('OK')) {
+                    this.activeWindow = windowTitleOrApp;
+                    this.expectedApplication = windowTitleOrApp;
+                    return {
+                        success: true,
+                        output: `Focused window matching: "${windowTitleOrApp}"`,
+                        action: 'switch_window',
+                        activeWindow: windowTitleOrApp,
+                        expectedWindow: windowTitleOrApp,
+                    };
+                }
+            }
+            catch { }
+            return {
+                success: false,
+                output: `Could not find or focus window: "${windowTitleOrApp}". Try computer_inspect_ui to list open windows.`,
+                action: 'switch_window',
+                recoverable: true,
+            };
+        }
+        this.activeWindow = windowTitleOrApp;
+        return { success: true, output: `Switched window to: "${windowTitleOrApp}"`, action: 'switch_window' };
+    }
+    async getWindowBounds(windowTitleOrApp) {
+        if (process.platform === 'win32') {
+            const res = await this.runDriverAction('window_bounds', [windowTitleOrApp]);
+            if (res && res.success && typeof res.x === 'number') {
+                return { x: res.x, y: res.y, width: res.width, height: res.height };
+            }
+        }
+        return null;
+    }
+    async inspectUI() {
+        if (process.platform === 'win32') {
+            const res = await this.runDriverAction('list_windows', []);
+            if (res && res.success && Array.isArray(res.windows)) {
+                return {
+                    activeWindow: res.activeWindow || 'Desktop',
+                    windows: res.windows,
+                };
+            }
+            try {
+                const { stdout } = await execAsync(`powershell -NoProfile -NonInteractive -Command "Get-Process | Where-Object { $_.MainWindowTitle -and $_.MainWindowTitle.Trim().Length -gt 0 } | Select-Object -ExpandProperty MainWindowTitle"`);
+                const titles = stdout
+                    .split('\n')
+                    .map((t) => t.trim())
+                    .filter((t) => t.length > 0);
+                const active = titles.length > 0 ? titles[0] : 'Desktop';
+                return {
+                    activeWindow: active,
+                    windows: titles,
+                };
+            }
+            catch {
+                return { activeWindow: 'Desktop', windows: ['Desktop'] };
+            }
+        }
+        return { activeWindow: 'Desktop', windows: ['Desktop', 'Terminal'] };
     }
 }
 //# sourceMappingURL=computerController.js.map