ft-scout 8.0.0 → 8.0.2

package/bin/src/engine/appControl.js

@@ -4,6 +4,10 @@ import http from 'http';
 import https from 'https';
 import { exec } from 'child_process';
 import { promisify } from 'util';
+import OpenAI from 'openai';
+import dotenv from 'dotenv';
+import chalk from 'chalk';
+dotenv.config({ quiet: true });
 const execAsync = promisify(exec);
 /**
  * Standardize app name aliases across platforms.
@@ -80,6 +84,16 @@ export function buildSocialUrl(platformOrAppName, target, text) {
     if (isLinkedIn) {
         return `https://www.linkedin.com/messaging/`;
     }
+    const isEmail = pLower.includes('gmail') || pLower.includes('mail') || pLower.includes('email') || (cleanTarget.includes('@') && cleanTarget.includes('.')) || cleanTarget.includes('mail.google.com');
+    if (isEmail) {
+        const to = cleanTarget.includes('@') && !cleanTarget.startsWith('http') ? cleanTarget : (parsedPayload.to || parsedPayload.recipient || '');
+        const subject = parsedPayload.subject || parsedPayload.su || 'Message from Scout';
+        const body = rawMsgText || parsedPayload.body || '';
+        if (to || body) {
+            return `https://mail.google.com/mail/?view=cm&fs=1${to ? `&to=${encodeURIComponent(to)}` : ''}${subject ? `&su=${encodeURIComponent(subject)}` : ''}${body ? `&body=${encodeURIComponent(body)}` : ''}`;
+        }
+        return 'https://mail.google.com';
+    }
     if (cleanTarget.startsWith('http://') || cleanTarget.startsWith('https://')) {
         return cleanTarget;
     }
@@ -261,7 +275,7 @@ export async function executeInApp(appName, action, payload) {
             output: res.output,
         };
     }
-    if (['capture_screen', 'screenshot', 'see_screen', 'inspect_screen', 'screen_shot', 'view_screen'].includes(actLower)) {
+    if (['capture_screen', 'screenshot', 'screen_shot'].includes(actLower)) {
         const res = await captureScreen(name || appName);
         return {
             success: res.success,
@@ -271,8 +285,9 @@ export async function executeInApp(appName, action, payload) {
             details: res.details,
         };
     }
-    if (['analyze_screen', 'inspect_desktop', 'visual_analysis', 'see_and_analyze'].includes(actLower)) {
-        const res = await analyzeScreen(name || appName);
+    if (['see_screen', 'inspect_screen', 'view_screen', 'analyze_screen', 'inspect_desktop', 'visual_analysis', 'see_and_analyze'].includes(actLower)) {
+        const query = typeof payload === 'string' ? payload : (payload?.query || payload?.target || payload?.text || undefined);
+        const res = await analyzeScreen(name || appName, query);
         return {
             success: res.success,
             app: name || appName,
@@ -293,9 +308,26 @@ export async function executeInApp(appName, action, payload) {
     }
     if (['type_text', 'send_keys', 'type', 'keystrokes', 'key_combo', 'press_hotkey', 'hotkey'].includes(actLower)) {
         const parsed = parsePayload(payload);
-        const textToType = typeof payload === 'string' ? payload : (parsed.text || parsed.content || parsed.message || '');
-        const pressEnter = typeof payload === 'object' ? parsed.enter !== false : true;
-        const keyCombo = typeof payload === 'object' ? (parsed.keyCombo || parsed.combo || parsed.hotkey || parsed.keys) : (typeof payload === 'string' && payload.includes('+') ? payload : undefined);
+        const isHotKeyAction = ['key_combo', 'press_hotkey', 'hotkey', 'send_keys'].includes(actLower);
+        let keyCombo = undefined;
+        let textToType = '';
+        if (typeof parsed === 'object') {
+            keyCombo = parsed.keyCombo || parsed.combo || parsed.hotkey || parsed.keys || parsed.key;
+            textToType = parsed.text || parsed.content || parsed.message || '';
+        }
+        if (!keyCombo && typeof payload === 'string') {
+            const trimmed = payload.trim();
+            if (isHotKeyAction || trimmed.includes('+') || ['enter', 'return', 'tab', 'esc', 'escape', 'backspace', 'space', 'up', 'down', 'left', 'right'].includes(trimmed.toLowerCase())) {
+                keyCombo = trimmed;
+            }
+            else {
+                textToType = trimmed;
+            }
+        }
+        else if (!textToType && typeof payload === 'string' && !payload.trim().startsWith('{')) {
+            textToType = payload;
+        }
+        const pressEnter = typeof parsed === 'object' && parsed.enter !== undefined ? parsed.enter : !isHotKeyAction;
         const ksRes = await sendKeystrokes(name || appName, textToType, pressEnter, keyCombo);
         return {
             success: ksRes.success,
@@ -305,14 +337,34 @@ export async function executeInApp(appName, action, payload) {
             details: { appName: name || appName, textToType, pressEnter, keyCombo },
         };
     }
-    if (['click', 'click_app', 'click_at', 'move_mouse', 'drag_mouse', 'scroll'].includes(actLower)) {
+    if (['click_element', 'click_ui', 'click_button', 'tap_element'].includes(actLower)) {
+        const parsed = parsePayload(payload);
+        const query = typeof parsed === 'object' ? (parsed.element || parsed.target || parsed.name || parsed.query || '') : String(parsed || payload);
+        const button = typeof parsed === 'object' && parsed.button ? parsed.button : 'left';
+        const res = await findAndClickElement(query, name || appName, button);
+        return {
+            success: res.success,
+            app: name || appName,
+            action: actLower,
+            output: res.output,
+            details: res.coordinates,
+        };
+    }
+    if (['click', 'click_app', 'click_at', 'move_mouse', 'drag_mouse'].includes(actLower)) {
         const parsedPayload = parsePayload(payload);
+        if (typeof parsedPayload === 'object' && parsedPayload.element && parsedPayload.x === undefined) {
+            const res = await findAndClickElement(parsedPayload.element, name || appName, parsedPayload.button || 'left');
+            return {
+                success: res.success,
+                app: name || appName,
+                action: actLower,
+                output: res.output,
+                details: res.coordinates,
+            };
+        }
         const x = typeof parsedPayload.x === 'number' ? parsedPayload.x : undefined;
         const y = typeof parsedPayload.y === 'number' ? parsedPayload.y : undefined;
         const button = ['right', 'double', 'middle', 'scroll'].includes(parsedPayload.button) ? parsedPayload.button : 'left';
-        if (name && name !== 'desktop' && name !== 'system') {
-            await openApp(name);
-        }
         const clickRes = await moveAndClickMouse(x, y, button, name || appName);
         return {
             success: clickRes.success,
@@ -322,6 +374,52 @@ export async function executeInApp(appName, action, payload) {
             details: { appName: name || appName, x, y, button },
         };
     }
+    if (['scroll', 'scroll_window', 'scroll_down', 'scroll_up', 'scroll_page'].includes(actLower)) {
+        const parsed = parsePayload(payload);
+        let direction = 'down';
+        let amount = 3;
+        if (typeof parsed === 'object') {
+            if (parsed.direction === 'up' || parsed.dir === 'up')
+                direction = 'up';
+            if (typeof parsed.amount === 'number')
+                amount = parsed.amount;
+            if (typeof parsed.clicks === 'number')
+                amount = parsed.clicks;
+        }
+        else if (typeof payload === 'string' && payload.toLowerCase().includes('up')) {
+            direction = 'up';
+        }
+        if (actLower === 'scroll_up')
+            direction = 'up';
+        const res = await scrollWindow(direction, amount);
+        return {
+            success: res.success,
+            app: name || appName,
+            action: actLower,
+            output: res.output,
+        };
+    }
+    if (['list_windows', 'get_windows', 'open_windows', 'show_windows'].includes(actLower)) {
+        const res = await listWindows();
+        return {
+            success: res.success,
+            app: name || appName,
+            action: actLower,
+            output: res.output,
+            details: res.windows,
+        };
+    }
+    if (['focus_window', 'switch_window', 'bring_to_front', 'select_window'].includes(actLower)) {
+        const parsed = parsePayload(payload);
+        const target = typeof parsed === 'object' ? (parsed.title || parsed.app || parsed.name || '') : String(parsed || payload || name || appName);
+        const res = await focusWindow(target);
+        return {
+            success: res.success,
+            app: target,
+            action: actLower,
+            output: res.output,
+        };
+    }
     if (['lock_app', 'block_interruption', 'lock_input', 'lock'].includes(actLower)) {
         const parsedPayload = parsePayload(payload);
         const durationMs = typeof parsedPayload.duration === 'number' ? parsedPayload.duration : 3000;
@@ -394,16 +492,27 @@ export async function executeInApp(appName, action, payload) {
                 details: { query, searchUrl, capture: capture.details },
             };
         }
-        if (['send_dm', 'open_dm', 'dm', 'send_message', 'message', 'social_dm'].includes(actLower)) {
-            const recipient = typeof payload === 'string' ? payload : (payload?.recipient || payload?.user || payload?.target || payload?.handle || payload?.username || '');
-            let msgText = typeof payload === 'object' ? (payload?.text || payload?.message || payload?.content) : undefined;
-            if (!msgText && typeof payload === 'string' && payload.length > 0 && !payload.startsWith('@') && !payload.startsWith('http')) {
+        if (['send_dm', 'open_dm', 'dm', 'send_message', 'message', 'social_dm', 'send_mail', 'send_email', 'mail', 'email'].includes(actLower)) {
+            const parsed = parsePayload(payload);
+            const recipient = typeof payload === 'string' && payload.includes('@') && !payload.startsWith('{')
+                ? payload
+                : (parsed.recipient || parsed.to || parsed.user || parsed.target || parsed.handle || parsed.username || '');
+            let msgText = typeof parsed === 'object' ? (parsed.text || parsed.message || parsed.content || parsed.body) : undefined;
+            if (!msgText && typeof payload === 'string' && payload.length > 0 && !payload.startsWith('@') && !payload.startsWith('http') && !payload.includes('@')) {
                 msgText = payload;
             }
-            const targetUrl = buildSocialUrl(name, recipient, msgText);
+            const isEmail = ['send_mail', 'send_email', 'mail', 'email'].includes(actLower) || recipient.includes('@');
+            const targetUrl = buildSocialUrl(isEmail ? 'gmail' : name, recipient, msgText);
             const openRes = await openApp('browser', targetUrl);
             let keystrokeOutput = '';
-            if (msgText) {
+            if (isEmail) {
+                // For Gmail direct compose: wait 2.5s for page & compose dialog to render, then press Ctrl+Enter to send!
+                await lockAppInput('browser', 3500);
+                await new Promise((r) => setTimeout(r, 2200));
+                const ksRes = await sendKeystrokes('browser', '', false, 'ctrl+enter');
+                keystrokeOutput = `\nAuto-Send Keystroke Dispatch (Ctrl+Enter): ${ksRes.output}`;
+            }
+            else if (msgText) {
                 // Lock user interruption during scratchpad keystroke dispatch so user clicks don't interrupt
                 await lockAppInput(name || 'browser', 3000);
                 await new Promise((r) => setTimeout(r, 1200));
@@ -414,8 +523,8 @@ export async function executeInApp(appName, action, payload) {
                 success: true,
                 app: name,
                 action: actLower,
-                output: `Opened ${name} Direct Message UI for target recipient "${recipient}" at ${targetUrl}.${keystrokeOutput}`,
-                details: { platform: name, recipient, msgText, targetUrl },
+                output: `Executed ${isEmail ? 'Gmail / Email Dispatch' : `${name} Direct Message`} for target recipient "${recipient}" at ${targetUrl}.${keystrokeOutput}`,
+                details: { platform: isEmail ? 'gmail' : name, recipient, msgText, targetUrl },
             };
         }
     }
@@ -562,8 +671,219 @@ $activeTitle = $sb.ToString()
         };
     }
 }
+/**
+ * Display a futuristic sky-blue aura animation on screen borders and corners,
+ * show the warning "Scout is on the screen.", and block external inputs
+ * during live screen capture and automation.
+ */
+export async function showScreenAuraOverlay(durationMs = 2200, message = "Scout is viewing the screen.", blockInput = false) {
+    const platform = process.platform;
+    if (platform !== 'win32')
+        return;
+    try {
+        const scriptsDir = path.join(process.cwd(), '.ft', 'scripts');
+        if (!fs.existsSync(scriptsDir)) {
+            fs.mkdirSync(scriptsDir, { recursive: true });
+        }
+        const psFile = path.join(scriptsDir, 'scout_aura_overlay.ps1');
+        const blockInputSnippet = blockInput ? `
+$sig = '[DllImport("user32.dll")] public static extern bool BlockInput(bool fBlock);'
+$b = Add-Type -memberDefinition $sig -name 'Win32AuraBlock' -namespace Win32Functions -passThru
+try { [void]$b::BlockInput($true) } catch {}
+` : '';
+        const unblockSnippet = blockInput ? `
+    try { [void]$b::BlockInput($false) } catch {}
+` : '';
+        const subtitleText = blockInput ? " • External Input Locked" : " • Perception Active";
+        const overlayPs = `
+Add-Type -AssemblyName PresentationFramework, PresentationCore, WindowsBase
+${blockInputSnippet}
+
+[xml]$xaml = @"
+<Window xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
+        xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
+        Title="Scout Aura Overlay"
+        WindowStyle="None"
+        AllowsTransparency="True"
+        Background="Transparent"
+        Topmost="True"
+        ShowInTaskbar="False"
+        ShowActivated="False"
+        IsHitTestVisible="False">
+    <Window.Resources>
+        <Storyboard x:Key="AuraPulse" RepeatBehavior="Forever" AutoReverse="True">
+            <DoubleAnimation Storyboard.TargetName="AuraBorder" Storyboard.TargetProperty="Opacity"
+                             From="0.4" To="1.0" Duration="0:0:0.6"/>
+            <DoubleAnimation Storyboard.TargetName="CornerTL" Storyboard.TargetProperty="Opacity"
+                             From="0.5" To="1.0" Duration="0:0:0.6"/>
+            <DoubleAnimation Storyboard.TargetName="CornerTR" Storyboard.TargetProperty="Opacity"
+                             From="0.5" To="1.0" Duration="0:0:0.6"/>
+            <DoubleAnimation Storyboard.TargetName="CornerBL" Storyboard.TargetProperty="Opacity"
+                             From="0.5" To="1.0" Duration="0:0:0.6"/>
+            <DoubleAnimation Storyboard.TargetName="CornerBR" Storyboard.TargetProperty="Opacity"
+                             From="0.5" To="1.0" Duration="0:0:0.6"/>
+            <DoubleAnimation Storyboard.TargetName="AuraGlow" Storyboard.TargetProperty="Opacity"
+                             From="0.25" To="0.9" Duration="0:0:0.6"/>
+        </Storyboard>
+    </Window.Resources>
+
+    <Grid IsHitTestVisible="False">
+        <!-- Outer Glowing Sky Blue Aura Border -->
+        <Border x:Name="AuraGlow" BorderThickness="6" BorderBrush="#00d8ff" Margin="0">
+            <Border.Effect>
+                <DropShadowEffect Color="#00e5ff" BlurRadius="45" ShadowDepth="0" Opacity="0.95"/>
+            </Border.Effect>
+        </Border>
+        <Border x:Name="AuraBorder" BorderThickness="3" BorderBrush="#38bdf8" Margin="0">
+            <Border.Effect>
+                <DropShadowEffect Color="#00e5ff" BlurRadius="25" ShadowDepth="0" Opacity="1"/>
+            </Border.Effect>
+        </Border>
+
+        <!-- Sky Blue Corner Brackets with Aura Glow -->
+        <!-- Top Left Corner -->
+        <Canvas HorizontalAlignment="Left" VerticalAlignment="Top" Width="120" Height="120" Margin="6,6,0,0">
+            <Path x:Name="CornerTL" Data="M 0,90 L 0,0 L 90,0" Stroke="#00e5ff" StrokeThickness="6">
+                <Path.Effect>
+                    <DropShadowEffect Color="#00e5ff" BlurRadius="30" ShadowDepth="0" Opacity="1"/>
+                </Path.Effect>
+            </Path>
+        </Canvas>
+
+        <!-- Top Right Corner -->
+        <Canvas HorizontalAlignment="Right" VerticalAlignment="Top" Width="120" Height="120" Margin="0,6,6,0">
+            <Path x:Name="CornerTR" Data="M 30,0 L 120,0 L 120,90" Stroke="#00e5ff" StrokeThickness="6">
+                <Path.Effect>
+                    <DropShadowEffect Color="#00e5ff" BlurRadius="30" ShadowDepth="0" Opacity="1"/>
+                </Path.Effect>
+            </Path>
+        </Canvas>
+
+        <!-- Bottom Left Corner -->
+        <Canvas HorizontalAlignment="Left" VerticalAlignment="Bottom" Width="120" Height="120" Margin="6,0,0,6">
+            <Path x:Name="CornerBL" Data="M 0,30 L 0,120 L 90,120" Stroke="#00e5ff" StrokeThickness="6">
+                <Path.Effect>
+                    <DropShadowEffect Color="#00e5ff" BlurRadius="30" ShadowDepth="0" Opacity="1"/>
+                </Path.Effect>
+            </Path>
+        </Canvas>
+
+        <!-- Bottom Right Corner -->
+        <Canvas HorizontalAlignment="Right" VerticalAlignment="Bottom" Width="120" Height="120" Margin="0,0,6,6">
+            <Path x:Name="CornerBR" Data="M 30,120 L 120,120 L 120,30" Stroke="#00e5ff" StrokeThickness="6">
+                <Path.Effect>
+                    <DropShadowEffect Color="#00e5ff" BlurRadius="30" ShadowDepth="0" Opacity="1"/>
+                </Path.Effect>
+            </Path>
+        </Canvas>
+
+        <!-- Top Floating Status Badge -->
+        <Border HorizontalAlignment="Center" VerticalAlignment="Top" Margin="0,22,0,0"
+                Background="#EE03121F" BorderBrush="#00e5ff" BorderThickness="1.8" CornerRadius="24" Padding="26,10">
+            <Border.Effect>
+                <DropShadowEffect Color="#00e5ff" BlurRadius="25" ShadowDepth="0" Opacity="0.95"/>
+            </Border.Effect>
+            <StackPanel Orientation="Horizontal" VerticalAlignment="Center">
+                <Ellipse Width="12" Height="12" Fill="#00e5ff" Margin="0,0,12,0">
+                    <Ellipse.Effect>
+                        <DropShadowEffect Color="#00e5ff" BlurRadius="12" ShadowDepth="0"/>
+                    </Ellipse.Effect>
+                </Ellipse>
+                <TextBlock Text="${message}" Foreground="#ffffff" FontSize="16" FontWeight="Bold" VerticalAlignment="Center"/>
+                <TextBlock Text="${subtitleText}" Foreground="#38bdf8" FontSize="13.5" FontWeight="SemiBold" Margin="10,0,0,0" VerticalAlignment="Center"/>
+            </StackPanel>
+        </Border>
+    </Grid>
+</Window>
+"@
+
+$reader = (New-Object System.Xml.XmlNodeReader $xaml)
+$window = [System.Windows.Markup.XamlReader]::Load($reader)
+
+$window.Left = [System.Windows.SystemParameters]::VirtualScreenLeft
+$window.Top = [System.Windows.SystemParameters]::VirtualScreenTop
+$window.Width = [System.Windows.SystemParameters]::VirtualScreenWidth
+$window.Height = [System.Windows.SystemParameters]::VirtualScreenHeight
+$window.Topmost = $true
+$window.ShowActivated = $false
+
+$storyboard = $window.Resources["AuraPulse"]
+$window.Add_Loaded({
+    $storyboard.Begin($window)
+})
+
+# Safety timer to close overlay and guarantee unblock
+$timer = New-Object System.Windows.Threading.DispatcherTimer
+$timer.Interval = [TimeSpan]::FromMilliseconds(${durationMs})
+$timer.Add_Tick({
+    $timer.Stop()
+    ${unblockSnippet}
+    $window.Close()
+})
+$timer.Start()
+
+[void]$window.ShowDialog()
+${unblockSnippet}
+`;
+        fs.writeFileSync(psFile, overlayPs, 'utf8');
+        await execAsync(`powershell -WindowStyle Hidden -NoProfile -Sta -ExecutionPolicy Bypass -File "${psFile}"`, { timeout: durationMs + 3000 });
+    }
+    catch {
+        if (blockInput) {
+            try {
+                await execAsync(`powershell -NoProfile -Command "$sig='[DllImport(\\"user32.dll\\")] public static extern bool BlockInput(bool f);';$b=Add-Type -m $sig -n B -p;[void]$b::BlockInput($false)"`);
+            }
+            catch { }
+        }
+    }
+}
+const capturedScreenshots = new Set();
+/**
+ * Register a captured screenshot file path for automatic post-task cleanup.
+ */
+export function registerCapturedScreenshot(filePath) {
+    if (filePath) {
+        capturedScreenshots.add(path.resolve(filePath));
+    }
+}
+/**
+ * Clean up all captured screenshots permanently from disk after task completion.
+ */
+export function cleanupScreenshots() {
+    let deletedCount = 0;
+    // 1. Delete all tracked screenshots
+    for (const filePath of capturedScreenshots) {
+        try {
+            if (fs.existsSync(filePath)) {
+                fs.unlinkSync(filePath);
+                deletedCount++;
+            }
+        }
+        catch { }
+    }
+    capturedScreenshots.clear();
+    // 2. Clean any remaining files in .ft/screenshots
+    try {
+        const screenshotsDir = path.join(process.cwd(), '.ft', 'screenshots');
+        if (fs.existsSync(screenshotsDir)) {
+            const files = fs.readdirSync(screenshotsDir);
+            for (const file of files) {
+                if (file.startsWith('screen_') && (file.endsWith('.png') || file.endsWith('.jpg'))) {
+                    try {
+                        fs.unlinkSync(path.join(screenshotsDir, file));
+                        deletedCount++;
+                    }
+                    catch { }
+                }
+            }
+        }
+    }
+    catch { }
+    return { success: true, deletedCount };
+}
 /**
  * Capture full desktop screen or specified window screenshot without running shell commands.
+ * Screenshots are tracked and will be deleted automatically upon task completion.
  */
 export async function captureScreen(appNameOrScreen) {
     const platform = process.platform;
@@ -608,9 +928,12 @@ $activeTitle = $sb.ToString()
             const encodedScript = Buffer.from(psScript, 'utf16le').toString('base64');
             const { stdout } = await execAsync(`powershell -NoProfile -EncodedCommand ${encodedScript}`);
             const meta = JSON.parse(stdout.trim());
+            registerCapturedScreenshot(outPath);
+            // Trigger visual sky-blue aura overlay on screen corners & borders
+            showScreenAuraOverlay(1800, "Scout is viewing the screen...", false).catch(() => { });
             return {
                 success: true,
-                output: `Screen Captured Successfully!\n- Screenshot Saved To: ${outPath}\n- Resolution: ${meta.width}x${meta.height}\n- Active Window: "${meta.activeWindow || 'Desktop'}"\n- Seeing Status: Screen is visible and analyzed. Mouse cursor & keyboard takeover active.`,
+                output: `Screen Captured Successfully!\n- Screenshot Saved To: ${outPath}\n- Resolution: ${meta.width}x${meta.height}\n- Active Window: "${meta.activeWindow || 'Desktop'}"\n- Seeing Status: Screen is visible and captured. (NOTE: This temporary screenshot will be automatically deleted upon task completion).`,
                 details: { ...meta, outPath },
             };
         }
@@ -624,9 +947,10 @@ $activeTitle = $sb.ToString()
     else if (platform === 'darwin') {
         try {
             await execAsync(`screencapture -x "${outPath}"`);
+            registerCapturedScreenshot(outPath);
             return {
                 success: true,
-                output: `macOS Screen Captured Successfully! Saved To: ${outPath}`,
+                output: `macOS Screen Captured Successfully! Saved To: ${outPath}\n(NOTE: This temporary screenshot will be automatically deleted upon task completion).`,
                 details: { outPath, platform: 'darwin' },
             };
         }
@@ -640,9 +964,10 @@ $activeTitle = $sb.ToString()
     else {
         try {
             await execAsync(`import -window root "${outPath}" || scrot "${outPath}"`);
+            registerCapturedScreenshot(outPath);
             return {
                 success: true,
-                output: `Linux Screen Captured Successfully! Saved To: ${outPath}`,
+                output: `Linux Screen Captured Successfully! Saved To: ${outPath}\n(NOTE: This temporary screenshot will be automatically deleted upon task completion).`,
                 details: { outPath, platform: 'linux' },
             };
         }
@@ -655,20 +980,62 @@ $activeTitle = $sb.ToString()
     }
 }
 /**
- * Analyze specified screen or active window visually without running shell commands.
+ * Visual screen analysis. Takes a full-screen screenshot and provides visual context
+ * (including AI Vision inspection if an API key is available).
+ * All temporary screenshots are automatically deleted upon task completion.
  */
-export async function analyzeScreen(appNameOrScreen) {
+export async function analyzeScreen(appNameOrScreen, targetQuery) {
     const capture = await captureScreen(appNameOrScreen);
     const info = await getScreenInfo(appNameOrScreen);
+    let visualAiAnalysis = '';
+    const outPath = capture.details?.outPath;
+    // Optional Vision Model Analysis if screenshot exists and API key is present
+    if (capture.success && outPath && fs.existsSync(outPath)) {
+        try {
+            const geminiKey = process.env.GEMINI_API_KEY;
+            const openaiKey = process.env.OPENAI_API_KEY;
+            const byokKey = process.env.BYOK_API_KEY;
+            const apiKey = geminiKey || openaiKey || byokKey;
+            if (apiKey) {
+                const imageBase64 = fs.readFileSync(outPath).toString('base64');
+                const isGemini = Boolean(geminiKey);
+                const baseURL = isGemini ? 'https://generativelanguage.googleapis.com/v1beta/openai' : undefined;
+                const model = isGemini ? 'gemini-2.5-flash' : 'gpt-4o-mini';
+                const client = new OpenAI({ apiKey, baseURL, timeout: 15000 });
+                const prompt = targetQuery
+                    ? `Describe what is visible on this screen specifically regarding: "${targetQuery}". Keep it concise, noting visible UI elements, open windows, tabs, buttons, or error messages.`
+                    : `Briefly describe what is currently visible on this desktop screen: active window, main content/tabs, buttons, and system state.`;
+                const resp = await client.chat.completions.create({
+                    model,
+                    messages: [
+                        {
+                            role: 'user',
+                            content: [
+                                { type: 'text', text: prompt },
+                                { type: 'image_url', image_url: { url: `data:image/png;base64,${imageBase64}` } },
+                            ],
+                        },
+                    ],
+                    max_tokens: 350,
+                });
+                const reply = resp.choices[0]?.message?.content;
+                if (reply && reply.trim()) {
+                    visualAiAnalysis = `\nAI Visual Screen Analysis:\n${reply.trim()}`;
+                }
+            }
+        }
+        catch {
+            // Vision API call is optional; fallback gracefully to system metadata
+        }
+    }
     const report = [
-        `=== SCOUT SCRATCHPAD DESKTOP SCREEN SEEING & VISUAL ANALYSIS ===`,
+        `=== SCOUT SCRATCHPAD DESKTOP SCREEN VISUAL INSPECTION ===`,
         `Target App / Screen Context: "${appNameOrScreen || 'Desktop'}"`,
-        `Screen Resolution: ${info.details?.screenWidth || 1920}x${info.details?.screenHeight || 1080}`,
-        `Active Window Focus: "${info.details?.activeWindow || appNameOrScreen || 'Active Desktop'}"`,
+        `Screen Resolution: ${info.details?.screenWidth || capture.details?.width || 1920}x${info.details?.screenHeight || capture.details?.height || 1080}`,
+        `Active Window Focus: "${capture.details?.activeWindow || info.details?.activeWindow || appNameOrScreen || 'Active Desktop'}"`,
         `Mouse Cursor Position: (${info.details?.cursorX || 0}, ${info.details?.cursorY || 0})`,
-        `Screenshot Artifact: ${capture.details?.outPath || 'Captured in memory'}`,
-        `Takeover Status: Full Mouse Cursor & Keyboard Input Takeover ACTIVE`,
-        `Visual Analysis: Screen successfully captured and verified. Target screen is ready for precise mouse clicks, cursor positioning, and direct keystrokes without shell command execution.`,
+        `Screenshot Status: Captured temporarily for visual inspection. (Will be automatically deleted upon task completion)`,
+        `Visual Status: Screen is visible and verified.${visualAiAnalysis}`,
     ].join('\n');
     return {
         success: capture.success,
@@ -683,14 +1050,19 @@ export async function analyzeScreen(appNameOrScreen) {
  */
 export async function takeoverControl(appName, durationMs = 3000, x, y) {
     const { category, name } = normalizeAppName(appName);
-    if (name && name !== 'desktop' && name !== 'system') {
+    if (name && name !== 'desktop' && name !== 'system' && category !== 'browser') {
         await openApp(name);
     }
+    // Trigger visual sky-blue aura overlay on screen borders & corners with warning badge & input block
+    try {
+        await showScreenAuraOverlay(Math.min(5000, Math.max(800, durationMs)), "Scout is on the screen.", true);
+    }
+    catch { }
     const mouseRes = await moveAndClickMouse(x, y, 'left', name || appName);
     const lockRes = await lockAppInput(name || appName, durationMs);
     return {
         success: lockRes.success && mouseRes.success,
-        output: `TAKEOVER EXECUTED: Scout Agent has taken over mouse cursor, window focus, and keyboard control on specified screen window ("${name || appName}"). User interruption locked for ${durationMs}ms. ${mouseRes.output} Agent can view, analyze, type, and click without shell commands.`,
+        output: `TAKEOVER EXECUTED: Scout Agent has taken over mouse cursor, window focus, and keyboard control on specified screen window ("${name || appName}"). Sky-blue aura HUD activated and user interruption locked for ${durationMs}ms. ${mouseRes.output} Agent can view, analyze, type, and click without shell commands.`,
         details: { appName: name || appName, durationMs, x, y, mouseOutput: mouseRes.output },
     };
 }
@@ -728,59 +1100,101 @@ export async function sendKeystrokes(appName, textToType, pressEnter = true, key
         const sendKeysText = sanitized.replace(/[{}+^~%()\[\]]/g, '{$&}');
         let sendKeysCombo = '';
         if (combo) {
-            if (combo.includes('ctrl+a') || combo === 'select_all')
+            const cLower = combo.toLowerCase().replace(/\s+/g, '');
+            if (cLower.includes('ctrl+enter') || cLower.includes('ctrl+return'))
+                sendKeysCombo = '^{ENTER}';
+            else if (cLower.includes('alt+enter') || cLower.includes('alt+return'))
+                sendKeysCombo = '%{ENTER}';
+            else if (cLower.includes('shift+enter') || cLower.includes('shift+return'))
+                sendKeysCombo = '+{ENTER}';
+            else if (cLower.includes('ctrl+a') || cLower === 'select_all')
                 sendKeysCombo = '^a';
-            else if (combo.includes('ctrl+c') || combo === 'copy')
+            else if (cLower.includes('ctrl+c') || cLower === 'copy')
                 sendKeysCombo = '^c';
-            else if (combo.includes('ctrl+v') || combo === 'paste')
+            else if (cLower.includes('ctrl+v') || cLower === 'paste')
                 sendKeysCombo = '^v';
-            else if (combo.includes('ctrl+z') || combo === 'undo')
+            else if (cLower.includes('ctrl+x') || cLower === 'cut')
+                sendKeysCombo = '^x';
+            else if (cLower.includes('ctrl+z') || cLower === 'undo')
                 sendKeysCombo = '^z';
-            else if (combo.includes('ctrl+s') || combo === 'save')
+            else if (cLower.includes('ctrl+s') || cLower === 'save')
                 sendKeysCombo = '^s';
-            else if (combo.includes('ctrl+l') || combo === 'address_bar')
+            else if (cLower.includes('ctrl+l') || cLower === 'address_bar')
                 sendKeysCombo = '^l';
-            else if (combo.includes('alt+d'))
+            else if (cLower.includes('ctrl+t') || cLower === 'new_tab')
+                sendKeysCombo = '^t';
+            else if (cLower.includes('ctrl+w') || cLower === 'close_tab')
+                sendKeysCombo = '^w';
+            else if (cLower.includes('ctrl+r') || cLower === 'reload')
+                sendKeysCombo = '^r';
+            else if (cLower.includes('ctrl+f') || cLower === 'find')
+                sendKeysCombo = '^f';
+            else if (cLower.includes('ctrl+n') || cLower === 'new_window')
+                sendKeysCombo = '^n';
+            else if (cLower.includes('ctrl+shift+i') || cLower.includes('devtools'))
+                sendKeysCombo = '^+i';
+            else if (cLower.includes('alt+d'))
                 sendKeysCombo = '%d';
-            else if (combo.includes('alt+tab'))
+            else if (cLower.includes('alt+tab'))
                 sendKeysCombo = '%{TAB}';
-            else if (combo.includes('alt+f4'))
+            else if (cLower.includes('alt+f4'))
                 sendKeysCombo = '%{F4}';
-            else if (combo === 'enter' || combo === 'return')
-                sendKeysCombo = '~';
-            else if (combo === 'esc' || combo === 'escape')
+            else if (cLower === 'enter' || cLower === 'return')
+                sendKeysCombo = '{ENTER}';
+            else if (cLower === 'esc' || cLower === 'escape')
                 sendKeysCombo = '{ESC}';
-            else if (combo === 'tab')
+            else if (cLower === 'tab')
                 sendKeysCombo = '{TAB}';
-            else if (combo === 'backspace')
+            else if (cLower === 'backspace')
                 sendKeysCombo = '{BACKSPACE}';
-            else if (combo === 'up')
+            else if (cLower === 'space')
+                sendKeysCombo = ' ';
+            else if (cLower === 'up')
                 sendKeysCombo = '{UP}';
-            else if (combo === 'down')
+            else if (cLower === 'down')
                 sendKeysCombo = '{DOWN}';
-            else if (combo === 'left')
+            else if (cLower === 'left')
                 sendKeysCombo = '{LEFT}';
-            else if (combo === 'right')
+            else if (cLower === 'right')
                 sendKeysCombo = '{RIGHT}';
             else
                 sendKeysCombo = combo;
         }
         const psScript = `
-$sig = '[DllImport("user32.dll")] public static extern bool BlockInput(bool fBlock); [DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr hWnd);'
-$b = Add-Type -memberDefinition $sig -name 'Win32Block' -namespace Win32Functions -passThru
+$sig = @'
+[DllImport("user32.dll")] public static extern bool BlockInput(bool fBlock);
+[DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr hWnd);
+[DllImport("user32.dll")] public static extern bool ShowWindow(IntPtr hWnd, int nCmdShow);
+[DllImport("user32.dll")] public static extern void keybd_event(byte bVk, byte bScan, uint dwFlags, UIntPtr dwExtraInfo);
+'@
+$b = Add-Type -memberDefinition $sig -name ('Win32Block_' + (Get-Random)) -namespace Win32Functions -passThru
 
 try { [void]$b::BlockInput($true) } catch {}
 
 Add-Type -AssemblyName System.Windows.Forms
 $ws = New-Object -ComObject WScript.Shell
-$activated = $ws.AppActivate("${appName}")
-if (-not $activated) { $activated = $ws.AppActivate("Chrome") }
-if (-not $activated) { $activated = $ws.AppActivate("Edge") }
-if (-not $activated) { $activated = $ws.AppActivate("Firefox") }
-if (-not $activated) { $activated = $ws.AppActivate("Brave") }
-if (-not $activated) { $activated = $ws.AppActivate("Notepad") }
-if (-not $activated) { $activated = $ws.AppActivate("Visual Studio Code") }
-Start-Sleep -Milliseconds 400
+
+try {
+    $proc = Get-Process -Name chrome, msedge, firefox, brave, opera, "${appName}" -ErrorAction SilentlyContinue | Where-Object { $_.MainWindowHandle -ne 0 } | Select-Object -First 1
+    if ($proc) {
+        $b::keybd_event(0x12, 0, 0, [UIntPtr]::Zero)
+        $b::keybd_event(0x12, 0, 2, [UIntPtr]::Zero)
+        [void]$b::ShowWindow($proc.MainWindowHandle, 9)
+        [void]$b::SetForegroundWindow($proc.MainWindowHandle)
+        Start-Sleep -Milliseconds 250
+    } else {
+        $activated = $ws.AppActivate("${appName}")
+        if (-not $activated) { $activated = $ws.AppActivate("Chrome") }
+        if (-not $activated) { $activated = $ws.AppActivate("Edge") }
+        if (-not $activated) { $activated = $ws.AppActivate("Firefox") }
+        if (-not $activated) { $activated = $ws.AppActivate("Brave") }
+        if (-not $activated) { $activated = $ws.AppActivate("Notepad") }
+        if (-not $activated) { $activated = $ws.AppActivate("Visual Studio Code") }
+        Start-Sleep -Milliseconds 300
+    }
+} catch {}
+
+Start-Sleep -Milliseconds 250
 
 ${sendKeysCombo ? `$ws.SendKeys("${sendKeysCombo}")` : ''}
 
@@ -875,9 +1289,20 @@ Add-Type -AssemblyName System.Windows.Forms
 $sig = '[DllImport("user32.dll")] public static extern void mouse_event(int dwFlags, int dx, int dy, int cButtons, int dwExtraInfo); [DllImport("user32.dll")] public static extern bool SetCursorPos(int X, int Y);'
 $m = Add-Type -memberDefinition $sig -name 'Win32Mouse' -namespace Win32Functions -passThru
 
-${appName && appName !== 'desktop' && appName !== 'system' ? `
-$ws = New-Object -ComObject WScript.Shell
-$ws.AppActivate("${appName}")
+${(appName && appName !== 'desktop' && appName !== 'system') ? `
+try {
+    $proc = Get-Process -Name chrome, msedge, firefox, brave, "${appName}" -ErrorAction SilentlyContinue | Where-Object { $_.MainWindowHandle -ne 0 } | Select-Object -First 1
+    if ($proc) {
+        $sigFw = '[DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr hWnd);'
+        $fw = Add-Type -memberDefinition $sigFw -name 'Win32FwMouse' -namespace Win32Functions -passThru
+        [void]$fw::SetForegroundWindow($proc.MainWindowHandle)
+        Start-Sleep -Milliseconds 250
+    } else {
+        $ws = New-Object -ComObject WScript.Shell
+        [void]$ws.AppActivate("${appName}")
+        Start-Sleep -Milliseconds 200
+    }
+} catch {}
 ` : ''}
 
 $screen = [System.Windows.Forms.Screen]::PrimaryScreen
@@ -1016,4 +1441,210 @@ end tell
         };
     }
 }
+/**
+ * List all active top-level application windows.
+ */
+export async function listWindows() {
+    const platform = process.platform;
+    if (platform === 'win32') {
+        const psScript = `
+Get-Process | Where-Object { $_.MainWindowTitle -ne '' } | Select-Object Id, ProcessName, MainWindowTitle, MainWindowHandle | ConvertTo-Json
+`;
+        try {
+            const encodedScript = Buffer.from(psScript, 'utf16le').toString('base64');
+            const { stdout } = await execAsync(`powershell -NoProfile -Sta -EncodedCommand ${encodedScript}`);
+            let rawList = [];
+            try {
+                rawList = JSON.parse(stdout.trim());
+            }
+            catch { }
+            if (!Array.isArray(rawList))
+                rawList = rawList ? [rawList] : [];
+            const windows = rawList.map((w) => ({
+                title: String(w.MainWindowTitle || ''),
+                processName: String(w.ProcessName || ''),
+                handle: String(w.MainWindowHandle || ''),
+            }));
+            const formatted = windows.map((w, idx) => ` ${idx + 1}. [${w.processName}] "${w.title}"`).join('\n');
+            return {
+                success: true,
+                output: `Active Application Windows (${windows.length}):\n${formatted || 'No active top-level windows with title found.'}`,
+                windows,
+            };
+        }
+        catch (err) {
+            return { success: false, output: `Failed to list windows: ${err?.message || String(err)}`, windows: [] };
+        }
+    }
+    return { success: true, output: 'Window listing only supported on Windows.', windows: [] };
+}
+/**
+ * Bring a target window to front and focus it.
+ */
+export async function focusWindow(appNameOrTitle) {
+    const platform = process.platform;
+    if (platform === 'win32') {
+        const psScript = `
+$target = "${appNameOrTitle.replace(/"/g, '`"')}"
+$proc = Get-Process | Where-Object { ($_.MainWindowTitle -like "*$target*" -or $_.ProcessName -like "*$target*") -and $_.MainWindowHandle -ne 0 } | Select-Object -First 1
+if ($proc) {
+    $sig = '[DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr hWnd); [DllImport("user32.dll")] public static extern bool ShowWindow(IntPtr hWnd, int nCmdShow);'
+    $w = Add-Type -memberDefinition $sig -name 'Win32Fw' -namespace Win32Functions -passThru
+    [void]$w::ShowWindow($proc.MainWindowHandle, 9)
+    [void]$w::SetForegroundWindow($proc.MainWindowHandle)
+    Write-Output "Focused '$($proc.MainWindowTitle)' ($($proc.ProcessName))"
+} else {
+    $ws = New-Object -ComObject WScript.Shell
+    $activated = $ws.AppActivate($target)
+    if ($activated) { Write-Output "Activated '$target' via WScript.Shell" } else { Write-Output "Window '$target' not found" }
+}
+`;
+        try {
+            const encodedScript = Buffer.from(psScript, 'utf16le').toString('base64');
+            const { stdout } = await execAsync(`powershell -NoProfile -Sta -EncodedCommand ${encodedScript}`);
+            return { success: true, output: stdout.trim() || `Focus request completed for "${appNameOrTitle}".` };
+        }
+        catch (err) {
+            return { success: false, output: `Could not focus window: ${err?.message || String(err)}` };
+        }
+    }
+    return { success: true, output: `Focus request sent for "${appNameOrTitle}".` };
+}
+/**
+ * Scroll mouse wheel up or down by specified notch amount.
+ */
+export async function scrollWindow(direction = 'down', amount = 3) {
+    const platform = process.platform;
+    if (platform === 'win32') {
+        const clicks = Math.max(1, Math.min(20, amount));
+        const delta = direction === 'up' ? 120 * clicks : -120 * clicks;
+        const psScript = `
+Add-Type -AssemblyName System.Windows.Forms
+$sig = '[DllImport("user32.dll")] public static extern void mouse_event(int dwFlags, int dx, int dy, int cButtons, int dwExtraInfo);'
+$m = Add-Type -memberDefinition $sig -name 'Win32Scroll' -namespace Win32Functions -passThru
+$m::mouse_event(0x0800, 0, 0, ${delta}, 0)
+`;
+        try {
+            const encodedScript = Buffer.from(psScript, 'utf16le').toString('base64');
+            await execAsync(`powershell -NoProfile -EncodedCommand ${encodedScript}`);
+            return { success: true, output: `Scrolled mouse wheel ${direction} by ${clicks} notch(es).` };
+        }
+        catch (err) {
+            return { success: false, output: `Failed to scroll: ${err?.message || String(err)}` };
+        }
+    }
+    return { success: true, output: `Scrolled ${direction}` };
+}
+/**
+ * Visually ground and click a target UI element by name or visual description.
+ */
+export async function findAndClickElement(elementQuery, appName, button = 'left') {
+    const capture = await captureScreen(appName);
+    if (!capture.success || !capture.details?.outPath) {
+        return { success: false, output: `Could not capture screen to locate element: ${capture.output}` };
+    }
+    const screenWidth = capture.details.width || 1920;
+    const screenHeight = capture.details.height || 1080;
+    const outPath = capture.details.outPath;
+    let targetX = undefined;
+    let targetY = undefined;
+    const geminiKey = process.env.GEMINI_API_KEY;
+    const openaiKey = process.env.OPENAI_API_KEY;
+    const byokKey = process.env.BYOK_API_KEY;
+    const apiKey = geminiKey || openaiKey || byokKey;
+    if (apiKey && fs.existsSync(outPath)) {
+        try {
+            const imageBase64 = fs.readFileSync(outPath).toString('base64');
+            const isGemini = Boolean(geminiKey);
+            const baseURL = isGemini ? 'https://generativelanguage.googleapis.com/v1beta/openai' : undefined;
+            const model = isGemini ? 'gemini-2.5-flash' : 'gpt-4o-mini';
+            const client = new OpenAI({ apiKey, baseURL, timeout: 15000 });
+            const prompt = `Look at this screenshot and locate the UI element: "${elementQuery}".
+Return ONLY a valid JSON object with the center point of this element in 0-1000 normalized coordinates:
+{"point": [y, x]}
+Where y is between 0 and 1000 (top to bottom) and x is between 0 and 1000 (left to right).
+If not found, return {"point": null}. Output nothing else.`;
+            const resp = await client.chat.completions.create({
+                model,
+                messages: [
+                    {
+                        role: 'user',
+                        content: [
+                            { type: 'text', text: prompt },
+                            { type: 'image_url', image_url: { url: `data:image/png;base64,${imageBase64}` } },
+                        ],
+                    },
+                ],
+                max_tokens: 80,
+            });
+            const reply = resp.choices[0]?.message?.content || '';
+            const match = reply.match(/\{[\s\S]*"point"[\s\S]*\}/);
+            if (match) {
+                const parsed = JSON.parse(match[0]);
+                if (Array.isArray(parsed.point) && parsed.point.length === 2) {
+                    const normY = Number(parsed.point[0]);
+                    const normX = Number(parsed.point[1]);
+                    if (!isNaN(normX) && !isNaN(normY)) {
+                        targetX = Math.round((normX / 1000) * screenWidth);
+                        targetY = Math.round((normY / 1000) * screenHeight);
+                    }
+                }
+            }
+        }
+        catch { }
+    }
+    if ((targetX === undefined || targetY === undefined) && process.platform === 'win32') {
+        const psScript = `
+Add-Type -AssemblyName UIAutomationClient
+Add-Type -AssemblyName UIAutomationTypes
+$target = "${elementQuery.replace(/"/g, '""')}"
+$root = [System.Windows.Automation.AutomationElement]::RootElement
+
+$condName = New-Object System.Windows.Automation.PropertyCondition([System.Windows.Automation.AutomationElement]::NameProperty, $target)
+$el = $root.FindFirst([System.Windows.Automation.TreeScope]::Descendants, $condName)
+
+if (-not $el) {
+    $all = $root.FindAll([System.Windows.Automation.TreeScope]::Descendants, [System.Windows.Automation.Condition]::TrueCondition)
+    foreach ($item in $all) {
+        if ($item.Current.Name -and $item.Current.Name.ToLower().Contains($target.ToLower())) {
+            $el = $item
+            break
+        }
+    }
+}
+
+if ($el) {
+    $rect = $el.Current.BoundingRectangle
+    if ($rect -and $rect.Width -gt 0 -and $rect.Height -gt 0) {
+        $cx = [int]($rect.X + ($rect.Width / 2))
+        $cy = [int]($rect.Y + ($rect.Height / 2))
+        @{ x = $cx; y = $cy; name = $el.Current.Name } | ConvertTo-Json
+    }
+}
+`;
+        try {
+            const { stdout } = await execAsync(`powershell -NoProfile -Command "${psScript.replace(/\r?\n/g, ' ')}"`);
+            if (stdout.trim()) {
+                const uia = JSON.parse(stdout.trim());
+                if (uia.x && uia.y) {
+                    targetX = uia.x;
+                    targetY = uia.y;
+                }
+            }
+        }
+        catch { }
+    }
+    if (targetX !== undefined && targetY !== undefined) {
+        const clickRes = await moveAndClickMouse(targetX, targetY, button, appName);
+        return {
+            success: clickRes.success,
+            output: `Visual Grounding Success: Located "${elementQuery}" at screen coordinates (${targetX}, ${targetY}). ${clickRes.output}`,
+            coordinates: { x: targetX, y: targetY },
+        };
+    }
+    return {
+        success: false,
+        output: `Could not visually or structurally locate UI element "${elementQuery}" on screen. Try calling analyze_screen to view current screen contents, or provide explicit (x, y) coordinates.`,
+    };
+}
 //# sourceMappingURL=appControl.js.map