ft-scout 8.0.2 → 8.0.4

package/bin/src/engine/appControl.js

@@ -2,11 +2,12 @@ import fs from 'fs';
 import path from 'path';
 import http from 'http';
 import https from 'https';
-import { exec } from 'child_process';
+import { exec, spawn, ChildProcess } from 'child_process';
 import { promisify } from 'util';
 import OpenAI from 'openai';
 import dotenv from 'dotenv';
 import chalk from 'chalk';
+import { startLiveScreenShare, stopLiveScreenShare, isLiveScreenShareActive, getLiveScreenFrame, analyzeLiveScreen, callVisionAiWithFrame, setLiveScreenTargetApp, getLiveScreenTargetApp, } from './liveScreenEngine.js';
 dotenv.config({ quiet: true });
 const execAsync = promisify(exec);
 /**
@@ -23,6 +24,9 @@ export function normalizeAppName(rawAppName) {
     if (['code', 'vscode', 'visual-studio-code', 'cursor', 'zed', 'sublime', 'notepad', 'notepad++', 'vim', 'nvim', 'nano', 'editor'].includes(appLower)) {
         return { category: 'editor', name: appLower || 'editor' };
     }
+    if (['game', 'games', 'gaming', 'play', 'launcher', 'steam', 'epic', 'epicgames', 'riot', 'battlenet', 'ea', 'ubisoft', 'gog'].includes(appLower) || appLower.startsWith('game:')) {
+        return { category: 'game', name: appLower };
+    }
     return { category: 'system', name: appLower };
 }
 export function parsePayload(payload) {
@@ -102,12 +106,87 @@ export function buildSocialUrl(platformOrAppName, target, text) {
     }
     return 'https://www.instagram.com/direct/inbox/';
 }
+/**
+ * Universally search and locate any installed app, game, or store package on the host OS without hardcoding.
+ * Automatically resolves Win32 executables, Microsoft Store games, Steam games, and desktop shortcuts.
+ */
+export async function findInstalledApp(query) {
+    const cleanQuery = (query || '').replace(/[^\w\s-]/g, '').trim().toLowerCase();
+    if (!cleanQuery)
+        return null;
+    const platform = process.platform;
+    if (platform === 'win32') {
+        try {
+            // Query Windows Get-StartApps with wildcard matching
+            const safePattern = `*${cleanQuery.replace(/[*_']/g, '')}*`;
+            const psCmd = `powershell.exe -NoProfile -NonInteractive -Command "Get-StartApps -Name '${safePattern}' | Select-Object -First 1 Name, AppID | ConvertTo-Json"`;
+            const { stdout } = await execAsync(psCmd, { timeout: 3500 });
+            if (stdout && stdout.trim()) {
+                const parsed = JSON.parse(stdout.trim());
+                if (parsed && (parsed.Name || parsed.AppID)) {
+                    return {
+                        name: String(parsed.Name || query),
+                        appId: String(parsed.AppID || ''),
+                        source: 'start_apps',
+                    };
+                }
+            }
+        }
+        catch { }
+        // Fallback: check registered URI scheme protocol (e.g. steam, roblox, spotify, discord, minecraft, riot)
+        if (!cleanQuery.includes(' ') && cleanQuery.length >= 3) {
+            try {
+                const checkReg = `powershell.exe -NoProfile -NonInteractive -Command "Test-Path 'Registry::HKEY_CLASSES_ROOT\\${cleanQuery}'"`;
+                const { stdout } = await execAsync(checkReg, { timeout: 1500 });
+                if (stdout && stdout.trim().toLowerCase() === 'true') {
+                    return {
+                        name: query,
+                        appId: `${cleanQuery}:`,
+                        source: 'protocol',
+                    };
+                }
+            }
+            catch { }
+        }
+    }
+    else if (platform === 'darwin') {
+        try {
+            const { stdout } = await execAsync(`mdfind "kMDItemKind == 'Application' && kMDItemDisplayName == '*${cleanQuery}*'c" | head -n 1`, { timeout: 3000 });
+            if (stdout && stdout.trim()) {
+                return {
+                    name: path.basename(stdout.trim(), '.app'),
+                    appId: stdout.trim(),
+                    source: 'path',
+                };
+            }
+        }
+        catch { }
+    }
+    else {
+        // Linux: check which or desktop application entries
+        try {
+            const { stdout } = await execAsync(`which "${cleanQuery}" || find /usr/share/applications -name "*${cleanQuery}*.desktop" | head -n 1`, { timeout: 2500 });
+            if (stdout && stdout.trim()) {
+                return {
+                    name: query,
+                    appId: stdout.trim(),
+                    source: 'path',
+                };
+            }
+        }
+        catch { }
+    }
+    return null;
+}
 /**
  * Cross-platform app opener.
  * Launches and opens target application on Windows, macOS, or Linux.
  */
 export async function openApp(appNameOrPath, target, options) {
     const { category, name } = normalizeAppName(appNameOrPath);
+    if (name && name !== 'desktop' && name !== 'system') {
+        setLiveScreenTargetApp(name);
+    }
     const platform = process.platform;
     let command = '';
     const cleanTarget = target ? target.trim() : '';
@@ -212,16 +291,42 @@ export async function openApp(appNameOrPath, target, options) {
         }
     }
     else {
-        // Custom system app / binary
-        const appExec = appNameOrPath;
-        if (platform === 'win32') {
-            command = cleanTarget ? `start "" "${appExec}" ${cleanTarget}` : `start "" "${appExec}"`;
-        }
-        else if (platform === 'darwin') {
-            command = cleanTarget ? `open -a "${appExec}" "${cleanTarget}"` : `open -a "${appExec}"`;
+        // Universal Application, Game & Desktop Launcher (Zero-Hardcode OS Discovery)
+        const targetQuery = name || appNameOrPath;
+        const installedApp = await findInstalledApp(targetQuery);
+        if (installedApp) {
+            if (platform === 'win32') {
+                if (installedApp.source === 'protocol') {
+                    const extraArg = cleanTarget ? ` "${cleanTarget}"` : '';
+                    command = `start ${installedApp.appId}${extraArg}`;
+                }
+                else {
+                    command = `explorer.exe "shell:AppsFolder\\${installedApp.appId}"`;
+                }
+            }
+            else if (platform === 'darwin') {
+                command = `open -a "${installedApp.appId || installedApp.name}"`;
+            }
+            else {
+                command = `"${installedApp.appId || installedApp.name}" &`;
+            }
         }
         else {
-            command = cleanTarget ? `${appExec} "${cleanTarget}" &` : `${appExec} &`;
+            // Protocol scheme check (e.g. steam:, roblox:, spotify:, discord:, minecraft:, etc.)
+            const isProtocol = /^[a-zA-Z0-9_\-]+:/.test(appNameOrPath) || (cleanTarget && /^[a-zA-Z0-9_\-]+:/.test(cleanTarget));
+            if (isProtocol && platform === 'win32') {
+                const proto = /^[a-zA-Z0-9_\-]+:/.test(appNameOrPath) ? appNameOrPath : cleanTarget;
+                command = `start ${proto}`;
+            }
+            else if (platform === 'win32') {
+                command = cleanTarget ? `start "" "${appNameOrPath}" ${cleanTarget}` : `start "" "${appNameOrPath}"`;
+            }
+            else if (platform === 'darwin') {
+                command = cleanTarget ? `open -a "${appNameOrPath}" "${cleanTarget}"` : `open -a "${appNameOrPath}"`;
+            }
+            else {
+                command = cleanTarget ? `${appNameOrPath} "${cleanTarget}" &` : `${appNameOrPath} &`;
+            }
         }
     }
     try {
@@ -251,6 +356,10 @@ export async function openApp(appNameOrPath, target, options) {
 export async function executeInApp(appName, action, payload) {
     const { category, name } = normalizeAppName(appName);
     const actLower = (action || '').trim().toLowerCase();
+    // Focus live screen sharing on the active app being operated
+    if (name && name !== 'desktop' && name !== 'system') {
+        setLiveScreenTargetApp(name);
+    }
     // Universal Scratchpad Desktop Automation Actions (Takeover, Screen Seeing, Keystrokes, Mouse Clicks, Focus Locks)
     if (['takeover', 'takeover_control', 'takeover_screen', 'takeover_app'].includes(actLower)) {
         const parsed = parsePayload(payload);
@@ -275,7 +384,38 @@ export async function executeInApp(appName, action, payload) {
             output: res.output,
         };
     }
+    if (['start_screen_share', 'live_screen_share', 'screen_share', 'start_live_stream'].includes(actLower)) {
+        const targetApp = (name && name !== 'desktop' && name !== 'system') ? name : undefined;
+        const res = await startLiveScreenShare({ targetApp });
+        return {
+            success: res.success,
+            app: name || appName,
+            action: actLower,
+            output: res.message,
+            details: { active: true, targetApp },
+        };
+    }
+    if (['stop_screen_share', 'end_screen_share', 'stop_live_stream'].includes(actLower)) {
+        const res = stopLiveScreenShare();
+        return {
+            success: res.success,
+            app: name || appName,
+            action: actLower,
+            output: res.summary,
+            details: { active: false },
+        };
+    }
     if (['capture_screen', 'screenshot', 'screen_shot'].includes(actLower)) {
+        if (isLiveScreenShareActive()) {
+            const liveRes = await analyzeLiveScreen();
+            return {
+                success: liveRes.success,
+                app: name || appName,
+                action: actLower,
+                output: liveRes.output,
+                details: liveRes.details,
+            };
+        }
         const res = await captureScreen(name || appName);
         return {
             success: res.success,
@@ -285,9 +425,9 @@ export async function executeInApp(appName, action, payload) {
             details: res.details,
         };
     }
-    if (['see_screen', 'inspect_screen', 'view_screen', 'analyze_screen', 'inspect_desktop', 'visual_analysis', 'see_and_analyze'].includes(actLower)) {
+    if (['see_screen', 'inspect_screen', 'view_screen', 'analyze_screen', 'inspect_desktop', 'visual_analysis', 'see_and_analyze', 'live_screen_stream', 'screen_stream'].includes(actLower)) {
         const query = typeof payload === 'string' ? payload : (payload?.query || payload?.target || payload?.text || undefined);
-        const res = await analyzeScreen(name || appName, query);
+        const res = await analyzeLiveScreen(query);
         return {
             success: res.success,
             app: name || appName,
@@ -435,7 +575,19 @@ export async function executeInApp(appName, action, payload) {
     if (category === 'browser') {
         if (['navigate', 'goto', 'open_url'].includes(actLower)) {
             const rawUrl = typeof payload === 'string' ? payload : (payload?.url || payload?.target || 'https://google.com');
-            const targetUrl = rawUrl.startsWith('http://') || rawUrl.startsWith('https://') ? rawUrl : `https://${rawUrl}`;
+            let targetUrl = rawUrl.trim();
+            let navigationNote = '';
+            // Universal URL vs Search Query Grounding:
+            // If the input is not a valid URL (contains spaces or lacks a dot / scheme), convert into a search
+            if (!targetUrl.startsWith('http://') && !targetUrl.startsWith('https://')) {
+                if (targetUrl.includes(' ') || !targetUrl.includes('.')) {
+                    navigationNote = ` [Note: Plain search term passed to navigate; converted to web search query]`;
+                    targetUrl = `https://www.google.com/search?q=${encodeURIComponent(targetUrl)}`;
+                }
+                else {
+                    targetUrl = `https://${targetUrl}`;
+                }
+            }
             await openApp(name || 'browser', targetUrl);
             await new Promise((r) => setTimeout(r, 800));
             const capture = await captureScreen(name || 'browser');
@@ -443,7 +595,7 @@ export async function executeInApp(appName, action, payload) {
                 success: true,
                 app: name || 'browser',
                 action: actLower,
-                output: `Browser Navigated Successfully to Target URL: ${targetUrl}\n- Screenshot Saved: ${capture.details?.outPath || 'Active'}`,
+                output: `Browser Navigated Successfully to Target URL: ${targetUrl}${navigationNote}\n- Screenshot Saved: ${capture.details?.outPath || 'Active'}`,
                 details: { url: targetUrl, capture: capture.details },
             };
         }
@@ -557,6 +709,10 @@ export async function executeInApp(appName, action, payload) {
             return await openApp(name, filePath);
         }
     }
+    if (category === 'game' || ['open', 'launch', 'start'].includes(actLower)) {
+        const rawTarget = typeof payload === 'string' ? payload : (payload?.target || payload?.game || payload?.url || payload?.query || payload?.id || '');
+        return await openApp(name || appName, rawTarget);
+    }
     // Generic fallback app action
     return {
         success: true,
@@ -617,49 +773,291 @@ export function getAppCapabilities() {
         defaultEditorLauncher: platform === 'win32' ? 'code / notepad' : platform === 'darwin' ? 'code / open -a' : 'code / xdg-open',
     };
 }
+const WIN_CAPTURE_SCRIPT_CONTENT = `param(
+    [string]$OutPath,
+    [switch]$InfoOnly
+)
+
+$source = @"
+using System;
+using System.Drawing;
+using System.Drawing.Imaging;
+using System.Runtime.InteropServices;
+using System.Text;
+using System.Threading;
+using System.Collections.Generic;
+
+public class WinCaptureResult {
+    public bool success;
+    public string savedPath;
+    public int width;
+    public int height;
+    public int cursorX;
+    public int cursorY;
+    public string activeWindow;
+    public string[] visibleWindows;
+    public string error;
+}
+
+public class WinCaptureHelper {
+    [DllImport("user32.dll", SetLastError = true)]
+    public static extern IntPtr OpenInputDesktop(uint dwFlags, bool fInherit, uint dwDesiredAccess);
+
+    [DllImport("user32.dll", SetLastError = true)]
+    public static extern IntPtr OpenDesktop(string lpszDesktop, uint dwFlags, bool fInherit, uint dwDesiredAccess);
+
+    [DllImport("user32.dll", SetLastError = true)]
+    public static extern bool SetThreadDesktop(IntPtr hDesktop);
+
+    [DllImport("user32.dll", SetLastError = true)]
+    public static extern bool CloseDesktop(IntPtr hDesktop);
+
+    [DllImport("user32.dll")]
+    public static extern IntPtr GetDC(IntPtr hWnd);
+
+    [DllImport("user32.dll")]
+    public static extern int ReleaseDC(IntPtr hWnd, IntPtr hDC);
+
+    [DllImport("gdi32.dll")]
+    public static extern IntPtr CreateCompatibleDC(IntPtr hdc);
+
+    [DllImport("gdi32.dll")]
+    public static extern IntPtr CreateCompatibleBitmap(IntPtr hdc, int nWidth, int nHeight);
+
+    [DllImport("gdi32.dll")]
+    public static extern IntPtr SelectObject(IntPtr hdc, IntPtr hgdiobj);
+
+    [DllImport("gdi32.dll")]
+    public static extern bool DeleteDC(IntPtr hdc);
+
+    [DllImport("gdi32.dll")]
+    public static extern bool DeleteObject(IntPtr hObject);
+
+    [DllImport("gdi32.dll")]
+    public static extern bool BitBlt(IntPtr hdcDest, int nXDest, int nYDest, int nWidth, int nHeight, IntPtr hdcSrc, int nXSrc, int nYSrc, int dwRop);
+
+    [DllImport("user32.dll")]
+    public static extern int GetSystemMetrics(int nIndex);
+
+    public delegate bool EnumWindowsProc(IntPtr hWnd, IntPtr lParam);
+
+    [DllImport("user32.dll")]
+    public static extern bool EnumWindows(EnumWindowsProc lpEnumFunc, IntPtr lParam);
+
+    [DllImport("user32.dll", CharSet = CharSet.Auto)]
+    public static extern int GetWindowText(IntPtr hWnd, StringBuilder lpString, int nMaxCount);
+
+    [DllImport("user32.dll")]
+    public static extern bool IsWindowVisible(IntPtr hWnd);
+
+    [DllImport("user32.dll")]
+    public static extern IntPtr GetForegroundWindow();
+
+    [StructLayout(LayoutKind.Sequential)]
+    public struct POINT {
+        public int X;
+        public int Y;
+    }
+
+    [DllImport("user32.dll")]
+    public static extern bool GetCursorPos(out POINT lpPoint);
+
+    const uint DESKTOP_ACCESS = 0x01FF | 0x00020000;
+    const int SM_CXSCREEN = 0;
+    const int SM_CYSCREEN = 1;
+    const int SRCCOPY = 0x00CC0020;
+    const int CAPTUREBLT = 0x40000000;
+
+    public static WinCaptureResult GetInfo() {
+        var res = new WinCaptureResult();
+        res.success = true;
+        res.error = "";
+        var visibleWindows = new List<string>();
+
+        Thread worker = new Thread(() => {
+            IntPtr hDesk = OpenInputDesktop(0, false, DESKTOP_ACCESS);
+            if (hDesk == IntPtr.Zero) {
+                hDesk = OpenDesktop("Default", 0, false, DESKTOP_ACCESS);
+            }
+            if (hDesk != IntPtr.Zero) {
+                SetThreadDesktop(hDesk);
+            }
+
+            res.width = GetSystemMetrics(SM_CXSCREEN);
+            res.height = GetSystemMetrics(SM_CYSCREEN);
+            if (res.width <= 0) res.width = 1920;
+            if (res.height <= 0) res.height = 1080;
+
+            POINT pt;
+            if (GetCursorPos(out pt)) {
+                res.cursorX = pt.X;
+                res.cursorY = pt.Y;
+            }
+
+            IntPtr fg = GetForegroundWindow();
+            if (fg != IntPtr.Zero) {
+                var sbFg = new StringBuilder(256);
+                GetWindowText(fg, sbFg, 256);
+                res.activeWindow = sbFg.ToString().Trim();
+            } else {
+                res.activeWindow = "";
+            }
+
+            EnumWindows((hWnd, lParam) => {
+                if (IsWindowVisible(hWnd)) {
+                    var title = new StringBuilder(256);
+                    GetWindowText(hWnd, title, 256);
+                    string t = title.ToString().Trim();
+                    if (t.Length > 0 && !visibleWindows.Contains(t) && t != "Program Manager" && t != "Windows Input Experience") {
+                        visibleWindows.Add(t);
+                    }
+                }
+                return true;
+            }, IntPtr.Zero);
+
+            if (hDesk != IntPtr.Zero) {
+                CloseDesktop(hDesk);
+            }
+        });
+
+        worker.SetApartmentState(ApartmentState.STA);
+        worker.Start();
+        worker.Join();
+
+        res.visibleWindows = visibleWindows.ToArray();
+        return res;
+    }
+
+    public static WinCaptureResult Capture(string filePath) {
+        var res = new WinCaptureResult();
+        res.savedPath = filePath;
+        res.error = "";
+        var visibleWindows = new List<string>();
+
+        Thread worker = new Thread(() => {
+            IntPtr hDesk = OpenInputDesktop(0, false, DESKTOP_ACCESS);
+            if (hDesk == IntPtr.Zero) {
+                hDesk = OpenDesktop("Default", 0, false, DESKTOP_ACCESS);
+            }
+            if (hDesk != IntPtr.Zero) {
+                SetThreadDesktop(hDesk);
+            }
+
+            res.width = GetSystemMetrics(SM_CXSCREEN);
+            res.height = GetSystemMetrics(SM_CYSCREEN);
+            if (res.width <= 0) res.width = 1920;
+            if (res.height <= 0) res.height = 1080;
+
+            POINT pt;
+            if (GetCursorPos(out pt)) {
+                res.cursorX = pt.X;
+                res.cursorY = pt.Y;
+            }
+
+            IntPtr fg = GetForegroundWindow();
+            if (fg != IntPtr.Zero) {
+                var sbFg = new StringBuilder(256);
+                GetWindowText(fg, sbFg, 256);
+                res.activeWindow = sbFg.ToString().Trim();
+            } else {
+                res.activeWindow = "";
+            }
+
+            EnumWindows((hWnd, lParam) => {
+                if (IsWindowVisible(hWnd)) {
+                    var title = new StringBuilder(256);
+                    GetWindowText(hWnd, title, 256);
+                    string t = title.ToString().Trim();
+                    if (t.Length > 0 && !visibleWindows.Contains(t) && t != "Program Manager" && t != "Windows Input Experience") {
+                        visibleWindows.Add(t);
+                    }
+                }
+                return true;
+            }, IntPtr.Zero);
+
+            try {
+                IntPtr hdcSrc = GetDC(IntPtr.Zero);
+                IntPtr hdcDest = CreateCompatibleDC(hdcSrc);
+                IntPtr hBitmap = CreateCompatibleBitmap(hdcSrc, res.width, res.height);
+                IntPtr hOld = SelectObject(hdcDest, hBitmap);
+                bool bltOk = BitBlt(hdcDest, 0, 0, res.width, res.height, hdcSrc, 0, 0, SRCCOPY | CAPTUREBLT);
+
+                SelectObject(hdcDest, hOld);
+                DeleteDC(hdcDest);
+                ReleaseDC(IntPtr.Zero, hdcSrc);
+
+                if (bltOk) {
+                    using (Bitmap bmp = Bitmap.FromHbitmap(hBitmap)) {
+                        bmp.Save(filePath, ImageFormat.Png);
+                    }
+                    res.success = true;
+                } else {
+                    res.error = "BitBlt failed";
+                }
+                DeleteObject(hBitmap);
+            } catch (Exception ex) {
+                res.error = ex.Message;
+            }
+
+            if (hDesk != IntPtr.Zero) {
+                CloseDesktop(hDesk);
+            }
+        });
+
+        worker.SetApartmentState(ApartmentState.STA);
+        worker.Start();
+        worker.Join();
+
+        res.visibleWindows = visibleWindows.ToArray();
+        return res;
+    }
+}
+"@
+
+Add-Type -TypeDefinition $source -ReferencedAssemblies System.Drawing, System.Windows.Forms
+
+if ($InfoOnly) {
+    [WinCaptureHelper]::GetInfo() | ConvertTo-Json -Compress
+} else {
+    [WinCaptureHelper]::Capture($OutPath) | ConvertTo-Json -Compress
+}
+`;
+export function ensureWinCaptureScript() {
+    const scriptsDir = path.join(process.cwd(), '.ft', 'scripts');
+    if (!fs.existsSync(scriptsDir)) {
+        fs.mkdirSync(scriptsDir, { recursive: true });
+    }
+    const scriptPath = path.join(scriptsDir, 'winCapture.ps1');
+    fs.writeFileSync(scriptPath, WIN_CAPTURE_SCRIPT_CONTENT, 'utf8');
+    return scriptPath;
+}
 /**
- * Get screen metrics, active window title, and cursor position.
+ * Get screen metrics, active window title, visible windows, and cursor position.
  */
 export async function getScreenInfo(appName) {
     const platform = process.platform;
     if (platform === 'win32') {
-        const psScript = `
-Add-Type -AssemblyName System.Windows.Forms
-Add-Type -AssemblyName System.Drawing
-
-$screen = [System.Windows.Forms.Screen]::PrimaryScreen
-$cursor = [System.Windows.Forms.Cursor]::Position
-
-$sig = '[DllImport("user32.dll")] public static extern IntPtr GetForegroundWindow(); [DllImport("user32.dll")] public static extern int GetWindowText(IntPtr hWnd, System.Text.StringBuilder text, int count);'
-$w = Add-Type -memberDefinition $sig -name 'Win32Window' -namespace Win32Functions -passThru
-$hwnd = $w::GetForegroundWindow()
-$sb = New-Object System.Text.StringBuilder 256
-[void]$w::GetWindowText($hwnd, $sb, 256)
-$activeTitle = $sb.ToString()
-
-@{
-    screenWidth = $screen.Bounds.Width
-    screenHeight = $screen.Bounds.Height
-    cursorX = $cursor.X
-    cursorY = $cursor.Y
-    activeWindow = $activeTitle
-} | ConvertTo-Json
-`;
         try {
-            const encodedScript = Buffer.from(psScript, 'utf16le').toString('base64');
-            const { stdout } = await execAsync(`powershell -NoProfile -EncodedCommand ${encodedScript}`);
+            const scriptPath = ensureWinCaptureScript();
+            const { stdout } = await execAsync(`powershell -NoProfile -ExecutionPolicy Bypass -File "${scriptPath}" -InfoOnly`);
             const data = JSON.parse(stdout.trim());
+            const visList = Array.isArray(data.visibleWindows) && data.visibleWindows.length > 0
+                ? ` | Visible Windows: ${data.visibleWindows.slice(0, 5).join(', ')}`
+                : '';
             return {
                 success: true,
-                output: `Screen Info: Resolution ${data.screenWidth}x${data.screenHeight} | Cursor at (${data.cursorX}, ${data.cursorY}) | Active Window: "${data.activeWindow || 'Desktop'}"`,
-                details: data,
+                output: `Screen Info: Resolution ${data.screenWidth}x${data.screenHeight} | Cursor at (${data.cursorX}, ${data.cursorY}) | Active Window: "${data.activeWindow || appName || 'Desktop'}"${visList}`,
+                details: {
+                    ...data,
+                    activeWindow: data.activeWindow || appName || 'Desktop',
+                },
             };
         }
-        catch (err) {
+        catch {
             return {
                 success: true,
                 output: `Screen Info (Default): Resolution 1920x1080 | Active Window: "${appName || 'Desktop'}"`,
-                details: { screenWidth: 1920, screenHeight: 1080, activeWindow: appName || 'Desktop' },
+                details: { screenWidth: 1920, screenHeight: 1080, activeWindow: appName || 'Desktop', cursorX: 0, cursorY: 0 },
             };
         }
     }
@@ -671,12 +1069,208 @@ $activeTitle = $sb.ToString()
         };
     }
 }
+let persistentAuraProcess = null;
+let persistentAuraFlagPath = null;
+/**
+ * Start persistent futuristic sky-blue aura overlay on screen borders and corners.
+ * Stays continuously active and animating as long as live screen sharing is running.
+ */
+export async function startScreenAuraOverlay(message = "Live Screen Share Active • Streaming to Model") {
+    const platform = process.platform;
+    if (platform !== 'win32')
+        return;
+    // Stop any existing overlay first
+    stopScreenAuraOverlay();
+    try {
+        const scriptsDir = path.join(process.cwd(), '.ft', 'scripts');
+        if (!fs.existsSync(scriptsDir)) {
+            fs.mkdirSync(scriptsDir, { recursive: true });
+        }
+        const psFile = path.join(scriptsDir, 'scout_aura_persistent.ps1');
+        const flagFile = path.join(scriptsDir, 'aura_active.flag');
+        fs.writeFileSync(flagFile, String(Date.now()), 'utf8');
+        persistentAuraFlagPath = flagFile;
+        const subtitleText = " Perception Active";
+        const parentPid = process.pid;
+        const overlayPs = `
+Add-Type -AssemblyName PresentationFramework, PresentationCore, WindowsBase
+
+[xml]$xaml = @"
+<Window xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
+        xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
+        Title="Scout Aura Overlay"
+        WindowStyle="None"
+        AllowsTransparency="True"
+        Background="Transparent"
+        Topmost="True"
+        ShowInTaskbar="False"
+        ShowActivated="False"
+        IsHitTestVisible="False">
+    <Window.Resources>
+        <Storyboard x:Key="AuraPulse" RepeatBehavior="Forever" AutoReverse="True">
+            <DoubleAnimation Storyboard.TargetName="AuraBorder" Storyboard.TargetProperty="Opacity"
+                             From="0.4" To="1.0" Duration="0:0:0.6"/>
+            <DoubleAnimation Storyboard.TargetName="CornerTL" Storyboard.TargetProperty="Opacity"
+                             From="0.5" To="1.0" Duration="0:0:0.6"/>
+            <DoubleAnimation Storyboard.TargetName="CornerTR" Storyboard.TargetProperty="Opacity"
+                             From="0.5" To="1.0" Duration="0:0:0.6"/>
+            <DoubleAnimation Storyboard.TargetName="CornerBL" Storyboard.TargetProperty="Opacity"
+                             From="0.5" To="1.0" Duration="0:0:0.6"/>
+            <DoubleAnimation Storyboard.TargetName="CornerBR" Storyboard.TargetProperty="Opacity"
+                             From="0.5" To="1.0" Duration="0:0:0.6"/>
+            <DoubleAnimation Storyboard.TargetName="AuraGlow" Storyboard.TargetProperty="Opacity"
+                             From="0.25" To="0.9" Duration="0:0:0.6"/>
+        </Storyboard>
+    </Window.Resources>
+
+    <Grid IsHitTestVisible="False">
+        <!-- Outer Glowing Sky Blue Aura Border -->
+        <Border x:Name="AuraGlow" BorderThickness="6" BorderBrush="#00d8ff" Margin="0">
+            <Border.Effect>
+                <DropShadowEffect Color="#00e5ff" BlurRadius="45" ShadowDepth="0" Opacity="0.95"/>
+            </Border.Effect>
+        </Border>
+        <Border x:Name="AuraBorder" BorderThickness="3" BorderBrush="#38bdf8" Margin="0">
+            <Border.Effect>
+                <DropShadowEffect Color="#00e5ff" BlurRadius="25" ShadowDepth="0" Opacity="1"/>
+            </Border.Effect>
+        </Border>
+
+        <!-- Sky Blue Corner Brackets with Aura Glow -->
+        <!-- Top Left Corner -->
+        <Canvas HorizontalAlignment="Left" VerticalAlignment="Top" Width="120" Height="120" Margin="6,6,0,0">
+            <Path x:Name="CornerTL" Data="M 0,90 L 0,0 L 90,0" Stroke="#00e5ff" StrokeThickness="6">
+                <Path.Effect>
+                    <DropShadowEffect Color="#00e5ff" BlurRadius="30" ShadowDepth="0" Opacity="1"/>
+                </Path.Effect>
+            </Path>
+        </Canvas>
+
+        <!-- Top Right Corner -->
+        <Canvas HorizontalAlignment="Right" VerticalAlignment="Top" Width="120" Height="120" Margin="0,6,6,0">
+            <Path x:Name="CornerTR" Data="M 30,0 L 120,0 L 120,90" Stroke="#00e5ff" StrokeThickness="6">
+                <Path.Effect>
+                    <DropShadowEffect Color="#00e5ff" BlurRadius="30" ShadowDepth="0" Opacity="1"/>
+                </Path.Effect>
+            </Path>
+        </Canvas>
+
+        <!-- Bottom Left Corner -->
+        <Canvas HorizontalAlignment="Left" VerticalAlignment="Bottom" Width="120" Height="120" Margin="6,0,0,6">
+            <Path x:Name="CornerBL" Data="M 0,30 L 0,120 L 90,120" Stroke="#00e5ff" StrokeThickness="6">
+                <Path.Effect>
+                    <DropShadowEffect Color="#00e5ff" BlurRadius="30" ShadowDepth="0" Opacity="1"/>
+                </Path.Effect>
+            </Path>
+        </Canvas>
+
+        <!-- Bottom Right Corner -->
+        <Canvas HorizontalAlignment="Right" VerticalAlignment="Bottom" Width="120" Height="120" Margin="0,0,6,6">
+            <Path x:Name="CornerBR" Data="M 30,120 L 120,120 L 120,30" Stroke="#00e5ff" StrokeThickness="6">
+                <Path.Effect>
+                    <DropShadowEffect Color="#00e5ff" BlurRadius="30" ShadowDepth="0" Opacity="1"/>
+                </Path.Effect>
+            </Path>
+        </Canvas>
+
+        <!-- Top Floating Status Badge -->
+        <Border HorizontalAlignment="Center" VerticalAlignment="Top" Margin="0,22,0,0"
+                Background="#EE03121F" BorderBrush="#00e5ff" BorderThickness="1.8" CornerRadius="24" Padding="26,10">
+            <Border.Effect>
+                <DropShadowEffect Color="#00e5ff" BlurRadius="25" ShadowDepth="0" Opacity="0.95"/>
+            </Border.Effect>
+            <StackPanel Orientation="Horizontal" VerticalAlignment="Center">
+                <Ellipse Width="12" Height="12" Fill="#00e5ff" Margin="0,0,12,0">
+                    <Ellipse.Effect>
+                        <DropShadowEffect Color="#00e5ff" BlurRadius="12" ShadowDepth="0"/>
+                    </Ellipse.Effect>
+                </Ellipse>
+                <TextBlock Text="${message}" Foreground="#ffffff" FontSize="16" FontWeight="Bold" VerticalAlignment="Center"/>
+                <TextBlock Text="${subtitleText}" Foreground="#38bdf8" FontSize="13.5" FontWeight="SemiBold" Margin="10,0,0,0" VerticalAlignment="Center"/>
+            </StackPanel>
+        </Border>
+    </Grid>
+</Window>
+"@
+
+$reader = (New-Object System.Xml.XmlNodeReader $xaml)
+$window = [System.Windows.Markup.XamlReader]::Load($reader)
+
+$window.Left = [System.Windows.SystemParameters]::VirtualScreenLeft
+$window.Top = [System.Windows.SystemParameters]::VirtualScreenTop
+$window.Width = [System.Windows.SystemParameters]::VirtualScreenWidth
+$window.Height = [System.Windows.SystemParameters]::VirtualScreenHeight
+$window.Topmost = $true
+$window.ShowActivated = $false
+
+$storyboard = $window.Resources["AuraPulse"]
+$window.Add_Loaded({
+    $storyboard.Begin($window)
+})
+
+$flagPath = "${flagFile.replace(/\\/g, '\\\\')}"
+$parentPid = ${parentPid}
+
+# Check if screen sharing is active or parent exited every 350ms
+$timer = New-Object System.Windows.Threading.DispatcherTimer
+$timer.Interval = [TimeSpan]::FromMilliseconds(350)
+$timer.Add_Tick({
+    $shouldClose = $false
+    if (-not (Test-Path $flagPath)) {
+        $shouldClose = $true
+    } else {
+        $proc = Get-Process -Id $parentPid -ErrorAction SilentlyContinue
+        if (-not $proc) {
+            $shouldClose = $true
+        }
+    }
+    if ($shouldClose) {
+        $timer.Stop()
+        $window.Close()
+    }
+})
+$timer.Start()
+
+[void]$window.ShowDialog()
+`;
+        fs.writeFileSync(psFile, overlayPs, 'utf8');
+        const child = spawn('powershell', ['-WindowStyle', 'Hidden', '-NoProfile', '-Sta', '-ExecutionPolicy', 'Bypass', '-File', psFile], {
+            detached: false,
+            stdio: 'ignore',
+        });
+        child.unref();
+        persistentAuraProcess = child;
+    }
+    catch { }
+}
+/**
+ * Stop persistent screen aura animation when live screen sharing stops.
+ */
+export function stopScreenAuraOverlay() {
+    if (persistentAuraFlagPath && fs.existsSync(persistentAuraFlagPath)) {
+        try {
+            fs.unlinkSync(persistentAuraFlagPath);
+        }
+        catch { }
+        persistentAuraFlagPath = null;
+    }
+    if (persistentAuraProcess) {
+        try {
+            persistentAuraProcess.kill();
+        }
+        catch { }
+        persistentAuraProcess = null;
+    }
+}
 /**
  * Display a futuristic sky-blue aura animation on screen borders and corners,
- * show the warning "Scout is on the screen.", and block external inputs
- * during live screen capture and automation.
+ * show the warning "Scout is on the screen.", and optionally block external inputs.
+ * If durationMs <= 0, aura stays on continuously until stopScreenAuraOverlay() is called.
  */
 export async function showScreenAuraOverlay(durationMs = 2200, message = "Scout is viewing the screen.", blockInput = false) {
+    if (durationMs <= 0) {
+        return await startScreenAuraOverlay(message);
+    }
     const platform = process.platform;
     if (platform !== 'win32')
         return;
@@ -694,7 +1288,7 @@ try { [void]$b::BlockInput($true) } catch {}
         const unblockSnippet = blockInput ? `
     try { [void]$b::BlockInput($false) } catch {}
 ` : '';
-        const subtitleText = blockInput ? " • External Input Locked" : " • Perception Active";
+        const subtitleText = blockInput ? " • External Input Locked" : " Perception Active";
         const overlayPs = `
 Add-Type -AssemblyName PresentationFramework, PresentationCore, WindowsBase
 ${blockInputSnippet}
@@ -897,51 +1491,56 @@ export async function captureScreen(appNameOrScreen) {
     const filename = `screen_${Date.now()}.png`;
     const outPath = path.join(screenshotsDir, filename);
     if (platform === 'win32') {
-        const psScript = `
+        try {
+            const scriptPath = ensureWinCaptureScript();
+            const { stdout } = await execAsync(`powershell -NoProfile -ExecutionPolicy Bypass -File "${scriptPath}" -OutPath "${outPath}"`);
+            const meta = JSON.parse(stdout.trim());
+            if (meta.success && fs.existsSync(outPath) && fs.statSync(outPath).size > 0) {
+                registerCapturedScreenshot(outPath);
+                showScreenAuraOverlay(1800, "Scout is viewing the screen...", false).catch(() => { });
+                const visibleList = Array.isArray(meta.visibleWindows) && meta.visibleWindows.length > 0
+                    ? `\n- Visible Windows: ${meta.visibleWindows.slice(0, 6).map((w) => `"${w}"`).join(', ')}`
+                    : '';
+                return {
+                    success: true,
+                    output: `Screen Captured Successfully!\n- Screenshot Saved To: ${outPath}\n- Resolution: ${meta.width}x${meta.height}\n- Active Window: "${meta.activeWindow || 'Desktop'}"${visibleList}\n- Seeing Status: Screen is visible and captured. (NOTE: This temporary screenshot will be automatically deleted upon task completion).`,
+                    details: { ...meta, outPath },
+                };
+            }
+            throw new Error(meta.error || 'Desktop capture returned false');
+        }
+        catch (err) {
+            // Fallback to Graphics.CopyFromScreen
+            const psScript = `
 Add-Type -AssemblyName System.Windows.Forms
 Add-Type -AssemblyName System.Drawing
-
 $screen = [System.Windows.Forms.Screen]::PrimaryScreen
 $bounds = $screen.Bounds
 $bmp = New-Object System.Drawing.Bitmap($bounds.Width, $bounds.Height)
 $graphics = [System.Drawing.Graphics]::FromImage($bmp)
-$graphics.CopyFromScreen($bounds.Location, [System.Drawing.Point]::Empty, $bounds.Size)
+try { $graphics.CopyFromScreen($bounds.Location, [System.Drawing.Point]::Empty, $bounds.Size) } catch {}
 $bmp.Save("${outPath.replace(/\\/g, '\\\\')}", [System.Drawing.Imaging.ImageFormat]::Png)
 $graphics.Dispose()
 $bmp.Dispose()
-
-$sig = '[DllImport("user32.dll")] public static extern IntPtr GetForegroundWindow(); [DllImport("user32.dll")] public static extern int GetWindowText(IntPtr hWnd, System.Text.StringBuilder text, int count);'
-$w = Add-Type -memberDefinition $sig -name 'Win32Window' -namespace Win32Functions -passThru
-$hwnd = $w::GetForegroundWindow()
-$sb = New-Object System.Text.StringBuilder 256
-[void]$w::GetWindowText($hwnd, $sb, 256)
-$activeTitle = $sb.ToString()
-
-@{
-    savedPath = "${outPath.replace(/\\/g, '\\\\')}"
-    width = $bounds.Width
-    height = $bounds.Height
-    activeWindow = $activeTitle
-} | ConvertTo-Json
+@{ savedPath = "${outPath.replace(/\\/g, '\\\\')}"; width = $bounds.Width; height = $bounds.Height; activeWindow = "${appNameOrScreen || 'Desktop'}" } | ConvertTo-Json
 `;
-        try {
-            const encodedScript = Buffer.from(psScript, 'utf16le').toString('base64');
-            const { stdout } = await execAsync(`powershell -NoProfile -EncodedCommand ${encodedScript}`);
-            const meta = JSON.parse(stdout.trim());
-            registerCapturedScreenshot(outPath);
-            // Trigger visual sky-blue aura overlay on screen corners & borders
-            showScreenAuraOverlay(1800, "Scout is viewing the screen...", false).catch(() => { });
-            return {
-                success: true,
-                output: `Screen Captured Successfully!\n- Screenshot Saved To: ${outPath}\n- Resolution: ${meta.width}x${meta.height}\n- Active Window: "${meta.activeWindow || 'Desktop'}"\n- Seeing Status: Screen is visible and captured. (NOTE: This temporary screenshot will be automatically deleted upon task completion).`,
-                details: { ...meta, outPath },
-            };
-        }
-        catch (err) {
-            return {
-                success: false,
-                output: `Failed to capture screen: ${err?.message || String(err)}`,
-            };
+            try {
+                const encodedScript = Buffer.from(psScript, 'utf16le').toString('base64');
+                const { stdout } = await execAsync(`powershell -NoProfile -EncodedCommand ${encodedScript}`);
+                const meta = JSON.parse(stdout.trim());
+                registerCapturedScreenshot(outPath);
+                return {
+                    success: true,
+                    output: `Screen Captured Successfully!\n- Screenshot Saved To: ${outPath}\n- Resolution: ${meta.width}x${meta.height}\n- Active Window: "${meta.activeWindow || 'Desktop'}"\n- Seeing Status: Screen is visible and captured.`,
+                    details: { ...meta, outPath },
+                };
+            }
+            catch (fallbackErr) {
+                return {
+                    success: false,
+                    output: `Failed to capture screen: ${err?.message || fallbackErr?.message || String(err)}`,
+                };
+            }
         }
     }
     else if (platform === 'darwin') {
@@ -979,6 +1578,64 @@ $activeTitle = $sb.ToString()
         }
     }
 }
+/**
+ * Call Vision AI with graceful multi-provider fallback.
+ * Tries NVIDIA Vision (meta/llama-3.2-11b-vision-instruct), Gemini (gemini-2.0-flash), OpenAI (gpt-4o-mini), and BYOK.
+ */
+export async function callVisionAi(outPath, prompt, maxTokens = 350) {
+    if (!outPath || !fs.existsSync(outPath))
+        return null;
+    let imageBase64;
+    try {
+        imageBase64 = fs.readFileSync(outPath).toString('base64');
+    }
+    catch {
+        return null;
+    }
+    const nvidiaKey = process.env.NVIDIA_API_KEY || process.env.NVCF_API_KEY;
+    const geminiKey = process.env.GEMINI_API_KEY;
+    const openaiKey = process.env.OPENAI_API_KEY;
+    const byokKey = process.env.BYOK_API_KEY;
+    const candidates = [];
+    if (nvidiaKey) {
+        candidates.push({ apiKey: nvidiaKey, baseURL: 'https://integrate.api.nvidia.com/v1', model: 'meta/llama-3.2-11b-vision-instruct', timeout: 35000 });
+    }
+    if (geminiKey) {
+        candidates.push({ apiKey: geminiKey, baseURL: 'https://generativelanguage.googleapis.com/v1beta/openai', model: 'gemini-2.0-flash', timeout: 20000 });
+    }
+    if (openaiKey) {
+        candidates.push({ apiKey: openaiKey, model: 'gpt-4o-mini', timeout: 20000 });
+    }
+    if (byokKey) {
+        candidates.push({ apiKey: byokKey, model: 'gpt-4o-mini', timeout: 20000 });
+    }
+    for (const cand of candidates) {
+        try {
+            const client = new OpenAI({ apiKey: cand.apiKey, baseURL: cand.baseURL, timeout: cand.timeout });
+            const resp = await client.chat.completions.create({
+                model: cand.model,
+                messages: [
+                    {
+                        role: 'user',
+                        content: [
+                            { type: 'text', text: prompt },
+                            { type: 'image_url', image_url: { url: `data:image/png;base64,${imageBase64}` } },
+                        ],
+                    },
+                ],
+                max_tokens: maxTokens,
+            });
+            const reply = resp.choices[0]?.message?.content?.trim();
+            if (reply) {
+                return reply;
+            }
+        }
+        catch {
+            continue;
+        }
+    }
+    return null;
+}
 /**
  * Visual screen analysis. Takes a full-screen screenshot and provides visual context
  * (including AI Vision inspection if an API key is available).
@@ -989,36 +1646,14 @@ export async function analyzeScreen(appNameOrScreen, targetQuery) {
     const info = await getScreenInfo(appNameOrScreen);
     let visualAiAnalysis = '';
     const outPath = capture.details?.outPath;
-    // Optional Vision Model Analysis if screenshot exists and API key is present
+    // Optional Vision Model Analysis if screenshot exists and has valid content
     if (capture.success && outPath && fs.existsSync(outPath)) {
         try {
-            const geminiKey = process.env.GEMINI_API_KEY;
-            const openaiKey = process.env.OPENAI_API_KEY;
-            const byokKey = process.env.BYOK_API_KEY;
-            const apiKey = geminiKey || openaiKey || byokKey;
-            if (apiKey) {
-                const imageBase64 = fs.readFileSync(outPath).toString('base64');
-                const isGemini = Boolean(geminiKey);
-                const baseURL = isGemini ? 'https://generativelanguage.googleapis.com/v1beta/openai' : undefined;
-                const model = isGemini ? 'gemini-2.5-flash' : 'gpt-4o-mini';
-                const client = new OpenAI({ apiKey, baseURL, timeout: 15000 });
+            if (fs.statSync(outPath).size > 0) {
                 const prompt = targetQuery
                     ? `Describe what is visible on this screen specifically regarding: "${targetQuery}". Keep it concise, noting visible UI elements, open windows, tabs, buttons, or error messages.`
-                    : `Briefly describe what is currently visible on this desktop screen: active window, main content/tabs, buttons, and system state.`;
-                const resp = await client.chat.completions.create({
-                    model,
-                    messages: [
-                        {
-                            role: 'user',
-                            content: [
-                                { type: 'text', text: prompt },
-                                { type: 'image_url', image_url: { url: `data:image/png;base64,${imageBase64}` } },
-                            ],
-                        },
-                    ],
-                    max_tokens: 350,
-                });
-                const reply = resp.choices[0]?.message?.content;
+                    : `Briefly describe what is currently visible on this desktop screen: active window, main content/tabs, buttons, and system state in 2-3 sentences.`;
+                const reply = await callVisionAi(outPath, prompt, 350);
                 if (reply && reply.trim()) {
                     visualAiAnalysis = `\nAI Visual Screen Analysis:\n${reply.trim()}`;
                 }
@@ -1028,12 +1663,19 @@ export async function analyzeScreen(appNameOrScreen, targetQuery) {
             // Vision API call is optional; fallback gracefully to system metadata
         }
     }
+    const activeWin = capture.details?.activeWindow || info.details?.activeWindow || appNameOrScreen || 'Active Desktop';
+    const visWins = Array.isArray(capture.details?.visibleWindows) && capture.details.visibleWindows.length > 0
+        ? capture.details.visibleWindows
+        : (Array.isArray(info.details?.visibleWindows) ? info.details.visibleWindows : []);
+    const visibleWindowsText = visWins.length > 0
+        ? `\nVisible Open Windows:\n${visWins.slice(0, 8).map((w) => `  - "${w}"`).join('\n')}`
+        : '';
     const report = [
         `=== SCOUT SCRATCHPAD DESKTOP SCREEN VISUAL INSPECTION ===`,
         `Target App / Screen Context: "${appNameOrScreen || 'Desktop'}"`,
         `Screen Resolution: ${info.details?.screenWidth || capture.details?.width || 1920}x${info.details?.screenHeight || capture.details?.height || 1080}`,
-        `Active Window Focus: "${capture.details?.activeWindow || info.details?.activeWindow || appNameOrScreen || 'Active Desktop'}"`,
-        `Mouse Cursor Position: (${info.details?.cursorX || 0}, ${info.details?.cursorY || 0})`,
+        `Active Window Focus: "${activeWin}"${visibleWindowsText}`,
+        `Mouse Cursor Position: (${info.details?.cursorX || capture.details?.cursorX || 0}, ${info.details?.cursorY || capture.details?.cursorY || 0})`,
         `Screenshot Status: Captured temporarily for visual inspection. (Will be automatically deleted upon task completion)`,
         `Visual Status: Screen is visible and verified.${visualAiAnalysis}`,
     ].join('\n');
@@ -1050,12 +1692,18 @@ export async function analyzeScreen(appNameOrScreen, targetQuery) {
  */
 export async function takeoverControl(appName, durationMs = 3000, x, y) {
     const { category, name } = normalizeAppName(appName);
+    const targetApp = (name && name !== 'desktop' && name !== 'system') ? name : undefined;
+    if (targetApp) {
+        setLiveScreenTargetApp(targetApp);
+    }
     if (name && name !== 'desktop' && name !== 'system' && category !== 'browser') {
         await openApp(name);
     }
     // Trigger visual sky-blue aura overlay on screen borders & corners with warning badge & input block
     try {
-        await showScreenAuraOverlay(Math.min(5000, Math.max(800, durationMs)), "Scout is on the screen.", true);
+        const auraLabel = targetApp ? `Scout is controlling ${targetApp}...` : "Scout is on the screen.";
+        await showScreenAuraOverlay(Math.min(5000, Math.max(800, durationMs)), auraLabel, true);
+        startLiveScreenShare({ targetApp }).catch(() => { });
     }
     catch { }
     const mouseRes = await moveAndClickMouse(x, y, 'left', name || appName);
@@ -1539,60 +2187,69 @@ $m::mouse_event(0x0800, 0, 0, ${delta}, 0)
  * Visually ground and click a target UI element by name or visual description.
  */
 export async function findAndClickElement(elementQuery, appName, button = 'left') {
-    const capture = await captureScreen(appName);
-    if (!capture.success || !capture.details?.outPath) {
-        return { success: false, output: `Could not capture screen to locate element: ${capture.output}` };
-    }
-    const screenWidth = capture.details.width || 1920;
-    const screenHeight = capture.details.height || 1080;
-    const outPath = capture.details.outPath;
+    let screenWidth = 1920;
+    let screenHeight = 1080;
     let targetX = undefined;
     let targetY = undefined;
-    const geminiKey = process.env.GEMINI_API_KEY;
-    const openaiKey = process.env.OPENAI_API_KEY;
-    const byokKey = process.env.BYOK_API_KEY;
-    const apiKey = geminiKey || openaiKey || byokKey;
-    if (apiKey && fs.existsSync(outPath)) {
+    // Use in-memory live screen frame first without writing screenshot files to disk
+    const liveFrame = await getLiveScreenFrame();
+    if (liveFrame) {
+        screenWidth = liveFrame.width;
+        screenHeight = liveFrame.height;
         try {
-            const imageBase64 = fs.readFileSync(outPath).toString('base64');
-            const isGemini = Boolean(geminiKey);
-            const baseURL = isGemini ? 'https://generativelanguage.googleapis.com/v1beta/openai' : undefined;
-            const model = isGemini ? 'gemini-2.5-flash' : 'gpt-4o-mini';
-            const client = new OpenAI({ apiKey, baseURL, timeout: 15000 });
-            const prompt = `Look at this screenshot and locate the UI element: "${elementQuery}".
+            const prompt = `Look at this screen and locate the UI element: "${elementQuery}".
 Return ONLY a valid JSON object with the center point of this element in 0-1000 normalized coordinates:
 {"point": [y, x]}
 Where y is between 0 and 1000 (top to bottom) and x is between 0 and 1000 (left to right).
 If not found, return {"point": null}. Output nothing else.`;
-            const resp = await client.chat.completions.create({
-                model,
-                messages: [
-                    {
-                        role: 'user',
-                        content: [
-                            { type: 'text', text: prompt },
-                            { type: 'image_url', image_url: { url: `data:image/png;base64,${imageBase64}` } },
-                        ],
-                    },
-                ],
-                max_tokens: 80,
-            });
-            const reply = resp.choices[0]?.message?.content || '';
-            const match = reply.match(/\{[\s\S]*"point"[\s\S]*\}/);
-            if (match) {
-                const parsed = JSON.parse(match[0]);
-                if (Array.isArray(parsed.point) && parsed.point.length === 2) {
-                    const normY = Number(parsed.point[0]);
-                    const normX = Number(parsed.point[1]);
-                    if (!isNaN(normX) && !isNaN(normY)) {
-                        targetX = Math.round((normX / 1000) * screenWidth);
-                        targetY = Math.round((normY / 1000) * screenHeight);
+            const reply = await callVisionAiWithFrame(liveFrame, prompt, 80);
+            if (reply) {
+                const match = reply.match(/\{[\s\S]*"point"[\s\S]*\}/);
+                if (match) {
+                    const parsed = JSON.parse(match[0]);
+                    if (Array.isArray(parsed.point) && parsed.point.length === 2) {
+                        const normY = Number(parsed.point[0]);
+                        const normX = Number(parsed.point[1]);
+                        if (!isNaN(normX) && !isNaN(normY)) {
+                            targetX = Math.round((normX / 1000) * screenWidth);
+                            targetY = Math.round((normY / 1000) * screenHeight);
+                        }
                     }
                 }
             }
         }
         catch { }
     }
+    else {
+        const capture = await captureScreen(appName);
+        if (capture.success && capture.details?.outPath && fs.existsSync(capture.details.outPath)) {
+            screenWidth = capture.details.width || 1920;
+            screenHeight = capture.details.height || 1080;
+            try {
+                const prompt = `Look at this screenshot and locate the UI element: "${elementQuery}".
+Return ONLY a valid JSON object with the center point of this element in 0-1000 normalized coordinates:
+{"point": [y, x]}
+Where y is between 0 and 1000 (top to bottom) and x is between 0 and 1000 (left to right).
+If not found, return {"point": null}. Output nothing else.`;
+                const reply = await callVisionAi(capture.details.outPath, prompt, 80);
+                if (reply) {
+                    const match = reply.match(/\{[\s\S]*"point"[\s\S]*\}/);
+                    if (match) {
+                        const parsed = JSON.parse(match[0]);
+                        if (Array.isArray(parsed.point) && parsed.point.length === 2) {
+                            const normY = Number(parsed.point[0]);
+                            const normX = Number(parsed.point[1]);
+                            if (!isNaN(normX) && !isNaN(normY)) {
+                                targetX = Math.round((normX / 1000) * screenWidth);
+                                targetY = Math.round((normY / 1000) * screenHeight);
+                            }
+                        }
+                    }
+                }
+            }
+            catch { }
+        }
+    }
     if ((targetX === undefined || targetY === undefined) && process.platform === 'win32') {
         const psScript = `
 Add-Type -AssemblyName UIAutomationClient