ft-scout 8.0.3 → 8.0.5

package/bin/src/engine/appControl.js

@@ -2,11 +2,12 @@ import fs from 'fs';
 import path from 'path';
 import http from 'http';
 import https from 'https';
-import { exec } from 'child_process';
+import { exec, spawn, ChildProcess } from 'child_process';
 import { promisify } from 'util';
 import OpenAI from 'openai';
 import dotenv from 'dotenv';
 import chalk from 'chalk';
+import { startLiveScreenShare, stopLiveScreenShare, isLiveScreenShareActive, getLiveScreenFrame, analyzeLiveScreen, callVisionAiWithFrame, setLiveScreenTargetApp, getLiveScreenTargetApp, } from './liveScreenEngine.js';
 dotenv.config({ quiet: true });
 const execAsync = promisify(exec);
 /**
@@ -183,6 +184,9 @@ export async function findInstalledApp(query) {
  */
 export async function openApp(appNameOrPath, target, options) {
     const { category, name } = normalizeAppName(appNameOrPath);
+    if (name && name !== 'desktop' && name !== 'system') {
+        setLiveScreenTargetApp(name);
+    }
     const platform = process.platform;
     let command = '';
     const cleanTarget = target ? target.trim() : '';
@@ -352,6 +356,10 @@ export async function openApp(appNameOrPath, target, options) {
 export async function executeInApp(appName, action, payload) {
     const { category, name } = normalizeAppName(appName);
     const actLower = (action || '').trim().toLowerCase();
+    // Focus live screen sharing on the active app being operated
+    if (name && name !== 'desktop' && name !== 'system') {
+        setLiveScreenTargetApp(name);
+    }
     // Universal Scratchpad Desktop Automation Actions (Takeover, Screen Seeing, Keystrokes, Mouse Clicks, Focus Locks)
     if (['takeover', 'takeover_control', 'takeover_screen', 'takeover_app'].includes(actLower)) {
         const parsed = parsePayload(payload);
@@ -376,7 +384,38 @@ export async function executeInApp(appName, action, payload) {
             output: res.output,
         };
     }
+    if (['start_screen_share', 'live_screen_share', 'screen_share', 'start_live_stream'].includes(actLower)) {
+        const targetApp = (name && name !== 'desktop' && name !== 'system') ? name : undefined;
+        const res = await startLiveScreenShare({ targetApp });
+        return {
+            success: res.success,
+            app: name || appName,
+            action: actLower,
+            output: res.message,
+            details: { active: true, targetApp },
+        };
+    }
+    if (['stop_screen_share', 'end_screen_share', 'stop_live_stream'].includes(actLower)) {
+        const res = stopLiveScreenShare();
+        return {
+            success: res.success,
+            app: name || appName,
+            action: actLower,
+            output: res.summary,
+            details: { active: false },
+        };
+    }
     if (['capture_screen', 'screenshot', 'screen_shot'].includes(actLower)) {
+        if (isLiveScreenShareActive()) {
+            const liveRes = await analyzeLiveScreen();
+            return {
+                success: liveRes.success,
+                app: name || appName,
+                action: actLower,
+                output: liveRes.output,
+                details: liveRes.details,
+            };
+        }
         const res = await captureScreen(name || appName);
         return {
             success: res.success,
@@ -386,9 +425,9 @@ export async function executeInApp(appName, action, payload) {
             details: res.details,
         };
     }
-    if (['see_screen', 'inspect_screen', 'view_screen', 'analyze_screen', 'inspect_desktop', 'visual_analysis', 'see_and_analyze'].includes(actLower)) {
+    if (['see_screen', 'inspect_screen', 'view_screen', 'analyze_screen', 'inspect_desktop', 'visual_analysis', 'see_and_analyze', 'live_screen_stream', 'screen_stream'].includes(actLower)) {
         const query = typeof payload === 'string' ? payload : (payload?.query || payload?.target || payload?.text || undefined);
-        const res = await analyzeScreen(name || appName, query);
+        const res = await analyzeLiveScreen(query);
         return {
             success: res.success,
             app: name || appName,
@@ -1030,12 +1069,208 @@ export async function getScreenInfo(appName) {
         };
     }
 }
+let persistentAuraProcess = null;
+let persistentAuraFlagPath = null;
+/**
+ * Start persistent futuristic sky-blue aura overlay on screen borders and corners.
+ * Stays continuously active and animating as long as live screen sharing is running.
+ */
+export async function startScreenAuraOverlay(message = "Live Screen Share Active • Streaming to Model") {
+    const platform = process.platform;
+    if (platform !== 'win32')
+        return;
+    // Stop any existing overlay first
+    stopScreenAuraOverlay();
+    try {
+        const scriptsDir = path.join(process.cwd(), '.ft', 'scripts');
+        if (!fs.existsSync(scriptsDir)) {
+            fs.mkdirSync(scriptsDir, { recursive: true });
+        }
+        const psFile = path.join(scriptsDir, 'scout_aura_persistent.ps1');
+        const flagFile = path.join(scriptsDir, 'aura_active.flag');
+        fs.writeFileSync(flagFile, String(Date.now()), 'utf8');
+        persistentAuraFlagPath = flagFile;
+        const subtitleText = " Perception Active";
+        const parentPid = process.pid;
+        const overlayPs = `
+Add-Type -AssemblyName PresentationFramework, PresentationCore, WindowsBase
+
+[xml]$xaml = @"
+<Window xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
+        xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
+        Title="Scout Aura Overlay"
+        WindowStyle="None"
+        AllowsTransparency="True"
+        Background="Transparent"
+        Topmost="True"
+        ShowInTaskbar="False"
+        ShowActivated="False"
+        IsHitTestVisible="False">
+    <Window.Resources>
+        <Storyboard x:Key="AuraPulse" RepeatBehavior="Forever" AutoReverse="True">
+            <DoubleAnimation Storyboard.TargetName="AuraBorder" Storyboard.TargetProperty="Opacity"
+                             From="0.4" To="1.0" Duration="0:0:0.6"/>
+            <DoubleAnimation Storyboard.TargetName="CornerTL" Storyboard.TargetProperty="Opacity"
+                             From="0.5" To="1.0" Duration="0:0:0.6"/>
+            <DoubleAnimation Storyboard.TargetName="CornerTR" Storyboard.TargetProperty="Opacity"
+                             From="0.5" To="1.0" Duration="0:0:0.6"/>
+            <DoubleAnimation Storyboard.TargetName="CornerBL" Storyboard.TargetProperty="Opacity"
+                             From="0.5" To="1.0" Duration="0:0:0.6"/>
+            <DoubleAnimation Storyboard.TargetName="CornerBR" Storyboard.TargetProperty="Opacity"
+                             From="0.5" To="1.0" Duration="0:0:0.6"/>
+            <DoubleAnimation Storyboard.TargetName="AuraGlow" Storyboard.TargetProperty="Opacity"
+                             From="0.25" To="0.9" Duration="0:0:0.6"/>
+        </Storyboard>
+    </Window.Resources>
+
+    <Grid IsHitTestVisible="False">
+        <!-- Outer Glowing Sky Blue Aura Border -->
+        <Border x:Name="AuraGlow" BorderThickness="6" BorderBrush="#00d8ff" Margin="0">
+            <Border.Effect>
+                <DropShadowEffect Color="#00e5ff" BlurRadius="45" ShadowDepth="0" Opacity="0.95"/>
+            </Border.Effect>
+        </Border>
+        <Border x:Name="AuraBorder" BorderThickness="3" BorderBrush="#38bdf8" Margin="0">
+            <Border.Effect>
+                <DropShadowEffect Color="#00e5ff" BlurRadius="25" ShadowDepth="0" Opacity="1"/>
+            </Border.Effect>
+        </Border>
+
+        <!-- Sky Blue Corner Brackets with Aura Glow -->
+        <!-- Top Left Corner -->
+        <Canvas HorizontalAlignment="Left" VerticalAlignment="Top" Width="120" Height="120" Margin="6,6,0,0">
+            <Path x:Name="CornerTL" Data="M 0,90 L 0,0 L 90,0" Stroke="#00e5ff" StrokeThickness="6">
+                <Path.Effect>
+                    <DropShadowEffect Color="#00e5ff" BlurRadius="30" ShadowDepth="0" Opacity="1"/>
+                </Path.Effect>
+            </Path>
+        </Canvas>
+
+        <!-- Top Right Corner -->
+        <Canvas HorizontalAlignment="Right" VerticalAlignment="Top" Width="120" Height="120" Margin="0,6,6,0">
+            <Path x:Name="CornerTR" Data="M 30,0 L 120,0 L 120,90" Stroke="#00e5ff" StrokeThickness="6">
+                <Path.Effect>
+                    <DropShadowEffect Color="#00e5ff" BlurRadius="30" ShadowDepth="0" Opacity="1"/>
+                </Path.Effect>
+            </Path>
+        </Canvas>
+
+        <!-- Bottom Left Corner -->
+        <Canvas HorizontalAlignment="Left" VerticalAlignment="Bottom" Width="120" Height="120" Margin="6,0,0,6">
+            <Path x:Name="CornerBL" Data="M 0,30 L 0,120 L 90,120" Stroke="#00e5ff" StrokeThickness="6">
+                <Path.Effect>
+                    <DropShadowEffect Color="#00e5ff" BlurRadius="30" ShadowDepth="0" Opacity="1"/>
+                </Path.Effect>
+            </Path>
+        </Canvas>
+
+        <!-- Bottom Right Corner -->
+        <Canvas HorizontalAlignment="Right" VerticalAlignment="Bottom" Width="120" Height="120" Margin="0,0,6,6">
+            <Path x:Name="CornerBR" Data="M 30,120 L 120,120 L 120,30" Stroke="#00e5ff" StrokeThickness="6">
+                <Path.Effect>
+                    <DropShadowEffect Color="#00e5ff" BlurRadius="30" ShadowDepth="0" Opacity="1"/>
+                </Path.Effect>
+            </Path>
+        </Canvas>
+
+        <!-- Top Floating Status Badge -->
+        <Border HorizontalAlignment="Center" VerticalAlignment="Top" Margin="0,22,0,0"
+                Background="#EE03121F" BorderBrush="#00e5ff" BorderThickness="1.8" CornerRadius="24" Padding="26,10">
+            <Border.Effect>
+                <DropShadowEffect Color="#00e5ff" BlurRadius="25" ShadowDepth="0" Opacity="0.95"/>
+            </Border.Effect>
+            <StackPanel Orientation="Horizontal" VerticalAlignment="Center">
+                <Ellipse Width="12" Height="12" Fill="#00e5ff" Margin="0,0,12,0">
+                    <Ellipse.Effect>
+                        <DropShadowEffect Color="#00e5ff" BlurRadius="12" ShadowDepth="0"/>
+                    </Ellipse.Effect>
+                </Ellipse>
+                <TextBlock Text="${message}" Foreground="#ffffff" FontSize="16" FontWeight="Bold" VerticalAlignment="Center"/>
+                <TextBlock Text="${subtitleText}" Foreground="#38bdf8" FontSize="13.5" FontWeight="SemiBold" Margin="10,0,0,0" VerticalAlignment="Center"/>
+            </StackPanel>
+        </Border>
+    </Grid>
+</Window>
+"@
+
+$reader = (New-Object System.Xml.XmlNodeReader $xaml)
+$window = [System.Windows.Markup.XamlReader]::Load($reader)
+
+$window.Left = [System.Windows.SystemParameters]::VirtualScreenLeft
+$window.Top = [System.Windows.SystemParameters]::VirtualScreenTop
+$window.Width = [System.Windows.SystemParameters]::VirtualScreenWidth
+$window.Height = [System.Windows.SystemParameters]::VirtualScreenHeight
+$window.Topmost = $true
+$window.ShowActivated = $false
+
+$storyboard = $window.Resources["AuraPulse"]
+$window.Add_Loaded({
+    $storyboard.Begin($window)
+})
+
+$flagPath = "${flagFile.replace(/\\/g, '\\\\')}"
+$parentPid = ${parentPid}
+
+# Check if screen sharing is active or parent exited every 350ms
+$timer = New-Object System.Windows.Threading.DispatcherTimer
+$timer.Interval = [TimeSpan]::FromMilliseconds(350)
+$timer.Add_Tick({
+    $shouldClose = $false
+    if (-not (Test-Path $flagPath)) {
+        $shouldClose = $true
+    } else {
+        $proc = Get-Process -Id $parentPid -ErrorAction SilentlyContinue
+        if (-not $proc) {
+            $shouldClose = $true
+        }
+    }
+    if ($shouldClose) {
+        $timer.Stop()
+        $window.Close()
+    }
+})
+$timer.Start()
+
+[void]$window.ShowDialog()
+`;
+        fs.writeFileSync(psFile, overlayPs, 'utf8');
+        const child = spawn('powershell', ['-WindowStyle', 'Hidden', '-NoProfile', '-Sta', '-ExecutionPolicy', 'Bypass', '-File', psFile], {
+            detached: false,
+            stdio: 'ignore',
+        });
+        child.unref();
+        persistentAuraProcess = child;
+    }
+    catch { }
+}
+/**
+ * Stop persistent screen aura animation when live screen sharing stops.
+ */
+export function stopScreenAuraOverlay() {
+    if (persistentAuraFlagPath && fs.existsSync(persistentAuraFlagPath)) {
+        try {
+            fs.unlinkSync(persistentAuraFlagPath);
+        }
+        catch { }
+        persistentAuraFlagPath = null;
+    }
+    if (persistentAuraProcess) {
+        try {
+            persistentAuraProcess.kill();
+        }
+        catch { }
+        persistentAuraProcess = null;
+    }
+}
 /**
  * Display a futuristic sky-blue aura animation on screen borders and corners,
- * show the warning "Scout is on the screen.", and block external inputs
- * during live screen capture and automation.
+ * show the warning "Scout is on the screen.", and optionally block external inputs.
+ * If durationMs <= 0, aura stays on continuously until stopScreenAuraOverlay() is called.
  */
 export async function showScreenAuraOverlay(durationMs = 2200, message = "Scout is viewing the screen.", blockInput = false) {
+    if (durationMs <= 0) {
+        return await startScreenAuraOverlay(message);
+    }
     const platform = process.platform;
     if (platform !== 'win32')
         return;
@@ -1053,7 +1288,7 @@ try { [void]$b::BlockInput($true) } catch {}
         const unblockSnippet = blockInput ? `
     try { [void]$b::BlockInput($false) } catch {}
 ` : '';
-        const subtitleText = blockInput ? " • External Input Locked" : " • Perception Active";
+        const subtitleText = blockInput ? " • External Input Locked" : " Perception Active";
         const overlayPs = `
 Add-Type -AssemblyName PresentationFramework, PresentationCore, WindowsBase
 ${blockInputSnippet}
@@ -1457,12 +1692,18 @@ export async function analyzeScreen(appNameOrScreen, targetQuery) {
  */
 export async function takeoverControl(appName, durationMs = 3000, x, y) {
     const { category, name } = normalizeAppName(appName);
+    const targetApp = (name && name !== 'desktop' && name !== 'system') ? name : undefined;
+    if (targetApp) {
+        setLiveScreenTargetApp(targetApp);
+    }
     if (name && name !== 'desktop' && name !== 'system' && category !== 'browser') {
         await openApp(name);
     }
     // Trigger visual sky-blue aura overlay on screen borders & corners with warning badge & input block
     try {
-        await showScreenAuraOverlay(Math.min(5000, Math.max(800, durationMs)), "Scout is on the screen.", true);
+        const auraLabel = targetApp ? `Scout is controlling ${targetApp}...` : "Scout is on the screen.";
+        await showScreenAuraOverlay(Math.min(5000, Math.max(800, durationMs)), auraLabel, true);
+        startLiveScreenShare({ targetApp }).catch(() => { });
     }
     catch { }
     const mouseRes = await moveAndClickMouse(x, y, 'left', name || appName);
@@ -1946,23 +2187,22 @@ $m::mouse_event(0x0800, 0, 0, ${delta}, 0)
  * Visually ground and click a target UI element by name or visual description.
  */
 export async function findAndClickElement(elementQuery, appName, button = 'left') {
-    const capture = await captureScreen(appName);
-    if (!capture.success || !capture.details?.outPath) {
-        return { success: false, output: `Could not capture screen to locate element: ${capture.output}` };
-    }
-    const screenWidth = capture.details.width || 1920;
-    const screenHeight = capture.details.height || 1080;
-    const outPath = capture.details.outPath;
+    let screenWidth = 1920;
+    let screenHeight = 1080;
     let targetX = undefined;
     let targetY = undefined;
-    if (fs.existsSync(outPath)) {
+    // Use in-memory live screen frame first without writing screenshot files to disk
+    const liveFrame = await getLiveScreenFrame();
+    if (liveFrame) {
+        screenWidth = liveFrame.width;
+        screenHeight = liveFrame.height;
         try {
-            const prompt = `Look at this screenshot and locate the UI element: "${elementQuery}".
+            const prompt = `Look at this screen and locate the UI element: "${elementQuery}".
 Return ONLY a valid JSON object with the center point of this element in 0-1000 normalized coordinates:
 {"point": [y, x]}
 Where y is between 0 and 1000 (top to bottom) and x is between 0 and 1000 (left to right).
 If not found, return {"point": null}. Output nothing else.`;
-            const reply = await callVisionAi(outPath, prompt, 80);
+            const reply = await callVisionAiWithFrame(liveFrame, prompt, 80);
             if (reply) {
                 const match = reply.match(/\{[\s\S]*"point"[\s\S]*\}/);
                 if (match) {
@@ -1980,6 +2220,36 @@ If not found, return {"point": null}. Output nothing else.`;
         }
         catch { }
     }
+    else {
+        const capture = await captureScreen(appName);
+        if (capture.success && capture.details?.outPath && fs.existsSync(capture.details.outPath)) {
+            screenWidth = capture.details.width || 1920;
+            screenHeight = capture.details.height || 1080;
+            try {
+                const prompt = `Look at this screenshot and locate the UI element: "${elementQuery}".
+Return ONLY a valid JSON object with the center point of this element in 0-1000 normalized coordinates:
+{"point": [y, x]}
+Where y is between 0 and 1000 (top to bottom) and x is between 0 and 1000 (left to right).
+If not found, return {"point": null}. Output nothing else.`;
+                const reply = await callVisionAi(capture.details.outPath, prompt, 80);
+                if (reply) {
+                    const match = reply.match(/\{[\s\S]*"point"[\s\S]*\}/);
+                    if (match) {
+                        const parsed = JSON.parse(match[0]);
+                        if (Array.isArray(parsed.point) && parsed.point.length === 2) {
+                            const normY = Number(parsed.point[0]);
+                            const normX = Number(parsed.point[1]);
+                            if (!isNaN(normX) && !isNaN(normY)) {
+                                targetX = Math.round((normX / 1000) * screenWidth);
+                                targetY = Math.round((normY / 1000) * screenHeight);
+                            }
+                        }
+                    }
+                }
+            }
+            catch { }
+        }
+    }
     if ((targetX === undefined || targetY === undefined) && process.platform === 'win32') {
         const psScript = `
 Add-Type -AssemblyName UIAutomationClient