ft-scout 9.0.2 → 9.0.3

package/bin/src/engine/appControl.js

@@ -62,7 +62,7 @@ export function buildSocialUrl(platformOrAppName, target, text) {
     const isLinkedIn = pLower.includes('linkedin') || cleanTarget.includes('linkedin.com');
     if (isInstagram) {
         if (handle && handle !== 'instagram' && !cleanTarget.includes('http')) {
-            return `https://www.instagram.com/direct/i/`;
+            return `https://www.instagram.com/${handle}/`;
         }
         return `https://www.instagram.com/direct/inbox/`;
     }
@@ -653,9 +653,23 @@ export async function executeInApp(appName, action, payload) {
             if (!msgText && typeof payload === 'string' && payload.length > 0 && !payload.startsWith('@') && !payload.startsWith('http') && !payload.includes('@')) {
                 msgText = payload;
             }
-            const isEmail = ['send_mail', 'send_email', 'mail', 'email'].includes(actLower) || recipient.includes('@');
-            const targetUrl = buildSocialUrl(isEmail ? 'gmail' : name, recipient, msgText);
-            const openRes = await openApp('browser', targetUrl);
+            const cleanRecipient = String(recipient || '').trim();
+            const cleanMsg = String(msgText || '').trim();
+            if (!cleanRecipient && !cleanMsg) {
+                const targetPlatform = name || 'browser';
+                return {
+                    success: false,
+                    app: targetPlatform,
+                    action: actLower,
+                    output: `Missing recipient username/handle and message text for ${targetPlatform}. You MUST call the ask_user tool to ask: "Who would you like to message on ${targetPlatform}, and what message should I send?" before completing the task.`,
+                    details: { platform: targetPlatform, recipient: '', msgText: '' },
+                };
+            }
+            const isEmail = ['send_mail', 'send_email', 'mail', 'email'].includes(actLower) || cleanRecipient.includes('@');
+            const targetUrl = buildSocialUrl(isEmail ? 'gmail' : name, cleanRecipient, cleanMsg);
+            await openApp('browser', targetUrl);
+            // Display aura HUD directly over the browser window
+            showScreenAuraOverlay(4500, `Scout is messaging on ${isEmail ? 'Gmail' : (name || 'browser')}...`, false, 'browser').catch(() => { });
             let keystrokeOutput = '';
             if (isEmail) {
                 // For Gmail direct compose: wait 2.5s for page & compose dialog to render, then press Ctrl+Enter to send!
@@ -664,19 +678,38 @@ export async function executeInApp(appName, action, payload) {
                 const ksRes = await sendKeystrokes('browser', '', false, 'ctrl+enter');
                 keystrokeOutput = `\nAuto-Send Keystroke Dispatch (Ctrl+Enter): ${ksRes.output}`;
             }
-            else if (msgText) {
+            else if (cleanRecipient && (name.toLowerCase().includes('instagram') || targetUrl.includes('instagram.com'))) {
+                // Real Instagram Web UI Automation:
+                // 1. Wait for profile / direct page to fully render
+                await new Promise((r) => setTimeout(r, 3500));
+                // 2. Focus browser window
+                await focusWindow('chrome');
+                await focusWindow('browser');
+                // 3. If message text is provided, click "Message" on profile and type text
+                if (cleanMsg) {
+                    await new Promise((r) => setTimeout(r, 1000));
+                    await findAndClickElement('Message', 'browser');
+                    await new Promise((r) => setTimeout(r, 2200));
+                    const ksRes = await sendKeystrokes('browser', cleanMsg, true);
+                    keystrokeOutput = `\nInstagram Message Dispatched to ${cleanRecipient}: ${ksRes.output}`;
+                }
+                else {
+                    keystrokeOutput = `\nNavigated to Instagram profile for ${cleanRecipient}. Message text was not specified. Call ask_user to get message content.`;
+                }
+            }
+            else if (cleanMsg) {
                 // Lock user interruption during scratchpad keystroke dispatch so user clicks don't interrupt
                 await lockAppInput(name || 'browser', 3000);
-                await new Promise((r) => setTimeout(r, 1200));
-                const ksRes = await sendKeystrokes(name || 'browser', msgText, true);
+                await new Promise((r) => setTimeout(r, 2000));
+                const ksRes = await sendKeystrokes(name || 'browser', cleanMsg, true);
                 keystrokeOutput = `\nAuto-Keystroke Automated Dispatch: ${ksRes.output}`;
             }
             return {
                 success: true,
                 app: name,
                 action: actLower,
-                output: `Executed ${isEmail ? 'Gmail / Email Dispatch' : `${name} Direct Message`} for target recipient "${recipient}" at ${targetUrl}.${keystrokeOutput}`,
-                details: { platform: isEmail ? 'gmail' : name, recipient, msgText, targetUrl },
+                output: `Executed ${isEmail ? 'Gmail / Email Dispatch' : `${name} Direct Message`} for target recipient "${cleanRecipient}" at ${targetUrl}.${keystrokeOutput}`,
+                details: { platform: isEmail ? 'gmail' : name, recipient: cleanRecipient, msgText: cleanMsg, targetUrl },
             };
         }
     }
@@ -1080,11 +1113,76 @@ function escapeXaml(text) {
 }
 let persistentAuraProcess = null;
 let persistentAuraFlagPath = null;
+function buildTargetWindowRectSnippet(targetApp) {
+    const clean = (targetApp || '').trim().replace(/["'`$\\]/g, '');
+    if (!clean || clean.toLowerCase() === 'desktop' || clean.toLowerCase() === 'system') {
+        return `
+$window.Left = [System.Windows.SystemParameters]::VirtualScreenLeft
+$window.Top = [System.Windows.SystemParameters]::VirtualScreenTop
+$window.Width = [System.Windows.SystemParameters]::VirtualScreenWidth
+$window.Height = [System.Windows.SystemParameters]::VirtualScreenHeight
+`;
+    }
+    return `
+$targetApp = "${clean}"
+$winLeft = [System.Windows.SystemParameters]::VirtualScreenLeft
+$winTop = [System.Windows.SystemParameters]::VirtualScreenTop
+$winWidth = [System.Windows.SystemParameters]::VirtualScreenWidth
+$winHeight = [System.Windows.SystemParameters]::VirtualScreenHeight
+
+$procNames = @($targetApp)
+if ($targetApp -match "browser|chrome|edge|web") {
+    $procNames = @("chrome", "msedge", "brave", "firefox")
+} elseif ($targetApp -match "editor|code|vscode") {
+    $procNames = @("Code", "cursor", "notepad")
+}
+
+$proc = Get-Process -Name $procNames -ErrorAction SilentlyContinue | Where-Object { $_.MainWindowHandle -ne 0 } | Select-Object -First 1
+if (-not $proc) {
+    $proc = Get-Process | Where-Object { ($_.MainWindowTitle -like "*$targetApp*" -or $_.ProcessName -like "*$targetApp*") -and $_.MainWindowHandle -ne 0 } | Select-Object -First 1
+}
+
+if ($proc) {
+    try {
+        $sigRect = '[DllImport("user32.dll")] public static extern bool GetWindowRect(IntPtr hWnd, out RECT lpRect); public struct RECT { public int Left; public int Top; public int Right; public int Bottom; }'
+        $wType = Add-Type -MemberDefinition $sigRect -Name ('WinRect_' + [Guid]::NewGuid().ToString('N').Substring(0,8)) -Namespace 'Win32' -PassThru
+        $rc = [Activator]::CreateInstance($wType.GetNestedType('RECT'))
+        [void]$wType::GetWindowRect($proc.MainWindowHandle, [ref]$rc)
+        $calcW = $rc.Right - $rc.Left
+        $calcH = $rc.Bottom - $rc.Top
+        if ($calcW -gt 120 -and $calcH -gt 120) {
+            $winLeft = [double]$rc.Left
+            $winTop = [double]$rc.Top
+            $winWidth = [double]$calcW
+            $winHeight = [double]$calcH
+        }
+    } catch {
+        try {
+            Add-Type -AssemblyName UIAutomationClient
+            $el = [System.Windows.Automation.AutomationElement]::FromHandle($proc.MainWindowHandle)
+            $b = $el.Current.BoundingRectangle
+            if ($b.Width -gt 120 -and $b.Height -gt 120) {
+                $winLeft = [double]$b.X
+                $winTop = [double]$b.Y
+                $winWidth = [double]$b.Width
+                $winHeight = [double]$b.Height
+            }
+        } catch {}
+    }
+}
+
+$window.Left = $winLeft
+$window.Top = $winTop
+$window.Width = $winWidth
+$window.Height = $winHeight
+`;
+}
 /**
  * Start persistent futuristic sky-blue aura overlay on screen borders and corners.
  * Stays continuously active and animating as long as live screen sharing is running.
+ * When targetApp is specified, bounds strictly to that application window.
  */
-export async function startScreenAuraOverlay(message = "Live Screen Share Active • Streaming to Model") {
+export async function startScreenAuraOverlay(message = "Live Screen Share Active • Streaming to Model", targetApp) {
     const platform = process.platform;
     if (platform !== 'win32')
         return;
@@ -1099,7 +1197,7 @@ export async function startScreenAuraOverlay(message = "Live Screen Share Active
         const flagFile = path.join(scriptsDir, 'aura_active.flag');
         fs.writeFileSync(flagFile, String(Date.now()), 'utf8');
         persistentAuraFlagPath = flagFile;
-        const subtitleText = " Perception Active";
+        const subtitleText = targetApp ? ` • Focused: ${targetApp}` : " Perception Active";
         const safeMessage = escapeXaml(message);
         const safeSubtitle = escapeXaml(subtitleText);
         const parentPid = process.pid;
@@ -1149,55 +1247,55 @@ Add-Type -AssemblyName PresentationFramework, PresentationCore, WindowsBase
 
         <!-- Sky Blue Corner Brackets with Aura Glow -->
         <!-- Top Left Corner -->
-        <Canvas HorizontalAlignment="Left" VerticalAlignment="Top" Width="120" Height="120" Margin="6,6,0,0">
-            <Path x:Name="CornerTL" Data="M 0,90 L 0,0 L 90,0" Stroke="#00e5ff" StrokeThickness="6">
+        <Canvas HorizontalAlignment="Left" VerticalAlignment="Top" Width="100" Height="100" Margin="4,4,0,0">
+            <Path x:Name="CornerTL" Data="M 0,70 L 0,0 L 70,0" Stroke="#00e5ff" StrokeThickness="5">
                 <Path.Effect>
-                    <DropShadowEffect Color="#00e5ff" BlurRadius="30" ShadowDepth="0" Opacity="1"/>
+                    <DropShadowEffect Color="#00e5ff" BlurRadius="25" ShadowDepth="0" Opacity="1"/>
                 </Path.Effect>
             </Path>
         </Canvas>
 
         <!-- Top Right Corner -->
-        <Canvas HorizontalAlignment="Right" VerticalAlignment="Top" Width="120" Height="120" Margin="0,6,6,0">
-            <Path x:Name="CornerTR" Data="M 30,0 L 120,0 L 120,90" Stroke="#00e5ff" StrokeThickness="6">
+        <Canvas HorizontalAlignment="Right" VerticalAlignment="Top" Width="100" Height="100" Margin="0,4,4,0">
+            <Path x:Name="CornerTR" Data="M 30,0 L 100,0 L 100,70" Stroke="#00e5ff" StrokeThickness="5">
                 <Path.Effect>
-                    <DropShadowEffect Color="#00e5ff" BlurRadius="30" ShadowDepth="0" Opacity="1"/>
+                    <DropShadowEffect Color="#00e5ff" BlurRadius="25" ShadowDepth="0" Opacity="1"/>
                 </Path.Effect>
             </Path>
         </Canvas>
 
         <!-- Bottom Left Corner -->
-        <Canvas HorizontalAlignment="Left" VerticalAlignment="Bottom" Width="120" Height="120" Margin="6,0,0,6">
-            <Path x:Name="CornerBL" Data="M 0,30 L 0,120 L 90,120" Stroke="#00e5ff" StrokeThickness="6">
+        <Canvas HorizontalAlignment="Left" VerticalAlignment="Bottom" Width="100" Height="100" Margin="4,0,0,4">
+            <Path x:Name="CornerBL" Data="M 0,30 L 0,100 L 70,100" Stroke="#00e5ff" StrokeThickness="5">
                 <Path.Effect>
-                    <DropShadowEffect Color="#00e5ff" BlurRadius="30" ShadowDepth="0" Opacity="1"/>
+                    <DropShadowEffect Color="#00e5ff" BlurRadius="25" ShadowDepth="0" Opacity="1"/>
                 </Path.Effect>
             </Path>
         </Canvas>
 
         <!-- Bottom Right Corner -->
-        <Canvas HorizontalAlignment="Right" VerticalAlignment="Bottom" Width="120" Height="120" Margin="0,0,6,6">
-            <Path x:Name="CornerBR" Data="M 30,120 L 120,120 L 120,30" Stroke="#00e5ff" StrokeThickness="6">
+        <Canvas HorizontalAlignment="Right" VerticalAlignment="Bottom" Width="100" Height="100" Margin="0,0,4,4">
+            <Path x:Name="CornerBR" Data="M 30,100 L 100,100 L 100,30" Stroke="#00e5ff" StrokeThickness="5">
                 <Path.Effect>
-                    <DropShadowEffect Color="#00e5ff" BlurRadius="30" ShadowDepth="0" Opacity="1"/>
+                    <DropShadowEffect Color="#00e5ff" BlurRadius="25" ShadowDepth="0" Opacity="1"/>
                 </Path.Effect>
             </Path>
         </Canvas>
 
         <!-- Top Floating Status Badge -->
-        <Border HorizontalAlignment="Center" VerticalAlignment="Top" Margin="0,22,0,0"
-                Background="#EE03121F" BorderBrush="#00e5ff" BorderThickness="1.8" CornerRadius="24" Padding="26,10">
+        <Border HorizontalAlignment="Center" VerticalAlignment="Top" Margin="0,16,0,0"
+                Background="#EE03121F" BorderBrush="#00e5ff" BorderThickness="1.8" CornerRadius="24" Padding="22,8">
             <Border.Effect>
                 <DropShadowEffect Color="#00e5ff" BlurRadius="25" ShadowDepth="0" Opacity="0.95"/>
             </Border.Effect>
             <StackPanel Orientation="Horizontal" VerticalAlignment="Center">
-                <Ellipse Width="12" Height="12" Fill="#00e5ff" Margin="0,0,12,0">
+                <Ellipse Width="10" Height="10" Fill="#00e5ff" Margin="0,0,10,0">
                     <Ellipse.Effect>
                         <DropShadowEffect Color="#00e5ff" BlurRadius="12" ShadowDepth="0"/>
                     </Ellipse.Effect>
                 </Ellipse>
-                <TextBlock Text="${safeMessage}" Foreground="#ffffff" FontSize="16" FontWeight="Bold" VerticalAlignment="Center"/>
-                <TextBlock Text="${safeSubtitle}" Foreground="#38bdf8" FontSize="13.5" FontWeight="SemiBold" Margin="10,0,0,0" VerticalAlignment="Center"/>
+                <TextBlock Text="${safeMessage}" Foreground="#ffffff" FontSize="15" FontWeight="Bold" VerticalAlignment="Center"/>
+                <TextBlock Text="${safeSubtitle}" Foreground="#38bdf8" FontSize="13" FontWeight="SemiBold" Margin="8,0,0,0" VerticalAlignment="Center"/>
             </StackPanel>
         </Border>
     </Grid>
@@ -1207,10 +1305,7 @@ Add-Type -AssemblyName PresentationFramework, PresentationCore, WindowsBase
 $reader = (New-Object System.Xml.XmlNodeReader $xaml)
 $window = [System.Windows.Markup.XamlReader]::Load($reader)
 
-$window.Left = [System.Windows.SystemParameters]::VirtualScreenLeft
-$window.Top = [System.Windows.SystemParameters]::VirtualScreenTop
-$window.Width = [System.Windows.SystemParameters]::VirtualScreenWidth
-$window.Height = [System.Windows.SystemParameters]::VirtualScreenHeight
+${buildTargetWindowRectSnippet(targetApp)}
 $window.Topmost = $true
 $window.ShowActivated = $false
 
@@ -1276,11 +1371,12 @@ export function stopScreenAuraOverlay() {
 /**
  * Display a futuristic sky-blue aura animation on screen borders and corners,
  * show the warning "Scout is on the screen.", and optionally block external inputs.
+ * When targetApp is specified, bounds strictly to that application window.
  * If durationMs <= 0, aura stays on continuously until stopScreenAuraOverlay() is called.
  */
-export async function showScreenAuraOverlay(durationMs = 2200, message = "Scout is viewing the screen.", blockInput = false) {
+export async function showScreenAuraOverlay(durationMs = 2200, message = "Scout is viewing the screen.", blockInput = false, targetApp) {
     if (durationMs <= 0) {
-        return await startScreenAuraOverlay(message);
+        return await startScreenAuraOverlay(message, targetApp);
     }
     const platform = process.platform;
     if (platform !== 'win32')
@@ -1299,7 +1395,7 @@ try { [void]$b::BlockInput($true) } catch {}
         const unblockSnippet = blockInput ? `
     try { [void]$b::BlockInput($false) } catch {}
 ` : '';
-        const subtitleText = blockInput ? " • External Input Locked" : " Perception Active";
+        const subtitleText = blockInput ? (targetApp ? ` • Controlling ${targetApp} (Input Locked)` : " • External Input Locked") : (targetApp ? ` • Focused: ${targetApp}` : " Perception Active");
         const safeMessage = escapeXaml(message);
         const safeSubtitle = escapeXaml(subtitleText);
         const overlayPs = `
@@ -1349,55 +1445,55 @@ ${blockInputSnippet}
 
         <!-- Sky Blue Corner Brackets with Aura Glow -->
         <!-- Top Left Corner -->
-        <Canvas HorizontalAlignment="Left" VerticalAlignment="Top" Width="120" Height="120" Margin="6,6,0,0">
-            <Path x:Name="CornerTL" Data="M 0,90 L 0,0 L 90,0" Stroke="#00e5ff" StrokeThickness="6">
+        <Canvas HorizontalAlignment="Left" VerticalAlignment="Top" Width="100" Height="100" Margin="4,4,0,0">
+            <Path x:Name="CornerTL" Data="M 0,70 L 0,0 L 70,0" Stroke="#00e5ff" StrokeThickness="5">
                 <Path.Effect>
-                    <DropShadowEffect Color="#00e5ff" BlurRadius="30" ShadowDepth="0" Opacity="1"/>
+                    <DropShadowEffect Color="#00e5ff" BlurRadius="25" ShadowDepth="0" Opacity="1"/>
                 </Path.Effect>
             </Path>
         </Canvas>
 
         <!-- Top Right Corner -->
-        <Canvas HorizontalAlignment="Right" VerticalAlignment="Top" Width="120" Height="120" Margin="0,6,6,0">
-            <Path x:Name="CornerTR" Data="M 30,0 L 120,0 L 120,90" Stroke="#00e5ff" StrokeThickness="6">
+        <Canvas HorizontalAlignment="Right" VerticalAlignment="Top" Width="100" Height="100" Margin="0,4,4,0">
+            <Path x:Name="CornerTR" Data="M 30,0 L 100,0 L 100,70" Stroke="#00e5ff" StrokeThickness="5">
                 <Path.Effect>
-                    <DropShadowEffect Color="#00e5ff" BlurRadius="30" ShadowDepth="0" Opacity="1"/>
+                    <DropShadowEffect Color="#00e5ff" BlurRadius="25" ShadowDepth="0" Opacity="1"/>
                 </Path.Effect>
             </Path>
         </Canvas>
 
         <!-- Bottom Left Corner -->
-        <Canvas HorizontalAlignment="Left" VerticalAlignment="Bottom" Width="120" Height="120" Margin="6,0,0,6">
-            <Path x:Name="CornerBL" Data="M 0,30 L 0,120 L 90,120" Stroke="#00e5ff" StrokeThickness="6">
+        <Canvas HorizontalAlignment="Left" VerticalAlignment="Bottom" Width="100" Height="100" Margin="4,0,0,4">
+            <Path x:Name="CornerBL" Data="M 0,30 L 0,100 L 70,100" Stroke="#00e5ff" StrokeThickness="5">
                 <Path.Effect>
-                    <DropShadowEffect Color="#00e5ff" BlurRadius="30" ShadowDepth="0" Opacity="1"/>
+                    <DropShadowEffect Color="#00e5ff" BlurRadius="25" ShadowDepth="0" Opacity="1"/>
                 </Path.Effect>
             </Path>
         </Canvas>
 
         <!-- Bottom Right Corner -->
-        <Canvas HorizontalAlignment="Right" VerticalAlignment="Bottom" Width="120" Height="120" Margin="0,0,6,6">
-            <Path x:Name="CornerBR" Data="M 30,120 L 120,120 L 120,30" Stroke="#00e5ff" StrokeThickness="6">
+        <Canvas HorizontalAlignment="Right" VerticalAlignment="Bottom" Width="100" Height="100" Margin="0,0,4,4">
+            <Path x:Name="CornerBR" Data="M 30,100 L 100,100 L 100,30" Stroke="#00e5ff" StrokeThickness="5">
                 <Path.Effect>
-                    <DropShadowEffect Color="#00e5ff" BlurRadius="30" ShadowDepth="0" Opacity="1"/>
+                    <DropShadowEffect Color="#00e5ff" BlurRadius="25" ShadowDepth="0" Opacity="1"/>
                 </Path.Effect>
             </Path>
         </Canvas>
 
         <!-- Top Floating Status Badge -->
-        <Border HorizontalAlignment="Center" VerticalAlignment="Top" Margin="0,22,0,0"
-                Background="#EE03121F" BorderBrush="#00e5ff" BorderThickness="1.8" CornerRadius="24" Padding="26,10">
+        <Border HorizontalAlignment="Center" VerticalAlignment="Top" Margin="0,16,0,0"
+                Background="#EE03121F" BorderBrush="#00e5ff" BorderThickness="1.8" CornerRadius="24" Padding="22,8">
             <Border.Effect>
                 <DropShadowEffect Color="#00e5ff" BlurRadius="25" ShadowDepth="0" Opacity="0.95"/>
             </Border.Effect>
             <StackPanel Orientation="Horizontal" VerticalAlignment="Center">
-                <Ellipse Width="12" Height="12" Fill="#00e5ff" Margin="0,0,12,0">
+                <Ellipse Width="10" Height="10" Fill="#00e5ff" Margin="0,0,10,0">
                     <Ellipse.Effect>
                         <DropShadowEffect Color="#00e5ff" BlurRadius="12" ShadowDepth="0"/>
                     </Ellipse.Effect>
                 </Ellipse>
-                <TextBlock Text="${safeMessage}" Foreground="#ffffff" FontSize="16" FontWeight="Bold" VerticalAlignment="Center"/>
-                <TextBlock Text="${safeSubtitle}" Foreground="#38bdf8" FontSize="13.5" FontWeight="SemiBold" Margin="10,0,0,0" VerticalAlignment="Center"/>
+                <TextBlock Text="${safeMessage}" Foreground="#ffffff" FontSize="15" FontWeight="Bold" VerticalAlignment="Center"/>
+                <TextBlock Text="${safeSubtitle}" Foreground="#38bdf8" FontSize="13" FontWeight="SemiBold" Margin="8,0,0,0" VerticalAlignment="Center"/>
             </StackPanel>
         </Border>
     </Grid>
@@ -1407,10 +1503,7 @@ ${blockInputSnippet}
 $reader = (New-Object System.Xml.XmlNodeReader $xaml)
 $window = [System.Windows.Markup.XamlReader]::Load($reader)
 
-$window.Left = [System.Windows.SystemParameters]::VirtualScreenLeft
-$window.Top = [System.Windows.SystemParameters]::VirtualScreenTop
-$window.Width = [System.Windows.SystemParameters]::VirtualScreenWidth
-$window.Height = [System.Windows.SystemParameters]::VirtualScreenHeight
+${buildTargetWindowRectSnippet(targetApp)}
 $window.Topmost = $true
 $window.ShowActivated = $false
 
@@ -1715,7 +1808,7 @@ export async function takeoverControl(appName, durationMs = 3000, x, y) {
     // Trigger visual sky-blue aura overlay on screen borders & corners with warning badge & input block
     try {
         const auraLabel = targetApp ? `Scout is controlling ${targetApp}...` : "Scout is on the screen.";
-        await showScreenAuraOverlay(Math.min(5000, Math.max(800, durationMs)), auraLabel, true);
+        await showScreenAuraOverlay(Math.min(5000, Math.max(800, durationMs)), auraLabel, true, targetApp);
         startLiveScreenShare({ targetApp }).catch(() => { });
     }
     catch { }