copilot-liku-cli 0.0.2 → 0.0.4

package/README.md

@@ -1,5 +1,9 @@
 # GitHub Copilot CLI: Liku Edition (Public Preview)
 
+[![npm version](https://img.shields.io/npm/v/copilot-liku-cli.svg)](https://www.npmjs.com/package/copilot-liku-cli)
+[![Node.js](https://img.shields.io/badge/node-%3E%3D22.0.0-brightgreen.svg)](https://nodejs.org/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE.md)
+
 The power of GitHub Copilot, now with visual-spatial awareness and advanced automation.
 
 GitHub Copilot-Liku CLI brings AI-powered coding assistance and UI automation directly to your terminal. This "Liku Edition" extends the standard Copilot experience with an ultra-thin Electron overlay, allowing the agent to "see" and interact with your screen through a coordinated grid system and native UI automation.
@@ -89,7 +93,7 @@ The Liku Edition moves beyond single-turn responses with a specialized team of a
 
 #### Global Installation (Recommended for Users)
 
-Once published to npm, install globally with:
+Install globally from npm:
 ```bash
 npm install -g copilot-liku-cli
 ```

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "copilot-liku-cli",
-  "version": "0.0.2",
+  "version": "0.0.4",
   "description": "GitHub Copilot CLI with headless agent + ultra-thin overlay architecture",
   "main": "src/main/index.js",
   "bin": {

package/src/cli/liku.js

@@ -1,4 +1,4 @@
-#!/usr/bin/env node
+#!/usr/bin/env node
 /**
  * liku - Copilot-Liku CLI
  * 

package/src/main/ai-service.js

@@ -10,9 +10,16 @@ const https = require('https');
 const http = require('http');
 const fs = require('fs');
 const path = require('path');
+const os = require('os');
 const { shell } = require('electron');
 const systemAutomation = require('./system-automation');
 
+// ===== ENVIRONMENT DETECTION =====
+const PLATFORM = process.platform; // 'win32', 'darwin', 'linux'
+const OS_NAME = PLATFORM === 'win32' ? 'Windows' : PLATFORM === 'darwin' ? 'macOS' : 'Linux';
+const OS_VERSION = os.release();
+const ARCHITECTURE = process.arch;
+
 // Lazy-load inspect service to avoid circular dependencies
 let inspectService = null;
 function getInspectService() {
@@ -22,6 +29,16 @@ function getInspectService() {
   return inspectService;
 }
 
+// Lazy-load UI watcher for live UI context
+let uiWatcher = null;
+function getUIWatcher() {
+  if (!uiWatcher) {
+    const { UIWatcher } = require('./ui-watcher');
+    uiWatcher = new UIWatcher();
+  }
+  return uiWatcher;
+}
+
 // ===== CONFIGURATION =====
 
 // Available models for GitHub Copilot (based on Copilot CLI changelog)
@@ -106,61 +123,118 @@ let visualContextBuffer = [];
 const MAX_VISUAL_CONTEXT = 5;
 
 // ===== SYSTEM PROMPT =====
+// Generate platform-specific context dynamically
+function getPlatformContext() {
+  if (PLATFORM === 'win32') {
+    return `
+## Platform: Windows ${OS_VERSION}
+
+### Windows-Specific Keyboard Shortcuts (USE THESE!)
+- **Open new terminal**: \`win+x\` then \`i\` (opens Windows Terminal) OR \`win+r\` then type \`wt\` then \`enter\`
+- **Open Run dialog**: \`win+r\`
+- **Open Start menu/Search**: \`win\` (Windows key alone)
+- **Switch windows**: \`alt+tab\`
+- **Show desktop**: \`win+d\`
+- **File Explorer**: \`win+e\`
+- **Settings**: \`win+i\`
+- **Lock screen**: \`win+l\`
+- **Clipboard history**: \`win+v\`
+- **Screenshot**: \`win+shift+s\`
+
+### Windows Terminal Shortcuts
+- **New tab**: \`ctrl+shift+t\`
+- **Close tab**: \`ctrl+shift+w\`
+- **Split pane**: \`alt+shift+d\`
+
+### IMPORTANT: On Windows, NEVER use:
+- \`cmd+space\` (that's macOS Spotlight)
+- \`ctrl+alt+t\` (that's Linux terminal shortcut)`;
+  } else if (PLATFORM === 'darwin') {
+    return `
+## Platform: macOS ${OS_VERSION}
+
+### macOS-Specific Keyboard Shortcuts
+- **Open terminal**: \`cmd+space\` then type "Terminal" then \`enter\`
+- **Spotlight search**: \`cmd+space\`
+- **Switch windows**: \`cmd+tab\`
+- **Switch windows same app**: \`cmd+\`\`
+- **Show desktop**: \`f11\` or \`cmd+mission control\`
+- **Finder**: \`cmd+shift+g\`
+- **Force quit**: \`cmd+option+esc\`
+- **Screenshot**: \`cmd+shift+4\``;
+  } else {
+    return `
+## Platform: Linux ${OS_VERSION}
+
+### Linux-Specific Keyboard Shortcuts
+- **Open terminal**: \`ctrl+alt+t\` (most distros)
+- **Application menu**: \`super\` (Windows key)
+- **Switch windows**: \`alt+tab\`
+- **Show desktop**: \`super+d\`
+- **File manager**: \`super+e\`
+- **Screenshot**: \`print\` or \`shift+print\``;
+  }
+}
+
 const SYSTEM_PROMPT = `You are Liku, an intelligent AGENTIC AI assistant integrated into a desktop overlay system with visual screen awareness AND the ability to control the user's computer.
 
+${getPlatformContext()}
+
 ## Your Core Capabilities
 
 1. **Screen Vision**: When the user captures their screen, you receive it as an image. ALWAYS analyze visible content immediately.
 
-2. **Grid Coordinate System**: The screen has a dot grid overlay:
+2. **SEMANTIC ELEMENT ACTIONS (PREFERRED!)**: You can interact with UI elements by their text/name - MORE RELIABLE than coordinates:
+   - \`{"type": "click_element", "text": "Submit", "reason": "Click Submit button"}\` - Finds and clicks element by text
+   - \`{"type": "find_element", "text": "Save", "reason": "Locate Save button"}\` - Finds element info
+
+3. **Grid Coordinate System**: The screen has a dot grid overlay:
    - **Columns**: Letters A, B, C, D... (left to right), spacing 100px
    - **Rows**: Numbers 0, 1, 2, 3... (top to bottom), spacing 100px
    - **Start**: Grid is centered, so A0 is at (50, 50)
-   - **Format**: "C3" = column C (index 2), row 3 = pixel (250, 350)
-   - **Formula**: x = 50 + col_index * 100, y = 50 + row_index * 100
-   - A0 ≈ (50, 50), B0 ≈ (150, 50), A1 ≈ (50, 150)
    - **Fine Grid**: Sub-labels like C3.12 refer to 25px subcells inside C3
 
-3. **SYSTEM CONTROL - AGENTIC ACTIONS**: You can execute actions on the user's computer:
-   - **Click**: Click at coordinates
+4. **SYSTEM CONTROL - AGENTIC ACTIONS**: You can execute actions on the user's computer:
+   - **Click**: Click at coordinates (use click_element when possible!)
    - **Type**: Type text into focused fields
-   - **Press Keys**: Press keyboard shortcuts (ctrl+c, enter, etc.)
+   - **Press Keys**: Press keyboard shortcuts (platform-specific - see above!)
    - **Scroll**: Scroll up/down
    - **Drag**: Drag from one point to another
 
 ## ACTION FORMAT - CRITICAL
 
-When the user asks you to DO something (click, type, interact), respond with a JSON action block:
+When the user asks you to DO something, respond with a JSON action block:
 
 \`\`\`json
 {
   "thought": "Brief explanation of what I'm about to do",
   "actions": [
-    {"type": "click", "x": 300, "y": 200, "reason": "Click the input field"},
-    {"type": "type", "text": "Hello world", "reason": "Type the requested text"},
-    {"type": "key", "key": "enter", "reason": "Submit the form"}
+    {"type": "key", "key": "win+x", "reason": "Open Windows power menu"},
+    {"type": "wait", "ms": 300},
+    {"type": "key", "key": "i", "reason": "Select Terminal option"}
   ],
-  "verification": "After these actions, the text field should show 'Hello world'"
+  "verification": "A new Windows Terminal window should open"
 }
 \`\`\`
 
 ### Action Types:
-- \`{"type": "click", "x": <number>, "y": <number>}\` - Left click at pixel coordinates
+- \`{"type": "click_element", "text": "<button text>"}\` - **PREFERRED**: Click element by text (uses Windows UI Automation)
+- \`{"type": "find_element", "text": "<search text>"}\` - Find element and return its info
+- \`{"type": "click", "x": <number>, "y": <number>}\` - Left click at pixel coordinates (use as fallback)
 - \`{"type": "double_click", "x": <number>, "y": <number>}\` - Double click
 - \`{"type": "right_click", "x": <number>, "y": <number>}\` - Right click
 - \`{"type": "type", "text": "<string>"}\` - Type text (types into currently focused element)
-- \`{"type": "key", "key": "<key combo>"}\` - Press key (e.g., "enter", "ctrl+c", "alt+tab", "f5")
-- \`{"type": "scroll", "direction": "up|down", "amount": <number>}\` - Scroll (amount = clicks)
+- \`{"type": "key", "key": "<key combo>"}\` - Press key (e.g., "enter", "ctrl+c", "win+r", "alt+tab")
+- \`{"type": "scroll", "direction": "up|down", "amount": <number>}\` - Scroll
 - \`{"type": "drag", "fromX": <n>, "fromY": <n>, "toX": <n>, "toY": <n>}\` - Drag
-- \`{"type": "wait", "ms": <number>}\` - Wait milliseconds
+- \`{"type": "wait", "ms": <number>}\` - Wait milliseconds (IMPORTANT: add waits between multi-step actions!)
 - \`{"type": "screenshot"}\` - Take screenshot to verify result
 
 ### Grid to Pixel Conversion:
 - A0 → (50, 50), B0 → (150, 50), C0 → (250, 50)
 - A1 → (50, 150), B1 → (150, 150), C1 → (250, 150)
 - Formula: x = 50 + col_index * 100, y = 50 + row_index * 100
-- Column A=0, B=1, C=2... so C3 = x: 50 + 2*100 = 250, y: 50 + 3*100 = 350
- - Fine labels: C3.12 = x: 12.5 + (2*4+1)*25 = 237.5, y: 12.5 + (3*4+2)*25 = 362.5
+- Fine labels: C3.12 = x: 12.5 + (2*4+1)*25 = 237.5, y: 12.5 + (3*4+2)*25 = 362.5
 
 ## Response Guidelines
 
@@ -170,21 +244,27 @@ When the user asks you to DO something (click, type, interact), respond with a J
 
 **For ACTION requests** (click here, type this, open that):
 - ALWAYS respond with the JSON action block
-- Include your thought process
-- Calculate coordinates precisely
+- Use PLATFORM-SPECIFIC shortcuts (see above!)
+- Prefer \`click_element\` over coordinate clicks when targeting named UI elements
+- Add \`wait\` actions between steps that need UI to update
 - Add verification step to confirm success
 
-**When executing a sequence**:
-1. First action: click to focus the target element
-2. Second action: perform the main task (type, etc.)
-3. Optional: verify with screenshot
-
-**IMPORTANT**: When asked to interact with something visible in the screenshot:
-1. Identify the element's approximate position
-2. Convert to pixel coordinates
-3. Return the action JSON
-
-Be precise, efficient, and execute actions confidently based on visual information.`;
+**Common Task Patterns**:
+${PLATFORM === 'win32' ? `
+- **Open new terminal**: Use \`win+x\` then \`i\` (or \`win+r\` → type "wt" → \`enter\`)
+- **Open application**: Use \`win\` key, type app name, press \`enter\`
+- **Save file**: \`ctrl+s\`
+- **Copy/Paste**: \`ctrl+c\` / \`ctrl+v\`` : PLATFORM === 'darwin' ? `
+- **Open terminal**: \`cmd+space\`, type "Terminal", \`enter\`
+- **Open application**: \`cmd+space\`, type app name, \`enter\`
+- **Save file**: \`cmd+s\`
+- **Copy/Paste**: \`cmd+c\` / \`cmd+v\`` : `
+- **Open terminal**: \`ctrl+alt+t\`
+- **Open application**: \`super\` key, type name, \`enter\`
+- **Save file**: \`ctrl+s\`
+- **Copy/Paste**: \`ctrl+c\` / \`ctrl+v\``}
+
+Be precise, use platform-correct shortcuts, and execute actions confidently!`;
 
 /**
  * Set the AI provider
@@ -331,8 +411,23 @@ ${inspectContext.regions.slice(0, 20).map((r, i) =>
     console.warn('[AI] Could not get inspect context:', e.message);
   }
   
-  const enhancedMessage = inspectContextText 
-    ? `${userMessage}${inspectContextText}` 
+  // Get live UI context from the UI watcher (always-on mirror)
+  let liveUIContextText = '';
+  try {
+    const watcher = getUIWatcher();
+    if (watcher && watcher.isRunning) {
+      const uiContext = watcher.getContextForAI();
+      if (uiContext && uiContext.trim()) {
+        liveUIContextText = `\n\n${uiContext}`;
+        console.log('[AI] Including live UI context from watcher');
+      }
+    }
+  } catch (e) {
+    console.warn('[AI] Could not get live UI context:', e.message);
+  }
+  
+  const enhancedMessage = inspectContextText || liveUIContextText
+    ? `${userMessage}${inspectContextText}${liveUIContextText}` 
     : userMessage;
 
   if (latestVisual && (currentProvider === 'copilot' || currentProvider === 'openai')) {

package/src/main/index.js

@@ -25,6 +25,9 @@ const aiService = require('./ai-service.js');
 // Visual awareness for advanced screen analysis
 const visualAwareness = require('./visual-awareness.js');
 
+// Live UI Watcher for continuous UI monitoring
+const { UIWatcher } = require('./ui-watcher.js');
+
 // Multi-agent system for advanced AI orchestration
 const { createAgentSystem } = require('./agents/index.js');
 
@@ -58,6 +61,9 @@ let overlayWindow = null;
 let chatWindow = null;
 let tray = null;
 
+// Live UI watcher instance
+let uiWatcher = null;
+
 // State management
 let overlayMode = 'selection'; // start in selection so the grid is visible immediately
 let isChatVisible = false;
@@ -112,9 +118,12 @@ function createOverlayWindow() {
   });
 
   // Pipe renderer console to main for debugging without DevTools
-  overlayWindow.webContents.on('console-message', (event) => {
-    const { level, message, line, sourceId } = event;
-    console.log(`[overlay console] (${level}) ${sourceId}:${line} - ${message}`);
+  overlayWindow.webContents.on('console-message', (event, level, message, line, sourceId) => {
+    const levelNames = ['verbose', 'info', 'warn', 'error'];
+    const levelStr = levelNames[level] || `level-${level}`;
+    const lineStr = line !== undefined ? `:${line}` : '';
+    const source = sourceId ? sourceId.split('/').pop() : 'overlay';
+    console.log(`[overlay console] (${levelStr}) ${source}${lineStr} - ${message}`);
   });
 
   // Prevent overlay from appearing in Dock/Taskbar
@@ -1068,6 +1077,17 @@ function setupIPC() {
       total: actionData.actions.length 
     });
     
+    // CRITICAL: Blur chat window before executing actions so keyboard/mouse
+    // input reaches the desktop instead of staying within Electron
+    if (chatWindow && !chatWindow.isDestroyed()) {
+      chatWindow.blur();
+    }
+    if (overlayWindow && !overlayWindow.isDestroyed()) {
+      overlayWindow.blur();
+      // Temporarily lower overlay z-index so popups (like Run dialog) appear above
+      overlayWindow.setAlwaysOnTop(true, 'pop-up-menu');
+    }
+    
     try {
       const results = await aiService.executeActions(
         actionData,
@@ -1167,6 +1187,11 @@ function setupIPC() {
         actionsCount: actionData.actions ? actionData.actions.length : 0,
         error: error.message
       });
+    } finally {
+      // Restore overlay z-index after action execution
+      if (overlayWindow && !overlayWindow.isDestroyed()) {
+        overlayWindow.setAlwaysOnTop(true, 'screen-saver');
+      }
     }
     
     pendingActions = null;
@@ -2148,6 +2173,25 @@ app.whenReady().then(() => {
   registerShortcuts();
   setupIPC();
   
+  // Start the UI watcher for live UI monitoring
+  try {
+    uiWatcher = new UIWatcher({
+      pollInterval: 400,
+      maxElements: 60,
+      includeInvisible: false
+    });
+    uiWatcher.on('ui-changed', (diff) => {
+      // Forward UI changes to overlay for live mirror updates
+      if (overlayWindow && !overlayWindow.isDestroyed()) {
+        overlayWindow.webContents.send('ui-watcher-update', diff);
+      }
+    });
+    uiWatcher.start();
+    console.log('[Main] UI Watcher started for live UI monitoring');
+  } catch (e) {
+    console.warn('[Main] Could not start UI watcher:', e.message);
+  }
+  
   // Set up Copilot OAuth callback to notify chat on auth completion
   aiService.setOAuthCallback((result) => {
     if (chatWindow && !chatWindow.isDestroyed()) {
@@ -2197,9 +2241,13 @@ app.on('window-all-closed', () => {
   }
 });
 
-// Clean up shortcuts on quit
+// Clean up shortcuts and UI watcher on quit
 app.on('will-quit', () => {
   globalShortcut.unregisterAll();
+  if (uiWatcher) {
+    uiWatcher.stop();
+    console.log('[Main] UI Watcher stopped');
+  }
 });
 
 // Prevent app from quitting when closing chat window

package/src/main/system-automation.js

@@ -88,6 +88,33 @@ function executePowerShell(command) {
   });
 }
 
+/**
+ * Focus the desktop / unfocus Electron windows before sending keyboard input
+ * This is critical for SendKeys/SendInput to reach the correct target
+ */
+async function focusDesktop() {
+  const script = `
+Add-Type @"
+using System;
+using System.Runtime.InteropServices;
+public class FocusHelper {
+    [DllImport("user32.dll")]
+    public static extern IntPtr GetDesktopWindow();
+    [DllImport("user32.dll")]
+    public static extern bool SetForegroundWindow(IntPtr hWnd);
+    [DllImport("user32.dll")]
+    public static extern IntPtr GetShellWindow();
+}
+"@
+# Focus shell window (explorer desktop) 
+$shell = [FocusHelper]::GetShellWindow()
+[FocusHelper]::SetForegroundWindow($shell)
+Start-Sleep -Milliseconds 50
+`;
+  await executePowerShell(script);
+  console.log('[AUTOMATION] Focused desktop before input');
+}
+
 /**
  * Move mouse to coordinates (Windows)
  */
@@ -443,15 +470,125 @@ Add-Type -AssemblyName System.Windows.Forms
 }
 
 /**
- * Press a key or key combination (e.g., "ctrl+c", "enter", "alt+tab")
+ * Press a key or key combination (e.g., "ctrl+c", "enter", "alt+tab", "win+r")
+ * Now supports Windows key using SendInput with virtual key codes
  */
 async function pressKey(keyCombo) {
-  let sendKeysStr = '';
-  
-  // Parse key combo
   const parts = keyCombo.toLowerCase().split('+').map(k => k.trim());
   
-  // Build SendKeys string
+  // Check if Windows key is involved - requires special handling
+  const hasWinKey = parts.includes('win') || parts.includes('windows') || parts.includes('super');
+  
+  if (hasWinKey) {
+    // Use SendInput for Windows key combos
+    const otherKeys = parts.filter(p => p !== 'win' && p !== 'windows' && p !== 'super');
+    const hasCtrl = otherKeys.includes('ctrl') || otherKeys.includes('control');
+    const hasAlt = otherKeys.includes('alt');
+    const hasShift = otherKeys.includes('shift');
+    const mainKey = otherKeys.find(p => !['ctrl', 'control', 'alt', 'shift'].includes(p)) || '';
+    
+    // Virtual key codes for common keys
+    const vkCodes = {
+      'a': 0x41, 'b': 0x42, 'c': 0x43, 'd': 0x44, 'e': 0x45, 'f': 0x46, 'g': 0x47, 'h': 0x48,
+      'i': 0x49, 'j': 0x4A, 'k': 0x4B, 'l': 0x4C, 'm': 0x4D, 'n': 0x4E, 'o': 0x4F, 'p': 0x50,
+      'q': 0x51, 'r': 0x52, 's': 0x53, 't': 0x54, 'u': 0x55, 'v': 0x56, 'w': 0x57, 'x': 0x58,
+      'y': 0x59, 'z': 0x5A,
+      '0': 0x30, '1': 0x31, '2': 0x32, '3': 0x33, '4': 0x34, '5': 0x35, '6': 0x36, '7': 0x37, '8': 0x38, '9': 0x39,
+      'enter': 0x0D, 'return': 0x0D, 'tab': 0x09, 'escape': 0x1B, 'esc': 0x1B,
+      'space': 0x20, 'backspace': 0x08, 'delete': 0x2E, 'del': 0x2E,
+      'up': 0x26, 'down': 0x28, 'left': 0x25, 'right': 0x27,
+      'home': 0x24, 'end': 0x23, 'pageup': 0x21, 'pagedown': 0x22,
+      'f1': 0x70, 'f2': 0x71, 'f3': 0x72, 'f4': 0x73, 'f5': 0x74, 'f6': 0x75,
+      'f7': 0x76, 'f8': 0x77, 'f9': 0x78, 'f10': 0x79, 'f11': 0x7A, 'f12': 0x7B,
+    };
+    
+    const mainKeyCode = mainKey ? (vkCodes[mainKey] || mainKey.charCodeAt(0)) : 0;
+    
+    const script = `
+Add-Type -TypeDefinition @"
+using System;
+using System.Runtime.InteropServices;
+
+public class WinKeyPress {
+    [StructLayout(LayoutKind.Sequential)]
+    public struct INPUT {
+        public uint type;
+        public InputUnion U;
+    }
+
+    [StructLayout(LayoutKind.Explicit)]
+    public struct InputUnion {
+        [FieldOffset(0)] public MOUSEINPUT mi;
+        [FieldOffset(0)] public KEYBDINPUT ki;
+    }
+
+    [StructLayout(LayoutKind.Sequential)]
+    public struct MOUSEINPUT {
+        public int dx, dy;
+        public uint mouseData, dwFlags, time;
+        public IntPtr dwExtraInfo;
+    }
+
+    [StructLayout(LayoutKind.Sequential)]
+    public struct KEYBDINPUT {
+        public ushort wVk;
+        public ushort wScan;
+        public uint dwFlags;
+        public uint time;
+        public IntPtr dwExtraInfo;
+    }
+
+    public const uint INPUT_KEYBOARD = 1;
+    public const uint KEYEVENTF_KEYUP = 0x0002;
+    public const ushort VK_LWIN = 0x5B;
+    public const ushort VK_CONTROL = 0x11;
+    public const ushort VK_SHIFT = 0x10;
+    public const ushort VK_MENU = 0x12; // Alt
+
+    [DllImport("user32.dll", SetLastError = true)]
+    public static extern uint SendInput(uint nInputs, INPUT[] pInputs, int cbSize);
+
+    public static void KeyDown(ushort vk) {
+        INPUT[] inputs = new INPUT[1];
+        inputs[0].type = INPUT_KEYBOARD;
+        inputs[0].U.ki.wVk = vk;
+        inputs[0].U.ki.dwFlags = 0;
+        SendInput(1, inputs, Marshal.SizeOf(typeof(INPUT)));
+    }
+
+    public static void KeyUp(ushort vk) {
+        INPUT[] inputs = new INPUT[1];
+        inputs[0].type = INPUT_KEYBOARD;
+        inputs[0].U.ki.wVk = vk;
+        inputs[0].U.ki.dwFlags = KEYEVENTF_KEYUP;
+        SendInput(1, inputs, Marshal.SizeOf(typeof(INPUT)));
+    }
+}
+"@
+
+# Press modifiers
+[WinKeyPress]::KeyDown([WinKeyPress]::VK_LWIN)
+${hasCtrl ? '[WinKeyPress]::KeyDown([WinKeyPress]::VK_CONTROL)' : ''}
+${hasAlt ? '[WinKeyPress]::KeyDown([WinKeyPress]::VK_MENU)' : ''}
+${hasShift ? '[WinKeyPress]::KeyDown([WinKeyPress]::VK_SHIFT)' : ''}
+
+# Press main key if any
+${mainKeyCode ? `[WinKeyPress]::KeyDown(${mainKeyCode})
+Start-Sleep -Milliseconds 50
+[WinKeyPress]::KeyUp(${mainKeyCode})` : 'Start-Sleep -Milliseconds 100'}
+
+# Release modifiers in reverse order
+${hasShift ? '[WinKeyPress]::KeyUp([WinKeyPress]::VK_SHIFT)' : ''}
+${hasAlt ? '[WinKeyPress]::KeyUp([WinKeyPress]::VK_MENU)' : ''}
+${hasCtrl ? '[WinKeyPress]::KeyUp([WinKeyPress]::VK_CONTROL)' : ''}
+[WinKeyPress]::KeyUp([WinKeyPress]::VK_LWIN)
+`;
+    await executePowerShell(script);
+    console.log(`[AUTOMATION] Pressed Windows key combo: ${keyCombo} (using SendInput)`);
+    return;
+  }
+  
+  // Non-Windows key combos use SendKeys
   let modifiers = '';
   let mainKey = '';
   
@@ -470,7 +607,7 @@ async function pressKey(keyCombo) {
     }
   }
   
-  sendKeysStr = modifiers + (mainKey ? `(${mainKey})` : '');
+  const sendKeysStr = modifiers + (mainKey ? `(${mainKey})` : '');
   
   if (!sendKeysStr) {
     throw new Error(`Invalid key combo: ${keyCombo}`);

package/src/main/ui-watcher.js

@@ -0,0 +1,503 @@
+/**
+ * UI Watcher Service - Live UI Mirror for AI Awareness
+ * 
+ * Provides continuous background monitoring of the Windows UI tree,
+ * enabling the AI to have real-time awareness without manual screenshots.
+ * 
+ * Architecture:
+ * - Polls Windows UI Automation every 300-500ms
+ * - Maintains an element cache with bounds, text, and roles
+ * - Sends incremental diffs to the overlay
+ * - Provides instant context to AI for every message
+ */
+
+const { exec, spawn } = require('child_process');
+const os = require('os');
+const path = require('path');
+const fs = require('fs');
+const EventEmitter = require('events');
+
+class UIWatcher extends EventEmitter {
+  constructor(options = {}) {
+    super();
+    
+    this.options = {
+      pollInterval: options.pollInterval || 400,      // ms between polls
+      focusedWindowOnly: options.focusedWindowOnly ?? true,  // only scan active window
+      maxElements: options.maxElements || 200,        // limit results for performance
+      minConfidence: options.minConfidence || 0.3,    // filter low-confidence elements
+      enabled: false,
+      ...options
+    };
+    
+    // Element cache
+    this.cache = {
+      elements: [],
+      activeWindow: null,
+      lastUpdate: 0,
+      updateCount: 0
+    };
+    
+    // Polling state
+    this.pollTimer = null;
+    this.isPolling = false;
+    this.pollInProgress = false;
+    
+    // Performance tracking
+    this.metrics = {
+      avgPollTime: 0,
+      pollCount: 0,
+      lastPollTime: 0,
+      errorCount: 0
+    };
+    
+    // Persistent PowerShell process for performance
+    this.psProcess = null;
+    this.psQueue = [];
+    this.psReady = false;
+  }
+  
+  /**
+   * Start continuous UI monitoring
+   */
+  start() {
+    if (this.isPolling) return;
+    
+    console.log('[UI-WATCHER] Starting continuous monitoring (interval:', this.options.pollInterval, 'ms)');
+    this.isPolling = true;
+    this.options.enabled = true;
+    
+    // Initial poll
+    this.poll();
+    
+    // Start polling loop
+    this.pollTimer = setInterval(() => {
+      if (!this.pollInProgress) {
+        this.poll();
+      }
+    }, this.options.pollInterval);
+    
+    this.emit('started');
+  }
+  
+  /**
+   * Stop monitoring
+   */
+  stop() {
+    if (!this.isPolling) return;
+    
+    console.log('[UI-WATCHER] Stopping monitoring');
+    this.isPolling = false;
+    this.options.enabled = false;
+    
+    if (this.pollTimer) {
+      clearInterval(this.pollTimer);
+      this.pollTimer = null;
+    }
+    
+    this.killPsProcess();
+    this.emit('stopped');
+  }
+  
+  /**
+   * Perform a single poll of the UI tree
+   */
+  async poll() {
+    if (this.pollInProgress) return;
+    this.pollInProgress = true;
+    
+    const startTime = Date.now();
+    
+    try {
+      // Get active window info
+      const activeWindow = await this.getActiveWindow();
+      
+      // Get UI elements (focused window only for performance)
+      const elements = await this.detectElements(activeWindow);
+      
+      // Calculate diff
+      const diff = this.calculateDiff(elements);
+      
+      // Update cache
+      const oldCache = { ...this.cache };
+      this.cache = {
+        elements,
+        activeWindow,
+        lastUpdate: Date.now(),
+        updateCount: this.cache.updateCount + 1
+      };
+      
+      // Track metrics
+      const pollTime = Date.now() - startTime;
+      this.metrics.pollCount++;
+      this.metrics.lastPollTime = pollTime;
+      this.metrics.avgPollTime = (this.metrics.avgPollTime * (this.metrics.pollCount - 1) + pollTime) / this.metrics.pollCount;
+      
+      // Emit events
+      if (diff.hasChanges) {
+        this.emit('ui-changed', {
+          added: diff.added,
+          removed: diff.removed,
+          changed: diff.changed,
+          activeWindow,
+          elementCount: elements.length
+        });
+      }
+      
+      this.emit('poll-complete', {
+        elements,
+        activeWindow,
+        pollTime,
+        hasChanges: diff.hasChanges
+      });
+      
+    } catch (error) {
+      this.metrics.errorCount++;
+      console.error('[UI-WATCHER] Poll error:', error.message);
+      this.emit('error', error);
+    } finally {
+      this.pollInProgress = false;
+    }
+  }
+  
+  /**
+   * Get the currently active/focused window
+   */
+  async getActiveWindow() {
+    const script = `
+Add-Type @"
+using System;
+using System.Runtime.InteropServices;
+using System.Text;
+public class ActiveWindow {
+    [DllImport("user32.dll")] public static extern IntPtr GetForegroundWindow();
+    [DllImport("user32.dll")] public static extern int GetWindowText(IntPtr hWnd, StringBuilder text, int count);
+    [DllImport("user32.dll")] public static extern int GetWindowThreadProcessId(IntPtr hWnd, out int processId);
+    [DllImport("user32.dll")] public static extern bool GetWindowRect(IntPtr hWnd, out RECT lpRect);
+    [StructLayout(LayoutKind.Sequential)] public struct RECT { public int Left, Top, Right, Bottom; }
+}
+"@
+$hwnd = [ActiveWindow]::GetForegroundWindow()
+$sb = New-Object System.Text.StringBuilder 256
+[ActiveWindow]::GetWindowText($hwnd, $sb, 256) | Out-Null
+$processId = 0
+[ActiveWindow]::GetWindowThreadProcessId($hwnd, [ref]$processId) | Out-Null
+$rect = New-Object ActiveWindow+RECT
+[ActiveWindow]::GetWindowRect($hwnd, [ref]$rect) | Out-Null
+$proc = Get-Process -Id $processId -ErrorAction SilentlyContinue
+@{
+    hwnd = [long]$hwnd
+    title = $sb.ToString()
+    processId = $processId
+    processName = if($proc){$proc.ProcessName}else{""}
+    bounds = @{ x = $rect.Left; y = $rect.Top; width = $rect.Right - $rect.Left; height = $rect.Bottom - $rect.Top }
+} | ConvertTo-Json -Compress
+`;
+    
+    return new Promise((resolve, reject) => {
+      exec(`powershell -NoProfile -Command "${script.replace(/"/g, '\\"').replace(/\n/g, ' ')}"`, 
+        { encoding: 'utf8', timeout: 2000 },
+        (error, stdout, stderr) => {
+          if (error) {
+            resolve(null);
+            return;
+          }
+          try {
+            resolve(JSON.parse(stdout.trim()));
+          } catch (e) {
+            resolve(null);
+          }
+        }
+      );
+    });
+  }
+  
+  /**
+   * Detect UI elements using Windows UI Automation
+   */
+  async detectElements(activeWindow) {
+    // Build scope filter based on active window
+    const windowFilter = this.options.focusedWindowOnly && activeWindow
+      ? `$targetWindow = "${(activeWindow.title || '').replace(/"/g, '\\"')}"`
+      : '$targetWindow = ""';
+    
+    const script = `
+Add-Type -AssemblyName UIAutomationClient
+Add-Type -AssemblyName UIAutomationTypes
+
+${windowFilter}
+$maxElements = ${this.options.maxElements}
+
+$root = [System.Windows.Automation.AutomationElement]::RootElement
+
+# If targeting specific window, find it first
+if ($targetWindow -ne "") {
+    $nameCondition = New-Object System.Windows.Automation.PropertyCondition(
+        [System.Windows.Automation.AutomationElement]::NameProperty, $targetWindow
+    )
+    $targetEl = $root.FindFirst([System.Windows.Automation.TreeScope]::Children, $nameCondition)
+    if ($targetEl) { $root = $targetEl }
+}
+
+$condition = [System.Windows.Automation.Condition]::TrueCondition
+$elements = $root.FindAll([System.Windows.Automation.TreeScope]::Descendants, $condition)
+
+$results = @()
+$count = 0
+
+foreach ($el in $elements) {
+    if ($count -ge $maxElements) { break }
+    try {
+        $rect = $el.Current.BoundingRectangle
+        if ($rect.Width -le 0 -or $rect.Height -le 0) { continue }
+        if ($rect.X -lt -10000 -or $rect.Y -lt -10000) { continue }
+        
+        $name = $el.Current.Name
+        $ctrlType = $el.Current.ControlType.ProgrammaticName -replace 'ControlType\\.', ''
+        $autoId = $el.Current.AutomationId
+        $className = $el.Current.ClassName
+        $isEnabled = $el.Current.IsEnabled
+        
+        # Skip elements with no useful identifying info
+        if ([string]::IsNullOrWhiteSpace($name) -and [string]::IsNullOrWhiteSpace($autoId)) { continue }
+        
+        # Generate a unique ID
+        $id = "$ctrlType|$name|$autoId|$([int]$rect.X)|$([int]$rect.Y)"
+        
+        $results += @{
+            id = $id
+            name = $name
+            type = $ctrlType
+            automationId = $autoId
+            className = $className
+            bounds = @{
+                x = [int]$rect.X
+                y = [int]$rect.Y
+                width = [int]$rect.Width
+                height = [int]$rect.Height
+            }
+            center = @{
+                x = [int]($rect.X + $rect.Width / 2)
+                y = [int]($rect.Y + $rect.Height / 2)
+            }
+            isEnabled = $isEnabled
+        }
+        $count++
+    } catch {}
+}
+
+$results | ConvertTo-Json -Depth 4 -Compress
+`;
+
+    return new Promise((resolve, reject) => {
+      exec(`powershell -NoProfile -Command "${script.replace(/"/g, '\\"').replace(/\n/g, ' ')}"`,
+        { encoding: 'utf8', timeout: 5000, maxBuffer: 10 * 1024 * 1024 },
+        (error, stdout, stderr) => {
+          if (error) {
+            resolve([]);
+            return;
+          }
+          try {
+            let elements = JSON.parse(stdout.trim() || '[]');
+            if (!Array.isArray(elements)) elements = elements ? [elements] : [];
+            resolve(elements);
+          } catch (e) {
+            resolve([]);
+          }
+        }
+      );
+    });
+  }
+  
+  /**
+   * Calculate diff between old and new element sets
+   */
+  calculateDiff(newElements) {
+    const oldElements = this.cache.elements || [];
+    const oldMap = new Map(oldElements.map(e => [e.id, e]));
+    const newMap = new Map(newElements.map(e => [e.id, e]));
+    
+    const added = newElements.filter(e => !oldMap.has(e.id));
+    const removed = oldElements.filter(e => !newMap.has(e.id));
+    const changed = newElements.filter(e => {
+      const old = oldMap.get(e.id);
+      if (!old) return false;
+      // Check if bounds or enabled state changed
+      return old.bounds.x !== e.bounds.x || 
+             old.bounds.y !== e.bounds.y ||
+             old.isEnabled !== e.isEnabled;
+    });
+    
+    return {
+      added,
+      removed,
+      changed,
+      hasChanges: added.length > 0 || removed.length > 0 || changed.length > 0
+    };
+  }
+  
+  /**
+   * Get current UI state for AI context
+   * This is called by ai-service before every API call
+   */
+  getContextForAI() {
+    if (!this.cache.elements || this.cache.elements.length === 0) {
+      return null;
+    }
+    
+    const { elements, activeWindow, lastUpdate } = this.cache;
+    const age = Date.now() - lastUpdate;
+    
+    // Group elements by type for cleaner context
+    const byType = {};
+    elements.forEach(el => {
+      const type = el.type || 'Unknown';
+      if (!byType[type]) byType[type] = [];
+      byType[type].push(el);
+    });
+    
+    // Build context string
+    let context = `\n## Live UI State (${age}ms ago)\n`;
+    
+    if (activeWindow) {
+      context += `**Active Window**: ${activeWindow.title || 'Unknown'} (${activeWindow.processName})\n`;
+      context += `**Window Bounds**: (${activeWindow.bounds.x}, ${activeWindow.bounds.y}) ${activeWindow.bounds.width}x${activeWindow.bounds.height}\n\n`;
+    }
+    
+    // List interactive elements (buttons, text fields, etc.)
+    const interactiveTypes = ['Button', 'Edit', 'ComboBox', 'CheckBox', 'RadioButton', 'MenuItem', 'ListItem', 'TabItem', 'Hyperlink'];
+    
+    context += `**Interactive Elements** (${elements.length} total):\n`;
+    
+    let listed = 0;
+    for (const type of interactiveTypes) {
+      const typeElements = byType[type] || [];
+      for (const el of typeElements.slice(0, 10)) { // Limit per type
+        if (listed >= 30) break; // Total limit
+        const name = el.name || el.automationId || '[unnamed]';
+        context += `- **${type}**: "${name}" at (${el.center.x}, ${el.center.y})${el.isEnabled ? '' : ' [disabled]'}\n`;
+        listed++;
+      }
+    }
+    
+    if (elements.length > listed) {
+      context += `... and ${elements.length - listed} more elements\n`;
+    }
+    
+    return context;
+  }
+  
+  /**
+   * Find element by text (for semantic actions)
+   */
+  findElementByText(searchText, options = {}) {
+    const { exact = false, type = null } = options;
+    const { elements } = this.cache;
+    
+    if (!elements) return null;
+    
+    const matches = elements.filter(el => {
+      const name = el.name || '';
+      const textMatch = exact 
+        ? name.toLowerCase() === searchText.toLowerCase()
+        : name.toLowerCase().includes(searchText.toLowerCase());
+      
+      if (!textMatch) return false;
+      if (type && el.type !== type) return false;
+      
+      return true;
+    });
+    
+    return matches.length > 0 ? matches[0] : null;
+  }
+  
+  /**
+   * Find all elements matching criteria
+   */
+  findElements(criteria = {}) {
+    const { elements } = this.cache;
+    if (!elements) return [];
+    
+    return elements.filter(el => {
+      if (criteria.text && !el.name?.toLowerCase().includes(criteria.text.toLowerCase())) return false;
+      if (criteria.type && el.type !== criteria.type) return false;
+      if (criteria.automationId && el.automationId !== criteria.automationId) return false;
+      if (criteria.enabledOnly && !el.isEnabled) return false;
+      return true;
+    });
+  }
+  
+  /**
+   * Get element at specific coordinates
+   */
+  getElementAtPoint(x, y) {
+    const { elements } = this.cache;
+    if (!elements) return null;
+    
+    // Find elements that contain the point, prefer smaller (more specific) elements
+    const containing = elements.filter(el => {
+      const { bounds } = el;
+      return x >= bounds.x && x <= bounds.x + bounds.width &&
+             y >= bounds.y && y <= bounds.y + bounds.height;
+    });
+    
+    if (containing.length === 0) return null;
+    
+    // Sort by area (smallest first - most specific element)
+    containing.sort((a, b) => {
+      const areaA = a.bounds.width * a.bounds.height;
+      const areaB = b.bounds.width * b.bounds.height;
+      return areaA - areaB;
+    });
+    
+    return containing[0];
+  }
+  
+  /**
+   * Get current metrics
+   */
+  getMetrics() {
+    return {
+      ...this.metrics,
+      cacheSize: this.cache.elements?.length || 0,
+      lastUpdate: this.cache.lastUpdate,
+      isPolling: this.isPolling
+    };
+  }
+  
+  /**
+   * Clean up
+   */
+  killPsProcess() {
+    if (this.psProcess) {
+      try {
+        this.psProcess.kill();
+      } catch (e) {}
+      this.psProcess = null;
+      this.psReady = false;
+    }
+  }
+  
+  /**
+   * Destroy watcher
+   */
+  destroy() {
+    this.stop();
+    this.removeAllListeners();
+  }
+}
+
+// Singleton instance
+let instance = null;
+
+function getUIWatcher(options) {
+  if (!instance) {
+    instance = new UIWatcher(options);
+  }
+  return instance;
+}
+
+module.exports = {
+  UIWatcher,
+  getUIWatcher
+};

package/src/renderer/overlay/overlay.js

@@ -24,7 +24,10 @@ let state = {
   inspectMode: false,
   inspectRegions: [],
   hoveredRegion: null,
-  selectedRegionId: null
+  selectedRegionId: null,
+  // Live UI mirror state
+  uiMirrorMode: false,
+  uiMirrorElements: []
 };
 
 // ===== CANVAS SETUP =====
@@ -448,6 +451,23 @@ if (window.electronAPI) {
     });
   }
   
+  // Listen for live UI watcher updates (background UI changes)
+  if (window.electronAPI.onUIWatcherUpdate) {
+    window.electronAPI.onUIWatcherUpdate((diff) => {
+      if (diff && (diff.added?.length || diff.changed?.length || diff.removed?.length)) {
+        console.log('[Overlay] UI watcher update:', {
+          added: diff.added?.length || 0,
+          changed: diff.changed?.length || 0,
+          removed: diff.removed?.length || 0
+        });
+        // Update UI mirror elements for subtle visual feedback
+        state.uiMirrorElements = diff.currentElements || [];
+        // Could trigger subtle highlight of changed elements here
+        // For now, just log - full visual rendering can be added later
+      }
+    });
+  }
+  
   // Identify
   console.log('Hooked electronAPI events');
 } else {

package/src/renderer/overlay/preload.js

@@ -86,5 +86,10 @@ contextBridge.exposeInMainWorld('electronAPI', {
   requestInspectRegions: () => ipcRenderer.send('request-inspect-regions'),
   
   // Toggle inspect mode
-  toggleInspectMode: () => ipcRenderer.send('toggle-inspect-mode')
+  toggleInspectMode: () => ipcRenderer.send('toggle-inspect-mode'),
+  
+  // ===== LIVE UI MIRROR API =====
+  
+  // Listen for live UI watcher updates (element changes in background)
+  onUIWatcherUpdate: (callback) => ipcRenderer.on('ui-watcher-update', (event, diff) => callback(diff))
 });