copilot-liku-cli 0.0.4 → 0.0.8

package/src/main/ui-watcher.js

@@ -16,6 +16,15 @@ const os = require('os');
 const path = require('path');
 const fs = require('fs');
 const EventEmitter = require('events');
+const { getSharedUIAHost } = require('./ui-automation/core/uia-host');
+
+// Watcher mode state machine
+const MODE = {
+  POLLING: 'POLLING',
+  STARTING_EVENTS: 'STARTING_EVENTS',
+  EVENT_MODE: 'EVENT_MODE',
+  FALLBACK: 'FALLBACK'       // polling after event failure, auto-retry after 30s
+};
 
 class UIWatcher extends EventEmitter {
   constructor(options = {}) {
@@ -23,8 +32,8 @@ class UIWatcher extends EventEmitter {
     
     this.options = {
       pollInterval: options.pollInterval || 400,      // ms between polls
-      focusedWindowOnly: options.focusedWindowOnly ?? true,  // only scan active window
-      maxElements: options.maxElements || 200,        // limit results for performance
+      focusedWindowOnly: options.focusedWindowOnly ?? false, // scan all visible windows by default
+      maxElements: options.maxElements || 300,        // increased limit for desktop scan
       minConfidence: options.minConfidence || 0.3,    // filter low-confidence elements
       enabled: false,
       ...options
@@ -55,6 +64,13 @@ class UIWatcher extends EventEmitter {
     this.psProcess = null;
     this.psQueue = [];
     this.psReady = false;
+
+    // Phase 4: event-driven mode
+    this._mode = MODE.POLLING;
+    this._healthCheckTimer = null;
+    this._lastEventTs = 0;
+    this._fallbackRetryTimer = null;
+    this._uiaEventHandler = null;
   }
   
   /**
@@ -162,6 +178,7 @@ class UIWatcher extends EventEmitter {
   
   /**
    * Get the currently active/focused window
+   * Uses file-based script execution for reliable parsing
    */
   async getActiveWindow() {
     const script = `
@@ -194,10 +211,24 @@ $proc = Get-Process -Id $processId -ErrorAction SilentlyContinue
 } | ConvertTo-Json -Compress
 `;
     
+    // Use file-based execution for reliable parsing
+    const tempFile = path.join(os.tmpdir(), `liku-activewin-${Date.now()}.ps1`);
+    
     return new Promise((resolve, reject) => {
-      exec(`powershell -NoProfile -Command "${script.replace(/"/g, '\\"').replace(/\n/g, ' ')}"`, 
-        { encoding: 'utf8', timeout: 2000 },
+      // Write script to temp file
+      try {
+        fs.writeFileSync(tempFile, script, 'utf8');
+      } catch (e) {
+        resolve(null);
+        return;
+      }
+      
+      exec(`powershell -NoProfile -ExecutionPolicy Bypass -File "${tempFile}"`, 
+        { encoding: 'utf8', timeout: 3000 },
         (error, stdout, stderr) => {
+          // Clean up temp file
+          try { fs.unlinkSync(tempFile); } catch {}
+          
           if (error) {
             resolve(null);
             return;
@@ -214,14 +245,16 @@ $proc = Get-Process -Id $processId -ErrorAction SilentlyContinue
   
   /**
    * Detect UI elements using Windows UI Automation
+   * Uses file-based script execution for reliable parsing
    */
   async detectElements(activeWindow) {
     // Build scope filter based on active window
     const windowFilter = this.options.focusedWindowOnly && activeWindow
-      ? `$targetWindow = "${(activeWindow.title || '').replace(/"/g, '\\"')}"`
+      ? `$targetWindow = "${(activeWindow.title || '').replace(/"/g, '`"')}"`
       : '$targetWindow = ""';
     
     const script = `
+[Console]::OutputEncoding = [System.Text.Encoding]::UTF8
 Add-Type -AssemblyName UIAutomationClient
 Add-Type -AssemblyName UIAutomationTypes
 
@@ -229,47 +262,33 @@ ${windowFilter}
 $maxElements = ${this.options.maxElements}
 
 $root = [System.Windows.Automation.AutomationElement]::RootElement
-
-# If targeting specific window, find it first
-if ($targetWindow -ne "") {
-    $nameCondition = New-Object System.Windows.Automation.PropertyCondition(
-        [System.Windows.Automation.AutomationElement]::NameProperty, $targetWindow
-    )
-    $targetEl = $root.FindFirst([System.Windows.Automation.TreeScope]::Children, $nameCondition)
-    if ($targetEl) { $root = $targetEl }
-}
-
 $condition = [System.Windows.Automation.Condition]::TrueCondition
-$elements = $root.FindAll([System.Windows.Automation.TreeScope]::Descendants, $condition)
-
 $results = @()
 $count = 0
 
-foreach ($el in $elements) {
-    if ($count -ge $maxElements) { break }
+function Add-Element($el, $rootHwnd) {
     try {
         $rect = $el.Current.BoundingRectangle
-        if ($rect.Width -le 0 -or $rect.Height -le 0) { continue }
-        if ($rect.X -lt -10000 -or $rect.Y -lt -10000) { continue }
+        if ($rect.Width -le 0 -or $rect.Height -le 0) { return $null }
+        if ($rect.X -lt -10000 -or $rect.Y -lt -10000) { return $null }
         
         $name = $el.Current.Name
-        $ctrlType = $el.Current.ControlType.ProgrammaticName -replace 'ControlType\\.', ''
+        if ($name) { $name = $name -replace '[\\r\\n\\t]', ' ' }
+        
+        $ctrlType = $el.Current.ControlType.ProgrammaticName -replace 'ControlType\\.',''
         $autoId = $el.Current.AutomationId
-        $className = $el.Current.ClassName
-        $isEnabled = $el.Current.IsEnabled
+        if ($autoId) { $autoId = $autoId -replace '[\\r\\n\\t]', ' ' }
         
         # Skip elements with no useful identifying info
-        if ([string]::IsNullOrWhiteSpace($name) -and [string]::IsNullOrWhiteSpace($autoId)) { continue }
+        if ([string]::IsNullOrWhiteSpace($name) -and [string]::IsNullOrWhiteSpace($autoId)) { return $null }
         
-        # Generate a unique ID
-        $id = "$ctrlType|$name|$autoId|$([int]$rect.X)|$([int]$rect.Y)"
-        
-        $results += @{
-            id = $id
+        return @{
+            id = "$ctrlType|$name|$autoId|$([int]$rect.X)|$([int]$rect.Y)"
             name = $name
             type = $ctrlType
             automationId = $autoId
-            className = $className
+            className = $el.Current.ClassName
+            windowHandle = $rootHwnd
             bounds = @{
                 x = [int]$rect.X
                 y = [int]$rect.Y
@@ -280,28 +299,99 @@ foreach ($el in $elements) {
                 x = [int]($rect.X + $rect.Width / 2)
                 y = [int]($rect.Y + $rect.Height / 2)
             }
-            isEnabled = $isEnabled
+            isEnabled = $el.Current.IsEnabled
+        }
+    } catch { return $null }
+}
+
+if ($targetWindow -ne "") {
+    # FOCUSED WINDOW MODE
+    $nameCondition = New-Object System.Windows.Automation.PropertyCondition(
+        [System.Windows.Automation.AutomationElement]::NameProperty, $targetWindow
+    )
+    $targetEl = $root.FindFirst([System.Windows.Automation.TreeScope]::Children, $nameCondition)
+    
+    if ($targetEl) {
+        $targetHwnd = 0
+        try { $targetHwnd = $targetEl.Current.NativeWindowHandle } catch {}
+        
+        $elements = $targetEl.FindAll([System.Windows.Automation.TreeScope]::Descendants, $condition)
+        foreach ($el in $elements) {
+            if ($count -ge $maxElements) { break }
+            $data = Add-Element $el $targetHwnd
+            if ($data) { $results += $data; $count++ }
+        }
+    }
+} else {
+    # GLOBAL DESKTOP MODE (Iterate Windows)
+    # Get all top-level windows first
+    $windows = $root.FindAll([System.Windows.Automation.TreeScope]::Children, $condition)
+    
+    foreach ($win in $windows) {
+        if ($count -ge $maxElements) { break }
+        
+        $winHwnd = 0
+        try { $winHwnd = $win.Current.NativeWindowHandle } catch {}
+        
+        # Add window itself
+        $winData = Add-Element $win $winHwnd
+        if ($winData) { $results += $winData; $count++ }
+        
+        # Only process descendants for visible windows that have size
+        if ($winData) {
+            # Limit descendants per window to avoid starving other windows
+            $winElements = $win.FindAll([System.Windows.Automation.TreeScope]::Descendants, $condition)
+            $winCount = 0
+            foreach ($el in $winElements) {
+                if ($count -ge $maxElements) { break }
+                # Limit per window (e.g. 50% of remaining budget or fixed 50)
+                if ($winCount -ge 50) { break } 
+                
+                $data = Add-Element $el $winHwnd
+                if ($data) { 
+                    $results += $data
+                    $count++
+                    $winCount++
+                }
+            }
         }
-        $count++
-    } catch {}
+    }
 }
 
 $results | ConvertTo-Json -Depth 4 -Compress
 `;
 
+    // Use file-based execution for reliable parsing (inline -Command breaks on complex scripts)
+    const tempFile = path.join(os.tmpdir(), `liku-detect-${Date.now()}.ps1`);
+    
     return new Promise((resolve, reject) => {
-      exec(`powershell -NoProfile -Command "${script.replace(/"/g, '\\"').replace(/\n/g, ' ')}"`,
-        { encoding: 'utf8', timeout: 5000, maxBuffer: 10 * 1024 * 1024 },
+      // Write script to temp file
+      try {
+        fs.writeFileSync(tempFile, script, 'utf8');
+      } catch (e) {
+        resolve([]);
+        return;
+      }
+      
+      exec(`powershell -NoProfile -ExecutionPolicy Bypass -File "${tempFile}"`,
+        { encoding: 'utf8', timeout: 8000, maxBuffer: 10 * 1024 * 1024 },
         (error, stdout, stderr) => {
+          // Clean up temp file
+          try { fs.unlinkSync(tempFile); } catch {}
+          
           if (error) {
+            console.error('[UI-WATCHER] PowerShell detection error:', error.message);
             resolve([]);
             return;
           }
+          
           try {
             let elements = JSON.parse(stdout.trim() || '[]');
             if (!Array.isArray(elements)) elements = elements ? [elements] : [];
             resolve(elements);
           } catch (e) {
+            console.error('[UI-WATCHER] JSON Parse failed:', e.message);
+            console.error('[UI-WATCHER] STDOUT preview:', stdout.trim().substring(0, 200));
             resolve([]);
           }
         }
@@ -348,40 +438,46 @@ $results | ConvertTo-Json -Depth 4 -Compress
     const { elements, activeWindow, lastUpdate } = this.cache;
     const age = Date.now() - lastUpdate;
     
-    // Group elements by type for cleaner context
-    const byType = {};
-    elements.forEach(el => {
-      const type = el.type || 'Unknown';
-      if (!byType[type]) byType[type] = [];
-      byType[type].push(el);
-    });
-    
-    // Build context string
+    // Build context string with window hierarchy
     let context = `\n## Live UI State (${age}ms ago)\n`;
     
     if (activeWindow) {
-      context += `**Active Window**: ${activeWindow.title || 'Unknown'} (${activeWindow.processName})\n`;
-      context += `**Window Bounds**: (${activeWindow.bounds.x}, ${activeWindow.bounds.y}) ${activeWindow.bounds.width}x${activeWindow.bounds.height}\n\n`;
+      context += `**Focused Window**: ${activeWindow.title || 'Unknown'} (${activeWindow.processName})\n`;
+      context += `**Cursor**: (${activeWindow.bounds.x}, ${activeWindow.bounds.y}) ${activeWindow.bounds.width}x${activeWindow.bounds.height}\n\n`;
     }
     
-    // List interactive elements (buttons, text fields, etc.)
-    const interactiveTypes = ['Button', 'Edit', 'ComboBox', 'CheckBox', 'RadioButton', 'MenuItem', 'ListItem', 'TabItem', 'Hyperlink'];
-    
-    context += `**Interactive Elements** (${elements.length} total):\n`;
+    context += `**Visible Context** (${elements.length} elements detected):\n`;
     
     let listed = 0;
-    for (const type of interactiveTypes) {
-      const typeElements = byType[type] || [];
-      for (const el of typeElements.slice(0, 10)) { // Limit per type
-        if (listed >= 30) break; // Total limit
-        const name = el.name || el.automationId || '[unnamed]';
-        context += `- **${type}**: "${name}" at (${el.center.x}, ${el.center.y})${el.isEnabled ? '' : ' [disabled]'}\n`;
+    const limit = 300; 
+    
+    // Important interactive types to highlight
+    const importantTypes = ['Button', 'Edit', 'ComboBox', 'CheckBox', 'RadioButton', 'MenuItem', 'ListItem', 'TabItem', 'Hyperlink', 'Window'];
+    
+    for (let i = 0; i < elements.length; i++) {
+      if (listed >= limit) break;
+      
+      const el = elements[i];
+      const name = el.name || el.automationId || '[unnamed]';
+      
+      // Handle Window headers
+      if (el.type === 'Window') {
+        context += `\n[WIN] **Window**: "${name}" (Handle: ${el.windowHandle || 0})\n`;
         listed++;
+        continue;
       }
+      
+      // Skip boring layout elements unless they have a name
+      if (!importantTypes.includes(el.type) && !name && name !== '[unnamed]') continue;
+      
+      // Format element line with index for robust referencing
+      const status = el.isEnabled ? '' : ' (disabled)';
+      context += `- [${i+1}] ${el.type}: "${name}" at (${el.center.x}, ${el.center.y})${status}\n`;
+      listed++;
     }
     
     if (elements.length > listed) {
-      context += `... and ${elements.length - listed} more elements\n`;
+      context += `\n... and ${elements.length - listed} more elements (showing first ${limit})\n`;
     }
     
     return context;
@@ -465,6 +561,13 @@ $results | ConvertTo-Json -Depth 4 -Compress
     };
   }
   
+  /**
+   * Check if watcher is running (alias for isPolling)
+   */
+  get isRunning() {
+    return this.isPolling;
+  }
+  
   /**
    * Clean up
    */
@@ -482,9 +585,249 @@ $results | ConvertTo-Json -Depth 4 -Compress
    * Destroy watcher
    */
   destroy() {
+    this.stopEventMode();
     this.stop();
     this.removeAllListeners();
   }
+
+  // ── Phase 4: Event-driven mode ──────────────────────────────────────
+
+  /** Current watcher mode */
+  get mode() { return this._mode; }
+
+  /**
+   * Switch to event-driven mode — subscribes to .NET UIA events,
+   * stops PowerShell polling, sets up health check timer.
+   */
+  async startEventMode() {
+    if (this._mode === MODE.EVENT_MODE || this._mode === MODE.STARTING_EVENTS) return;
+
+    console.log('[UI-WATCHER] Switching to EVENT mode');
+    this._mode = MODE.STARTING_EVENTS;
+
+    // Stop polling — events will drive updates
+    if (this.pollTimer) {
+      clearInterval(this.pollTimer);
+      this.pollTimer = null;
+    }
+
+    try {
+      const host = getSharedUIAHost();
+
+      // Attach event handler (idempotent — remove first if exists)
+      this._detachEventHandler();
+      this._uiaEventHandler = (evt) => this._onUiaEvent(evt);
+      host.on('uia-event', this._uiaEventHandler);
+
+      const resp = await host.subscribeEvents();
+
+      // Seed cache with initial snapshot
+      if (resp.initial) {
+        const elements = resp.initial.elements || [];
+        const activeWindow = resp.initial.activeWindow || null;
+
+        const diff = this.calculateDiff(elements);
+        this.cache = {
+          elements,
+          activeWindow,
+          lastUpdate: Date.now(),
+          updateCount: this.cache.updateCount + 1
+        };
+
+        this.emit('poll-complete', {
+          elements,
+          activeWindow,
+          pollTime: 0,
+          hasChanges: diff.hasChanges,
+          source: 'event-initial'
+        });
+      }
+
+      this._mode = MODE.EVENT_MODE;
+      this._lastEventTs = Date.now();
+      this._startHealthCheck();
+
+      console.log('[UI-WATCHER] EVENT mode active');
+      this.emit('mode-changed', MODE.EVENT_MODE);
+    } catch (err) {
+      console.error('[UI-WATCHER] Failed to start event mode:', err.message);
+      this._mode = MODE.POLLING;
+      // Fall back to polling
+      this._restartPolling();
+    }
+  }
+
+  /**
+   * Switch back to polling mode — unsubscribes events, restarts poll timer.
+   */
+  async stopEventMode() {
+    if (this._mode !== MODE.EVENT_MODE && this._mode !== MODE.STARTING_EVENTS && this._mode !== MODE.FALLBACK) return;
+
+    console.log('[UI-WATCHER] Switching back to POLLING mode');
+
+    this._stopHealthCheck();
+    this._detachEventHandler();
+
+    if (this._fallbackRetryTimer) {
+      clearTimeout(this._fallbackRetryTimer);
+      this._fallbackRetryTimer = null;
+    }
+
+    try {
+      const host = getSharedUIAHost();
+      await host.unsubscribeEvents();
+    } catch { /* ignore — host may be dead */ }
+
+    this._mode = MODE.POLLING;
+
+    // Restart polling if watcher should be active
+    if (this.isPolling || this.options.enabled) {
+      this._restartPolling();
+    }
+
+    this.emit('mode-changed', MODE.POLLING);
+  }
+
+  /** Handle incoming UIA event from the .NET host */
+  _onUiaEvent(evt) {
+    this._lastEventTs = Date.now();
+
+    switch (evt.event) {
+      case 'focusChanged': {
+        // New window — update active window, await structureChanged for elements
+        if (evt.data?.activeWindow) {
+          this.cache.activeWindow = evt.data.activeWindow;
+        }
+        break;
+      }
+      case 'structureChanged': {
+        // Full element refresh
+        const elements = evt.data?.elements || [];
+        const diff = this.calculateDiff(elements);
+        this.cache = {
+          elements,
+          activeWindow: this.cache.activeWindow,
+          lastUpdate: Date.now(),
+          updateCount: this.cache.updateCount + 1
+        };
+
+        if (diff.hasChanges) {
+          this.emit('ui-changed', {
+            added: diff.added,
+            removed: diff.removed,
+            changed: diff.changed,
+            activeWindow: this.cache.activeWindow,
+            elementCount: elements.length
+          });
+        }
+
+        this.emit('poll-complete', {
+          elements,
+          activeWindow: this.cache.activeWindow,
+          pollTime: 0,
+          hasChanges: diff.hasChanges,
+          source: 'event-structure'
+        });
+        break;
+      }
+      case 'propertyChanged': {
+        // Incremental property patches — merge into cache
+        const changed = evt.data?.elements || [];
+        if (changed.length === 0) break;
+
+        const map = new Map(this.cache.elements.map(e => [e.id, e]));
+        let patchCount = 0;
+
+        for (const patch of changed) {
+          if (map.has(patch.id)) {
+            Object.assign(map.get(patch.id), patch);
+            patchCount++;
+          } else {
+            // New element appeared via property event — add it
+            map.set(patch.id, patch);
+            patchCount++;
+          }
+        }
+
+        if (patchCount > 0) {
+          const elements = Array.from(map.values());
+          this.cache.elements = elements;
+          this.cache.lastUpdate = Date.now();
+
+          this.emit('poll-complete', {
+            elements,
+            activeWindow: this.cache.activeWindow,
+            pollTime: 0,
+            hasChanges: true,
+            source: 'event-property'
+          });
+        }
+        break;
+      }
+      case 'error':
+        console.error('[UI-WATCHER] .NET event error:', evt.data?.error);
+        break;
+    }
+  }
+
+  /** Health check: if no events for 10s while in event mode, fall back to polling */
+  _startHealthCheck() {
+    this._stopHealthCheck();
+    this._healthCheckTimer = setInterval(() => {
+      if (this._mode !== MODE.EVENT_MODE) return;
+      const elapsed = Date.now() - this._lastEventTs;
+      if (elapsed > 10000) {
+        console.warn('[UI-WATCHER] No events for 10s — falling back to polling');
+        this._fallbackToPolling();
+      }
+    }, 5000);
+  }
+
+  _stopHealthCheck() {
+    if (this._healthCheckTimer) {
+      clearInterval(this._healthCheckTimer);
+      this._healthCheckTimer = null;
+    }
+  }
+
+  /** Fall back to polling and schedule a retry */
+  _fallbackToPolling() {
+    this._stopHealthCheck();
+    this._mode = MODE.FALLBACK;
+    this._restartPolling();
+    this.emit('mode-changed', MODE.FALLBACK);
+
+    // Auto-retry event mode after 30s
+    this._fallbackRetryTimer = setTimeout(async () => {
+      this._fallbackRetryTimer = null;
+      if (this._mode === MODE.FALLBACK) {
+        console.log('[UI-WATCHER] Retrying event mode after fallback');
+        await this.startEventMode();
+      }
+    }, 30000);
+  }
+
+  _restartPolling() {
+    if (this.pollTimer) {
+      clearInterval(this.pollTimer);
+      this.pollTimer = null;
+    }
+    this.isPolling = true;
+    this.options.enabled = true;
+    this.pollTimer = setInterval(() => {
+      if (!this.pollInProgress) this.poll();
+    }, this.options.pollInterval);
+  }
+
+  _detachEventHandler() {
+    if (this._uiaEventHandler) {
+      try {
+        const host = getSharedUIAHost();
+        host.removeListener('uia-event', this._uiaEventHandler);
+      } catch { /* ignore */ }
+      this._uiaEventHandler = null;
+    }
+  }
 }
 
 // Singleton instance

package/src/main/visual-awareness.js

@@ -7,6 +7,7 @@ const { exec } = require('child_process');
 const path = require('path');
 const fs = require('fs');
 const os = require('os');
+const { getSharedUIAHost } = require('./ui-automation/core/uia-host');
 
 // ===== STATE =====
 let previousScreenshot = null;
@@ -457,13 +458,29 @@ $elements | ConvertTo-Json -Depth 10
 }
 
 /**
- * Find UI element at specific coordinates
+ * Find UI element at specific coordinates.
+ * Fast path: persistent .NET UIA host (~5-20ms).
+ * Fallback: PowerShell one-shot (~200-500ms).
  */
 async function findElementAtPoint(x, y) {
   if (process.platform !== 'win32') {
     return { error: 'UI Automation only available on Windows' };
   }
 
+  // Fast path — .NET host (persistent process, JSONL protocol)
+  try {
+    const host = getSharedUIAHost();
+    const el = await host.elementFromPoint(x, y);
+    return {
+      ...el,
+      queryPoint: { x, y },
+      timestamp: Date.now()
+    };
+  } catch (hostErr) {
+    // Fall through to PowerShell path
+  }
+
+  // Fallback — PowerShell (spawns new process each call)
   const psScript = `
 Add-Type -AssemblyName UIAutomationClient
 Add-Type -AssemblyName UIAutomationTypes