@phuetz/code-buddy 0.3.0 → 1.0.0-rc.8

package/dist/desktop-automation/smart-snapshot.js

@@ -1,7 +1,7 @@
 /**
  * Smart Snapshot System
  *
- * OpenClaw-inspired UI element referencing system.
+ * Enterprise-grade UI element referencing system.
  * Assigns numeric references [1], [2], [3] to interactive elements
  * instead of using brittle CSS selectors or coordinates.
  *
@@ -15,8 +15,13 @@
 import { EventEmitter } from 'events';
 import { execSync, exec } from 'child_process';
 import { promisify } from 'util';
+import path from 'path';
+import fs from 'fs';
+import { fileURLToPath } from 'url';
 import { logger } from '../utils/logger.js';
 const execAsync = promisify(exec);
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
 const DEFAULT_CONFIG = {
     method: 'accessibility',
     defaultTtl: 5000,
@@ -42,6 +47,12 @@ export class SmartSnapshotManager extends EventEmitter {
         super();
         this.config = { ...DEFAULT_CONFIG, ...config };
     }
+    async runPowerShellEncoded(script, options = {}) {
+        const executable = options.executable ?? 'powershell.exe';
+        const encoded = Buffer.from(script, 'utf16le').toString('base64');
+        const { stdout } = await execAsync(`${executable} -NoProfile -EncodedCommand ${encoded}`, { timeout: options.timeout ?? 15000 });
+        return stdout.trim();
+    }
     // ============================================================================
     // Main API
     // ============================================================================
@@ -272,7 +283,7 @@ export class SmartSnapshotManager extends EventEmitter {
     /**
      * Generate annotated screenshot with ref badges overlaid on each element
      */
-    async toAnnotatedScreenshot() {
+    async toAnnotatedScreenshot(opts = {}) {
         if (!this.currentSnapshot?.valid) {
             return null;
         }
@@ -287,19 +298,28 @@ export class SmartSnapshotManager extends EventEmitter {
                 return null;
             }
             const screenshotPath = captureData.path;
-            const elements = this.currentSnapshot.elements;
+            // Set-of-marks grounding needs a LEGIBLE image: badge only interactive elements
+            // (a full snapshot can be 80+ overlapping badges) and optionally crop to the
+            // window content region so a vision model isn't handed the whole desktop.
+            const allElements = this.currentSnapshot.elements;
+            const badgeElements = opts.interactiveOnly
+                ? allElements.filter((e) => e.interactive)
+                : allElements;
+            const cropRegion = opts.crop
+                ? this.computeContentBounds(allElements, 24, this.currentSnapshot.screenSize.width, this.currentSnapshot.screenSize.height)
+                : null;
             // Try sharp first for compositing ref badges
             try {
-                const annotatedResult = await this.annotateWithSharp(screenshotPath, elements);
+                const annotatedResult = await this.annotateWithSharp(screenshotPath, badgeElements, cropRegion);
                 if (annotatedResult)
                     return annotatedResult;
             }
             catch (err) {
                 logger.debug('sharp annotation failed, trying ffmpeg fallback', { error: err });
             }
-            // Fallback: ffmpeg drawtext filter
+            // Fallback: ffmpeg drawtext filter (no crop; badges only)
             try {
-                const annotatedResult = await this.annotateWithFFmpeg(screenshotPath, elements);
+                const annotatedResult = await this.annotateWithFFmpeg(screenshotPath, badgeElements);
                 if (annotatedResult)
                     return annotatedResult;
             }
@@ -322,7 +342,32 @@ export class SmartSnapshotManager extends EventEmitter {
             return null;
         }
     }
-    async annotateWithSharp(screenshotPath, elements) {
+    /**
+     * Bounding box (with padding, clamped to the screen) that encloses every visible
+     * element — used to crop the annotated screenshot to the window content region so
+     * set-of-marks grounding isn't handed the whole desktop. Returns null when there
+     * is nothing visible to enclose.
+     */
+    computeContentBounds(elements, pad, imgWidth, imgHeight) {
+        const visible = elements.filter((e) => e.visible && e.bounds.width > 0 && e.bounds.height > 0);
+        if (visible.length === 0)
+            return null;
+        let minX = Infinity, minY = Infinity, maxX = -Infinity, maxY = -Infinity;
+        for (const e of visible) {
+            minX = Math.min(minX, e.bounds.x);
+            minY = Math.min(minY, e.bounds.y);
+            maxX = Math.max(maxX, e.bounds.x + e.bounds.width);
+            maxY = Math.max(maxY, e.bounds.y + e.bounds.height);
+        }
+        const left = Math.max(0, Math.floor(minX - pad));
+        const top = Math.max(0, Math.floor(minY - pad));
+        const width = Math.min(imgWidth - left, Math.ceil(maxX - minX + pad * 2));
+        const height = Math.min(imgHeight - top, Math.ceil(maxY - minY + pad * 2));
+        if (width <= 0 || height <= 0)
+            return null;
+        return { left, top, width, height };
+    }
+    async annotateWithSharp(screenshotPath, elements, cropRegion = null) {
         const sharp = (await import('sharp')).default;
         const image = sharp(screenshotPath);
         const metadata = await image.metadata();
@@ -343,12 +388,29 @@ export class SmartSnapshotManager extends EventEmitter {
         }
         svgParts.push('</svg>');
         const svgOverlay = Buffer.from(svgParts.join('\n'));
-        const outputBuffer = await image
+        const composited = await image
             .composite([{ input: svgOverlay, top: 0, left: 0 }])
             .png()
             .toBuffer();
+        // Optional crop to the content region (second pass so order is unambiguous).
+        if (cropRegion) {
+            const left = Math.max(0, Math.min(cropRegion.left, imgWidth - 1));
+            const top = Math.max(0, Math.min(cropRegion.top, imgHeight - 1));
+            const width = Math.min(cropRegion.width, imgWidth - left);
+            const height = Math.min(cropRegion.height, imgHeight - top);
+            if (width > 0 && height > 0) {
+                const cropped = await sharp(composited).extract({ left, top, width, height }).png().toBuffer();
+                return {
+                    image: cropped.toString('base64'),
+                    format: 'png',
+                    width,
+                    height,
+                    snapshot: this.currentSnapshot,
+                };
+            }
+        }
         return {
-            image: outputBuffer.toString('base64'),
+            image: composited.toString('base64'),
             format: 'png',
             width: imgWidth,
             height: imgHeight,
@@ -358,7 +420,8 @@ export class SmartSnapshotManager extends EventEmitter {
     async annotateWithFFmpeg(screenshotPath, elements) {
         const { mkdtempSync, readFileSync, unlinkSync } = await import('fs');
         const { join } = await import('path');
-        const tmpDir = mkdtempSync(join(require('os').tmpdir(), 'cb-annotate-'));
+        const { tmpdir } = await import('os');
+        const tmpDir = mkdtempSync(join(tmpdir(), 'cb-annotate-'));
         const outputPath = join(tmpDir, 'annotated.png');
         // Build drawtext filter chain
         const filters = elements
@@ -418,39 +481,9 @@ export class SmartSnapshotManager extends EventEmitter {
                 return [];
         }
     }
-    async detectMacOSElements(options) {
+    async detectMacOSElements(_options) {
         const elements = [];
         try {
-            // Use AppleScript to query accessibility elements
-            const script = `
-        tell application "System Events"
-          set frontApp to first application process whose frontmost is true
-          set appName to name of frontApp
-          set allElements to {}
-
-          try
-            set allWindows to windows of frontApp
-            repeat with w in allWindows
-              set windowElements to entire contents of w
-              repeat with elem in windowElements
-                try
-                  set elemRole to role of elem
-                  set elemName to name of elem
-                  set elemDesc to description of elem
-                  set elemPos to position of elem
-                  set elemSize to size of elem
-                  set elemEnabled to enabled of elem
-                  set elemFocused to focused of elem
-
-                  set end of allElements to {elemRole, elemName, elemDesc, elemPos, elemSize, elemEnabled, elemFocused}
-                end try
-              end repeat
-            end repeat
-          end try
-
-          return allElements
-        end tell
-      `;
             // This is a simplified version - real implementation would need
             // proper AppleScript output parsing
             const output = execSync(`osascript -e 'tell application "System Events" to get name of every UI element of window 1 of (first application process whose frontmost is true)' 2>/dev/null`, { encoding: 'utf-8', timeout: 10000 });
@@ -470,6 +503,9 @@ export class SmartSnapshotManager extends EventEmitter {
      * Detect if running inside WSL2
      */
     isWSL() {
+        if (process.platform !== 'linux') {
+            return false;
+        }
         try {
             const release = execSync('uname -r', { encoding: 'utf-8' });
             return /microsoft|wsl/i.test(release);
@@ -494,14 +530,70 @@ export class SmartSnapshotManager extends EventEmitter {
         // Native Linux: try AT-SPI
         return this.detectLinuxATSPIElements(options);
     }
+    ensureBridgeCompiled(wsl = false) {
+        const srcPath = path.join(__dirname, 'CodeBuddyDesktopBridge.cs');
+        const exePath = path.join(__dirname, 'CodeBuddyDesktopBridge.exe');
+        if (process.env.NODE_ENV === 'test' || process.env.VITEST === 'true' || !!process.env.JEST_WORKER_ID) {
+            return ''; // Test mode, skip compiling
+        }
+        try {
+            let needsCompile = false;
+            if (!fs.existsSync(exePath)) {
+                needsCompile = true;
+            }
+            else {
+                const srcStat = fs.statSync(srcPath);
+                const exeStat = fs.statSync(exePath);
+                if (srcStat.mtime > exeStat.mtime) {
+                    needsCompile = true;
+                }
+            }
+            if (needsCompile) {
+                logger.info('Compiling CodeBuddyDesktopBridge for SmartSnapshotManager...');
+                const cscPath = wsl
+                    ? '/mnt/c/Windows/Microsoft.NET/Framework64/v4.0.30319/csc.exe'
+                    : 'C:\\Windows\\Microsoft.NET\\Framework64\\v4.0.30319\\csc.exe';
+                let winSrcPath = srcPath;
+                let winExePath = exePath;
+                if (wsl) {
+                    winSrcPath = execSync(`wslpath -w "${srcPath}"`, { encoding: 'utf8' }).trim();
+                    winExePath = execSync(`wslpath -w "${exePath}"`, { encoding: 'utf8' }).trim();
+                }
+                const cmd = `"${cscPath}" /r:"System.dll" /r:"System.Drawing.dll" /r:"System.Windows.Forms.dll" /r:"System.Web.Extensions.dll" /r:"C:\\Windows\\Microsoft.NET\\Framework64\\v4.0.30319\\WPF\\UIAutomationClient.dll" /r:"C:\\Windows\\Microsoft.NET\\Framework64\\v4.0.30319\\WPF\\UIAutomationTypes.dll" /r:"C:\\Windows\\Microsoft.NET\\Framework64\\v4.0.30319\\WPF\\WindowsBase.dll" /out:"${winExePath}" "${winSrcPath}"`;
+                execSync(cmd, { stdio: 'ignore' });
+                logger.info('CodeBuddyDesktopBridge compiled successfully.');
+            }
+            return exePath;
+        }
+        catch (err) {
+            logger.warn('Failed to compile CodeBuddyDesktopBridge in SmartSnapshotManager, using legacy PowerShell fallback:', { error: err });
+            return '';
+        }
+    }
     /**
      * Detect elements on WSL2 via PowerShell UIAutomation (same script as detectWindowsElements)
      */
-    async detectWSLElements(options) {
+    async detectWSLElements(_options) {
         const elements = [];
-        // Check powershell.exe is available
-        execSync('which powershell.exe', { stdio: 'ignore' });
-        const script = `
+        const maxDepth = _options.maxDepth ?? 5;
+        try {
+            const exePath = this.ensureBridgeCompiled(true);
+            let stdout = '';
+            if (exePath && fs.existsSync(exePath)) {
+                try {
+                    const winExePath = execSync(`wslpath -w "${exePath}"`, { encoding: 'utf8' }).trim();
+                    const { stdout: bridgeOut } = await execAsync(`powershell.exe -NoProfile -NonInteractive -Command "& \\"${winExePath}\\" get_uia_tree ${maxDepth}"`, { timeout: 15000 });
+                    stdout = bridgeOut.trim();
+                }
+                catch (err) {
+                    logger.warn('Failed to query C# bridge for WSL snapshot, falling back to legacy PowerShell:', { error: err });
+                    stdout = '';
+                }
+            }
+            if (!stdout) {
+                // Check powershell.exe is available
+                execSync('which powershell.exe', { stdio: 'ignore' });
+                const script = `
 Add-Type -AssemblyName UIAutomationClient
 $automation = [System.Windows.Automation.AutomationElement]::RootElement
 $condition = [System.Windows.Automation.Condition]::TrueCondition
@@ -519,10 +611,15 @@ function Get-Elements($element, $depth) {
         $results += @{
             name = $name
             role = $role
+            automationId = $element.Current.AutomationId
+            className = $element.Current.ClassName
+            runtimeId = (($element.GetRuntimeId()) -join '.')
             x = [int]$rect.X
             y = [int]$rect.Y
             width = [int]$rect.Width
             height = [int]$rect.Height
+            focused = $element.Current.HasKeyboardFocus
+            enabled = $element.Current.IsEnabled
         }
 
         $child = $walker.GetFirstChild($element)
@@ -530,7 +627,7 @@ function Get-Elements($element, $depth) {
             $results += Get-Elements $child ($depth + 1)
             $child = $walker.GetNextSibling($child)
         }
-    } catch {}
+    } catch [System.Exception] { $_ | Out-Null }
 
     return $results
 }
@@ -545,33 +642,46 @@ if ($focused) {
         Get-Elements $parent 0 | ConvertTo-Json -Depth 10
     }
 }
-    `;
-        const { stdout } = await execAsync(`powershell.exe -NoProfile -Command "${script.replace(/\n/g, '; ')}"`, { timeout: 15000 });
-        const parsed = JSON.parse(stdout);
-        const items = Array.isArray(parsed) ? parsed : [parsed];
-        for (const item of items.slice(0, this.config.maxElements)) {
-            const role = this.mapWindowsRole(item.role);
-            elements.push({
-                ref: this.nextRef++,
-                role,
-                name: item.name || 'Unknown',
-                bounds: { x: item.x, y: item.y, width: item.width, height: item.height },
-                center: {
-                    x: item.x + item.width / 2,
-                    y: item.y + item.height / 2,
-                },
-                interactive: this.isInteractiveRole(role),
-                focused: false,
-                enabled: true,
-                visible: item.width > 0 && item.height > 0,
-            });
+        `;
+                stdout = await this.runPowerShellEncoded(script, {
+                    executable: 'powershell.exe',
+                    timeout: 15000,
+                });
+            }
+            const parsed = JSON.parse(stdout);
+            const items = Array.isArray(parsed) ? parsed : [parsed];
+            for (const item of items.slice(0, this.config.maxElements)) {
+                const role = this.mapWindowsRole(item.role, item);
+                elements.push({
+                    ref: this.nextRef++,
+                    role,
+                    name: item.name || 'Unknown',
+                    bounds: { x: item.x, y: item.y, width: item.width, height: item.height },
+                    center: {
+                        x: item.x + item.width / 2,
+                        y: item.y + item.height / 2,
+                    },
+                    interactive: this.isInteractiveRole(role),
+                    focused: Boolean(item.focused),
+                    enabled: item.enabled !== false,
+                    visible: item.width > 0 && item.height > 0,
+                    controlType: typeof item.role === 'string' ? item.role : undefined,
+                    automationId: item.automationId || undefined,
+                    runtimeId: item.runtimeId || undefined,
+                    className: item.className || undefined,
+                });
+            }
+        }
+        catch (error) {
+            logger.debug('WSL elements detection failed', { error });
+            elements.push(...this.getMockElements());
         }
         return elements;
     }
     /**
      * Detect elements on native Linux via AT-SPI
      */
-    async detectLinuxATSPIElements(options) {
+    async detectLinuxATSPIElements(_options) {
         const elements = [];
         try {
             // Use AT-SPI via python-atspi or accerciser
@@ -658,11 +768,25 @@ print(json.dumps(all_elements[:100]))
         }
         return elements;
     }
-    async detectWindowsElements(options) {
+    async detectWindowsElements(_options) {
         const elements = [];
+        const maxDepth = _options.maxDepth ?? 5;
         try {
-            // Use UIAutomation via PowerShell
-            const script = `
+            const exePath = this.ensureBridgeCompiled(false);
+            let stdout = '';
+            if (exePath && fs.existsSync(exePath)) {
+                try {
+                    const { stdout: bridgeOut } = await execAsync(`"${exePath}" get_uia_tree ${maxDepth}`, { timeout: 15000 });
+                    stdout = bridgeOut.trim();
+                }
+                catch (err) {
+                    logger.warn('Failed to query C# bridge for snapshot, falling back to legacy PowerShell:', { error: err });
+                    stdout = '';
+                }
+            }
+            if (!stdout) {
+                // Use UIAutomation via PowerShell
+                const script = `
 Add-Type -AssemblyName UIAutomationClient
 $automation = [System.Windows.Automation.AutomationElement]::RootElement
 $condition = [System.Windows.Automation.Condition]::TrueCondition
@@ -680,10 +804,15 @@ function Get-Elements($element, $depth) {
         $results += @{
             name = $name
             role = $role
+            automationId = $element.Current.AutomationId
+            className = $element.Current.ClassName
+            runtimeId = (($element.GetRuntimeId()) -join '.')
             x = [int]$rect.X
             y = [int]$rect.Y
             width = [int]$rect.Width
             height = [int]$rect.Height
+            focused = $element.Current.HasKeyboardFocus
+            enabled = $element.Current.IsEnabled
         }
 
         $child = $walker.GetFirstChild($element)
@@ -691,7 +820,7 @@ function Get-Elements($element, $depth) {
             $results += Get-Elements $child ($depth + 1)
             $child = $walker.GetNextSibling($child)
         }
-    } catch {}
+    } catch [System.Exception] { $_ | Out-Null }
 
     return $results
 }
@@ -706,50 +835,132 @@ if ($focused) {
         Get-Elements $parent 0 | ConvertTo-Json -Depth 10
     }
 }
-      `;
-            try {
-                const { stdout } = await execAsync(`powershell -Command "${script.replace(/\n/g, '; ')}"`, { timeout: 15000 });
-                const parsed = JSON.parse(stdout);
-                const items = Array.isArray(parsed) ? parsed : [parsed];
-                for (const item of items.slice(0, this.config.maxElements)) {
-                    const role = this.mapWindowsRole(item.role);
+        `;
+                stdout = await this.runPowerShellEncoded(script, {
+                    executable: 'powershell.exe',
+                    timeout: 15000,
+                });
+            }
+            const parsed = JSON.parse(stdout);
+            const items = Array.isArray(parsed) ? parsed : [parsed];
+            for (const item of items.slice(0, this.config.maxElements)) {
+                const role = this.mapWindowsRole(item.role, item);
+                elements.push({
+                    ref: this.nextRef++,
+                    role,
+                    name: item.name || 'Unknown',
+                    bounds: { x: item.x, y: item.y, width: item.width, height: item.height },
+                    center: {
+                        x: item.x + item.width / 2,
+                        y: item.y + item.height / 2,
+                    },
+                    interactive: this.isInteractiveRole(role),
+                    focused: Boolean(item.focused),
+                    enabled: item.enabled !== false,
+                    visible: item.width > 0 && item.height > 0,
+                    controlType: typeof item.role === 'string' ? item.role : undefined,
+                    automationId: item.automationId || undefined,
+                    runtimeId: item.runtimeId || undefined,
+                    className: item.className || undefined,
+                });
+            }
+        }
+        catch (error) {
+            logger.debug('Windows accessibility detection failed', { error });
+            elements.push(...this.getMockElements());
+        }
+        return elements;
+    }
+    // ============================================================================
+    // Element Detection - OCR Fallback
+    // ============================================================================
+    async detectOCRElements(_options) {
+        const elements = [];
+        try {
+            const { ScreenshotTool } = await import('../tools/screenshot-tool.js');
+            const screenshotTool = new ScreenshotTool();
+            const screenshot = await screenshotTool.capture({
+                fullscreen: true,
+                format: 'png',
+            });
+            const screenshotData = screenshot.data;
+            if (!screenshot.success || !screenshotData?.path) {
+                logger.debug('OCR detection skipped: screenshot capture failed');
+                return [];
+            }
+            const { OCRTool } = await import('../tools/ocr-tool.js');
+            const ocrTool = new OCRTool();
+            const ocrResult = await ocrTool.extractText(screenshotData.path, {
+                language: process.env.OCR_LANGUAGE || 'eng',
+            });
+            if (!ocrResult.success) {
+                logger.debug('OCR detection skipped: OCR extraction failed', { error: ocrResult.error });
+                return [];
+            }
+            const data = ocrResult.data;
+            // Prefer word/line-level OCR blocks when available.
+            if (Array.isArray(data?.blocks) && data.blocks.length > 0) {
+                for (const block of data.blocks.slice(0, this.config.maxElements)) {
+                    const text = block.text?.trim();
+                    const box = block.boundingBox;
+                    if (!text || !box || box.width <= 0 || box.height <= 0) {
+                        continue;
+                    }
+                    const role = this.mapOCRTextToRole(text, box);
                     elements.push({
                         ref: this.nextRef++,
                         role,
-                        name: item.name || 'Unknown',
-                        bounds: { x: item.x, y: item.y, width: item.width, height: item.height },
+                        name: text,
+                        bounds: { x: box.x, y: box.y, width: box.width, height: box.height },
                         center: {
-                            x: item.x + item.width / 2,
-                            y: item.y + item.height / 2,
+                            x: box.x + box.width / 2,
+                            y: box.y + box.height / 2,
                         },
                         interactive: this.isInteractiveRole(role),
                         focused: false,
                         enabled: true,
-                        visible: item.width > 0 && item.height > 0,
+                        visible: true,
+                        attributes: {
+                            source: 'ocr',
+                            confidence: block.confidence,
+                        },
                     });
                 }
             }
-            catch (_err) {
-                // Intentionally ignored: UIAutomation JSON parsing may fail, fallback to mock elements
-                elements.push(...this.getMockElements());
+            else if (data?.text?.trim()) {
+                // Fallback when OCR backend does not return boxes:
+                // preserve at least textual affordances as lightweight references.
+                const lines = data.text
+                    .split(/\r?\n/)
+                    .map((line) => line.trim())
+                    .filter(Boolean)
+                    .slice(0, this.config.maxElements);
+                let y = 20;
+                for (const line of lines) {
+                    const role = this.mapOCRTextToRole(line, { x: 20, y, width: 240, height: 20 });
+                    elements.push({
+                        ref: this.nextRef++,
+                        role,
+                        name: line,
+                        bounds: { x: 20, y, width: 240, height: 20 },
+                        center: { x: 140, y: y + 10 },
+                        interactive: this.isInteractiveRole(role),
+                        focused: false,
+                        enabled: true,
+                        visible: true,
+                        attributes: { source: 'ocr-fallback' },
+                    });
+                    y += 24;
+                }
             }
         }
         catch (error) {
-            logger.debug('Windows accessibility detection failed', { error });
-            elements.push(...this.getMockElements());
+            logger.debug('OCR element detection failed', { error });
+            return [];
         }
         return elements;
     }
     // ============================================================================
-    // Element Detection - OCR Fallback
-    // ============================================================================
-    async detectOCRElements(options) {
-        // OCR-based detection would use the OCR tool to find text elements
-        // This is a placeholder - actual implementation would integrate with ocr-tool.ts
-        logger.debug('OCR element detection not yet implemented');
-        return [];
-    }
-    // ============================================================================
     // Helpers
     // ============================================================================
     filterElements(elements, options) {
@@ -781,6 +992,9 @@ if ($focused) {
             'menu item': 'menu-item',
             'page tab': 'tab',
             'list item': 'list-item',
+            'slider': 'slider',
+            'tree': 'tree',
+            'tree item': 'tree-item',
             'image': 'image',
             'label': 'text',
             'panel': 'container',
@@ -788,7 +1002,7 @@ if ($focused) {
         };
         return roleMap[atspiRole.toLowerCase()] || 'unknown';
     }
-    mapWindowsRole(controlType) {
+    mapWindowsRole(controlType, item) {
         const roleMap = {
             'ControlType.Button': 'button',
             'ControlType.Hyperlink': 'link',
@@ -801,13 +1015,56 @@ if ($focused) {
             'ControlType.Tab': 'tab',
             'ControlType.TabItem': 'tab',
             'ControlType.ListItem': 'list-item',
+            'ControlType.Slider': 'slider',
+            'ControlType.Tree': 'tree',
+            'ControlType.TreeItem': 'tree-item',
             'ControlType.Image': 'image',
             'ControlType.Text': 'text',
             'ControlType.Pane': 'container',
             'ControlType.Window': 'window',
         };
+        if (controlType === 'ControlType.Pane' && item) {
+            const bounds = {
+                x: 0,
+                y: 0,
+                width: item.width ?? 0,
+                height: item.height ?? 0,
+            };
+            if (item.focused && bounds.width >= 80 && bounds.height <= 48) {
+                return 'text-field';
+            }
+            const inferred = this.mapOCRTextToRole(item.name ?? '', bounds);
+            if (inferred !== 'text') {
+                return inferred;
+            }
+            if ((item.name ?? '').trim()) {
+                return 'text';
+            }
+        }
         return roleMap[controlType] || 'unknown';
     }
+    mapOCRTextToRole(text, bounds) {
+        const lower = text.toLowerCase();
+        if (/(https?:\/\/|www\.|\.com\b|\.io\b|\.dev\b)/.test(lower)) {
+            return 'link';
+        }
+        if (/\b(ok|cancel|submit|save|continue|next|back|close|apply|search|login|sign in|sign up)\b/.test(lower)) {
+            return 'button';
+        }
+        if (/\b(remember me|accept terms|agree|enable|disable)\b/.test(lower)) {
+            return 'checkbox';
+        }
+        if (/\b(select|choose|option|dropdown)\b/.test(lower)) {
+            return 'dropdown';
+        }
+        if (/\b(tab|overview|details|settings)\b/.test(lower) && bounds.height <= 48) {
+            return 'tab';
+        }
+        if (/\b(type|enter|input|search)\b/.test(lower) && bounds.width >= 120 && bounds.height <= 64) {
+            return 'text-field';
+        }
+        return 'text';
+    }
     isInteractiveRole(role) {
         const interactiveRoles = [
             'button',
@@ -827,19 +1084,29 @@ if ($focused) {
             if (process.platform === 'darwin') {
                 const output = execSync(`system_profiler SPDisplaysDataType | grep Resolution | head -1`, { encoding: 'utf-8' });
                 const match = output.match(/(\d+)\s*x\s*(\d+)/);
-                if (match) {
+                if (match && match[1] !== undefined && match[2] !== undefined) {
                     return { width: parseInt(match[1], 10), height: parseInt(match[2], 10) };
                 }
             }
             else if (process.platform === 'linux') {
-                const output = execSync(`xdpyinfo | grep dimensions`, { encoding: 'utf-8' });
-                const match = output.match(/(\d+)x(\d+)/);
-                if (match) {
-                    return { width: parseInt(match[1], 10), height: parseInt(match[2], 10) };
+                try {
+                    const output = execSync(`xdpyinfo | grep dimensions`, { encoding: 'utf-8' });
+                    const match = output.match(/(\d+)x(\d+)/);
+                    if (match && match[1] !== undefined && match[2] !== undefined) {
+                        return { width: parseInt(match[1], 10), height: parseInt(match[2], 10) };
+                    }
+                }
+                catch (_error) {
+                    const output = execSync(`xrandr --query`, { encoding: 'utf-8' });
+                    const match = output.match(/current\s+(\d+)\s+x\s+(\d+)/i);
+                    if (match && match[1] !== undefined && match[2] !== undefined) {
+                        return { width: parseInt(match[1], 10), height: parseInt(match[2], 10) };
+                    }
                 }
             }
             else if (process.platform === 'win32') {
-                const output = execSync(`powershell -Command "[System.Windows.Forms.Screen]::PrimaryScreen.Bounds | ConvertTo-Json"`, { encoding: 'utf-8' });
+                const script = `Add-Type -AssemblyName System.Windows.Forms; [System.Windows.Forms.Screen]::PrimaryScreen.Bounds | ConvertTo-Json -Compress`;
+                const output = execSync(`powershell -NoProfile -NonInteractive -Command "${script}"`, { encoding: 'utf-8' });
                 const data = JSON.parse(output);
                 return { width: data.Width, height: data.Height };
             }