@lardexx/arenacode 2.7.0 → 2.8.0

package/README.md

@@ -148,11 +148,17 @@ from the same bridge request the desktop app answers:
 | `ui apps` | the Start menu (Windows), `/Applications` (macOS), `.desktop` entries (Linux) |
 | `ui multi-select`, `ui multi-edit` | built from the primitives above |
 
-No extra install on **Windows** (PowerShell + `user32.dll`). **macOS**: grant Accessibility to your terminal, and
-`brew install cliclick` for the mouse. **Linux** (X11): `sudo apt install xdotool wmctrl imagemagick`; on Wayland
-`xdotool` cannot inject input, so Computer Agent needs XWayland. What this client does *not* have is the desktop's UI
-Automation tree: `ui state` returns windows and pixels, with `interactive_elements` empty, and `--annotated`
-screenshots are plain screenshots.
+No extra install on **Windows**: PowerShell + `user32.dll` + the .NET UI Automation assemblies. The desktop client
+ships a `windows-use.dll` (exports `wu_call_json`, `wu_free`, `wu_version`) for the same job; the terminal client
+reproduces its behaviour instead of depending on it — the same JSON keys (`bounding_box`, `center`, `control_type`,
+`interactive_elements_text`, `scrollable_elements`, `screenshot_png_base64`), the same ideas: the UI Automation tree of
+the focused window for `ui state`, **unicode typing through `SendInput`** so the clipboard is untouched, extended-key
+flags for arrows/navigation, fuzzy window matching (`ui app --name chrome` finds *"Google Chrome — New Tab"*), and
+numbered labels drawn on the elements for `ui shot --annotated`.
+
+**macOS**: grant Accessibility to your terminal, and `brew install cliclick` for the mouse. **Linux** (X11):
+`sudo apt install xdotool wmctrl imagemagick`; on Wayland `xdotool` cannot inject input, so Computer Agent needs
+XWayland. On macOS/Linux the accessibility tree is not read: `ui state` returns windows and geometry as elements.
 
 **Ask and Proceed / Proceed.** The desktop's Settings switch, `ctrl+x a` here (also `/ask`, `/proceed`,
 `--proceed`, and `arenacode config set approval proceed`). The default, **Ask and Proceed**, asks before commands,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lardexx/arenacode",
-  "version": "2.7.0",
+  "version": "2.8.0",
   "description": "Terminal client for ArenaCode: talks to an arena-bridge, streams the Agent Mode timeline, and runs local commands and file operations on your PC. Node port of the ArenaCode desktop client.",
   "license": "MIT",
   "type": "commonjs",

package/src/automation.js

@@ -55,6 +55,8 @@ $ErrorActionPreference = 'Stop'
 $req = ([Console]::In.ReadToEnd() | ConvertFrom-Json)
 Add-Type -AssemblyName System.Drawing
 Add-Type -AssemblyName System.Windows.Forms
+Add-Type -AssemblyName UIAutomationClient
+Add-Type -AssemblyName UIAutomationTypes
 Add-Type -TypeDefinition @'
 using System;
 using System.Text;
@@ -67,48 +69,66 @@ public class AC {
   [DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr h);
   [DllImport("user32.dll")] public static extern bool ShowWindow(IntPtr h, int cmd);
   [DllImport("user32.dll")] public static extern bool MoveWindow(IntPtr h, int x, int y, int w, int ht, bool repaint);
-  [DllImport("user32.dll")] public static extern bool EnumWindows(EnumProc cb, IntPtr p);
-  [DllImport("user32.dll")] public static extern bool IsWindowVisible(IntPtr h);
-  [DllImport("user32.dll")] public static extern int GetWindowText(IntPtr h, StringBuilder s, int n);
-  [DllImport("user32.dll")] public static extern uint GetWindowThreadProcessId(IntPtr h, out uint pid);
   [DllImport("user32.dll")] public static extern bool GetWindowRect(IntPtr h, out RECT r);
+  [DllImport("user32.dll")] public static extern bool IsWindowVisible(IntPtr h);
   [DllImport("user32.dll")] public static extern bool IsIconic(IntPtr h);
-  public delegate bool EnumProc(IntPtr h, IntPtr p);
   [StructLayout(LayoutKind.Sequential)] public struct RECT { public int Left, Top, Right, Bottom; }
 }
 '@
 
+$AC_TYPES = @(
+  [System.Windows.Automation.ControlType]::Button,
+  [System.Windows.Automation.ControlType]::CheckBox,
+  [System.Windows.Automation.ControlType]::ComboBox,
+  [System.Windows.Automation.ControlType]::Custom,
+  [System.Windows.Automation.ControlType]::DataItem,
+  [System.Windows.Automation.ControlType]::Edit,
+  [System.Windows.Automation.ControlType]::Hyperlink,
+  [System.Windows.Automation.ControlType]::ListItem,
+  [System.Windows.Automation.ControlType]::MenuItem,
+  [System.Windows.Automation.ControlType]::RadioButton,
+  [System.Windows.Automation.ControlType]::ScrollBar,
+  [System.Windows.Automation.ControlType]::Slider,
+  [System.Windows.Automation.ControlType]::Spinner,
+  [System.Windows.Automation.ControlType]::SplitButton,
+  [System.Windows.Automation.ControlType]::Tab,
+  [System.Windows.Automation.ControlType]::TabItem,
+  [System.Windows.Automation.ControlType]::Text,
+  [System.Windows.Automation.ControlType]::ToolBar,
+  [System.Windows.Automation.ControlType]::TreeItem
+)
+
+function Get-ACWindow($h) {
+  $hh = [IntPtr][int64]$h
+  $r = New-Object AC+RECT
+  [void][AC]::GetWindowRect($hh, [ref]$r)
+  $sb = New-Object System.Text.StringBuilder 1024
+  [void][AC]::GetWindowText($hh, $sb, 1024)
+  return [pscustomobject]@{
+    name = $sb.ToString()
+    handle = $hh.ToInt64()
+    hwnd = $hh.ToInt64()
+    x = $r.Left; y = $r.Top
+    width = [Math]::Max(0, $r.Right - $r.Left); height = [Math]::Max(0, $r.Bottom - $r.Top)
+    status = $(if ([AC]::IsIconic($hh)) { 'minimized' } else { 'normal' })
+  }
+}
+
 function Get-ACWindows {
   $list = New-Object System.Collections.ArrayList
   foreach ($p in (Get-Process | Where-Object { $_.MainWindowHandle -ne 0 })) {
     try {
-      $h = [IntPtr]$p.MainWindowHandle
-      $r = New-Object AC+RECT
-      [void][AC]::GetWindowRect($h, [ref]$r)
       $title = [string]$p.MainWindowTitle
       if ($title.Trim().Length -eq 0) { continue }
-      [void]$list.Add([pscustomobject]@{
-        name = $title
-        app = [string]$p.ProcessName
-        handle = $h.ToInt64()
-        x = $r.Left; y = $r.Top
-        width = [Math]::Max(0, $r.Right - $r.Left); height = [Math]::Max(0, $r.Bottom - $r.Top)
-        process_id = [int]$p.Id
-        status = $(if ($p.MainWindowHandle -eq 0) { 'hidden' } else { 'normal' })
-      })
+      $w = Get-ACWindow $p.MainWindowHandle
+      $w | Add-Member -NotePropertyName process_id -NotePropertyValue ([int]$p.Id)
+      $w | Add-Member -NotePropertyName app -NotePropertyValue ([string]$p.ProcessName)
+      [void]$list.Add($w)
     } catch { }
   }
   return $list
 }
 
-function Find-ACWindow($name) {
-  if (-not $name) { return $null }
-  $all = Get-ACWindows
-  $exact = $all | Where-Object { $_.name -eq $name } | Select-Object -First 1
-  if ($exact) { return $exact }
-  return ($all | Where-Object { $_.name -like "*$name*" } | Select-Object -First 1)
-}
-
 function Get-ACScreen {
   $b = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds
   return @{ width = $b.Width; height = $b.Height }
@@ -116,37 +136,119 @@ function Get-ACScreen {
 
 function Send-ACKeys($pairs) {
   foreach ($k in $pairs) {
-    if ($k.flags -eq 0) { [AC]::keybd_event([byte]$k.vk, 0, 0, [UIntPtr]::Zero) }
-    else { [AC]::keybd_event([byte]$k.vk, 0, 2, [UIntPtr]::Zero) }
+    $flags = 0
+    if ($k.flags -eq 2) { $flags = $flags -bor 2 }
+    if ($k.ext) { $flags = $flags -bor 1 }
+    [AC]::keybd_event([byte]$k.vk, 0, $flags, [UIntPtr]::Zero)
     Start-Sleep -Milliseconds 12
   }
 }
 
+# windows-use types through SendInput unicode events: the clipboard is never touched
+function Send-ACUnicode($text) {
+  $typed = 0
+  foreach ($ch in $text.ToCharArray()) {
+    if ($ch -eq [char]10) {
+      [AC]::keybd_event(0x0D, 0, 0, [UIntPtr]::Zero)
+      [AC]::keybd_event(0x0D, 0, 2, [UIntPtr]::Zero)
+      $typed++
+      continue
+    }
+    if ($ch -eq [char]13) { continue }
+    [AC]::keybd_event(0, [byte][char]$ch, 4, [UIntPtr]::Zero)
+    [AC]::keybd_event(0, [byte][char]$ch, 6, [UIntPtr]::Zero)
+    $typed++
+    Start-Sleep -Milliseconds 3
+  }
+  return $typed
+}
+
+function Get-ACElements($hwnd, $interactiveOnly) {
+  $handle = [IntPtr][int64]$hwnd
+  $root = $null
+  if ($handle -ne [IntPtr]::Zero) {
+    $root = [System.Windows.Automation.AutomationElement]::FromHandle($handle)
+  } else {
+    $root = [System.Windows.Automation.AutomationElement]::FocusedElement
+    while ($root -ne $null -and $root.Current.NativeWindowHandle -eq 0) {
+      $walker = [System.Windows.Automation.TreeWalker]::ControlViewWalker
+      $root = $walker.GetParent($root)
+    }
+  }
+  if ($root -eq $null) { return @() }
+
+  $conditions = @()
+  foreach ($ct in $AC_TYPES) {
+    $conditions += New-Object System.Windows.Automation.PropertyCondition(
+      [System.Windows.Automation.AutomationElement]::ControlTypeProperty, $ct)
+  }
+  $cond = New-Object System.Windows.Automation.OrCondition $conditions
+  $found = $root.FindAll([System.Windows.Automation.TreeScope]::Descendants, $cond)
+
+  $out = New-Object System.Collections.ArrayList
+  $n = 0
+  foreach ($el in $found) {
+    if ($n -ge 120) { break }
+    try {
+      $cur = $el.Current
+      if ($cur.IsOffscreen) { continue }
+      $rect = $cur.BoundingRectangle
+      if ($rect.Width -le 0 -or $rect.Height -le 0) { continue }
+      $name = [string]$cur.Name
+      $ctype = [string]$cur.ControlType.ProgrammaticName -replace '^ControlType\.', ''
+      if ($interactiveOnly -and $ctype -eq 'Text' -and $name.Trim().Length -eq 0) { continue }
+      $val = ''
+      try {
+        $vp = $null
+        if ($el.TryGetCurrentPattern([System.Windows.Automation.ValuePattern]::Pattern, [ref]$vp)) { $val = [string]$vp.Current.Value }
+      } catch { $val = '' }
+      $scrollable = $false
+      try {
+        $sp = $null
+        if ($el.TryGetCurrentPattern([System.Windows.Automation.ScrollPattern]::Pattern, [ref]$sp)) {
+          $scrollable = [bool]$sp.Current.VerticallyScrollable -or [bool]$sp.Current.HorizontallyScrollable
+        }
+      } catch { }
+      [void]$out.Add([pscustomobject]@{
+        name = $name
+        control_type = $ctype
+        kind = $ctype
+        bounding_box = @{ left = [int]$rect.X; top = [int]$rect.Y; right = [int]($rect.X + $rect.Width); bottom = [int]($rect.Y + $rect.Height) }
+        center = @{ x = [int]($rect.X + $rect.Width / 2); y = [int]($rect.Y + $rect.Height / 2) }
+        value = $val
+        scrollable = $scrollable
+        enabled = [bool]$cur.IsEnabled
+        automation_id = [string]$cur.AutomationId
+      })
+      $n++
+    } catch { }
+  }
+  return $out
+}
+
 $out = @{ ok = $true }
 switch ($req.action) {
   'screen' { $s = Get-ACScreen; $out.width = $s.width; $out.height = $s.height }
-  'windows' { $out.windows = @(Get-ACWindows); $out.active_window = (Get-ACWindows | Select-Object -First 1) }
+  'windows' { $out.windows = @(Get-ACWindows) }
   'active' {
-    $h = [AC]::GetForegroundWindow()
-    $sb = New-Object System.Text.StringBuilder 1024
-    [void][AC]::GetWindowText($h, $sb, 1024)
-    $out.name = $sb.ToString()
+    $hh = [AC]::GetForegroundWindow()
+    $w = Get-ACWindow $hh.ToInt64()
+    $out.name = $w.name
+    $out.handle = $w.handle
   }
   'focus' {
-    $w = Find-ACWindow $req.name
-    if ($w) {
-      [void][AC]::ShowWindow([IntPtr]$w.handle, 9)
-      [void][AC]::SetForegroundWindow([IntPtr]$w.handle)
-      Start-Sleep -Milliseconds 150
-      $out.name = $w.name
-    } else { $out = @{ ok = $false; error = "window not found: $($req.name)" } }
+    $hh = [IntPtr][int64]$req.handle
+    [void][AC]::ShowWindow($hh, 9)
+    [void][AC]::SetForegroundWindow($hh)
+    Start-Sleep -Milliseconds 200
+    $w = Get-ACWindow $req.handle
+    $out.name = $w.name
   }
   'resize' {
-    $w = Find-ACWindow $req.name
-    if ($w) {
-      [void][AC]::MoveWindow([IntPtr]$w.handle, [int]$req.x, [int]$req.y, [int]$req.width, [int]$req.height, $true)
-      $out.name = $w.name
-    } else { $out = @{ ok = $false; error = "window not found: $($req.name)" } }
+    $hh = [IntPtr][int64]$req.handle
+    [void][AC]::MoveWindow($hh, [int]$req.x, [int]$req.y, [int]$req.width, [int]$req.height, $true)
+    $w = Get-ACWindow $req.handle
+    $out.name = $w.name
   }
   'launch' {
     $name = [string]$req.name
@@ -161,7 +263,7 @@ switch ($req.action) {
         if ($exe) { Start-Process $exe.Source; $done = $true }
       }
     }
-    if ($done) { Start-Sleep -Milliseconds 700; $out.name = $name } else { $out = @{ ok = $false; error = "app not found: $name" } }
+    if ($done) { Start-Sleep -Milliseconds 900; $out.name = $name } else { $out = @{ ok = $false; error = "app not found: $name" } }
   }
   'move' { [void][AC]::SetCursorPos([int]$req.x, [int]$req.y) }
   'click' {
@@ -185,7 +287,7 @@ switch ($req.action) {
   }
   'up' {
     $flag = 4; if ($req.button -eq 'right') { $flag = 16 } elseif ($req.button -eq 'middle') { $flag = 64 }
-    if ($req.x -ne $null) { [void][AC]::SetCursorPos([int]$req.x, [int]$req.y) }
+    if ($null -ne $req.x) { [void][AC]::SetCursorPos([int]$req.x, [int]$req.y) }
     [AC]::mouse_event($flag, 0, 0, 0, [UIntPtr]::Zero)
   }
   'drag' {
@@ -202,7 +304,7 @@ switch ($req.action) {
     [AC]::mouse_event(4, 0, 0, 0, [UIntPtr]::Zero)
   }
   'scroll' {
-    if ($req.x -ne $null) { [void][AC]::SetCursorPos([int]$req.x, [int]$req.y); Start-Sleep -Milliseconds 40 }
+    if ($null -ne $req.x -and $null -ne $req.y) { [void][AC]::SetCursorPos([int]$req.x, [int]$req.y); Start-Sleep -Milliseconds 40 }
     $amount = [int]$req.amount
     $dir = 120; if ($amount -lt 0) { $dir = -120 }
     for ($i = 0; $i -lt [Math]::Abs($amount); $i++) {
@@ -212,24 +314,38 @@ switch ($req.action) {
   }
   'keys' { Send-ACKeys $req.sequence }
   'type' {
-    if ($req.x -ne $null) {
+    if ($null -ne $req.x -and $null -ne $req.y) {
       [void][AC]::SetCursorPos([int]$req.x, [int]$req.y)
       Start-Sleep -Milliseconds 60
       [AC]::mouse_event(2, 0, 0, 0, [UIntPtr]::Zero)
       [AC]::mouse_event(4, 0, 0, 0, [UIntPtr]::Zero)
-      Start-Sleep -Milliseconds 120
+      Start-Sleep -Milliseconds 150
     }
-    Set-Clipboard -Value ([string]$req.text)
-    Start-Sleep -Milliseconds 80
-    [System.Windows.Forms.SendKeys]::SendWait('^v')
-    Start-Sleep -Milliseconds 150
-    $out.typed = ([string]$req.text).Length
+    $out.typed = (Send-ACUnicode ([string]$req.text))
   }
   'shot' {
     $s = Get-ACScreen
     $bmp = New-Object System.Drawing.Bitmap $s.width, $s.height
     $g = [System.Drawing.Graphics]::FromImage($bmp)
     $g.CopyFromScreen(0, 0, 0, 0, $bmp.Size)
+    if ($req.annotate) {
+      $fg = $null
+      try { $fg = [AC]::GetForegroundWindow().ToInt64() } catch { $fg = 0 }
+      $els = @(Get-ACElements $fg $true)
+      $font = New-Object System.Drawing.Font('Segoe UI', 9, [System.Drawing.FontStyle]::Bold)
+      $brush = New-Object System.Drawing.SolidBrush ([System.Drawing.Color]::FromArgb(230, 250, 178, 131))
+      $pen = New-Object System.Drawing.Pen ([System.Drawing.Color]::FromArgb(200, 250, 178, 131)), 1
+      $i = 1
+      foreach ($e in $els) {
+        $bb = $e.bounding_box
+        $g.DrawRectangle($pen, $bb.left, $bb.top, [Math]::Max(1, $bb.right - $bb.left), [Math]::Max(1, $bb.bottom - $bb.top))
+        $g.FillRectangle($brush, $bb.left, [Math]::Max(0, $bb.top - 14), 18, 14)
+        $g.DrawString([string]$i, $font, (New-Object System.Drawing.SolidBrush ([System.Drawing.Color]::Black)), $bb.left + 2, [Math]::Max(0, $bb.top - 14))
+        $i++
+      }
+      $pen.Dispose(); $brush.Dispose(); $font.Dispose()
+      $out.elements = $els.Count
+    }
     $bmp.Save($req.path, [System.Drawing.Imaging.ImageFormat]::Png)
     $g.Dispose(); $bmp.Dispose()
     $out.width = $s.width; $out.height = $s.height; $out.file = $req.path
@@ -238,6 +354,35 @@ switch ($req.action) {
     try { $out.apps = @(Get-StartApps | ForEach-Object { @{ name = $_.Name; app_id = $_.AppID } }) }
     catch { $out.apps = @() }
   }
+  'state' {
+    $hh = $null
+    if ($null -ne $req.handle -and [int64]$req.handle -ne 0) { $hh = [int64]$req.handle } else { $hh = [AC]::GetForegroundWindow().ToInt64() }
+    $els = @(Get-ACElements $hh $true)
+    $interactive = @()
+    $scrollable = @()
+    $lines = New-Object System.Collections.ArrayList
+    $i = 1
+    foreach ($e in $els) {
+      if ($e.scrollable) { $scrollable += $e } else { $interactive += $e }
+      [void]$lines.Add("$i. $($e.control_type) '$($e.name)' at ($($e.center.x), $($e.center.y))")
+      $i++
+    }
+    $out.windows = @(Get-ACWindows)
+    $out.active_window = (Get-ACWindow $hh)
+    $out.interactive_elements = $interactive
+    $out.scrollable_elements = $scrollable
+    $out.interactive_elements_text = ($lines -join [Environment]::NewLine)
+    $out.scrollable_elements_text = (($scrollable | ForEach-Object { "$($_.control_type) '$($_.name)' at ($($_.center.x), $($_.center.y))" }) -join [Environment]::NewLine)
+    $out.system = @{
+      windows_version = [string](Get-CimInstance Win32_OperatingSystem).Caption
+      os_version = [Environment]::OSVersion.Version.ToString()
+      screen_size = (Get-ACScreen)
+      user_account_type = 'Local Account'
+      default_language = (Get-Culture).Name
+      dpi_scaling = 1
+    }
+    $out.metadata = @{ kind = 'ui-automation'; depth = 'descendants'; all_desktops = $false }
+  }
   'sysinfo' {
     $out.system = @{
       windows_version = [string](Get-CimInstance Win32_OperatingSystem).Caption
@@ -250,9 +395,8 @@ switch ($req.action) {
   }
   default { $out = @{ ok = $false; error = "unknown action: $($req.action)" } }
 }
-$out | ConvertTo-Json -Depth 6 -Compress
+$out | ConvertTo-Json -Depth 8 -Compress
 `;
-
 let psEnginePath = null;
 function psEngine() {
   if (psEnginePath && fs.existsSync(psEnginePath)) return psEnginePath;
@@ -384,54 +528,103 @@ const IMAGE_TOOLS = [
   ['spectacle', (p) => ['-b', '-n', '-o', p]],
 ];
 
+const VK = {
+  ctrl: 0x11, control: 0x11, alt: 0x12, option: 0x12, shift: 0x10, win: 0x5b, super: 0x5b,
+  meta: 0x5b, cmd: 0x5b, command: 0x5b,
+  enter: 0x0d, return: 0x0d, tab: 0x09, esc: 0x1b, escape: 0x1b, space: 0x20, spacebar: 0x20,
+  backspace: 0x08, bs: 0x08, delete: 0x2e, del: 0x2e, insert: 0x2d, ins: 0x2d,
+  home: 0x24, end: 0x23, pageup: 0x21, pagedown: 0x22, pgup: 0x21, pgdn: 0x22,
+  up: 0x26, down: 0x28, left: 0x25, right: 0x27, arrowup: 0x26, arrowdown: 0x28,
+  arrowleft: 0x25, arrowright: 0x27,
+  capslock: 0x14, numlock: 0x90, scrolllock: 0x91, printscreen: 0x2c, prtsc: 0x2c,
+  pause: 0x13, menu: 0x5d, contextmenu: 0x5d, apps: 0x5d,
+  plus: 0xbb, minus: 0xbd, comma: 0xbc, period: 0xbe, slash: 0xbf, backslash: 0xdc,
+  semicolon: 0xba, quote: 0xde, bracketleft: 0xdb, bracketright: 0xdd, backtick: 0xc0,
+  numpad0: 0x60, numpad1: 0x61, numpad2: 0x62, numpad3: 0x63, numpad4: 0x64,
+  numpad5: 0x65, numpad6: 0x66, numpad7: 0x67, numpad8: 0x68, numpad9: 0x69,
+};
+for (let i = 1; i <= 24; i += 1) VK[`f${i}`] = 0x6f + i;
+
+// keys that need KEYEVENTF_EXTENDEDKEY, like windows-use's kScExtendedVks
+const EXTENDED = new Set([
+  0x2e, 0x2d, 0x24, 0x23, 0x21, 0x22, 0x26, 0x28, 0x25, 0x27, 0x5b, 0x5d, 0x2c, 0x90, 0x5d,
+  0x6f + 11, 0x6f + 12,
+]);
+const MODIFIERS = new Set(['ctrl', 'control', 'alt', 'option', 'shift', 'win', 'super', 'meta', 'cmd', 'command']);
+
 function keyboardSequence(combo) {
-  // "ctrl+shift+s" -> [{vk, flags}] using Windows virtual-key codes, which the
-  // PowerShell engine sends through keybd_event. Unknown names are ignored.
-  const VK = {
-    ctrl: 0x11, control: 0x11, alt: 0x12, shift: 0x10, win: 0x5b, meta: 0x5b, cmd: 0x5b,
-    enter: 0x0d, return: 0x0d, tab: 0x09, esc: 0x1b, escape: 0x1b, space: 0x20,
-    backspace: 0x08, delete: 0x2e, del: 0x2e, insert: 0x2d, home: 0x24, end: 0x23,
-    pageup: 0x21, pagedown: 0x22, pgup: 0x21, pgdn: 0x22, up: 0x26, down: 0x28,
-    left: 0x25, right: 0x27, printscreen: 0x2c, capslock: 0x14, menu: 0x5d,
-  };
-  for (let i = 1; i <= 24; i += 1) VK[`f${i}`] = 0x6f + i;
+  // "ctrl+shift+s" -> [{ vk, flags, ext }] for keybd_event, the call windows-use makes too
   const parts = String(combo || '')
     .toLowerCase()
     .split('+')
     .map((p) => p.trim())
     .filter(Boolean);
   if (!parts.length) return [];
-  const seq = [];
-  const repeat = parts.filter((p) => p === 'repeat').length;
-  const keys = parts.filter((p) => p !== 'repeat');
+  let repeat = 1;
   const modVks = [];
   const mainVks = [];
-  for (const key of keys) {
-    if (VK[key]) {
-      if (['ctrl', 'control', 'alt', 'shift', 'win', 'meta', 'cmd'].includes(key)) modVks.push(VK[key]);
-      else mainVks.push(VK[key]);
-    } else if (key.length === 1) {
-      mainVks.push(key.toUpperCase().charCodeAt(0));
+  for (const key of parts) {
+    if (key === 'repeat' || /^x\d+$/.test(key)) {
+      repeat = key === 'repeat' ? 2 : Number(key.slice(1));
+      continue;
     }
+    let vk = VK[key];
+    if (vk === undefined && key.length === 1) vk = key.toUpperCase().charCodeAt(0);
+    if (vk === undefined) continue;
+    if (MODIFIERS.has(key)) modVks.push(vk);
+    else mainVks.push(vk);
   }
-  const times = Math.max(1, repeat || 1);
+  const seq = [];
+  const times = Math.max(1, Math.min(50, repeat));
   for (let i = 0; i < times; i += 1) {
     if (!mainVks.length && modVks.length) {
       // "win" on its own: press and release the modifier itself
       const only = modVks[modVks.length - 1];
-      seq.push({ vk: only, flags: 0 }, { vk: only, flags: 2 });
+      seq.push({ vk: only, flags: 0, ext: EXTENDED.has(only) });
+      seq.push({ vk: only, flags: 2, ext: EXTENDED.has(only) });
       continue;
     }
-    for (const vk of modVks) seq.push({ vk, flags: 0 });
+    for (const vk of modVks) seq.push({ vk, flags: 0, ext: EXTENDED.has(vk) });
     for (const vk of mainVks) {
-      seq.push({ vk, flags: 0 });
-      seq.push({ vk, flags: 2 });
+      seq.push({ vk, flags: 0, ext: EXTENDED.has(vk) });
+      seq.push({ vk, flags: 2, ext: EXTENDED.has(vk) });
     }
-    for (const vk of [...modVks].reverse()) seq.push({ vk, flags: 2 });
+    for (const vk of [...modVks].reverse()) seq.push({ vk, flags: 2, ext: EXTENDED.has(vk) });
   }
   return seq;
 }
 
+// windows-use matches window names with fuzzy scores (wu::fuzzy::WRatio); the same
+// idea keeps `ui app --name chrome` working when the title is "Google Chrome — …".
+const fuzzyScore = (needle, haystack) => {
+  const a = String(needle || '').toLowerCase().trim();
+  const b = String(haystack || '').toLowerCase().trim();
+  if (!a || !b) return 0;
+  if (a === b) return 1;
+  if (b === a || b.startsWith(a)) return 0.95;
+  if (b.includes(a)) return 0.85;
+  const tokens = a.split(/\s+/).filter(Boolean);
+  const words = b.split(/\s+/).filter(Boolean);
+  const hits = tokens.filter((t) => words.some((w) => w.startsWith(t) || t.startsWith(w))).length;
+  if (hits) return 0.6 + (0.2 * hits) / tokens.length;
+  // character-level overlap, same spirit as partial_ratio
+  const chars = new Set(b.replace(/\s+/g, ''));
+  const overlap = [...a.replace(/\s+/g, '')].filter((c) => chars.has(c)).length;
+  return 0.4 * (overlap / Math.max(1, a.replace(/\s+/g, '').length));
+};
+
+const AVOIDED = [/arenacode/i, /arena code/i];
+
+function pickWindow(list, name) {
+  const candidates = (list || []).filter((w) => w && w.name && !AVOIDED.some((re) => re.test(w.name)));
+  let best = null;
+  for (const w of candidates) {
+    const score = fuzzyScore(name, w.name);
+    if (score >= 0.6 && (!best || score > best.score)) best = { ...w, score };
+  }
+  return best;
+}
+
 const automation = {
   platform: process.platform,
 
@@ -471,16 +664,31 @@ const automation = {
 
   windows() {
     if (isWin) {
-      const r = winRun({ action: 'windows' });
-      return r.ok ? { ok: true, windows: r.windows || [], active_window: r.active_window || null } : fail(r.error);
+      const r = winRun({ action: 'windows' }, 45000);
+      if (!r.ok) return fail(r.error);
+      return { ok: true, windows: r.windows || [] };
     }
     const list = isMac ? macWindows() : linuxWindows();
-    return { ok: true, windows: list, active_window: list[0] || null };
+    return { ok: true, windows: list };
   },
 
   activeWindow() {
+    if (isWin) {
+      const r = winRun({ action: 'active' });
+      return r.ok ? { ok: true, name: r.name, handle: r.handle } : fail(r.error);
+    }
     const w = this.windows();
-    return w.ok ? { ok: true, name: (w.active_window && w.active_window.name) || '' } : w;
+    const first = w.ok && w.windows.length ? w.windows[0] : null;
+    return { ok: true, name: first ? first.name : '', handle: first ? first.handle : null };
+  },
+
+  /** Finds the window the user means, the way windows-use does: best fuzzy match. */
+  findWindow(name) {
+    const list = this.windows();
+    if (!list.ok) return list;
+    const hit = pickWindow(list.windows, name);
+    if (!hit) return { ok: false, error: `window not found: ${name}` };
+    return { ok: true, window: hit };
   },
 
   launchApp(name) {
@@ -501,8 +709,10 @@ const automation = {
 
   focusWindow(name) {
     if (isWin) {
-      const r = winRun({ action: 'focus', name });
-      return r.ok ? { ok: true, name: r.name, app: r.name } : fail(r.error);
+      const found = this.findWindow(name);
+      if (!found.ok) return found;
+      const r = winRun({ action: 'focus', handle: found.window.handle });
+      return r.ok ? { ok: true, name: r.name, handle: found.window.handle, matched: found.window.name } : fail(r.error);
     }
     if (isMac) {
       const res = osa(`tell application "System Events" to set frontmost of process ${JSON.stringify(name)} to true`);
@@ -514,7 +724,16 @@ const automation = {
 
   resizeWindow(name, x, y, width, height) {
     if (isWin) {
-      const r = winRun({ action: 'resize', name, x, y, width, height });
+      const found = this.findWindow(name);
+      if (!found.ok) return found;
+      const r = winRun({
+        action: 'resize',
+        handle: found.window.handle,
+        x,
+        y,
+        width,
+        height,
+      });
       return r.ok ? { ok: true, name: r.name, x, y, width, height } : fail(r.error);
     }
     if (isMac) {
@@ -527,6 +746,52 @@ const automation = {
     return res.ok ? { ok: true, name, x, y, width, height } : fail(res.stderr.trim() || 'resize failed');
   },
 
+  /** The UI Automation tree of the focused window: what the desktop's windows-use returns. */
+  state(handle = null) {
+    if (isWin) {
+      const r = winRun({ action: 'state', handle }, 60000);
+      if (!r.ok) return fail(r.error);
+      return {
+        ok: true,
+        windows: r.windows || [],
+        active_window: r.active_window || null,
+        interactive_elements: r.interactive_elements || [],
+        scrollable_elements: r.scrollable_elements || [],
+        interactive_elements_text: r.interactive_elements_text || '',
+        scrollable_elements_text: r.scrollable_elements_text || '',
+        system: r.system || {},
+        metadata: r.metadata || {},
+      };
+    }
+    // no accessibility tree on the other platforms: windows, geometry and pixels
+    const wins = this.windows();
+    const screen = this.screenSize();
+    const active = wins.ok && wins.windows.length ? wins.windows[0] : null;
+    const elements = (wins.ok ? wins.windows : []).map((w) => ({
+      name: w.name,
+      control_type: 'Window',
+      kind: 'Window',
+      bounding_box: { left: w.x, top: w.y, right: w.x + w.width, bottom: w.y + w.height },
+      center: { x: w.x + Math.floor(w.width / 2), y: w.y + Math.floor(w.height / 2) },
+      value: '',
+      enabled: true,
+      automation_id: '',
+    }));
+    return {
+      ok: true,
+      windows: wins.ok ? wins.windows : [],
+      active_window: active,
+      interactive_elements: elements,
+      scrollable_elements: [],
+      interactive_elements_text: elements
+        .map((e, i) => `${i + 1}. ${e.control_type} '${e.name}' at (${e.center.x}, ${e.center.y})`)
+        .join('\n'),
+      scrollable_elements_text: '',
+      system: this.sysinfo().system,
+      metadata: { kind: 'windows-only-list', depth: 'top-level', all_desktops: false },
+    };
+  },
+
   click(x, y, { button = 'left', count = 1 } = {}) {
     if (isWin) {
       const r = winRun({ action: 'click', x, y, button, count });
@@ -644,11 +909,13 @@ const automation = {
     return res.ok ? { ok: true, keys: combo } : fail(res.stderr.trim() || 'xdotool key failed');
   },
 
-  screenshot(dest) {
+  screenshot(dest, { annotate = false } = {}) {
     fs.mkdirSync(path.dirname(dest), { recursive: true });
     if (isWin) {
-      const r = winRun({ action: 'shot', path: dest }, 60000);
-      return r.ok ? { ok: true, file: dest, width: r.width, height: r.height, mime: 'image/png' } : fail(r.error);
+      const r = winRun({ action: 'shot', path: dest, annotate }, 90000);
+      return r.ok
+        ? { ok: true, file: dest, width: r.width, height: r.height, mime: 'image/png', annotated: Boolean(annotate), elements: r.elements || 0 }
+        : fail(r.error);
     }
     if (isMac) {
       const res = run('screencapture', ['-x', '-t', 'png', dest], { timeout: 30000 });
@@ -723,4 +990,4 @@ const automation = {
   },
 };
 
-module.exports = { ...automation, keyboardSequence };
+module.exports = { ...automation, keyboardSequence, fuzzyScore, pickWindow };

package/src/local.js

@@ -531,22 +531,10 @@ async function runUiAction(bridge, taskId, request, opts) {
   };
 
   if (op === 'state') {
-    const wins = automation.windows();
-    const screen = automation.screenSize();
-    const info = automation.sysinfo();
-    if (!wins.ok) return wins;
-    return {
-      ok: true,
-      windows: wins.windows,
-      active_window: wins.active_window,
-      interactive_elements: [],
-      scrollable_elements: [],
-      interactive_elements_text: '',
-      system: info.system,
-      screen: { width: screen.width, height: screen.height },
-      // the terminal client reads windows and pixels, not the accessibility tree
-      note: 'interactive_elements are empty: this client does not read the UI Automation tree',
-    };
+    // the desktop's windows-use returns the UI Automation tree; so do we on Windows
+    const state = automation.state();
+    if (!state.ok) return state;
+    return state;
   }
 
   if (op === 'sysinfo') {
@@ -626,7 +614,7 @@ async function runUiAction(bridge, taskId, request, opts) {
     const dir = path.join(os.tmpdir(), 'arenacode-shots');
     fs.mkdirSync(dir, { recursive: true });
     const file = path.join(dir, `shot-${stamp()}.png`);
-    const shot = automation.screenshot(file);
+    const shot = automation.screenshot(file, { annotate: Boolean(request.annotated) });
     if (!shot.ok) return shot;
     const buf = fs.readFileSync(shot.file);
     // the agent runs in the sandbox: the image travels through the bridge workspace
@@ -639,6 +627,7 @@ async function runUiAction(bridge, taskId, request, opts) {
       height: shot.height,
       file: remote || shot.file,
       local_file: shot.file,
+      annotated: Boolean(request.annotated),
     };
     if (buf.length <= 2 * 1024 * 1024) out.image = buf.toString('base64');
     return out;