@lardexx/arenacode 2.6.2 → 2.8.0

package/README.md

@@ -134,6 +134,32 @@ so the port reproduces it byte for byte:
 
 The exact strings are checked by `test/parity.js` against the ones extracted from `ArenaCode-x64.exe`.
 
+**Computer Agent: mouse, keyboard and screen.** The three modes' `ui` commands now run in the terminal client too,
+from the same bridge request the desktop app answers:
+
+| Command | What it does here |
+|---|---|
+| `ui control` | `move`, `click`, `dblclick`, `rightclick`, `down`, `up`, `drag` (smooth, 12 steps), `scroll` |
+| `ui type` | clicks the field first when the task gives `x,y`, otherwise types into the focused field; the text is pasted, so accents and emoji survive |
+| `ui keys` | `ctrl+s`, `alt+f4`, `win`, `enter`, `f5`, `ctrl+shift+esc` … |
+| `ui app` | `launch` (Start menu on Windows, `open -a` on macOS, `gtk-launch` on Linux), `switch`, `resize` |
+| `ui shot` | full-screen PNG, uploaded to the bridge workspace and returned as `file` (+ `image` base64 when small) |
+| `ui state` | windows with position, size and process id, plus screen and system |
+| `ui apps` | the Start menu (Windows), `/Applications` (macOS), `.desktop` entries (Linux) |
+| `ui multi-select`, `ui multi-edit` | built from the primitives above |
+
+No extra install on **Windows**: PowerShell + `user32.dll` + the .NET UI Automation assemblies. The desktop client
+ships a `windows-use.dll` (exports `wu_call_json`, `wu_free`, `wu_version`) for the same job; the terminal client
+reproduces its behaviour instead of depending on it — the same JSON keys (`bounding_box`, `center`, `control_type`,
+`interactive_elements_text`, `scrollable_elements`, `screenshot_png_base64`), the same ideas: the UI Automation tree of
+the focused window for `ui state`, **unicode typing through `SendInput`** so the clipboard is untouched, extended-key
+flags for arrows/navigation, fuzzy window matching (`ui app --name chrome` finds *"Google Chrome — New Tab"*), and
+numbered labels drawn on the elements for `ui shot --annotated`.
+
+**macOS**: grant Accessibility to your terminal, and `brew install cliclick` for the mouse. **Linux** (X11):
+`sudo apt install xdotool wmctrl imagemagick`; on Wayland `xdotool` cannot inject input, so Computer Agent needs
+XWayland. On macOS/Linux the accessibility tree is not read: `ui state` returns windows and geometry as elements.
+
 **Ask and Proceed / Proceed.** The desktop's Settings switch, `ctrl+x a` here (also `/ask`, `/proceed`,
 `--proceed`, and `arenacode config set approval proceed`). The default, **Ask and Proceed**, asks before commands,
 file changes and plans; in Computer mode it asks once, on the plan, and after that the commands run on their own

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lardexx/arenacode",
-  "version": "2.6.2",
+  "version": "2.8.0",
   "description": "Terminal client for ArenaCode: talks to an arena-bridge, streams the Agent Mode timeline, and runs local commands and file operations on your PC. Node port of the ArenaCode desktop client.",
   "license": "MIT",
   "type": "commonjs",
@@ -19,7 +19,7 @@
     "LICENSE"
   ],
   "scripts": {
-    "test": "node test/integration.js && node test/parity.js"
+    "test": "node test/integration.js && node test/parity.js && node test/automation.js"
   },
   "keywords": [
     "arenacode",

package/src/automation.js

@@ -0,0 +1,993 @@
+'use strict';
+
+// Computer Agent automation for the terminal client: mouse, keyboard, screen,
+// windows and Start-menu apps. One small interface, three backends:
+//
+//   Windows  PowerShell + user32 (no extra install)
+//   macOS    osascript (System Events) + screencapture, cliclick for the mouse
+//   Linux    xdotool + ImageMagick/scrot/grim for screenshots
+//
+// Every function resolves to plain JSON, the same shapes the desktop client
+// returns to the bridge: { ok: true, ... } or { ok: false, error }.
+
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+const { spawnSync } = require('node:child_process');
+
+const isWin = process.platform === 'win32';
+const isMac = process.platform === 'darwin';
+const isLinux = process.platform === 'linux';
+
+function fail(error, extra = {}) {
+  return { ok: false, error, ...extra };
+}
+
+function which(cmd) {
+  const probe = isWin ? 'where' : 'which';
+  const res = spawnSync(probe, [cmd], { encoding: 'utf8', windowsHide: true });
+  return res.status === 0 && String(res.stdout).trim() !== '';
+}
+
+function run(cmd, args, opts = {}) {
+  const res = spawnSync(cmd, args, {
+    input: opts.input,
+    encoding: 'utf8',
+    timeout: opts.timeout || 30000,
+    maxBuffer: 64 * 1024 * 1024,
+    windowsHide: true,
+  });
+  return {
+    ok: res.status === 0,
+    status: res.status,
+    stdout: res.stdout || '',
+    stderr: res.stderr || '',
+    error: res.error ? res.error.message : '',
+  };
+}
+
+// --------------------------------------------------------------------------- //
+// Windows
+// --------------------------------------------------------------------------- //
+
+const PS_ENGINE = String.raw`
+$ErrorActionPreference = 'Stop'
+$req = ([Console]::In.ReadToEnd() | ConvertFrom-Json)
+Add-Type -AssemblyName System.Drawing
+Add-Type -AssemblyName System.Windows.Forms
+Add-Type -AssemblyName UIAutomationClient
+Add-Type -AssemblyName UIAutomationTypes
+Add-Type -TypeDefinition @'
+using System;
+using System.Text;
+using System.Runtime.InteropServices;
+public class AC {
+  [DllImport("user32.dll")] public static extern bool SetCursorPos(int x, int y);
+  [DllImport("user32.dll")] public static extern void mouse_event(uint flags, int dx, int dy, int data, UIntPtr extra);
+  [DllImport("user32.dll")] public static extern void keybd_event(byte vk, byte scan, uint flags, UIntPtr extra);
+  [DllImport("user32.dll")] public static extern IntPtr GetForegroundWindow();
+  [DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr h);
+  [DllImport("user32.dll")] public static extern bool ShowWindow(IntPtr h, int cmd);
+  [DllImport("user32.dll")] public static extern bool MoveWindow(IntPtr h, int x, int y, int w, int ht, bool repaint);
+  [DllImport("user32.dll")] public static extern bool GetWindowRect(IntPtr h, out RECT r);
+  [DllImport("user32.dll")] public static extern bool IsWindowVisible(IntPtr h);
+  [DllImport("user32.dll")] public static extern bool IsIconic(IntPtr h);
+  [StructLayout(LayoutKind.Sequential)] public struct RECT { public int Left, Top, Right, Bottom; }
+}
+'@
+
+$AC_TYPES = @(
+  [System.Windows.Automation.ControlType]::Button,
+  [System.Windows.Automation.ControlType]::CheckBox,
+  [System.Windows.Automation.ControlType]::ComboBox,
+  [System.Windows.Automation.ControlType]::Custom,
+  [System.Windows.Automation.ControlType]::DataItem,
+  [System.Windows.Automation.ControlType]::Edit,
+  [System.Windows.Automation.ControlType]::Hyperlink,
+  [System.Windows.Automation.ControlType]::ListItem,
+  [System.Windows.Automation.ControlType]::MenuItem,
+  [System.Windows.Automation.ControlType]::RadioButton,
+  [System.Windows.Automation.ControlType]::ScrollBar,
+  [System.Windows.Automation.ControlType]::Slider,
+  [System.Windows.Automation.ControlType]::Spinner,
+  [System.Windows.Automation.ControlType]::SplitButton,
+  [System.Windows.Automation.ControlType]::Tab,
+  [System.Windows.Automation.ControlType]::TabItem,
+  [System.Windows.Automation.ControlType]::Text,
+  [System.Windows.Automation.ControlType]::ToolBar,
+  [System.Windows.Automation.ControlType]::TreeItem
+)
+
+function Get-ACWindow($h) {
+  $hh = [IntPtr][int64]$h
+  $r = New-Object AC+RECT
+  [void][AC]::GetWindowRect($hh, [ref]$r)
+  $sb = New-Object System.Text.StringBuilder 1024
+  [void][AC]::GetWindowText($hh, $sb, 1024)
+  return [pscustomobject]@{
+    name = $sb.ToString()
+    handle = $hh.ToInt64()
+    hwnd = $hh.ToInt64()
+    x = $r.Left; y = $r.Top
+    width = [Math]::Max(0, $r.Right - $r.Left); height = [Math]::Max(0, $r.Bottom - $r.Top)
+    status = $(if ([AC]::IsIconic($hh)) { 'minimized' } else { 'normal' })
+  }
+}
+
+function Get-ACWindows {
+  $list = New-Object System.Collections.ArrayList
+  foreach ($p in (Get-Process | Where-Object { $_.MainWindowHandle -ne 0 })) {
+    try {
+      $title = [string]$p.MainWindowTitle
+      if ($title.Trim().Length -eq 0) { continue }
+      $w = Get-ACWindow $p.MainWindowHandle
+      $w | Add-Member -NotePropertyName process_id -NotePropertyValue ([int]$p.Id)
+      $w | Add-Member -NotePropertyName app -NotePropertyValue ([string]$p.ProcessName)
+      [void]$list.Add($w)
+    } catch { }
+  }
+  return $list
+}
+
+function Get-ACScreen {
+  $b = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds
+  return @{ width = $b.Width; height = $b.Height }
+}
+
+function Send-ACKeys($pairs) {
+  foreach ($k in $pairs) {
+    $flags = 0
+    if ($k.flags -eq 2) { $flags = $flags -bor 2 }
+    if ($k.ext) { $flags = $flags -bor 1 }
+    [AC]::keybd_event([byte]$k.vk, 0, $flags, [UIntPtr]::Zero)
+    Start-Sleep -Milliseconds 12
+  }
+}
+
+# windows-use types through SendInput unicode events: the clipboard is never touched
+function Send-ACUnicode($text) {
+  $typed = 0
+  foreach ($ch in $text.ToCharArray()) {
+    if ($ch -eq [char]10) {
+      [AC]::keybd_event(0x0D, 0, 0, [UIntPtr]::Zero)
+      [AC]::keybd_event(0x0D, 0, 2, [UIntPtr]::Zero)
+      $typed++
+      continue
+    }
+    if ($ch -eq [char]13) { continue }
+    [AC]::keybd_event(0, [byte][char]$ch, 4, [UIntPtr]::Zero)
+    [AC]::keybd_event(0, [byte][char]$ch, 6, [UIntPtr]::Zero)
+    $typed++
+    Start-Sleep -Milliseconds 3
+  }
+  return $typed
+}
+
+function Get-ACElements($hwnd, $interactiveOnly) {
+  $handle = [IntPtr][int64]$hwnd
+  $root = $null
+  if ($handle -ne [IntPtr]::Zero) {
+    $root = [System.Windows.Automation.AutomationElement]::FromHandle($handle)
+  } else {
+    $root = [System.Windows.Automation.AutomationElement]::FocusedElement
+    while ($root -ne $null -and $root.Current.NativeWindowHandle -eq 0) {
+      $walker = [System.Windows.Automation.TreeWalker]::ControlViewWalker
+      $root = $walker.GetParent($root)
+    }
+  }
+  if ($root -eq $null) { return @() }
+
+  $conditions = @()
+  foreach ($ct in $AC_TYPES) {
+    $conditions += New-Object System.Windows.Automation.PropertyCondition(
+      [System.Windows.Automation.AutomationElement]::ControlTypeProperty, $ct)
+  }
+  $cond = New-Object System.Windows.Automation.OrCondition $conditions
+  $found = $root.FindAll([System.Windows.Automation.TreeScope]::Descendants, $cond)
+
+  $out = New-Object System.Collections.ArrayList
+  $n = 0
+  foreach ($el in $found) {
+    if ($n -ge 120) { break }
+    try {
+      $cur = $el.Current
+      if ($cur.IsOffscreen) { continue }
+      $rect = $cur.BoundingRectangle
+      if ($rect.Width -le 0 -or $rect.Height -le 0) { continue }
+      $name = [string]$cur.Name
+      $ctype = [string]$cur.ControlType.ProgrammaticName -replace '^ControlType\.', ''
+      if ($interactiveOnly -and $ctype -eq 'Text' -and $name.Trim().Length -eq 0) { continue }
+      $val = ''
+      try {
+        $vp = $null
+        if ($el.TryGetCurrentPattern([System.Windows.Automation.ValuePattern]::Pattern, [ref]$vp)) { $val = [string]$vp.Current.Value }
+      } catch { $val = '' }
+      $scrollable = $false
+      try {
+        $sp = $null
+        if ($el.TryGetCurrentPattern([System.Windows.Automation.ScrollPattern]::Pattern, [ref]$sp)) {
+          $scrollable = [bool]$sp.Current.VerticallyScrollable -or [bool]$sp.Current.HorizontallyScrollable
+        }
+      } catch { }
+      [void]$out.Add([pscustomobject]@{
+        name = $name
+        control_type = $ctype
+        kind = $ctype
+        bounding_box = @{ left = [int]$rect.X; top = [int]$rect.Y; right = [int]($rect.X + $rect.Width); bottom = [int]($rect.Y + $rect.Height) }
+        center = @{ x = [int]($rect.X + $rect.Width / 2); y = [int]($rect.Y + $rect.Height / 2) }
+        value = $val
+        scrollable = $scrollable
+        enabled = [bool]$cur.IsEnabled
+        automation_id = [string]$cur.AutomationId
+      })
+      $n++
+    } catch { }
+  }
+  return $out
+}
+
+$out = @{ ok = $true }
+switch ($req.action) {
+  'screen' { $s = Get-ACScreen; $out.width = $s.width; $out.height = $s.height }
+  'windows' { $out.windows = @(Get-ACWindows) }
+  'active' {
+    $hh = [AC]::GetForegroundWindow()
+    $w = Get-ACWindow $hh.ToInt64()
+    $out.name = $w.name
+    $out.handle = $w.handle
+  }
+  'focus' {
+    $hh = [IntPtr][int64]$req.handle
+    [void][AC]::ShowWindow($hh, 9)
+    [void][AC]::SetForegroundWindow($hh)
+    Start-Sleep -Milliseconds 200
+    $w = Get-ACWindow $req.handle
+    $out.name = $w.name
+  }
+  'resize' {
+    $hh = [IntPtr][int64]$req.handle
+    [void][AC]::MoveWindow($hh, [int]$req.x, [int]$req.y, [int]$req.width, [int]$req.height, $true)
+    $w = Get-ACWindow $req.handle
+    $out.name = $w.name
+  }
+  'launch' {
+    $name = [string]$req.name
+    $done = $false
+    try { Start-Process -FilePath $name -ErrorAction Stop; $done = $true } catch { $done = $false }
+    if (-not $done) {
+      $hit = $null
+      try { $hit = Get-StartApps | Where-Object { $_.Name -like "*$name*" } | Select-Object -First 1 } catch { $hit = $null }
+      if ($hit) { Start-Process 'explorer.exe' "shell:AppsFolder\$($hit.AppID)"; $done = $true }
+      else {
+        $exe = Get-Command $name -ErrorAction SilentlyContinue
+        if ($exe) { Start-Process $exe.Source; $done = $true }
+      }
+    }
+    if ($done) { Start-Sleep -Milliseconds 900; $out.name = $name } else { $out = @{ ok = $false; error = "app not found: $name" } }
+  }
+  'move' { [void][AC]::SetCursorPos([int]$req.x, [int]$req.y) }
+  'click' {
+    [void][AC]::SetCursorPos([int]$req.x, [int]$req.y)
+    Start-Sleep -Milliseconds 40
+    $down = 2; $up = 4
+    if ($req.button -eq 'right') { $down = 8; $up = 16 }
+    elseif ($req.button -eq 'middle') { $down = 32; $up = 64 }
+    $times = [int]$req.count; if ($times -lt 1) { $times = 1 }
+    for ($i = 0; $i -lt $times; $i++) {
+      [AC]::mouse_event($down, 0, 0, 0, [UIntPtr]::Zero)
+      Start-Sleep -Milliseconds 20
+      [AC]::mouse_event($up, 0, 0, 0, [UIntPtr]::Zero)
+      Start-Sleep -Milliseconds 60
+    }
+  }
+  'down' {
+    [void][AC]::SetCursorPos([int]$req.x, [int]$req.y)
+    $flag = 2; if ($req.button -eq 'right') { $flag = 8 } elseif ($req.button -eq 'middle') { $flag = 32 }
+    [AC]::mouse_event($flag, 0, 0, 0, [UIntPtr]::Zero)
+  }
+  'up' {
+    $flag = 4; if ($req.button -eq 'right') { $flag = 16 } elseif ($req.button -eq 'middle') { $flag = 64 }
+    if ($null -ne $req.x) { [void][AC]::SetCursorPos([int]$req.x, [int]$req.y) }
+    [AC]::mouse_event($flag, 0, 0, 0, [UIntPtr]::Zero)
+  }
+  'drag' {
+    [void][AC]::SetCursorPos([int]$req.x, [int]$req.y)
+    Start-Sleep -Milliseconds 60
+    [AC]::mouse_event(2, 0, 0, 0, [UIntPtr]::Zero)
+    $steps = 12
+    for ($i = 1; $i -le $steps; $i++) {
+      $nx = [int]($req.x + (($req.x2 - $req.x) * $i / $steps))
+      $ny = [int]($req.y + (($req.y2 - $req.y) * $i / $steps))
+      [void][AC]::SetCursorPos($nx, $ny)
+      Start-Sleep -Milliseconds 15
+    }
+    [AC]::mouse_event(4, 0, 0, 0, [UIntPtr]::Zero)
+  }
+  'scroll' {
+    if ($null -ne $req.x -and $null -ne $req.y) { [void][AC]::SetCursorPos([int]$req.x, [int]$req.y); Start-Sleep -Milliseconds 40 }
+    $amount = [int]$req.amount
+    $dir = 120; if ($amount -lt 0) { $dir = -120 }
+    for ($i = 0; $i -lt [Math]::Abs($amount); $i++) {
+      [AC]::mouse_event(0x0800, 0, 0, $dir, [UIntPtr]::Zero)
+      Start-Sleep -Milliseconds 25
+    }
+  }
+  'keys' { Send-ACKeys $req.sequence }
+  'type' {
+    if ($null -ne $req.x -and $null -ne $req.y) {
+      [void][AC]::SetCursorPos([int]$req.x, [int]$req.y)
+      Start-Sleep -Milliseconds 60
+      [AC]::mouse_event(2, 0, 0, 0, [UIntPtr]::Zero)
+      [AC]::mouse_event(4, 0, 0, 0, [UIntPtr]::Zero)
+      Start-Sleep -Milliseconds 150
+    }
+    $out.typed = (Send-ACUnicode ([string]$req.text))
+  }
+  'shot' {
+    $s = Get-ACScreen
+    $bmp = New-Object System.Drawing.Bitmap $s.width, $s.height
+    $g = [System.Drawing.Graphics]::FromImage($bmp)
+    $g.CopyFromScreen(0, 0, 0, 0, $bmp.Size)
+    if ($req.annotate) {
+      $fg = $null
+      try { $fg = [AC]::GetForegroundWindow().ToInt64() } catch { $fg = 0 }
+      $els = @(Get-ACElements $fg $true)
+      $font = New-Object System.Drawing.Font('Segoe UI', 9, [System.Drawing.FontStyle]::Bold)
+      $brush = New-Object System.Drawing.SolidBrush ([System.Drawing.Color]::FromArgb(230, 250, 178, 131))
+      $pen = New-Object System.Drawing.Pen ([System.Drawing.Color]::FromArgb(200, 250, 178, 131)), 1
+      $i = 1
+      foreach ($e in $els) {
+        $bb = $e.bounding_box
+        $g.DrawRectangle($pen, $bb.left, $bb.top, [Math]::Max(1, $bb.right - $bb.left), [Math]::Max(1, $bb.bottom - $bb.top))
+        $g.FillRectangle($brush, $bb.left, [Math]::Max(0, $bb.top - 14), 18, 14)
+        $g.DrawString([string]$i, $font, (New-Object System.Drawing.SolidBrush ([System.Drawing.Color]::Black)), $bb.left + 2, [Math]::Max(0, $bb.top - 14))
+        $i++
+      }
+      $pen.Dispose(); $brush.Dispose(); $font.Dispose()
+      $out.elements = $els.Count
+    }
+    $bmp.Save($req.path, [System.Drawing.Imaging.ImageFormat]::Png)
+    $g.Dispose(); $bmp.Dispose()
+    $out.width = $s.width; $out.height = $s.height; $out.file = $req.path
+  }
+  'apps' {
+    try { $out.apps = @(Get-StartApps | ForEach-Object { @{ name = $_.Name; app_id = $_.AppID } }) }
+    catch { $out.apps = @() }
+  }
+  'state' {
+    $hh = $null
+    if ($null -ne $req.handle -and [int64]$req.handle -ne 0) { $hh = [int64]$req.handle } else { $hh = [AC]::GetForegroundWindow().ToInt64() }
+    $els = @(Get-ACElements $hh $true)
+    $interactive = @()
+    $scrollable = @()
+    $lines = New-Object System.Collections.ArrayList
+    $i = 1
+    foreach ($e in $els) {
+      if ($e.scrollable) { $scrollable += $e } else { $interactive += $e }
+      [void]$lines.Add("$i. $($e.control_type) '$($e.name)' at ($($e.center.x), $($e.center.y))")
+      $i++
+    }
+    $out.windows = @(Get-ACWindows)
+    $out.active_window = (Get-ACWindow $hh)
+    $out.interactive_elements = $interactive
+    $out.scrollable_elements = $scrollable
+    $out.interactive_elements_text = ($lines -join [Environment]::NewLine)
+    $out.scrollable_elements_text = (($scrollable | ForEach-Object { "$($_.control_type) '$($_.name)' at ($($_.center.x), $($_.center.y))" }) -join [Environment]::NewLine)
+    $out.system = @{
+      windows_version = [string](Get-CimInstance Win32_OperatingSystem).Caption
+      os_version = [Environment]::OSVersion.Version.ToString()
+      screen_size = (Get-ACScreen)
+      user_account_type = 'Local Account'
+      default_language = (Get-Culture).Name
+      dpi_scaling = 1
+    }
+    $out.metadata = @{ kind = 'ui-automation'; depth = 'descendants'; all_desktops = $false }
+  }
+  'sysinfo' {
+    $out.system = @{
+      windows_version = [string](Get-CimInstance Win32_OperatingSystem).Caption
+      os_version = [Environment]::OSVersion.Version.ToString()
+      screen_size = (Get-ACScreen)
+      user_account_type = 'Local Account'
+      default_language = (Get-Culture).Name
+      dpi_scaling = 1
+    }
+  }
+  default { $out = @{ ok = $false; error = "unknown action: $($req.action)" } }
+}
+$out | ConvertTo-Json -Depth 8 -Compress
+`;
+let psEnginePath = null;
+function psEngine() {
+  if (psEnginePath && fs.existsSync(psEnginePath)) return psEnginePath;
+  const dir = path.join(os.tmpdir(), 'arenacode-automation');
+  fs.mkdirSync(dir, { recursive: true });
+  const file = path.join(dir, 'engine.ps1');
+  fs.writeFileSync(file, PS_ENGINE, 'utf8');
+  psEnginePath = file;
+  return file;
+}
+
+function winRun(request, timeout = 30000) {
+  const shell = which('powershell') ? 'powershell' : 'pwsh';
+  if (!which(shell)) return fail('PowerShell not found');
+  const res = run(
+    shell,
+    ['-NoProfile', '-NonInteractive', '-STA', '-ExecutionPolicy', 'Bypass', '-File', psEngine()],
+    { input: JSON.stringify(request), timeout }
+  );
+  if (!res.ok && !res.stdout.trim()) {
+    return fail(res.stderr.trim() || res.error || `PowerShell exit ${res.status}`);
+  }
+  try {
+    return JSON.parse(res.stdout.trim().split('\n').pop());
+  } catch {
+    return fail(`unreadable answer from PowerShell: ${res.stdout.slice(0, 200)}`);
+  }
+}
+
+// --------------------------------------------------------------------------- //
+// macOS
+// --------------------------------------------------------------------------- //
+
+function osa(script, timeout = 20000) {
+  const res = run('osascript', ['-e', script], { timeout });
+  return { ok: res.ok, out: res.stdout.trim(), error: res.stderr.trim() || res.error };
+}
+
+function macScreen() {
+  const res = osa('tell application "Finder" to get bounds of window of desktop');
+  const nums = String(res.out)
+    .split(',')
+    .map((n) => Number(n.trim()));
+  if (nums.length === 4 && nums.every((n) => Number.isFinite(n))) {
+    return { width: nums[2] - nums[0], height: nums[3] - nums[1] };
+  }
+  return { width: 1440, height: 900 };
+}
+
+function macWindows() {
+  const res = osa(
+    'tell application "System Events" to set out to ""\n' +
+      'repeat with p in (every process whose background only is false)\n' +
+      '  repeat with w in (every window of p)\n' +
+      '    set out to out & (name of p) & tab & (name of w) & tab & (item 1 of (position of w)) & tab & (item 2 of (position of w)) & tab & (item 1 of (size of w)) & tab & (item 2 of (size of w)) & linefeed\n' +
+      '  end repeat\n' +
+      'end repeat\n' +
+      'return out'
+  );
+  if (!res.ok) return [];
+  return String(res.out)
+    .split('\n')
+    .filter(Boolean)
+    .map((line) => {
+      const [app, title, x, y, w, h] = line.split('\t');
+      return {
+        name: title || app,
+        app,
+        x: Number(x) || 0,
+        y: Number(y) || 0,
+        width: Number(w) || 0,
+        height: Number(h) || 0,
+        process_id: 0,
+        status: 'normal',
+      };
+    });
+}
+
+function macKeys(sequence, text) {
+  // sequence comes as vk pairs; on macOS it is easier to send the original combo
+  const script = text
+    ? `tell application "System Events" to keystroke ${JSON.stringify(text)}`
+    : `tell application "System Events" to keystroke ${JSON.stringify(sequence)}`;
+  return osa(script);
+}
+
+// --------------------------------------------------------------------------- //
+// Linux
+// --------------------------------------------------------------------------- //
+
+function linuxScreen() {
+  const res = run('xdotool', ['getdisplaygeometry']);
+  if (res.ok) {
+    const [w, h] = res.stdout.trim().split(/\s+/).map(Number);
+    if (w && h) return { width: w, height: h };
+  }
+  const xr = run('xrandr', ['--current']);
+  const m = /current (\d+) x (\d+)/.exec(xr.stdout);
+  if (m) return { width: Number(m[1]), height: Number(m[2]) };
+  return { width: 1280, height: 800 };
+}
+
+function linuxWindows() {
+  const res = run('wmctrl', ['-lG']);
+  const out = [];
+  if (res.ok) {
+    for (const line of res.stdout.split('\n').filter(Boolean)) {
+      const parts = line.split(/\s+/);
+      const x = Number(parts[2]);
+      const y = Number(parts[3]);
+      const width = Number(parts[4]);
+      const height = Number(parts[5]);
+      const name = parts.slice(7).join(' ');
+      out.push({ name, x, y, width, height, process_id: 0, status: 'normal' });
+    }
+  }
+  return out;
+}
+
+// --------------------------------------------------------------------------- //
+// public interface
+// --------------------------------------------------------------------------- //
+
+const IMAGE_TOOLS = [
+  ['gnome-screenshot', (p) => ['-f', p]],
+  ['scrot', (p) => [p]],
+  ['import', (p) => ['-window', 'root', p]],
+  ['grim', (p) => [p]],
+  ['spectacle', (p) => ['-b', '-n', '-o', p]],
+];
+
+const VK = {
+  ctrl: 0x11, control: 0x11, alt: 0x12, option: 0x12, shift: 0x10, win: 0x5b, super: 0x5b,
+  meta: 0x5b, cmd: 0x5b, command: 0x5b,
+  enter: 0x0d, return: 0x0d, tab: 0x09, esc: 0x1b, escape: 0x1b, space: 0x20, spacebar: 0x20,
+  backspace: 0x08, bs: 0x08, delete: 0x2e, del: 0x2e, insert: 0x2d, ins: 0x2d,
+  home: 0x24, end: 0x23, pageup: 0x21, pagedown: 0x22, pgup: 0x21, pgdn: 0x22,
+  up: 0x26, down: 0x28, left: 0x25, right: 0x27, arrowup: 0x26, arrowdown: 0x28,
+  arrowleft: 0x25, arrowright: 0x27,
+  capslock: 0x14, numlock: 0x90, scrolllock: 0x91, printscreen: 0x2c, prtsc: 0x2c,
+  pause: 0x13, menu: 0x5d, contextmenu: 0x5d, apps: 0x5d,
+  plus: 0xbb, minus: 0xbd, comma: 0xbc, period: 0xbe, slash: 0xbf, backslash: 0xdc,
+  semicolon: 0xba, quote: 0xde, bracketleft: 0xdb, bracketright: 0xdd, backtick: 0xc0,
+  numpad0: 0x60, numpad1: 0x61, numpad2: 0x62, numpad3: 0x63, numpad4: 0x64,
+  numpad5: 0x65, numpad6: 0x66, numpad7: 0x67, numpad8: 0x68, numpad9: 0x69,
+};
+for (let i = 1; i <= 24; i += 1) VK[`f${i}`] = 0x6f + i;
+
+// keys that need KEYEVENTF_EXTENDEDKEY, like windows-use's kScExtendedVks
+const EXTENDED = new Set([
+  0x2e, 0x2d, 0x24, 0x23, 0x21, 0x22, 0x26, 0x28, 0x25, 0x27, 0x5b, 0x5d, 0x2c, 0x90, 0x5d,
+  0x6f + 11, 0x6f + 12,
+]);
+const MODIFIERS = new Set(['ctrl', 'control', 'alt', 'option', 'shift', 'win', 'super', 'meta', 'cmd', 'command']);
+
+function keyboardSequence(combo) {
+  // "ctrl+shift+s" -> [{ vk, flags, ext }] for keybd_event, the call windows-use makes too
+  const parts = String(combo || '')
+    .toLowerCase()
+    .split('+')
+    .map((p) => p.trim())
+    .filter(Boolean);
+  if (!parts.length) return [];
+  let repeat = 1;
+  const modVks = [];
+  const mainVks = [];
+  for (const key of parts) {
+    if (key === 'repeat' || /^x\d+$/.test(key)) {
+      repeat = key === 'repeat' ? 2 : Number(key.slice(1));
+      continue;
+    }
+    let vk = VK[key];
+    if (vk === undefined && key.length === 1) vk = key.toUpperCase().charCodeAt(0);
+    if (vk === undefined) continue;
+    if (MODIFIERS.has(key)) modVks.push(vk);
+    else mainVks.push(vk);
+  }
+  const seq = [];
+  const times = Math.max(1, Math.min(50, repeat));
+  for (let i = 0; i < times; i += 1) {
+    if (!mainVks.length && modVks.length) {
+      // "win" on its own: press and release the modifier itself
+      const only = modVks[modVks.length - 1];
+      seq.push({ vk: only, flags: 0, ext: EXTENDED.has(only) });
+      seq.push({ vk: only, flags: 2, ext: EXTENDED.has(only) });
+      continue;
+    }
+    for (const vk of modVks) seq.push({ vk, flags: 0, ext: EXTENDED.has(vk) });
+    for (const vk of mainVks) {
+      seq.push({ vk, flags: 0, ext: EXTENDED.has(vk) });
+      seq.push({ vk, flags: 2, ext: EXTENDED.has(vk) });
+    }
+    for (const vk of [...modVks].reverse()) seq.push({ vk, flags: 2, ext: EXTENDED.has(vk) });
+  }
+  return seq;
+}
+
+// windows-use matches window names with fuzzy scores (wu::fuzzy::WRatio); the same
+// idea keeps `ui app --name chrome` working when the title is "Google Chrome — …".
+const fuzzyScore = (needle, haystack) => {
+  const a = String(needle || '').toLowerCase().trim();
+  const b = String(haystack || '').toLowerCase().trim();
+  if (!a || !b) return 0;
+  if (a === b) return 1;
+  if (b === a || b.startsWith(a)) return 0.95;
+  if (b.includes(a)) return 0.85;
+  const tokens = a.split(/\s+/).filter(Boolean);
+  const words = b.split(/\s+/).filter(Boolean);
+  const hits = tokens.filter((t) => words.some((w) => w.startsWith(t) || t.startsWith(w))).length;
+  if (hits) return 0.6 + (0.2 * hits) / tokens.length;
+  // character-level overlap, same spirit as partial_ratio
+  const chars = new Set(b.replace(/\s+/g, ''));
+  const overlap = [...a.replace(/\s+/g, '')].filter((c) => chars.has(c)).length;
+  return 0.4 * (overlap / Math.max(1, a.replace(/\s+/g, '').length));
+};
+
+const AVOIDED = [/arenacode/i, /arena code/i];
+
+function pickWindow(list, name) {
+  const candidates = (list || []).filter((w) => w && w.name && !AVOIDED.some((re) => re.test(w.name)));
+  let best = null;
+  for (const w of candidates) {
+    const score = fuzzyScore(name, w.name);
+    if (score >= 0.6 && (!best || score > best.score)) best = { ...w, score };
+  }
+  return best;
+}
+
+const automation = {
+  platform: process.platform,
+
+  available() {
+    if (isWin) return { ok: true, backend: 'windows', detail: 'PowerShell + user32' };
+    if (isMac) {
+      const mouse = which('cliclick');
+      return {
+        ok: true,
+        backend: 'macos',
+        detail: mouse ? 'osascript + cliclick' : 'osascript (mouse needs cliclick: brew install cliclick)',
+        mouse: Boolean(mouse),
+      };
+    }
+    if (isLinux) {
+      const xdotool = which('xdotool');
+      if (!xdotool) {
+        return { ok: false, backend: 'linux', error: 'xdotool not installed: sudo apt install xdotool (or dnf/pacman)' };
+      }
+      return { ok: true, backend: 'linux', detail: 'xdotool' };
+    }
+    return { ok: false, backend: null, error: `unsupported platform: ${process.platform}` };
+  },
+
+  screenSize() {
+    if (isWin) {
+      const r = winRun({ action: 'screen' });
+      return r.ok ? { ok: true, width: r.width, height: r.height } : fail(r.error);
+    }
+    if (isMac) {
+      const s = macScreen();
+      return { ok: true, ...s };
+    }
+    const s = linuxScreen();
+    return { ok: true, ...s };
+  },
+
+  windows() {
+    if (isWin) {
+      const r = winRun({ action: 'windows' }, 45000);
+      if (!r.ok) return fail(r.error);
+      return { ok: true, windows: r.windows || [] };
+    }
+    const list = isMac ? macWindows() : linuxWindows();
+    return { ok: true, windows: list };
+  },
+
+  activeWindow() {
+    if (isWin) {
+      const r = winRun({ action: 'active' });
+      return r.ok ? { ok: true, name: r.name, handle: r.handle } : fail(r.error);
+    }
+    const w = this.windows();
+    const first = w.ok && w.windows.length ? w.windows[0] : null;
+    return { ok: true, name: first ? first.name : '', handle: first ? first.handle : null };
+  },
+
+  /** Finds the window the user means, the way windows-use does: best fuzzy match. */
+  findWindow(name) {
+    const list = this.windows();
+    if (!list.ok) return list;
+    const hit = pickWindow(list.windows, name);
+    if (!hit) return { ok: false, error: `window not found: ${name}` };
+    return { ok: true, window: hit };
+  },
+
+  launchApp(name) {
+    if (isWin) {
+      const r = winRun({ action: 'launch', name }, 45000);
+      return r.ok ? { ok: true, app: name } : fail(r.error);
+    }
+    if (isMac) {
+      const res = run('open', ['-a', name], { timeout: 30000 });
+      return res.ok ? { ok: true, app: name } : fail(res.stderr.trim() || `open -a ${name} failed`);
+    }
+    const direct = run('sh', ['-c', `command -v ${JSON.stringify(name)} >/dev/null && setsid ${JSON.stringify(name)} >/dev/null 2>&1 &`]);
+    if (direct.ok) return { ok: true, app: name };
+    const app = run('gtk-launch', [name]);
+    if (app.ok) return { ok: true, app: name };
+    return fail(`app not found: ${name}`);
+  },
+
+  focusWindow(name) {
+    if (isWin) {
+      const found = this.findWindow(name);
+      if (!found.ok) return found;
+      const r = winRun({ action: 'focus', handle: found.window.handle });
+      return r.ok ? { ok: true, name: r.name, handle: found.window.handle, matched: found.window.name } : fail(r.error);
+    }
+    if (isMac) {
+      const res = osa(`tell application "System Events" to set frontmost of process ${JSON.stringify(name)} to true`);
+      return res.ok ? { ok: true, name } : fail(res.error);
+    }
+    const res = run('wmctrl', ['-a', name]);
+    return res.ok ? { ok: true, name } : fail(res.stderr.trim() || `window not found: ${name}`);
+  },
+
+  resizeWindow(name, x, y, width, height) {
+    if (isWin) {
+      const found = this.findWindow(name);
+      if (!found.ok) return found;
+      const r = winRun({
+        action: 'resize',
+        handle: found.window.handle,
+        x,
+        y,
+        width,
+        height,
+      });
+      return r.ok ? { ok: true, name: r.name, x, y, width, height } : fail(r.error);
+    }
+    if (isMac) {
+      const res = osa(
+        `tell application "System Events" to tell process ${JSON.stringify(name)} to set {position, size} of window 1 to {{${x}, ${y}}, {${width}, ${height}}}`
+      );
+      return res.ok ? { ok: true, name, x, y, width, height } : fail(res.error);
+    }
+    const res = run('wmctrl', ['-r', name, '-e', `0,${x},${y},${width},${height}`]);
+    return res.ok ? { ok: true, name, x, y, width, height } : fail(res.stderr.trim() || 'resize failed');
+  },
+
+  /** The UI Automation tree of the focused window: what the desktop's windows-use returns. */
+  state(handle = null) {
+    if (isWin) {
+      const r = winRun({ action: 'state', handle }, 60000);
+      if (!r.ok) return fail(r.error);
+      return {
+        ok: true,
+        windows: r.windows || [],
+        active_window: r.active_window || null,
+        interactive_elements: r.interactive_elements || [],
+        scrollable_elements: r.scrollable_elements || [],
+        interactive_elements_text: r.interactive_elements_text || '',
+        scrollable_elements_text: r.scrollable_elements_text || '',
+        system: r.system || {},
+        metadata: r.metadata || {},
+      };
+    }
+    // no accessibility tree on the other platforms: windows, geometry and pixels
+    const wins = this.windows();
+    const screen = this.screenSize();
+    const active = wins.ok && wins.windows.length ? wins.windows[0] : null;
+    const elements = (wins.ok ? wins.windows : []).map((w) => ({
+      name: w.name,
+      control_type: 'Window',
+      kind: 'Window',
+      bounding_box: { left: w.x, top: w.y, right: w.x + w.width, bottom: w.y + w.height },
+      center: { x: w.x + Math.floor(w.width / 2), y: w.y + Math.floor(w.height / 2) },
+      value: '',
+      enabled: true,
+      automation_id: '',
+    }));
+    return {
+      ok: true,
+      windows: wins.ok ? wins.windows : [],
+      active_window: active,
+      interactive_elements: elements,
+      scrollable_elements: [],
+      interactive_elements_text: elements
+        .map((e, i) => `${i + 1}. ${e.control_type} '${e.name}' at (${e.center.x}, ${e.center.y})`)
+        .join('\n'),
+      scrollable_elements_text: '',
+      system: this.sysinfo().system,
+      metadata: { kind: 'windows-only-list', depth: 'top-level', all_desktops: false },
+    };
+  },
+
+  click(x, y, { button = 'left', count = 1 } = {}) {
+    if (isWin) {
+      const r = winRun({ action: 'click', x, y, button, count });
+      return r.ok ? { ok: true, x, y, button, count } : fail(r.error);
+    }
+    if (isMac) {
+      if (!which('cliclick')) return fail('mouse needs cliclick: brew install cliclick');
+      const arg = `${button === 'right' ? 'rc' : 'c'}:${x},${y}`;
+      const rounds = Math.max(1, count);
+      for (let i = 0; i < rounds; i += 1) {
+        const res = run('cliclick', [arg]);
+        if (!res.ok) return fail(res.stderr.trim() || 'cliclick failed');
+      }
+      return { ok: true, x, y, button, count };
+    }
+    const btn = button === 'right' ? '3' : button === 'middle' ? '2' : '1';
+    const res = run('xdotool', ['mousemove', String(x), String(y), 'click', '--repeat', String(Math.max(1, count)), btn]);
+    return res.ok ? { ok: true, x, y, button, count } : fail(res.stderr.trim() || 'xdotool click failed');
+  },
+
+  moveMouse(x, y) {
+    if (isWin) {
+      const r = winRun({ action: 'move', x, y });
+      return r.ok ? { ok: true, x, y } : fail(r.error);
+    }
+    if (isMac) {
+      if (!which('cliclick')) return fail('mouse needs cliclick: brew install cliclick');
+      const res = run('cliclick', [`m:${x},${y}`]);
+      return res.ok ? { ok: true, x, y } : fail(res.stderr.trim() || 'cliclick failed');
+    }
+    const res = run('xdotool', ['mousemove', String(x), String(y)]);
+    return res.ok ? { ok: true, x, y } : fail(res.stderr.trim() || 'xdotool failed');
+  },
+
+  mouseButton(x, y, button, down) {
+    if (isWin) {
+      const r = winRun({ action: down ? 'down' : 'up', x, y, button });
+      return r.ok ? { ok: true, x, y, button } : fail(r.error);
+    }
+    if (isMac) {
+      if (!which('cliclick')) return fail('mouse needs cliclick: brew install cliclick');
+      const arg = `dd:${x},${y}`;
+      if (down) return run('cliclick', [arg]).ok ? { ok: true } : fail('cliclick failed');
+      return run('cliclick', [`du:${x},${y}`]).ok ? { ok: true } : fail('cliclick failed');
+    }
+    const btn = button === 'right' ? '3' : button === 'middle' ? '2' : '1';
+    const res = run('xdotool', [down ? 'mousedown' : 'mouseup', btn]);
+    return res.ok ? { ok: true } : fail(res.stderr.trim() || 'xdotool failed');
+  },
+
+  drag(x, y, x2, y2) {
+    if (isWin) {
+      const r = winRun({ action: 'drag', x, y, x2, y2 });
+      return r.ok ? { ok: true, from: [x, y], to: [x2, y2] } : fail(r.error);
+    }
+    if (isMac) {
+      if (!which('cliclick')) return fail('mouse needs cliclick: brew install cliclick');
+      const res = run('cliclick', [`dd:${x},${y}`, `dm:${x2},${y2}`, `du:${x2},${y2}`]);
+      return res.ok ? { ok: true, from: [x, y], to: [x2, y2] } : fail(res.stderr.trim() || 'cliclick failed');
+    }
+    const res = run('xdotool', ['mousemove', String(x), String(y), 'mousedown', '1', 'mousemove', String(x2), String(y2), 'mouseup', '1']);
+    return res.ok ? { ok: true, from: [x, y], to: [x2, y2] } : fail(res.stderr.trim() || 'xdotool failed');
+  },
+
+  scroll(amount, x = null, y = null) {
+    const clicks = Math.trunc(amount) || 1;
+    if (isWin) {
+      const r = winRun({ action: 'scroll', amount: clicks, x, y });
+      return r.ok ? { ok: true, amount: clicks } : fail(r.error);
+    }
+    const button = clicks > 0 ? '4' : '5';
+    const args = [];
+    if (x !== null && y !== null) args.push('mousemove', String(x), String(y));
+    args.push('click', '--repeat', String(Math.abs(clicks)), button);
+    if (isMac) {
+      if (!which('cliclick')) return fail('mouse needs cliclick: brew install cliclick');
+      const res = run('cliclick', [`w:${clicks > 0 ? 'up' : 'down'}`]);
+      return res.ok ? { ok: true, amount: clicks } : fail(res.stderr.trim());
+    }
+    const res = run('xdotool', args);
+    return res.ok ? { ok: true, amount: clicks } : fail(res.stderr.trim() || 'xdotool failed');
+  },
+
+  typeText(text, x = null, y = null) {
+    if (isWin) {
+      const r = winRun({ action: 'type', text, x, y }, 60000);
+      return r.ok ? { ok: true, chars: text.length, typed: r.typed ?? text.length } : fail(r.error);
+    }
+    if (x !== null && y !== null) this.click(x, y);
+    if (isMac) {
+      const res = osa(`tell application "System Events" to keystroke ${JSON.stringify(text)}`);
+      return res.ok ? { ok: true, chars: text.length, typed: text.length } : fail(res.error);
+    }
+    const res = run('xdotool', ['type', '--clearmodifiers', '--delay', '12', text], { timeout: 120000 });
+    return res.ok ? { ok: true, chars: text.length, typed: text.length } : fail(res.stderr.trim() || 'xdotool type failed');
+  },
+
+  pressKeys(combo) {
+    const sequence = keyboardSequence(combo);
+    if (!sequence.length) return fail(`unreadable key combination: ${combo}`);
+    if (isWin) {
+      const r = winRun({ action: 'keys', sequence });
+      return r.ok ? { ok: true, keys: combo } : fail(r.error);
+    }
+    if (isMac) {
+      const res = macKeys(combo);
+      return res.ok ? { ok: true, keys: combo } : fail(res.error);
+    }
+    const mapped = String(combo)
+      .toLowerCase()
+      .split('+')
+      .map((p) => (p === 'ctrl' ? 'ctrl' : p))
+      .join('+');
+    const res = run('xdotool', ['key', '--clearmodifiers', mapped]);
+    return res.ok ? { ok: true, keys: combo } : fail(res.stderr.trim() || 'xdotool key failed');
+  },
+
+  screenshot(dest, { annotate = false } = {}) {
+    fs.mkdirSync(path.dirname(dest), { recursive: true });
+    if (isWin) {
+      const r = winRun({ action: 'shot', path: dest, annotate }, 90000);
+      return r.ok
+        ? { ok: true, file: dest, width: r.width, height: r.height, mime: 'image/png', annotated: Boolean(annotate), elements: r.elements || 0 }
+        : fail(r.error);
+    }
+    if (isMac) {
+      const res = run('screencapture', ['-x', '-t', 'png', dest], { timeout: 30000 });
+      if (!res.ok) return fail(res.stderr.trim() || 'screencapture failed');
+      const size = this.screenSize();
+      return { ok: true, file: dest, width: size.width, height: size.height, mime: 'image/png' };
+    }
+    for (const [tool, args] of IMAGE_TOOLS) {
+      if (!which(tool)) continue;
+      const res = run(tool, args(dest), { timeout: 30000 });
+      if (res.ok && fs.existsSync(dest)) {
+        const size = this.screenSize();
+        return { ok: true, file: dest, width: size.width, height: size.height, mime: 'image/png' };
+      }
+    }
+    return fail('no screenshot tool found: install imagemagick (import), scrot, grim or gnome-screenshot');
+  },
+
+  listApps() {
+    if (isWin) {
+      const r = winRun({ action: 'apps' }, 60000);
+      return r.ok ? { ok: true, apps: r.apps || [] } : fail(r.error);
+    }
+    const out = [];
+    if (isMac) {
+      for (const dir of ['/Applications', '/System/Applications']) {
+        try {
+          for (const entry of fs.readdirSync(dir)) {
+            if (entry.endsWith('.app')) out.push({ name: entry.replace(/\.app$/, ''), app_id: path.join(dir, entry) });
+          }
+        } catch {
+          /* ignore */
+        }
+      }
+      return { ok: true, apps: out };
+    }
+    const dirs = ['/usr/share/applications', '/usr/local/share/applications', path.join(os.homedir(), '.local/share/applications')];
+    for (const dir of dirs) {
+      let files = [];
+      try {
+        files = fs.readdirSync(dir).filter((f) => f.endsWith('.desktop'));
+      } catch {
+        continue;
+      }
+      for (const f of files) {
+        try {
+          const text = fs.readFileSync(path.join(dir, f), 'utf8');
+          if (/^NoDisplay=true/m.test(text)) continue;
+          const name = /^Name=(.+)$/m.exec(text);
+          if (name) out.push({ name: name[1].trim(), app_id: f.replace(/\.desktop$/, '') });
+        } catch {
+          /* ignore */
+        }
+      }
+    }
+    return { ok: true, apps: out };
+  },
+
+  sysinfo() {
+    const screen = this.screenSize();
+    return {
+      ok: true,
+      system: {
+        windows_version: isWin ? os.release() : `${process.platform} ${os.release()}`,
+        os_version: os.release(),
+        screen_size: { width: screen.width, height: screen.height },
+        user_account_type: 'Local Account',
+        default_language: process.env.LANG || 'en',
+        dpi_scaling: 1,
+      },
+    };
+  },
+};
+
+module.exports = { ...automation, keyboardSequence, fuzzyScore, pickWindow };

package/src/local.js

@@ -1,6 +1,8 @@
 'use strict';
 
 const { decide } = require('./approval');
+const automation = require('./automation');
+const { stamp } = require('./clipboard');
 const session = require('./session');
 
 const fs = require('node:fs');
@@ -501,17 +503,161 @@ async function handleLocalUi(bridge, taskId, request, opts) {
   }
 
   // control / type / keystroke / screenshot / app / state / apps / multi_*
-  const error = t(
-    lang,
-    `operazione "${op}" non disponibile nel client da terminale: serve il client grafico ArenaCode`,
-    `operation "${op}" is not available in the terminal client: the ArenaCode desktop client is required`,
-    `la operación "${op}" no está disponible en el cliente de terminal: requiere el cliente gráfico ArenaCode`,
-  );
-  await bridge.localResult(taskId, { rid, output: JSON.stringify({ ok: false, error }), code: 1 });
-  write(`  ${c.red('✗')} ${c.gray(op)} ${c.gray('— ' + error)}`);
+  const result = await runUiAction(bridge, taskId, request, opts);
+  const ok = result && result.ok !== false;
+  await bridge.localResult(taskId, {
+    rid,
+    output: JSON.stringify(result),
+    code: ok ? 0 : 1,
+  });
+  const line = ok
+    ? `${c.green('✓')} ${c.gray(op)}${request.action ? c.gray(` ${request.action}`) : ''}`
+    : `${c.red('✗')} ${c.gray(op)} ${c.gray('— ' + (result && result.error ? result.error : 'failed'))}`;
+  write(`  ${line}`);
   return true;
 }
 
+/** Computer Agent actions: mouse, keyboard, screen, windows, apps. */
+async function runUiAction(bridge, taskId, request, opts) {
+  const { op, rid } = request;
+  const lang = opts.lang;
+  const toolkit = automation.available();
+  const json = (text) => {
+    try {
+      return JSON.parse(text || '{}');
+    } catch {
+      return {};
+    }
+  };
+
+  if (op === 'state') {
+    // the desktop's windows-use returns the UI Automation tree; so do we on Windows
+    const state = automation.state();
+    if (!state.ok) return state;
+    return state;
+  }
+
+  if (op === 'sysinfo') {
+    const info = automation.sysinfo();
+    return {
+      ...info,
+      os: `${process.platform} ${os.release()}`,
+      hostname: os.hostname(),
+      arch: os.arch(),
+      cpus: os.cpus().length,
+      memory_mb: Math.round(os.totalmem() / 1048576),
+      node: process.version,
+    };
+  }
+
+  if (op === 'apps') return automation.listApps();
+
+  if (!toolkit.ok) {
+    return {
+      ok: false,
+      error:
+        toolkit.error ||
+        t(lang, 'automazione non disponibile su questo sistema', 'automation is not available on this system', 'la automatización no está disponible en este sistema'),
+    };
+  }
+
+  if (op === 'control') {
+    const { action, x, y, x2, y2, amount, button } = request;
+    switch (action) {
+      case 'move':
+        return automation.moveMouse(x, y);
+      case 'click':
+        return automation.click(x, y, { button: button || 'left', count: 1 });
+      case 'dblclick':
+        return automation.click(x, y, { button: button || 'left', count: 2 });
+      case 'rightclick':
+        return automation.click(x, y, { button: 'right', count: 1 });
+      case 'down':
+        return automation.mouseButton(x, y, button || 'left', true);
+      case 'up':
+        return automation.mouseButton(x, y, button || 'left', false);
+      case 'drag':
+        return automation.drag(x, y, x2, y2);
+      case 'scroll':
+        return automation.scroll(amount ?? 1, x ?? null, y ?? null);
+      default:
+        return { ok: false, error: `unknown control action: ${action}` };
+    }
+  }
+
+  if (op === 'keystroke') {
+    return automation.pressKeys(request.keys || '');
+  }
+
+  if (op === 'type') {
+    let text = request.text;
+    if (request.payload || text === undefined) {
+      const res = await bridge.uiPayload(taskId, rid);
+      text = res.status === 200 ? (res.json && (res.json.text ?? res.json.payload ?? '')) : '';
+      if (!text) return { ok: false, error: 'the typed text could not be read from the bridge' };
+    }
+    return automation.typeText(String(text), request.x ?? null, request.y ?? null);
+  }
+
+  if (op === 'app') {
+    const name = request.text || request.name || '';
+    const action = request.action || 'launch';
+    if (action === 'launch') return automation.launchApp(name);
+    if (action === 'switch') return automation.focusWindow(name);
+    if (action === 'resize') {
+      return automation.resizeWindow(name, request.x ?? 0, request.y ?? 0, request.x2 ?? 800, request.y2 ?? 600);
+    }
+    return { ok: false, error: `unknown app action: ${action}` };
+  }
+
+  if (op === 'screenshot') {
+    const dir = path.join(os.tmpdir(), 'arenacode-shots');
+    fs.mkdirSync(dir, { recursive: true });
+    const file = path.join(dir, `shot-${stamp()}.png`);
+    const shot = automation.screenshot(file, { annotate: Boolean(request.annotated) });
+    if (!shot.ok) return shot;
+    const buf = fs.readFileSync(shot.file);
+    // the agent runs in the sandbox: the image travels through the bridge workspace
+    const up = await bridge.upload(`uploads/${stamp()}/shot-${stamp()}.png`, buf);
+    const remote = up.status === 200 ? up.json.path : null;
+    const out = {
+      ok: true,
+      mime: 'image/png',
+      width: shot.width,
+      height: shot.height,
+      file: remote || shot.file,
+      local_file: shot.file,
+      annotated: Boolean(request.annotated),
+    };
+    if (buf.length <= 2 * 1024 * 1024) out.image = buf.toString('base64');
+    return out;
+  }
+
+  if (op === 'multi_select') {
+    const { elements = [], press_ctrl = true } = json(request.args_json);
+    const modifier = press_ctrl ? automation.pressKeys('ctrl') : null;
+    if (modifier && !modifier.ok) return modifier;
+    for (const [x, y] of elements) {
+      const r = automation.click(x, y, { button: 'left', count: 1 });
+      if (!r.ok) return r;
+    }
+    return { ok: true, selected: elements.length };
+  }
+
+  if (op === 'multi_edit') {
+    const { elements = [] } = json(request.args_json);
+    let typed = 0;
+    for (const [x, y, text] of elements) {
+      const r = automation.typeText(String(text), x, y);
+      if (!r.ok) return r;
+      typed += 1;
+    }
+    return { ok: true, fields: typed };
+  }
+
+  return { ok: false, error: `unknown ui operation: ${op}` };
+}
+
 function available() {
   return os.release();
 }
@@ -536,6 +682,7 @@ async function handleLocalFile(bridge, taskId, request, opts) {
 }
 
 module.exports = {
+  runUiAction,
   REFUSED,
   resolveInRoot,
   runShell,

package/src/tui/app.js

@@ -1048,10 +1048,17 @@ class App {
         item.state = 'done';
         return;
       }
-      const error = `operation "${op}" is not available in the terminal client: the ArenaCode desktop client is required`;
-      await this.bridge.localResult(taskId, { rid: payload.rid, output: JSON.stringify({ ok: false, error }), code: 1 });
-      item.state = 'failed';
-      item.meta = 'needs the desktop client';
+      // mouse, keyboard, screen, windows and apps: the automation backends
+      const { runUiAction } = require('../local');
+      const result = await runUiAction(this.bridge, taskId, payload, { lang: this.lang });
+      const ok = result && result.ok !== false;
+      await this.bridge.localResult(taskId, {
+        rid: payload.rid,
+        output: JSON.stringify(result),
+        code: ok ? 0 : 1,
+      });
+      item.state = ok ? 'done' : 'failed';
+      if (!ok && result && result.error) item.detail = String(result.error);
     }
   }
 
@@ -1276,13 +1283,14 @@ class App {
         })
         .map((line) => R.spaces(2) + line);
 
-      const bodyH = Math.max(1, H - prompt.lines.length - 2);
+      const bodyH = Math.max(1, H - prompt.lines.length - 3);
       const maxScroll = Math.max(0, chat.length - bodyH);
       st.scroll = Math.min(Math.max(0, st.scroll), maxScroll);
       const start = Math.max(0, chat.length - bodyH - st.scroll);
       const visible = chat.slice(start, start + bodyH);
       while (visible.length < bodyH) visible.unshift('');
       frame.push(...visible);
+      frame.push('');
       const promptStart = frame.length;
       frame.push(...prompt.lines);
       frame.push('');
@@ -1308,7 +1316,11 @@ class App {
     this.term.draw(frame);
 
     if (!cursor || st.overlay || st.approval || st.prompt) this.term.hideCursor();
-    else this.term.placeCursor(cursor.row, Math.min(W - 1, Math.max(0, cursor.col)));
+    else {
+      // frame rows/cols are 0-based, the terminal's CUP escape is 1-based
+      const col = Math.min(W - 1, Math.max(0, cursor.col));
+      this.term.placeCursor(cursor.row + 1, col + 1);
+    }
 
     this.rendering = false;
     if (this.dirty) setTimeout(() => this.maybeRender(), 16);