@lardexx/arenacode 2.6.2 → 2.7.0

package/README.md

@@ -134,6 +134,26 @@ so the port reproduces it byte for byte:
 
 The exact strings are checked by `test/parity.js` against the ones extracted from `ArenaCode-x64.exe`.
 
+**Computer Agent: mouse, keyboard and screen.** The three modes' `ui` commands now run in the terminal client too,
+from the same bridge request the desktop app answers:
+
+| Command | What it does here |
+|---|---|
+| `ui control` | `move`, `click`, `dblclick`, `rightclick`, `down`, `up`, `drag` (smooth, 12 steps), `scroll` |
+| `ui type` | clicks the field first when the task gives `x,y`, otherwise types into the focused field; the text is pasted, so accents and emoji survive |
+| `ui keys` | `ctrl+s`, `alt+f4`, `win`, `enter`, `f5`, `ctrl+shift+esc` … |
+| `ui app` | `launch` (Start menu on Windows, `open -a` on macOS, `gtk-launch` on Linux), `switch`, `resize` |
+| `ui shot` | full-screen PNG, uploaded to the bridge workspace and returned as `file` (+ `image` base64 when small) |
+| `ui state` | windows with position, size and process id, plus screen and system |
+| `ui apps` | the Start menu (Windows), `/Applications` (macOS), `.desktop` entries (Linux) |
+| `ui multi-select`, `ui multi-edit` | built from the primitives above |
+
+No extra install on **Windows** (PowerShell + `user32.dll`). **macOS**: grant Accessibility to your terminal, and
+`brew install cliclick` for the mouse. **Linux** (X11): `sudo apt install xdotool wmctrl imagemagick`; on Wayland
+`xdotool` cannot inject input, so Computer Agent needs XWayland. What this client does *not* have is the desktop's UI
+Automation tree: `ui state` returns windows and pixels, with `interactive_elements` empty, and `--annotated`
+screenshots are plain screenshots.
+
 **Ask and Proceed / Proceed.** The desktop's Settings switch, `ctrl+x a` here (also `/ask`, `/proceed`,
 `--proceed`, and `arenacode config set approval proceed`). The default, **Ask and Proceed**, asks before commands,
 file changes and plans; in Computer mode it asks once, on the plan, and after that the commands run on their own

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lardexx/arenacode",
-  "version": "2.6.2",
+  "version": "2.7.0",
   "description": "Terminal client for ArenaCode: talks to an arena-bridge, streams the Agent Mode timeline, and runs local commands and file operations on your PC. Node port of the ArenaCode desktop client.",
   "license": "MIT",
   "type": "commonjs",
@@ -19,7 +19,7 @@
     "LICENSE"
   ],
   "scripts": {
-    "test": "node test/integration.js && node test/parity.js"
+    "test": "node test/integration.js && node test/parity.js && node test/automation.js"
   },
   "keywords": [
     "arenacode",

package/src/automation.js

@@ -0,0 +1,726 @@
+'use strict';
+
+// Computer Agent automation for the terminal client: mouse, keyboard, screen,
+// windows and Start-menu apps. One small interface, three backends:
+//
+//   Windows  PowerShell + user32 (no extra install)
+//   macOS    osascript (System Events) + screencapture, cliclick for the mouse
+//   Linux    xdotool + ImageMagick/scrot/grim for screenshots
+//
+// Every function resolves to plain JSON, the same shapes the desktop client
+// returns to the bridge: { ok: true, ... } or { ok: false, error }.
+
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+const { spawnSync } = require('node:child_process');
+
+const isWin = process.platform === 'win32';
+const isMac = process.platform === 'darwin';
+const isLinux = process.platform === 'linux';
+
+function fail(error, extra = {}) {
+  return { ok: false, error, ...extra };
+}
+
+function which(cmd) {
+  const probe = isWin ? 'where' : 'which';
+  const res = spawnSync(probe, [cmd], { encoding: 'utf8', windowsHide: true });
+  return res.status === 0 && String(res.stdout).trim() !== '';
+}
+
+function run(cmd, args, opts = {}) {
+  const res = spawnSync(cmd, args, {
+    input: opts.input,
+    encoding: 'utf8',
+    timeout: opts.timeout || 30000,
+    maxBuffer: 64 * 1024 * 1024,
+    windowsHide: true,
+  });
+  return {
+    ok: res.status === 0,
+    status: res.status,
+    stdout: res.stdout || '',
+    stderr: res.stderr || '',
+    error: res.error ? res.error.message : '',
+  };
+}
+
+// --------------------------------------------------------------------------- //
+// Windows
+// --------------------------------------------------------------------------- //
+
+const PS_ENGINE = String.raw`
+$ErrorActionPreference = 'Stop'
+$req = ([Console]::In.ReadToEnd() | ConvertFrom-Json)
+Add-Type -AssemblyName System.Drawing
+Add-Type -AssemblyName System.Windows.Forms
+Add-Type -TypeDefinition @'
+using System;
+using System.Text;
+using System.Runtime.InteropServices;
+public class AC {
+  [DllImport("user32.dll")] public static extern bool SetCursorPos(int x, int y);
+  [DllImport("user32.dll")] public static extern void mouse_event(uint flags, int dx, int dy, int data, UIntPtr extra);
+  [DllImport("user32.dll")] public static extern void keybd_event(byte vk, byte scan, uint flags, UIntPtr extra);
+  [DllImport("user32.dll")] public static extern IntPtr GetForegroundWindow();
+  [DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr h);
+  [DllImport("user32.dll")] public static extern bool ShowWindow(IntPtr h, int cmd);
+  [DllImport("user32.dll")] public static extern bool MoveWindow(IntPtr h, int x, int y, int w, int ht, bool repaint);
+  [DllImport("user32.dll")] public static extern bool EnumWindows(EnumProc cb, IntPtr p);
+  [DllImport("user32.dll")] public static extern bool IsWindowVisible(IntPtr h);
+  [DllImport("user32.dll")] public static extern int GetWindowText(IntPtr h, StringBuilder s, int n);
+  [DllImport("user32.dll")] public static extern uint GetWindowThreadProcessId(IntPtr h, out uint pid);
+  [DllImport("user32.dll")] public static extern bool GetWindowRect(IntPtr h, out RECT r);
+  [DllImport("user32.dll")] public static extern bool IsIconic(IntPtr h);
+  public delegate bool EnumProc(IntPtr h, IntPtr p);
+  [StructLayout(LayoutKind.Sequential)] public struct RECT { public int Left, Top, Right, Bottom; }
+}
+'@
+
+function Get-ACWindows {
+  $list = New-Object System.Collections.ArrayList
+  foreach ($p in (Get-Process | Where-Object { $_.MainWindowHandle -ne 0 })) {
+    try {
+      $h = [IntPtr]$p.MainWindowHandle
+      $r = New-Object AC+RECT
+      [void][AC]::GetWindowRect($h, [ref]$r)
+      $title = [string]$p.MainWindowTitle
+      if ($title.Trim().Length -eq 0) { continue }
+      [void]$list.Add([pscustomobject]@{
+        name = $title
+        app = [string]$p.ProcessName
+        handle = $h.ToInt64()
+        x = $r.Left; y = $r.Top
+        width = [Math]::Max(0, $r.Right - $r.Left); height = [Math]::Max(0, $r.Bottom - $r.Top)
+        process_id = [int]$p.Id
+        status = $(if ($p.MainWindowHandle -eq 0) { 'hidden' } else { 'normal' })
+      })
+    } catch { }
+  }
+  return $list
+}
+
+function Find-ACWindow($name) {
+  if (-not $name) { return $null }
+  $all = Get-ACWindows
+  $exact = $all | Where-Object { $_.name -eq $name } | Select-Object -First 1
+  if ($exact) { return $exact }
+  return ($all | Where-Object { $_.name -like "*$name*" } | Select-Object -First 1)
+}
+
+function Get-ACScreen {
+  $b = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds
+  return @{ width = $b.Width; height = $b.Height }
+}
+
+function Send-ACKeys($pairs) {
+  foreach ($k in $pairs) {
+    if ($k.flags -eq 0) { [AC]::keybd_event([byte]$k.vk, 0, 0, [UIntPtr]::Zero) }
+    else { [AC]::keybd_event([byte]$k.vk, 0, 2, [UIntPtr]::Zero) }
+    Start-Sleep -Milliseconds 12
+  }
+}
+
+$out = @{ ok = $true }
+switch ($req.action) {
+  'screen' { $s = Get-ACScreen; $out.width = $s.width; $out.height = $s.height }
+  'windows' { $out.windows = @(Get-ACWindows); $out.active_window = (Get-ACWindows | Select-Object -First 1) }
+  'active' {
+    $h = [AC]::GetForegroundWindow()
+    $sb = New-Object System.Text.StringBuilder 1024
+    [void][AC]::GetWindowText($h, $sb, 1024)
+    $out.name = $sb.ToString()
+  }
+  'focus' {
+    $w = Find-ACWindow $req.name
+    if ($w) {
+      [void][AC]::ShowWindow([IntPtr]$w.handle, 9)
+      [void][AC]::SetForegroundWindow([IntPtr]$w.handle)
+      Start-Sleep -Milliseconds 150
+      $out.name = $w.name
+    } else { $out = @{ ok = $false; error = "window not found: $($req.name)" } }
+  }
+  'resize' {
+    $w = Find-ACWindow $req.name
+    if ($w) {
+      [void][AC]::MoveWindow([IntPtr]$w.handle, [int]$req.x, [int]$req.y, [int]$req.width, [int]$req.height, $true)
+      $out.name = $w.name
+    } else { $out = @{ ok = $false; error = "window not found: $($req.name)" } }
+  }
+  'launch' {
+    $name = [string]$req.name
+    $done = $false
+    try { Start-Process -FilePath $name -ErrorAction Stop; $done = $true } catch { $done = $false }
+    if (-not $done) {
+      $hit = $null
+      try { $hit = Get-StartApps | Where-Object { $_.Name -like "*$name*" } | Select-Object -First 1 } catch { $hit = $null }
+      if ($hit) { Start-Process 'explorer.exe' "shell:AppsFolder\$($hit.AppID)"; $done = $true }
+      else {
+        $exe = Get-Command $name -ErrorAction SilentlyContinue
+        if ($exe) { Start-Process $exe.Source; $done = $true }
+      }
+    }
+    if ($done) { Start-Sleep -Milliseconds 700; $out.name = $name } else { $out = @{ ok = $false; error = "app not found: $name" } }
+  }
+  'move' { [void][AC]::SetCursorPos([int]$req.x, [int]$req.y) }
+  'click' {
+    [void][AC]::SetCursorPos([int]$req.x, [int]$req.y)
+    Start-Sleep -Milliseconds 40
+    $down = 2; $up = 4
+    if ($req.button -eq 'right') { $down = 8; $up = 16 }
+    elseif ($req.button -eq 'middle') { $down = 32; $up = 64 }
+    $times = [int]$req.count; if ($times -lt 1) { $times = 1 }
+    for ($i = 0; $i -lt $times; $i++) {
+      [AC]::mouse_event($down, 0, 0, 0, [UIntPtr]::Zero)
+      Start-Sleep -Milliseconds 20
+      [AC]::mouse_event($up, 0, 0, 0, [UIntPtr]::Zero)
+      Start-Sleep -Milliseconds 60
+    }
+  }
+  'down' {
+    [void][AC]::SetCursorPos([int]$req.x, [int]$req.y)
+    $flag = 2; if ($req.button -eq 'right') { $flag = 8 } elseif ($req.button -eq 'middle') { $flag = 32 }
+    [AC]::mouse_event($flag, 0, 0, 0, [UIntPtr]::Zero)
+  }
+  'up' {
+    $flag = 4; if ($req.button -eq 'right') { $flag = 16 } elseif ($req.button -eq 'middle') { $flag = 64 }
+    if ($req.x -ne $null) { [void][AC]::SetCursorPos([int]$req.x, [int]$req.y) }
+    [AC]::mouse_event($flag, 0, 0, 0, [UIntPtr]::Zero)
+  }
+  'drag' {
+    [void][AC]::SetCursorPos([int]$req.x, [int]$req.y)
+    Start-Sleep -Milliseconds 60
+    [AC]::mouse_event(2, 0, 0, 0, [UIntPtr]::Zero)
+    $steps = 12
+    for ($i = 1; $i -le $steps; $i++) {
+      $nx = [int]($req.x + (($req.x2 - $req.x) * $i / $steps))
+      $ny = [int]($req.y + (($req.y2 - $req.y) * $i / $steps))
+      [void][AC]::SetCursorPos($nx, $ny)
+      Start-Sleep -Milliseconds 15
+    }
+    [AC]::mouse_event(4, 0, 0, 0, [UIntPtr]::Zero)
+  }
+  'scroll' {
+    if ($req.x -ne $null) { [void][AC]::SetCursorPos([int]$req.x, [int]$req.y); Start-Sleep -Milliseconds 40 }
+    $amount = [int]$req.amount
+    $dir = 120; if ($amount -lt 0) { $dir = -120 }
+    for ($i = 0; $i -lt [Math]::Abs($amount); $i++) {
+      [AC]::mouse_event(0x0800, 0, 0, $dir, [UIntPtr]::Zero)
+      Start-Sleep -Milliseconds 25
+    }
+  }
+  'keys' { Send-ACKeys $req.sequence }
+  'type' {
+    if ($req.x -ne $null) {
+      [void][AC]::SetCursorPos([int]$req.x, [int]$req.y)
+      Start-Sleep -Milliseconds 60
+      [AC]::mouse_event(2, 0, 0, 0, [UIntPtr]::Zero)
+      [AC]::mouse_event(4, 0, 0, 0, [UIntPtr]::Zero)
+      Start-Sleep -Milliseconds 120
+    }
+    Set-Clipboard -Value ([string]$req.text)
+    Start-Sleep -Milliseconds 80
+    [System.Windows.Forms.SendKeys]::SendWait('^v')
+    Start-Sleep -Milliseconds 150
+    $out.typed = ([string]$req.text).Length
+  }
+  'shot' {
+    $s = Get-ACScreen
+    $bmp = New-Object System.Drawing.Bitmap $s.width, $s.height
+    $g = [System.Drawing.Graphics]::FromImage($bmp)
+    $g.CopyFromScreen(0, 0, 0, 0, $bmp.Size)
+    $bmp.Save($req.path, [System.Drawing.Imaging.ImageFormat]::Png)
+    $g.Dispose(); $bmp.Dispose()
+    $out.width = $s.width; $out.height = $s.height; $out.file = $req.path
+  }
+  'apps' {
+    try { $out.apps = @(Get-StartApps | ForEach-Object { @{ name = $_.Name; app_id = $_.AppID } }) }
+    catch { $out.apps = @() }
+  }
+  'sysinfo' {
+    $out.system = @{
+      windows_version = [string](Get-CimInstance Win32_OperatingSystem).Caption
+      os_version = [Environment]::OSVersion.Version.ToString()
+      screen_size = (Get-ACScreen)
+      user_account_type = 'Local Account'
+      default_language = (Get-Culture).Name
+      dpi_scaling = 1
+    }
+  }
+  default { $out = @{ ok = $false; error = "unknown action: $($req.action)" } }
+}
+$out | ConvertTo-Json -Depth 6 -Compress
+`;
+
+let psEnginePath = null;
+function psEngine() {
+  if (psEnginePath && fs.existsSync(psEnginePath)) return psEnginePath;
+  const dir = path.join(os.tmpdir(), 'arenacode-automation');
+  fs.mkdirSync(dir, { recursive: true });
+  const file = path.join(dir, 'engine.ps1');
+  fs.writeFileSync(file, PS_ENGINE, 'utf8');
+  psEnginePath = file;
+  return file;
+}
+
+function winRun(request, timeout = 30000) {
+  const shell = which('powershell') ? 'powershell' : 'pwsh';
+  if (!which(shell)) return fail('PowerShell not found');
+  const res = run(
+    shell,
+    ['-NoProfile', '-NonInteractive', '-STA', '-ExecutionPolicy', 'Bypass', '-File', psEngine()],
+    { input: JSON.stringify(request), timeout }
+  );
+  if (!res.ok && !res.stdout.trim()) {
+    return fail(res.stderr.trim() || res.error || `PowerShell exit ${res.status}`);
+  }
+  try {
+    return JSON.parse(res.stdout.trim().split('\n').pop());
+  } catch {
+    return fail(`unreadable answer from PowerShell: ${res.stdout.slice(0, 200)}`);
+  }
+}
+
+// --------------------------------------------------------------------------- //
+// macOS
+// --------------------------------------------------------------------------- //
+
+function osa(script, timeout = 20000) {
+  const res = run('osascript', ['-e', script], { timeout });
+  return { ok: res.ok, out: res.stdout.trim(), error: res.stderr.trim() || res.error };
+}
+
+function macScreen() {
+  const res = osa('tell application "Finder" to get bounds of window of desktop');
+  const nums = String(res.out)
+    .split(',')
+    .map((n) => Number(n.trim()));
+  if (nums.length === 4 && nums.every((n) => Number.isFinite(n))) {
+    return { width: nums[2] - nums[0], height: nums[3] - nums[1] };
+  }
+  return { width: 1440, height: 900 };
+}
+
+function macWindows() {
+  const res = osa(
+    'tell application "System Events" to set out to ""\n' +
+      'repeat with p in (every process whose background only is false)\n' +
+      '  repeat with w in (every window of p)\n' +
+      '    set out to out & (name of p) & tab & (name of w) & tab & (item 1 of (position of w)) & tab & (item 2 of (position of w)) & tab & (item 1 of (size of w)) & tab & (item 2 of (size of w)) & linefeed\n' +
+      '  end repeat\n' +
+      'end repeat\n' +
+      'return out'
+  );
+  if (!res.ok) return [];
+  return String(res.out)
+    .split('\n')
+    .filter(Boolean)
+    .map((line) => {
+      const [app, title, x, y, w, h] = line.split('\t');
+      return {
+        name: title || app,
+        app,
+        x: Number(x) || 0,
+        y: Number(y) || 0,
+        width: Number(w) || 0,
+        height: Number(h) || 0,
+        process_id: 0,
+        status: 'normal',
+      };
+    });
+}
+
+function macKeys(sequence, text) {
+  // sequence comes as vk pairs; on macOS it is easier to send the original combo
+  const script = text
+    ? `tell application "System Events" to keystroke ${JSON.stringify(text)}`
+    : `tell application "System Events" to keystroke ${JSON.stringify(sequence)}`;
+  return osa(script);
+}
+
+// --------------------------------------------------------------------------- //
+// Linux
+// --------------------------------------------------------------------------- //
+
+function linuxScreen() {
+  const res = run('xdotool', ['getdisplaygeometry']);
+  if (res.ok) {
+    const [w, h] = res.stdout.trim().split(/\s+/).map(Number);
+    if (w && h) return { width: w, height: h };
+  }
+  const xr = run('xrandr', ['--current']);
+  const m = /current (\d+) x (\d+)/.exec(xr.stdout);
+  if (m) return { width: Number(m[1]), height: Number(m[2]) };
+  return { width: 1280, height: 800 };
+}
+
+function linuxWindows() {
+  const res = run('wmctrl', ['-lG']);
+  const out = [];
+  if (res.ok) {
+    for (const line of res.stdout.split('\n').filter(Boolean)) {
+      const parts = line.split(/\s+/);
+      const x = Number(parts[2]);
+      const y = Number(parts[3]);
+      const width = Number(parts[4]);
+      const height = Number(parts[5]);
+      const name = parts.slice(7).join(' ');
+      out.push({ name, x, y, width, height, process_id: 0, status: 'normal' });
+    }
+  }
+  return out;
+}
+
+// --------------------------------------------------------------------------- //
+// public interface
+// --------------------------------------------------------------------------- //
+
+const IMAGE_TOOLS = [
+  ['gnome-screenshot', (p) => ['-f', p]],
+  ['scrot', (p) => [p]],
+  ['import', (p) => ['-window', 'root', p]],
+  ['grim', (p) => [p]],
+  ['spectacle', (p) => ['-b', '-n', '-o', p]],
+];
+
+function keyboardSequence(combo) {
+  // "ctrl+shift+s" -> [{vk, flags}] using Windows virtual-key codes, which the
+  // PowerShell engine sends through keybd_event. Unknown names are ignored.
+  const VK = {
+    ctrl: 0x11, control: 0x11, alt: 0x12, shift: 0x10, win: 0x5b, meta: 0x5b, cmd: 0x5b,
+    enter: 0x0d, return: 0x0d, tab: 0x09, esc: 0x1b, escape: 0x1b, space: 0x20,
+    backspace: 0x08, delete: 0x2e, del: 0x2e, insert: 0x2d, home: 0x24, end: 0x23,
+    pageup: 0x21, pagedown: 0x22, pgup: 0x21, pgdn: 0x22, up: 0x26, down: 0x28,
+    left: 0x25, right: 0x27, printscreen: 0x2c, capslock: 0x14, menu: 0x5d,
+  };
+  for (let i = 1; i <= 24; i += 1) VK[`f${i}`] = 0x6f + i;
+  const parts = String(combo || '')
+    .toLowerCase()
+    .split('+')
+    .map((p) => p.trim())
+    .filter(Boolean);
+  if (!parts.length) return [];
+  const seq = [];
+  const repeat = parts.filter((p) => p === 'repeat').length;
+  const keys = parts.filter((p) => p !== 'repeat');
+  const modVks = [];
+  const mainVks = [];
+  for (const key of keys) {
+    if (VK[key]) {
+      if (['ctrl', 'control', 'alt', 'shift', 'win', 'meta', 'cmd'].includes(key)) modVks.push(VK[key]);
+      else mainVks.push(VK[key]);
+    } else if (key.length === 1) {
+      mainVks.push(key.toUpperCase().charCodeAt(0));
+    }
+  }
+  const times = Math.max(1, repeat || 1);
+  for (let i = 0; i < times; i += 1) {
+    if (!mainVks.length && modVks.length) {
+      // "win" on its own: press and release the modifier itself
+      const only = modVks[modVks.length - 1];
+      seq.push({ vk: only, flags: 0 }, { vk: only, flags: 2 });
+      continue;
+    }
+    for (const vk of modVks) seq.push({ vk, flags: 0 });
+    for (const vk of mainVks) {
+      seq.push({ vk, flags: 0 });
+      seq.push({ vk, flags: 2 });
+    }
+    for (const vk of [...modVks].reverse()) seq.push({ vk, flags: 2 });
+  }
+  return seq;
+}
+
+const automation = {
+  platform: process.platform,
+
+  available() {
+    if (isWin) return { ok: true, backend: 'windows', detail: 'PowerShell + user32' };
+    if (isMac) {
+      const mouse = which('cliclick');
+      return {
+        ok: true,
+        backend: 'macos',
+        detail: mouse ? 'osascript + cliclick' : 'osascript (mouse needs cliclick: brew install cliclick)',
+        mouse: Boolean(mouse),
+      };
+    }
+    if (isLinux) {
+      const xdotool = which('xdotool');
+      if (!xdotool) {
+        return { ok: false, backend: 'linux', error: 'xdotool not installed: sudo apt install xdotool (or dnf/pacman)' };
+      }
+      return { ok: true, backend: 'linux', detail: 'xdotool' };
+    }
+    return { ok: false, backend: null, error: `unsupported platform: ${process.platform}` };
+  },
+
+  screenSize() {
+    if (isWin) {
+      const r = winRun({ action: 'screen' });
+      return r.ok ? { ok: true, width: r.width, height: r.height } : fail(r.error);
+    }
+    if (isMac) {
+      const s = macScreen();
+      return { ok: true, ...s };
+    }
+    const s = linuxScreen();
+    return { ok: true, ...s };
+  },
+
+  windows() {
+    if (isWin) {
+      const r = winRun({ action: 'windows' });
+      return r.ok ? { ok: true, windows: r.windows || [], active_window: r.active_window || null } : fail(r.error);
+    }
+    const list = isMac ? macWindows() : linuxWindows();
+    return { ok: true, windows: list, active_window: list[0] || null };
+  },
+
+  activeWindow() {
+    const w = this.windows();
+    return w.ok ? { ok: true, name: (w.active_window && w.active_window.name) || '' } : w;
+  },
+
+  launchApp(name) {
+    if (isWin) {
+      const r = winRun({ action: 'launch', name }, 45000);
+      return r.ok ? { ok: true, app: name } : fail(r.error);
+    }
+    if (isMac) {
+      const res = run('open', ['-a', name], { timeout: 30000 });
+      return res.ok ? { ok: true, app: name } : fail(res.stderr.trim() || `open -a ${name} failed`);
+    }
+    const direct = run('sh', ['-c', `command -v ${JSON.stringify(name)} >/dev/null && setsid ${JSON.stringify(name)} >/dev/null 2>&1 &`]);
+    if (direct.ok) return { ok: true, app: name };
+    const app = run('gtk-launch', [name]);
+    if (app.ok) return { ok: true, app: name };
+    return fail(`app not found: ${name}`);
+  },
+
+  focusWindow(name) {
+    if (isWin) {
+      const r = winRun({ action: 'focus', name });
+      return r.ok ? { ok: true, name: r.name, app: r.name } : fail(r.error);
+    }
+    if (isMac) {
+      const res = osa(`tell application "System Events" to set frontmost of process ${JSON.stringify(name)} to true`);
+      return res.ok ? { ok: true, name } : fail(res.error);
+    }
+    const res = run('wmctrl', ['-a', name]);
+    return res.ok ? { ok: true, name } : fail(res.stderr.trim() || `window not found: ${name}`);
+  },
+
+  resizeWindow(name, x, y, width, height) {
+    if (isWin) {
+      const r = winRun({ action: 'resize', name, x, y, width, height });
+      return r.ok ? { ok: true, name: r.name, x, y, width, height } : fail(r.error);
+    }
+    if (isMac) {
+      const res = osa(
+        `tell application "System Events" to tell process ${JSON.stringify(name)} to set {position, size} of window 1 to {{${x}, ${y}}, {${width}, ${height}}}`
+      );
+      return res.ok ? { ok: true, name, x, y, width, height } : fail(res.error);
+    }
+    const res = run('wmctrl', ['-r', name, '-e', `0,${x},${y},${width},${height}`]);
+    return res.ok ? { ok: true, name, x, y, width, height } : fail(res.stderr.trim() || 'resize failed');
+  },
+
+  click(x, y, { button = 'left', count = 1 } = {}) {
+    if (isWin) {
+      const r = winRun({ action: 'click', x, y, button, count });
+      return r.ok ? { ok: true, x, y, button, count } : fail(r.error);
+    }
+    if (isMac) {
+      if (!which('cliclick')) return fail('mouse needs cliclick: brew install cliclick');
+      const arg = `${button === 'right' ? 'rc' : 'c'}:${x},${y}`;
+      const rounds = Math.max(1, count);
+      for (let i = 0; i < rounds; i += 1) {
+        const res = run('cliclick', [arg]);
+        if (!res.ok) return fail(res.stderr.trim() || 'cliclick failed');
+      }
+      return { ok: true, x, y, button, count };
+    }
+    const btn = button === 'right' ? '3' : button === 'middle' ? '2' : '1';
+    const res = run('xdotool', ['mousemove', String(x), String(y), 'click', '--repeat', String(Math.max(1, count)), btn]);
+    return res.ok ? { ok: true, x, y, button, count } : fail(res.stderr.trim() || 'xdotool click failed');
+  },
+
+  moveMouse(x, y) {
+    if (isWin) {
+      const r = winRun({ action: 'move', x, y });
+      return r.ok ? { ok: true, x, y } : fail(r.error);
+    }
+    if (isMac) {
+      if (!which('cliclick')) return fail('mouse needs cliclick: brew install cliclick');
+      const res = run('cliclick', [`m:${x},${y}`]);
+      return res.ok ? { ok: true, x, y } : fail(res.stderr.trim() || 'cliclick failed');
+    }
+    const res = run('xdotool', ['mousemove', String(x), String(y)]);
+    return res.ok ? { ok: true, x, y } : fail(res.stderr.trim() || 'xdotool failed');
+  },
+
+  mouseButton(x, y, button, down) {
+    if (isWin) {
+      const r = winRun({ action: down ? 'down' : 'up', x, y, button });
+      return r.ok ? { ok: true, x, y, button } : fail(r.error);
+    }
+    if (isMac) {
+      if (!which('cliclick')) return fail('mouse needs cliclick: brew install cliclick');
+      const arg = `dd:${x},${y}`;
+      if (down) return run('cliclick', [arg]).ok ? { ok: true } : fail('cliclick failed');
+      return run('cliclick', [`du:${x},${y}`]).ok ? { ok: true } : fail('cliclick failed');
+    }
+    const btn = button === 'right' ? '3' : button === 'middle' ? '2' : '1';
+    const res = run('xdotool', [down ? 'mousedown' : 'mouseup', btn]);
+    return res.ok ? { ok: true } : fail(res.stderr.trim() || 'xdotool failed');
+  },
+
+  drag(x, y, x2, y2) {
+    if (isWin) {
+      const r = winRun({ action: 'drag', x, y, x2, y2 });
+      return r.ok ? { ok: true, from: [x, y], to: [x2, y2] } : fail(r.error);
+    }
+    if (isMac) {
+      if (!which('cliclick')) return fail('mouse needs cliclick: brew install cliclick');
+      const res = run('cliclick', [`dd:${x},${y}`, `dm:${x2},${y2}`, `du:${x2},${y2}`]);
+      return res.ok ? { ok: true, from: [x, y], to: [x2, y2] } : fail(res.stderr.trim() || 'cliclick failed');
+    }
+    const res = run('xdotool', ['mousemove', String(x), String(y), 'mousedown', '1', 'mousemove', String(x2), String(y2), 'mouseup', '1']);
+    return res.ok ? { ok: true, from: [x, y], to: [x2, y2] } : fail(res.stderr.trim() || 'xdotool failed');
+  },
+
+  scroll(amount, x = null, y = null) {
+    const clicks = Math.trunc(amount) || 1;
+    if (isWin) {
+      const r = winRun({ action: 'scroll', amount: clicks, x, y });
+      return r.ok ? { ok: true, amount: clicks } : fail(r.error);
+    }
+    const button = clicks > 0 ? '4' : '5';
+    const args = [];
+    if (x !== null && y !== null) args.push('mousemove', String(x), String(y));
+    args.push('click', '--repeat', String(Math.abs(clicks)), button);
+    if (isMac) {
+      if (!which('cliclick')) return fail('mouse needs cliclick: brew install cliclick');
+      const res = run('cliclick', [`w:${clicks > 0 ? 'up' : 'down'}`]);
+      return res.ok ? { ok: true, amount: clicks } : fail(res.stderr.trim());
+    }
+    const res = run('xdotool', args);
+    return res.ok ? { ok: true, amount: clicks } : fail(res.stderr.trim() || 'xdotool failed');
+  },
+
+  typeText(text, x = null, y = null) {
+    if (isWin) {
+      const r = winRun({ action: 'type', text, x, y }, 60000);
+      return r.ok ? { ok: true, chars: text.length, typed: r.typed ?? text.length } : fail(r.error);
+    }
+    if (x !== null && y !== null) this.click(x, y);
+    if (isMac) {
+      const res = osa(`tell application "System Events" to keystroke ${JSON.stringify(text)}`);
+      return res.ok ? { ok: true, chars: text.length, typed: text.length } : fail(res.error);
+    }
+    const res = run('xdotool', ['type', '--clearmodifiers', '--delay', '12', text], { timeout: 120000 });
+    return res.ok ? { ok: true, chars: text.length, typed: text.length } : fail(res.stderr.trim() || 'xdotool type failed');
+  },
+
+  pressKeys(combo) {
+    const sequence = keyboardSequence(combo);
+    if (!sequence.length) return fail(`unreadable key combination: ${combo}`);
+    if (isWin) {
+      const r = winRun({ action: 'keys', sequence });
+      return r.ok ? { ok: true, keys: combo } : fail(r.error);
+    }
+    if (isMac) {
+      const res = macKeys(combo);
+      return res.ok ? { ok: true, keys: combo } : fail(res.error);
+    }
+    const mapped = String(combo)
+      .toLowerCase()
+      .split('+')
+      .map((p) => (p === 'ctrl' ? 'ctrl' : p))
+      .join('+');
+    const res = run('xdotool', ['key', '--clearmodifiers', mapped]);
+    return res.ok ? { ok: true, keys: combo } : fail(res.stderr.trim() || 'xdotool key failed');
+  },
+
+  screenshot(dest) {
+    fs.mkdirSync(path.dirname(dest), { recursive: true });
+    if (isWin) {
+      const r = winRun({ action: 'shot', path: dest }, 60000);
+      return r.ok ? { ok: true, file: dest, width: r.width, height: r.height, mime: 'image/png' } : fail(r.error);
+    }
+    if (isMac) {
+      const res = run('screencapture', ['-x', '-t', 'png', dest], { timeout: 30000 });
+      if (!res.ok) return fail(res.stderr.trim() || 'screencapture failed');
+      const size = this.screenSize();
+      return { ok: true, file: dest, width: size.width, height: size.height, mime: 'image/png' };
+    }
+    for (const [tool, args] of IMAGE_TOOLS) {
+      if (!which(tool)) continue;
+      const res = run(tool, args(dest), { timeout: 30000 });
+      if (res.ok && fs.existsSync(dest)) {
+        const size = this.screenSize();
+        return { ok: true, file: dest, width: size.width, height: size.height, mime: 'image/png' };
+      }
+    }
+    return fail('no screenshot tool found: install imagemagick (import), scrot, grim or gnome-screenshot');
+  },
+
+  listApps() {
+    if (isWin) {
+      const r = winRun({ action: 'apps' }, 60000);
+      return r.ok ? { ok: true, apps: r.apps || [] } : fail(r.error);
+    }
+    const out = [];
+    if (isMac) {
+      for (const dir of ['/Applications', '/System/Applications']) {
+        try {
+          for (const entry of fs.readdirSync(dir)) {
+            if (entry.endsWith('.app')) out.push({ name: entry.replace(/\.app$/, ''), app_id: path.join(dir, entry) });
+          }
+        } catch {
+          /* ignore */
+        }
+      }
+      return { ok: true, apps: out };
+    }
+    const dirs = ['/usr/share/applications', '/usr/local/share/applications', path.join(os.homedir(), '.local/share/applications')];
+    for (const dir of dirs) {
+      let files = [];
+      try {
+        files = fs.readdirSync(dir).filter((f) => f.endsWith('.desktop'));
+      } catch {
+        continue;
+      }
+      for (const f of files) {
+        try {
+          const text = fs.readFileSync(path.join(dir, f), 'utf8');
+          if (/^NoDisplay=true/m.test(text)) continue;
+          const name = /^Name=(.+)$/m.exec(text);
+          if (name) out.push({ name: name[1].trim(), app_id: f.replace(/\.desktop$/, '') });
+        } catch {
+          /* ignore */
+        }
+      }
+    }
+    return { ok: true, apps: out };
+  },
+
+  sysinfo() {
+    const screen = this.screenSize();
+    return {
+      ok: true,
+      system: {
+        windows_version: isWin ? os.release() : `${process.platform} ${os.release()}`,
+        os_version: os.release(),
+        screen_size: { width: screen.width, height: screen.height },
+        user_account_type: 'Local Account',
+        default_language: process.env.LANG || 'en',
+        dpi_scaling: 1,
+      },
+    };
+  },
+};
+
+module.exports = { ...automation, keyboardSequence };

package/src/local.js

@@ -1,6 +1,8 @@
 'use strict';
 
 const { decide } = require('./approval');
+const automation = require('./automation');
+const { stamp } = require('./clipboard');
 const session = require('./session');
 
 const fs = require('node:fs');
@@ -501,17 +503,172 @@ async function handleLocalUi(bridge, taskId, request, opts) {
   }
 
   // control / type / keystroke / screenshot / app / state / apps / multi_*
-  const error = t(
-    lang,
-    `operazione "${op}" non disponibile nel client da terminale: serve il client grafico ArenaCode`,
-    `operation "${op}" is not available in the terminal client: the ArenaCode desktop client is required`,
-    `la operación "${op}" no está disponible en el cliente de terminal: requiere el cliente gráfico ArenaCode`,
-  );
-  await bridge.localResult(taskId, { rid, output: JSON.stringify({ ok: false, error }), code: 1 });
-  write(`  ${c.red('✗')} ${c.gray(op)} ${c.gray('— ' + error)}`);
+  const result = await runUiAction(bridge, taskId, request, opts);
+  const ok = result && result.ok !== false;
+  await bridge.localResult(taskId, {
+    rid,
+    output: JSON.stringify(result),
+    code: ok ? 0 : 1,
+  });
+  const line = ok
+    ? `${c.green('✓')} ${c.gray(op)}${request.action ? c.gray(` ${request.action}`) : ''}`
+    : `${c.red('✗')} ${c.gray(op)} ${c.gray('— ' + (result && result.error ? result.error : 'failed'))}`;
+  write(`  ${line}`);
   return true;
 }
 
+/** Computer Agent actions: mouse, keyboard, screen, windows, apps. */
+async function runUiAction(bridge, taskId, request, opts) {
+  const { op, rid } = request;
+  const lang = opts.lang;
+  const toolkit = automation.available();
+  const json = (text) => {
+    try {
+      return JSON.parse(text || '{}');
+    } catch {
+      return {};
+    }
+  };
+
+  if (op === 'state') {
+    const wins = automation.windows();
+    const screen = automation.screenSize();
+    const info = automation.sysinfo();
+    if (!wins.ok) return wins;
+    return {
+      ok: true,
+      windows: wins.windows,
+      active_window: wins.active_window,
+      interactive_elements: [],
+      scrollable_elements: [],
+      interactive_elements_text: '',
+      system: info.system,
+      screen: { width: screen.width, height: screen.height },
+      // the terminal client reads windows and pixels, not the accessibility tree
+      note: 'interactive_elements are empty: this client does not read the UI Automation tree',
+    };
+  }
+
+  if (op === 'sysinfo') {
+    const info = automation.sysinfo();
+    return {
+      ...info,
+      os: `${process.platform} ${os.release()}`,
+      hostname: os.hostname(),
+      arch: os.arch(),
+      cpus: os.cpus().length,
+      memory_mb: Math.round(os.totalmem() / 1048576),
+      node: process.version,
+    };
+  }
+
+  if (op === 'apps') return automation.listApps();
+
+  if (!toolkit.ok) {
+    return {
+      ok: false,
+      error:
+        toolkit.error ||
+        t(lang, 'automazione non disponibile su questo sistema', 'automation is not available on this system', 'la automatización no está disponible en este sistema'),
+    };
+  }
+
+  if (op === 'control') {
+    const { action, x, y, x2, y2, amount, button } = request;
+    switch (action) {
+      case 'move':
+        return automation.moveMouse(x, y);
+      case 'click':
+        return automation.click(x, y, { button: button || 'left', count: 1 });
+      case 'dblclick':
+        return automation.click(x, y, { button: button || 'left', count: 2 });
+      case 'rightclick':
+        return automation.click(x, y, { button: 'right', count: 1 });
+      case 'down':
+        return automation.mouseButton(x, y, button || 'left', true);
+      case 'up':
+        return automation.mouseButton(x, y, button || 'left', false);
+      case 'drag':
+        return automation.drag(x, y, x2, y2);
+      case 'scroll':
+        return automation.scroll(amount ?? 1, x ?? null, y ?? null);
+      default:
+        return { ok: false, error: `unknown control action: ${action}` };
+    }
+  }
+
+  if (op === 'keystroke') {
+    return automation.pressKeys(request.keys || '');
+  }
+
+  if (op === 'type') {
+    let text = request.text;
+    if (request.payload || text === undefined) {
+      const res = await bridge.uiPayload(taskId, rid);
+      text = res.status === 200 ? (res.json && (res.json.text ?? res.json.payload ?? '')) : '';
+      if (!text) return { ok: false, error: 'the typed text could not be read from the bridge' };
+    }
+    return automation.typeText(String(text), request.x ?? null, request.y ?? null);
+  }
+
+  if (op === 'app') {
+    const name = request.text || request.name || '';
+    const action = request.action || 'launch';
+    if (action === 'launch') return automation.launchApp(name);
+    if (action === 'switch') return automation.focusWindow(name);
+    if (action === 'resize') {
+      return automation.resizeWindow(name, request.x ?? 0, request.y ?? 0, request.x2 ?? 800, request.y2 ?? 600);
+    }
+    return { ok: false, error: `unknown app action: ${action}` };
+  }
+
+  if (op === 'screenshot') {
+    const dir = path.join(os.tmpdir(), 'arenacode-shots');
+    fs.mkdirSync(dir, { recursive: true });
+    const file = path.join(dir, `shot-${stamp()}.png`);
+    const shot = automation.screenshot(file);
+    if (!shot.ok) return shot;
+    const buf = fs.readFileSync(shot.file);
+    // the agent runs in the sandbox: the image travels through the bridge workspace
+    const up = await bridge.upload(`uploads/${stamp()}/shot-${stamp()}.png`, buf);
+    const remote = up.status === 200 ? up.json.path : null;
+    const out = {
+      ok: true,
+      mime: 'image/png',
+      width: shot.width,
+      height: shot.height,
+      file: remote || shot.file,
+      local_file: shot.file,
+    };
+    if (buf.length <= 2 * 1024 * 1024) out.image = buf.toString('base64');
+    return out;
+  }
+
+  if (op === 'multi_select') {
+    const { elements = [], press_ctrl = true } = json(request.args_json);
+    const modifier = press_ctrl ? automation.pressKeys('ctrl') : null;
+    if (modifier && !modifier.ok) return modifier;
+    for (const [x, y] of elements) {
+      const r = automation.click(x, y, { button: 'left', count: 1 });
+      if (!r.ok) return r;
+    }
+    return { ok: true, selected: elements.length };
+  }
+
+  if (op === 'multi_edit') {
+    const { elements = [] } = json(request.args_json);
+    let typed = 0;
+    for (const [x, y, text] of elements) {
+      const r = automation.typeText(String(text), x, y);
+      if (!r.ok) return r;
+      typed += 1;
+    }
+    return { ok: true, fields: typed };
+  }
+
+  return { ok: false, error: `unknown ui operation: ${op}` };
+}
+
 function available() {
   return os.release();
 }
@@ -536,6 +693,7 @@ async function handleLocalFile(bridge, taskId, request, opts) {
 }
 
 module.exports = {
+  runUiAction,
   REFUSED,
   resolveInRoot,
   runShell,

package/src/tui/app.js

@@ -1048,10 +1048,17 @@ class App {
         item.state = 'done';
         return;
       }
-      const error = `operation "${op}" is not available in the terminal client: the ArenaCode desktop client is required`;
-      await this.bridge.localResult(taskId, { rid: payload.rid, output: JSON.stringify({ ok: false, error }), code: 1 });
-      item.state = 'failed';
-      item.meta = 'needs the desktop client';
+      // mouse, keyboard, screen, windows and apps: the automation backends
+      const { runUiAction } = require('../local');
+      const result = await runUiAction(this.bridge, taskId, payload, { lang: this.lang });
+      const ok = result && result.ok !== false;
+      await this.bridge.localResult(taskId, {
+        rid: payload.rid,
+        output: JSON.stringify(result),
+        code: ok ? 0 : 1,
+      });
+      item.state = ok ? 'done' : 'failed';
+      if (!ok && result && result.error) item.detail = String(result.error);
     }
   }
 
@@ -1276,13 +1283,14 @@ class App {
         })
         .map((line) => R.spaces(2) + line);
 
-      const bodyH = Math.max(1, H - prompt.lines.length - 2);
+      const bodyH = Math.max(1, H - prompt.lines.length - 3);
       const maxScroll = Math.max(0, chat.length - bodyH);
       st.scroll = Math.min(Math.max(0, st.scroll), maxScroll);
       const start = Math.max(0, chat.length - bodyH - st.scroll);
       const visible = chat.slice(start, start + bodyH);
       while (visible.length < bodyH) visible.unshift('');
       frame.push(...visible);
+      frame.push('');
       const promptStart = frame.length;
       frame.push(...prompt.lines);
       frame.push('');
@@ -1308,7 +1316,11 @@ class App {
     this.term.draw(frame);
 
     if (!cursor || st.overlay || st.approval || st.prompt) this.term.hideCursor();
-    else this.term.placeCursor(cursor.row, Math.min(W - 1, Math.max(0, cursor.col)));
+    else {
+      // frame rows/cols are 0-based, the terminal's CUP escape is 1-based
+      const col = Math.min(W - 1, Math.max(0, cursor.col));
+      this.term.placeCursor(cursor.row + 1, col + 1);
+    }
 
     this.rendering = false;
     if (this.dirty) setTimeout(() => this.maybeRender(), 16);