@lardexx/arenacode 2.11.11 → 2.11.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +42 -5
- package/dist/arenacode-uinput.rules +8 -0
- package/package.json +3 -2
- package/src/automation/wayland.js +947 -0
- package/src/automation.js +18 -2
- package/src/prompts/agent-prompt.md +1 -0
- package/src/tui/app.js +43 -0
- package/src/tui/widgets.js +24 -4
package/README.md
CHANGED
|
@@ -121,7 +121,7 @@ How the copy is built and tested: [`docs/TUI.md`](docs/TUI.md).
|
|
|
121
121
|
| `wheel` | scroll the transcript (three lines a notch) |
|
|
122
122
|
| `pgup` / `pgdn` (or `ctrl+u` / `ctrl+j`) | scroll a page; `alt+↑` / `alt+↓` a line |
|
|
123
123
|
| `home` / `end` | jump to the top / back to the bottom (when the prompt is empty) |
|
|
124
|
-
| `ctrl+x` then | `q` quit · `n` new session · `l` switch session · `t` theme · `a` Ask and Proceed / Proceed · `d` tool output · `s` stop |
|
|
124
|
+
| `ctrl+x` then | `q` quit · `n` new session · `l` switch session · `e` expand the last diff · `t` theme · `a` Ask and Proceed / Proceed · `d` tool output · `s` stop |
|
|
125
125
|
| `ctrl+c` | interrupt, then quit |
|
|
126
126
|
|
|
127
127
|
**The same four flows as the desktop client.** The Windows app is the reference for how a task reaches the bridge,
|
|
@@ -172,8 +172,39 @@ outside the keymap borrow an unused keycode with `XChangeKeyboardMapping` and gi
|
|
|
172
172
|
from `XQueryTree` + `XGetGeometry` + `XTranslateCoordinates` + `_NET_WM_NAME`, focus from `XSetInputFocus`, and
|
|
173
173
|
screenshots are `XGetImage` + `XGetPixel` encoded to PNG by the client itself (zlib). This is what makes it testable:
|
|
174
174
|
`node test/native.js` runs it against a real X server (Xvfb is enough) — a window is created, the client types into it
|
|
175
|
-
and the X events the server delivered are compared with the requested keycodes, chords, buttons and pixels.
|
|
176
|
-
|
|
175
|
+
and the X events the server delivered are compared with the requested keycodes, chords, buttons and pixels.
|
|
176
|
+
`ui app launch` still starts the program's own `.desktop` command; nothing else is spawned.
|
|
177
|
+
|
|
178
|
+
**Wayland** is a different backend (`src/automation/wayland.js`), picked automatically when `XDG_SESSION_TYPE=wayland`
|
|
179
|
+
and `/dev/uinput` is writable. XTest cannot reach a native Wayland client — the compositor has no such entry point — so
|
|
180
|
+
input goes through the kernel instead: a virtual pointer and keyboard are declared on `/dev/uinput`
|
|
181
|
+
(`EV_KEY`/`EV_REL`, `UI_SET_*`, `UI_DEV_CREATE`) and the events written there are the same ones libinput reads, which
|
|
182
|
+
means they work in *every* native Wayland window regardless of toolkit. Typing maps characters through
|
|
183
|
+
`libxkbcommon` (`xkb_state_key_get_utf32`), so the layout in use is respected and shifted characters are typed with
|
|
184
|
+
shift instead of guessed; if the library is missing a complete US map is used instead, and characters that the keymap
|
|
185
|
+
cannot produce are refused by name rather than typed as something else. Screenshots take the two routes that Wayland
|
|
186
|
+
allows: `org.freedesktop.portal.Screenshot` first (xdg-desktop-portal, so GNOME, KDE and wlroots all work; the desktop
|
|
187
|
+
asks for confirmation once) and the `org.gnome.Shell.Screenshot` D-Bus service afterwards, with the XWayland root window
|
|
188
|
+
as the last resort. Both D-Bus paths are tested for real against a private `dbus-daemon` and a stub of each service
|
|
189
|
+
(`test/wayland.js`), the uinput path by reading the injected events back from `/dev/input/eventN` — the same node
|
|
190
|
+
libinput opens.
|
|
191
|
+
|
|
192
|
+
Wayland hides things that X11 exposes, and the client says so instead of failing quietly: `ui state` lists the XWayland
|
|
193
|
+
windows and marks them as such (native windows cannot be enumerated by any client), `ui app switch`/`resize` are
|
|
194
|
+
refused because a client may not focus or resize another client's window, and an absolute `ui move`/`ui click x y` needs
|
|
195
|
+
the pointer position, which comes from XWayland — without it the answer is an explicit error rather than a click in the
|
|
196
|
+
wrong place. Wheel, keys, typing and clicking at the current position work everywhere.
|
|
197
|
+
|
|
198
|
+
To let your user open `/dev/uinput`, install the udev rule that ships in this package:
|
|
199
|
+
|
|
200
|
+
```bash
|
|
201
|
+
sudo cp dist/arenacode-uinput.rules /etc/udev/rules.d/
|
|
202
|
+
sudo udevadm control --reload-rules && sudo udevadm trigger
|
|
203
|
+
sudo usermod -aG input $USER # then log out and back in
|
|
204
|
+
```
|
|
205
|
+
|
|
206
|
+
Without it the client stays on the X11 backend, keeps working for XWayland windows and reports
|
|
207
|
+
`native.degraded` with the reason.
|
|
177
208
|
|
|
178
209
|
**macOS** is native too, again with no helper binaries: mouse with `CGEventCreateMouseEvent` + `CGEventPost`
|
|
179
210
|
(left/right/middle, double and triple click through the click-state field, drag interpolated), keyboard with
|
|
@@ -317,8 +348,10 @@ number, new line number, the sign, the text — painted on the diff backgrounds
|
|
|
317
348
|
- A new file is all additions, a deleted one all deletions; text that did not change draws no card at all.
|
|
318
349
|
- The diff is computed **locally for the display** and never travels to the bridge: the agent still receives the
|
|
319
350
|
same `{ ok, path, size }` it always received.
|
|
320
|
-
-
|
|
321
|
-
|
|
351
|
+
- A long change is drawn as 20 rows and then a `⋯ n more lines` row. **That row is clickable**: the card opens
|
|
352
|
+
and shows every row, ending with `− collapse diff` (also clickable) to close it again. Without a mouse,
|
|
353
|
+
`ctrl+x e` does the same on the last diff of the session.
|
|
354
|
+
- `ctrl+x d` hides the blocks entirely (it toggles every card), and a binary file gets no diff.
|
|
322
355
|
|
|
323
356
|
### Sessions
|
|
324
357
|
|
|
@@ -443,6 +476,10 @@ Environment variables override the file: `ARENACODE_URL`, `ARENACODE_KEY`, `AREN
|
|
|
443
476
|
| `bridge unreachable` | tunnel down or wrong URL — check `/health` on the bridge, then `/settings` in the interface |
|
|
444
477
|
| agent never answers | the agent is not listening: run `bash listen.sh` in the bridge folder |
|
|
445
478
|
| everything hangs after a turn | `--idle-timeout` closes the session; the task state is in the interface |
|
|
479
|
+
| Wayland: `ui` does nothing in native windows | `/dev/uinput` is not writable — install `dist/arenacode-uinput.rules` (see *What runs on your machine*) and log in again |
|
|
480
|
+
| Wayland: `ui state` shows no windows | only XWayland windows can be listed; open the app with its X11 backend or use `ui app` |
|
|
481
|
+
| Wayland: `ui shot` asks for confirmation | that is the desktop's own portal dialog — it appears once per session |
|
|
482
|
+
| Wayland: `ui click 100 200` refuses | absolute positioning needs the XWayland pointer; use `ui keys`/`ui scroll`, or run the app under XWayland |
|
|
446
483
|
|
|
447
484
|
## Development
|
|
448
485
|
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
# Let members of the "input" group create virtual input devices, so the Computer Agent
|
|
2
|
+
# can drive native Wayland windows through /dev/uinput instead of XTest (which Wayland
|
|
3
|
+
# does not accept).
|
|
4
|
+
#
|
|
5
|
+
# sudo cp arenacode-uinput.rules /etc/udev/rules.d/
|
|
6
|
+
# sudo udevadm control --reload-rules && sudo udevadm trigger
|
|
7
|
+
# sudo usermod -aG input "$USER" # log out and back in afterwards
|
|
8
|
+
KERNEL=="uinput", SUBSYSTEM=="misc", MODE="0660", GROUP="input", OPTIONS+="static_node=uinput"
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lardexx/arenacode",
|
|
3
|
-
"version": "2.11.
|
|
3
|
+
"version": "2.11.13",
|
|
4
4
|
"description": "Terminal client for ArenaCode: talks to an arena-bridge, streams the Agent Mode timeline, and runs local commands, file operations and computer automation on your PC (windows-use.dll on Windows, Xlib/XTest on Linux, osascript on macOS). Node port of the ArenaCode desktop client.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "commonjs",
|
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
"node": ">=18.0.0"
|
|
13
13
|
},
|
|
14
14
|
"files": [
|
|
15
|
+
"dist/",
|
|
15
16
|
"bin/",
|
|
16
17
|
"src/",
|
|
17
18
|
"docs/",
|
|
@@ -20,7 +21,7 @@
|
|
|
20
21
|
"vendor/"
|
|
21
22
|
],
|
|
22
23
|
"scripts": {
|
|
23
|
-
"test": "node test/cli-surface.js && node test/sessions.js && node test/diff.js && node test/shell.js && node test/context.js && node test/integration.js && node test/parity.js && node test/automation.js && node test/native.js"
|
|
24
|
+
"test": "node test/cli-surface.js && node test/sessions.js && node test/diff.js && node test/shell.js && node test/context.js && node test/integration.js && node test/parity.js && node test/automation.js && node test/native.js && node test/wayland.js"
|
|
24
25
|
},
|
|
25
26
|
"keywords": [
|
|
26
27
|
"arenacode",
|
|
@@ -0,0 +1,947 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// Wayland support for the Computer Agent.
|
|
4
|
+
//
|
|
5
|
+
// Wayland has no XTest, no global window list and no `XGetImage` of another client's
|
|
6
|
+
// surface, so the X11 backend only ever reaches XWayland applications. This module adds
|
|
7
|
+
// the pieces that do work on a native Wayland session, without helper binaries:
|
|
8
|
+
//
|
|
9
|
+
// input /dev/uinput — a virtual mouse + keyboard created by the kernel itself.
|
|
10
|
+
// Works on every compositor (and on X11 too), no cooperation needed.
|
|
11
|
+
// text libxkbcommon reads the active keymap, so `ui type` sends the right keys
|
|
12
|
+
// for the layout in use instead of assuming US.
|
|
13
|
+
// screenshot org.gnome.Shell.Screenshot over D-Bus (GNOME Wayland), then the X11 root
|
|
14
|
+
// window (XWayland), then an honest error.
|
|
15
|
+
// windows XWayland windows through libX11; native Wayland windows cannot be listed
|
|
16
|
+
// by design, and the answer says so instead of pretending.
|
|
17
|
+
//
|
|
18
|
+
// Everything here is FFI: libc, libxkbcommon and libdbus, no external commands.
|
|
19
|
+
|
|
20
|
+
const fs = require('node:fs');
|
|
21
|
+
const os = require('node:os');
|
|
22
|
+
const path = require('node:path');
|
|
23
|
+
const { requireFfi } = require('./ffi');
|
|
24
|
+
|
|
25
|
+
const isLinux = process.platform === 'linux';
|
|
26
|
+
|
|
27
|
+
// ---------------------------------------------------------------- session detection --
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Tells whether the desktop session is Wayland.
|
|
31
|
+
* XDG_SESSION_TYPE wins (set by the display manager); WAYLAND_DISPLAY is the fallback a
|
|
32
|
+
* lot of shells set; XDG_CURRENT_DESKTOP only says which desktop it is.
|
|
33
|
+
*/
|
|
34
|
+
function detect(env = process.env) {
|
|
35
|
+
const type = String(env.XDG_SESSION_TYPE || '').toLowerCase();
|
|
36
|
+
const display = String(env.WAYLAND_DISPLAY || '');
|
|
37
|
+
const desktop = String(env.XDG_CURRENT_DESKTOP || '').toLowerCase();
|
|
38
|
+
const x11 = Boolean(env.DISPLAY);
|
|
39
|
+
const wayland = type === 'wayland' || (type !== 'x11' && display !== '');
|
|
40
|
+
return {
|
|
41
|
+
wayland,
|
|
42
|
+
x11,
|
|
43
|
+
compositor: desktop || (wayland ? 'wayland' : 'x11'),
|
|
44
|
+
// XWayland is there when an X display is exported next to the Wayland one
|
|
45
|
+
xwayland: wayland && x11,
|
|
46
|
+
display: display || null,
|
|
47
|
+
xdisplay: env.DISPLAY || null,
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function uinputPath() {
|
|
52
|
+
return process.env.ARENACODE_UINPUT || '/dev/uinput';
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/** Can we create a virtual device right now? */
|
|
56
|
+
function uinputAccess() {
|
|
57
|
+
const file = uinputPath();
|
|
58
|
+
try {
|
|
59
|
+
const fd = fs.openSync(file, 'r+');
|
|
60
|
+
fs.closeSync(fd);
|
|
61
|
+
return { ok: true, path: file };
|
|
62
|
+
} catch (err) {
|
|
63
|
+
return {
|
|
64
|
+
ok: false,
|
|
65
|
+
path: file,
|
|
66
|
+
error: err.code === 'EACCES' || err.code === 'EPERM'
|
|
67
|
+
? `no access to ${file}: add your user to the "input" group or run: sudo chmod 660 ${file}`
|
|
68
|
+
: `${file} unusable: ${err.message}`,
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
// ---------------------------------------------------------------- key names / layout --
|
|
74
|
+
|
|
75
|
+
// Linux input event codes (linux/input-event-codes.h), the ones a virtual device can send.
|
|
76
|
+
const KEYS = {
|
|
77
|
+
esc: 1, escape: 1, '1': 2, '2': 3, '3': 4, '4': 5, '5': 6, '6': 7, '7': 8, '8': 9, '9': 10, '0': 11,
|
|
78
|
+
minus: 12, equal: 13, backspace: 14, tab: 15,
|
|
79
|
+
q: 16, w: 17, e: 18, r: 19, t: 20, y: 21, u: 22, i: 23, o: 24, p: 25, bracketleft: 26, bracketright: 27,
|
|
80
|
+
enter: 28, return: 28, ctrl: 29, control: 29, a: 30, s: 31, d: 32, f: 33, g: 34, h: 35, j: 36, k: 37, l: 38,
|
|
81
|
+
semicolon: 39, apostrophe: 40, quote: 40, grave: 41, backtick: 41, shift: 42, backslash: 43,
|
|
82
|
+
z: 44, x: 45, c: 46, v: 47, b: 48, n: 49, m: 50, comma: 51, period: 52, dot: 52, slash: 53, shiftright: 54,
|
|
83
|
+
alt: 56, option: 56, space: 57, spacebar: 57, capslock: 58,
|
|
84
|
+
f1: 59, f2: 60, f3: 61, f4: 62, f5: 63, f6: 64, f7: 65, f8: 66, f9: 67, f10: 68, f11: 87, f12: 88,
|
|
85
|
+
home: 102, up: 103, arrowup: 103, pageup: 104, pgup: 104, left: 105, arrowleft: 105,
|
|
86
|
+
right: 106, arrowright: 106, end: 107, down: 108, arrowdown: 108, pagedown: 109, pgdn: 109,
|
|
87
|
+
insert: 110, ins: 110, delete: 111, del: 111, meta: 125, super: 125, win: 125, cmd: 125, command: 125,
|
|
88
|
+
menu: 127, printscreen: 99, scrolllock: 70, pause: 119, numlock: 69,
|
|
89
|
+
numpad0: 82, numpad1: 79, numpad2: 80, numpad3: 81, numpad4: 75, numpad5: 76, numpad6: 77,
|
|
90
|
+
numpad7: 71, numpad8: 72, numpad9: 73, numpadadd: 78, numpadsub: 74, numpadmul: 55, numpaddiv: 98,
|
|
91
|
+
numpadenter: 96, numpadperiod: 83,
|
|
92
|
+
};
|
|
93
|
+
const MODIFIERS = new Set(['ctrl', 'control', 'alt', 'option', 'shift', 'meta', 'super', 'win', 'cmd', 'command', 'shiftright']);
|
|
94
|
+
|
|
95
|
+
// US layout, used when libxkbcommon is not installed: char -> [keycode, shift]
|
|
96
|
+
const US_BASE = {
|
|
97
|
+
'-': 12, '=': 13, '[': 26, ']': 27, ';': 39, "'": 40, '`': 41, '\\': 43, ',': 51, '.': 52, '/': 53, ' ': 57,
|
|
98
|
+
};
|
|
99
|
+
const US_PAIRS = { '!': '1', '@': '2', '#': '3', $: '4', '%': '5', '^': '6', '&': '7', '*': '8', '(': '9', ')': '0',
|
|
100
|
+
_: '-', '+': '=', '{': '[', '}': ']', ':': ';', '"': "'", '~': '`', '|': '\\', '<': ',', '>': '.', '?': '/' };
|
|
101
|
+
|
|
102
|
+
function usCharMap() {
|
|
103
|
+
const map = new Map();
|
|
104
|
+
for (const [ch, code] of Object.entries(US_BASE)) map.set(ch, { code, shift: false });
|
|
105
|
+
for (const [ch, base] of Object.entries(US_PAIRS)) map.set(ch, { code: US_BASE[base] || KEYS[base], shift: true });
|
|
106
|
+
for (let i = 0; i < 26; i += 1) {
|
|
107
|
+
const lower = String.fromCharCode(97 + i);
|
|
108
|
+
const code = 30 + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, -1, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24][i];
|
|
109
|
+
map.set(lower, { code, shift: false });
|
|
110
|
+
map.set(lower.toUpperCase(), { code, shift: true });
|
|
111
|
+
}
|
|
112
|
+
for (let d = 0; d <= 9; d += 1) map.set(String(d), { code: 2 + d, shift: false });
|
|
113
|
+
map.set('\n', { code: KEYS.enter, shift: false });
|
|
114
|
+
map.set('\t', { code: KEYS.tab, shift: false });
|
|
115
|
+
return map;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* char -> { code, shift } from the keymap libxkbcommon reports (the one the desktop is
|
|
120
|
+
* actually using). Falls back to the US table when the library or the keymap is missing.
|
|
121
|
+
*/
|
|
122
|
+
function charMap() {
|
|
123
|
+
try {
|
|
124
|
+
const koffi = requireFfi();
|
|
125
|
+
const xkb = koffi.load('libxkbcommon.so.0');
|
|
126
|
+
const context_new = xkb.func('void *xkb_context_new(int flags)');
|
|
127
|
+
const keymap_new = xkb.func('void *xkb_keymap_new_from_names(void *ctx, void *names, int flags)');
|
|
128
|
+
const state_new = xkb.func('void *xkb_state_new(void *keymap)');
|
|
129
|
+
const key_get_utf32 = xkb.func('unsigned int xkb_state_key_get_utf32(void *state, unsigned int keycode)');
|
|
130
|
+
const mod_index = xkb.func('unsigned int xkb_keymap_mod_get_index(void *keymap, const char *name)');
|
|
131
|
+
const update_mask = xkb.func('int xkb_state_update_mask(void *state, unsigned int depressed, unsigned int latched, unsigned int locked, unsigned int g1, unsigned int g2, unsigned int g3)');
|
|
132
|
+
|
|
133
|
+
const ctx = context_new(0);
|
|
134
|
+
const keymap = keymap_new(ctx, null, 0);
|
|
135
|
+
if (!keymap) throw new Error('no keymap');
|
|
136
|
+
const plain = state_new(keymap);
|
|
137
|
+
const shifted = state_new(keymap);
|
|
138
|
+
const shiftIndex = Number(mod_index(keymap, 'Shift')) || 0;
|
|
139
|
+
update_mask(shifted, 2 ** shiftIndex, 0, 0, 0, 0, 0);
|
|
140
|
+
|
|
141
|
+
const map = new Map();
|
|
142
|
+
// xkb keycodes are evdev codes + 8
|
|
143
|
+
for (let code = 1; code <= 255; code += 1) {
|
|
144
|
+
const kc = code + 8;
|
|
145
|
+
const a = key_get_utf32(plain, kc);
|
|
146
|
+
const b = key_get_utf32(shifted, kc);
|
|
147
|
+
if (a) map.set(String.fromCodePoint(a), { code, shift: false });
|
|
148
|
+
if (b && !map.has(String.fromCodePoint(b))) map.set(String.fromCodePoint(b), { code, shift: true });
|
|
149
|
+
}
|
|
150
|
+
if (!map.size) throw new Error('empty keymap');
|
|
151
|
+
map.source = 'xkbcommon';
|
|
152
|
+
return map;
|
|
153
|
+
} catch (err) {
|
|
154
|
+
const map = usCharMap();
|
|
155
|
+
map.source = 'us-fallback';
|
|
156
|
+
map.reason = err.message;
|
|
157
|
+
return map;
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/** `ctrl+shift+s` -> [{code, down}] using the Linux keycodes. */
|
|
162
|
+
function keySequence(combo) {
|
|
163
|
+
// "ctrl+shift+s" and "tab x3" both work: plus or space separates the parts
|
|
164
|
+
const parts = String(combo || '').toLowerCase().split(/[+\s]+/).map((p) => p.trim()).filter(Boolean);
|
|
165
|
+
if (!parts.length) return [];
|
|
166
|
+
let repeat = 1;
|
|
167
|
+
const mods = [];
|
|
168
|
+
const main = [];
|
|
169
|
+
for (const part of parts) {
|
|
170
|
+
if (part === 'repeat' || /^x\d+$/.test(part)) {
|
|
171
|
+
repeat = part === 'repeat' ? 2 : Number(part.slice(1));
|
|
172
|
+
continue;
|
|
173
|
+
}
|
|
174
|
+
const code = KEYS[part] !== undefined ? KEYS[part] : (part.length === 1 ? KEYS[part] || null : null);
|
|
175
|
+
if (code === null || code === undefined) continue;
|
|
176
|
+
if (MODIFIERS.has(part)) mods.push(code);
|
|
177
|
+
else main.push(code);
|
|
178
|
+
}
|
|
179
|
+
const out = [];
|
|
180
|
+
const times = Math.max(1, Math.min(50, repeat));
|
|
181
|
+
for (let i = 0; i < times; i += 1) {
|
|
182
|
+
if (!main.length && mods.length) {
|
|
183
|
+
const only = mods[mods.length - 1];
|
|
184
|
+
out.push({ code: only, down: true }, { code: only, down: false });
|
|
185
|
+
continue;
|
|
186
|
+
}
|
|
187
|
+
for (const code of mods) out.push({ code, down: true });
|
|
188
|
+
for (const code of main) out.push({ code, down: true }, { code, down: false });
|
|
189
|
+
for (const code of [...mods].reverse()) out.push({ code, down: false });
|
|
190
|
+
}
|
|
191
|
+
return out;
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
// ---------------------------------------------------------------- uinput device --
|
|
195
|
+
|
|
196
|
+
const EV_SYN = 0x00;
|
|
197
|
+
const EV_KEY = 0x01;
|
|
198
|
+
const EV_REL = 0x02;
|
|
199
|
+
const EV_ABS = 0x03;
|
|
200
|
+
const REL_X = 0x00;
|
|
201
|
+
const REL_Y = 0x01;
|
|
202
|
+
const REL_WHEEL = 0x08;
|
|
203
|
+
const REL_HWHEEL = 0x06;
|
|
204
|
+
const ABS_X = 0x00;
|
|
205
|
+
const ABS_Y = 0x01;
|
|
206
|
+
const BTN_LEFT = 0x110;
|
|
207
|
+
const BTN_RIGHT = 0x111;
|
|
208
|
+
const BTN_MIDDLE = 0x112;
|
|
209
|
+
const SYN_REPORT = 0;
|
|
210
|
+
|
|
211
|
+
const UI_SET_EVBIT = 0x40045564;
|
|
212
|
+
const UI_SET_KEYBIT = 0x40045565;
|
|
213
|
+
const UI_SET_RELBIT = 0x40045566;
|
|
214
|
+
const UI_SET_ABSBIT = 0x40045567;
|
|
215
|
+
const UI_DEV_CREATE = 0x5501;
|
|
216
|
+
const UI_DEV_DESTROY = 0x5502;
|
|
217
|
+
|
|
218
|
+
const O_WRONLY = 1;
|
|
219
|
+
|
|
220
|
+
/** struct input_event on x86_64/arm64: timeval (16) + type (2) + code (2) + value (4). */
|
|
221
|
+
const EVENT_SIZE = 24;
|
|
222
|
+
|
|
223
|
+
/** Builds the buffers a caller writes to /dev/uinput. Pure, so it can be tested. */
|
|
224
|
+
function eventBuffer(events) {
|
|
225
|
+
const buffer = Buffer.alloc(EVENT_SIZE * (events.length + 1));
|
|
226
|
+
let at = 0;
|
|
227
|
+
for (const ev of events) {
|
|
228
|
+
// time left at zero: the kernel stamps the event on write
|
|
229
|
+
buffer.writeUInt16LE(ev.type & 0xffff, at + 16);
|
|
230
|
+
buffer.writeUInt16LE(ev.code & 0xffff, at + 18);
|
|
231
|
+
buffer.writeInt32LE(ev.value | 0, at + 20);
|
|
232
|
+
at += EVENT_SIZE;
|
|
233
|
+
}
|
|
234
|
+
buffer.writeUInt16LE(EV_SYN, at + 16);
|
|
235
|
+
buffer.writeUInt16LE(SYN_REPORT, at + 18);
|
|
236
|
+
buffer.writeInt32LE(0, at + 20);
|
|
237
|
+
return buffer;
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
let device = null; // { fd, kind }
|
|
241
|
+
|
|
242
|
+
/** ioctl returns -1 on failure; without this check the next write fails with a bare EINVAL. */
|
|
243
|
+
function check(rc, what) {
|
|
244
|
+
if (rc === -1) throw new Error(`${what} failed (errno ${process.errno || 'unknown'})`);
|
|
245
|
+
return rc;
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
function ioctlInt(koffi, fd, request, value) {
|
|
249
|
+
const libc = koffi.load('libc.so.6');
|
|
250
|
+
const ioctl = libc.func('int ioctl(int fd, unsigned long request, int arg)');
|
|
251
|
+
return ioctl(fd, request, value);
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
function ioctlNone(koffi, fd, request) {
|
|
255
|
+
const libc = koffi.load('libc.so.6');
|
|
256
|
+
const ioctl = libc.func('int ioctl(int fd, unsigned long request)');
|
|
257
|
+
return ioctl(fd, request);
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
/**
|
|
261
|
+
* Opens /dev/uinput and declares a pointer + keyboard device.
|
|
262
|
+
* Kept open for the life of the process so the compositor sees one stable device.
|
|
263
|
+
*/
|
|
264
|
+
function ensureDevice({ keyboard = true, pointer = true } = {}) {
|
|
265
|
+
if (device && device.keyboard === keyboard && device.pointer === pointer) return device;
|
|
266
|
+
if (device) destroyDevice();
|
|
267
|
+
const access = uinputAccess();
|
|
268
|
+
if (!access.ok) throw new Error(access.error);
|
|
269
|
+
const koffi = requireFfi();
|
|
270
|
+
const file = access.path;
|
|
271
|
+
const nodesBefore = listEventNodes();
|
|
272
|
+
const fd = fs.openSync(file, O_WRONLY);
|
|
273
|
+
const setup = () => {
|
|
274
|
+
if (pointer) {
|
|
275
|
+
check(ioctlInt(koffi, fd, UI_SET_EVBIT, EV_REL), 'UI_SET_EVBIT EV_REL');
|
|
276
|
+
check(ioctlInt(koffi, fd, UI_SET_RELBIT, REL_X), 'UI_SET_RELBIT REL_X');
|
|
277
|
+
check(ioctlInt(koffi, fd, UI_SET_RELBIT, REL_Y), 'UI_SET_RELBIT REL_Y');
|
|
278
|
+
check(ioctlInt(koffi, fd, UI_SET_RELBIT, REL_WHEEL), 'UI_SET_RELBIT REL_WHEEL');
|
|
279
|
+
check(ioctlInt(koffi, fd, UI_SET_RELBIT, REL_HWHEEL), 'UI_SET_RELBIT REL_HWHEEL');
|
|
280
|
+
check(ioctlInt(koffi, fd, UI_SET_EVBIT, EV_KEY), 'UI_SET_EVBIT EV_KEY');
|
|
281
|
+
for (const btn of [BTN_LEFT, BTN_RIGHT, BTN_MIDDLE]) check(ioctlInt(koffi, fd, UI_SET_KEYBIT, btn), `UI_SET_KEYBIT ${btn}`);
|
|
282
|
+
}
|
|
283
|
+
if (keyboard) {
|
|
284
|
+
check(ioctlInt(koffi, fd, UI_SET_EVBIT, EV_KEY), 'UI_SET_EVBIT EV_KEY');
|
|
285
|
+
for (let code = 1; code <= 127; code += 1) check(ioctlInt(koffi, fd, UI_SET_KEYBIT, code), `UI_SET_KEYBIT ${code}`);
|
|
286
|
+
}
|
|
287
|
+
// name the device so it shows up recognisably in `libinput debug-events`
|
|
288
|
+
const libc = koffi.load('libc.so.6');
|
|
289
|
+
const write = libc.func('long write(int fd, const unsigned char *buf, long count)');
|
|
290
|
+
const name = Buffer.from('arenacode-virtual-input\0', 'utf8');
|
|
291
|
+
// struct uinput_user_dev: name[80], input_id (4 x u16), ff_effects_max (u32),
|
|
292
|
+
// then absmax/absmin/absfuzz/absflat, 64 s32 each. ff_effects_max must stay 0:
|
|
293
|
+
// a non-zero value makes the kernel refuse UI_DEV_CREATE with EINVAL.
|
|
294
|
+
const uinput_user_dev = Buffer.alloc(1116);
|
|
295
|
+
name.copy(uinput_user_dev, 0);
|
|
296
|
+
uinput_user_dev.writeUInt16LE(0x03, 80); // bustype BUS_USB
|
|
297
|
+
uinput_user_dev.writeUInt16LE(0x1209, 82); // vendor (pid.codes)
|
|
298
|
+
uinput_user_dev.writeUInt16LE(0x0001, 84); // product
|
|
299
|
+
uinput_user_dev.writeUInt16LE(1, 86); // version
|
|
300
|
+
uinput_user_dev.writeUInt32LE(0, 88); // ff_effects_max
|
|
301
|
+
// no axes are declared, so the four abs tables stay zeroed on purpose
|
|
302
|
+
const wrote = write(fd, uinput_user_dev, uinput_user_dev.length);
|
|
303
|
+
if (wrote !== uinput_user_dev.length) throw new Error(`short write of the device description (${wrote})`);
|
|
304
|
+
return check(ioctlNone(koffi, fd, UI_DEV_CREATE), 'UI_DEV_CREATE');
|
|
305
|
+
};
|
|
306
|
+
try {
|
|
307
|
+
setup();
|
|
308
|
+
} catch (err) {
|
|
309
|
+
try {
|
|
310
|
+
fs.closeSync(fd);
|
|
311
|
+
} catch {
|
|
312
|
+
/* already gone */
|
|
313
|
+
}
|
|
314
|
+
throw new Error(`uinput device creation failed: ${err.message}`);
|
|
315
|
+
}
|
|
316
|
+
const eventNode = waitForDeviceNode(nodesBefore);
|
|
317
|
+
device = { fd, keyboard, pointer, path: file, eventNode };
|
|
318
|
+
return device;
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
/** Event nodes the kernel currently exposes, newest last. */
|
|
322
|
+
function listEventNodes() {
|
|
323
|
+
try {
|
|
324
|
+
return fs
|
|
325
|
+
.readdirSync('/dev/input')
|
|
326
|
+
.filter((name) => /^event\d+$/.test(name))
|
|
327
|
+
.sort();
|
|
328
|
+
} catch {
|
|
329
|
+
return [];
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
/**
|
|
334
|
+
* The kernel exposes the new node a moment after UI_DEV_CREATE. Returns the node this
|
|
335
|
+
* device created, not just "some node exists": a crashed process can leave a device
|
|
336
|
+
* behind, and picking the wrong one makes every later read look empty.
|
|
337
|
+
*/
|
|
338
|
+
function waitForDeviceNode(before, timeoutMs = 1500) {
|
|
339
|
+
const deadline = Date.now() + timeoutMs;
|
|
340
|
+
while (Date.now() < deadline) {
|
|
341
|
+
const fresh = listEventNodes().find((name) => !before.includes(name));
|
|
342
|
+
if (fresh) return `/dev/input/${fresh}`;
|
|
343
|
+
const until = Date.now() + 20;
|
|
344
|
+
while (Date.now() < until) {
|
|
345
|
+
/* spin: this runs once per device, not per event */
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
return null;
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
function destroyDevice() {
|
|
352
|
+
if (!device) return;
|
|
353
|
+
try {
|
|
354
|
+
const koffi = requireFfi();
|
|
355
|
+
ioctlNone(koffi, device.fd, UI_DEV_DESTROY);
|
|
356
|
+
} catch {
|
|
357
|
+
/* the device goes away with the process anyway */
|
|
358
|
+
}
|
|
359
|
+
try {
|
|
360
|
+
fs.closeSync(device.fd);
|
|
361
|
+
} catch {
|
|
362
|
+
/* already closed */
|
|
363
|
+
}
|
|
364
|
+
device = null;
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
function send(events) {
|
|
368
|
+
const dev = ensureDevice({ keyboard: true, pointer: true });
|
|
369
|
+
fs.writeSync(dev.fd, eventBuffer(events));
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
// ---------------------------------------------------------------- D-Bus (screenshot) --
|
|
373
|
+
|
|
374
|
+
// --- libdbus, loaded once ------------------------------------------------------
|
|
375
|
+
|
|
376
|
+
let dbusCache = null;
|
|
377
|
+
|
|
378
|
+
/**
|
|
379
|
+
* libdbus bindings, with the two details that cost the most time to find out:
|
|
380
|
+
* · DBusError.name / .message are struct fields, there is no dbus_error_message()
|
|
381
|
+
* · the value argument of the iter API is dereferenced for strings
|
|
382
|
+
* (`*(const char **)value`), so strings go in as a pointer to a pointer
|
|
383
|
+
*/
|
|
384
|
+
function loadDbus() {
|
|
385
|
+
if (dbusCache) return dbusCache;
|
|
386
|
+
const koffi = requireFfi();
|
|
387
|
+
const dbus = koffi.load('libdbus-1.so.3');
|
|
388
|
+
const DBusError = koffi.struct('ArenaCodeDBusError', {
|
|
389
|
+
name: 'const char *',
|
|
390
|
+
message: 'const char *',
|
|
391
|
+
dummy: 'unsigned int',
|
|
392
|
+
padding: 'void *',
|
|
393
|
+
});
|
|
394
|
+
const f = {
|
|
395
|
+
error_init: dbus.func('void dbus_error_init(void *err)'),
|
|
396
|
+
error_is_set: dbus.func('int dbus_error_is_set(void *err)'),
|
|
397
|
+
bus_get: dbus.func('void *dbus_bus_get(int type, void *err)'),
|
|
398
|
+
add_match: dbus.func('void dbus_bus_add_match(void *conn, const char *rule, void *err)'),
|
|
399
|
+
new_call: dbus.func('void *dbus_message_new_method_call(const char *dest, const char *path, const char *iface, const char *method)'),
|
|
400
|
+
iter_init_append: dbus.func('void dbus_message_iter_init_append(void *msg, void *iter)'),
|
|
401
|
+
iter_init: dbus.func('int dbus_message_iter_init(void *msg, void *iter)'),
|
|
402
|
+
iter_append: dbus.func('int dbus_message_iter_append_basic(void *iter, int type, void *value)'),
|
|
403
|
+
iter_get: dbus.func('void dbus_message_iter_get_basic(void *iter, void *value)'),
|
|
404
|
+
iter_arg_type: dbus.func('int dbus_message_iter_get_arg_type(void *iter)'),
|
|
405
|
+
iter_next: dbus.func('int dbus_message_iter_next(void *iter)'),
|
|
406
|
+
iter_recurse: dbus.func('void dbus_message_iter_recurse(void *iter, void *child)'),
|
|
407
|
+
open_container: dbus.func('int dbus_message_iter_open_container(void *iter, int type, const char *contained_signature, void *sub)'),
|
|
408
|
+
close_container: dbus.func('int dbus_message_iter_close_container(void *iter, void *sub)'),
|
|
409
|
+
send_block: dbus.func('void *dbus_connection_send_with_reply_and_block(void *conn, void *msg, int timeout, void *err)'),
|
|
410
|
+
send: dbus.func('int dbus_connection_send(void *conn, void *msg, unsigned int *serial)'),
|
|
411
|
+
flush: dbus.func('int dbus_connection_flush(void *conn)'),
|
|
412
|
+
dispatch: dbus.func('int dbus_connection_read_write_dispatch(void *conn, int timeout)'),
|
|
413
|
+
pop: dbus.func('void *dbus_connection_pop_message(void *conn)'),
|
|
414
|
+
is_signal: dbus.func('int dbus_message_is_signal(void *msg, const char *iface, const char *name)'),
|
|
415
|
+
get_type: dbus.func('int dbus_message_get_type(void *msg)'),
|
|
416
|
+
get_path: dbus.func('const char *dbus_message_get_path(void *msg)'),
|
|
417
|
+
get_error: dbus.func('const char *dbus_message_get_error_name(void *msg)'),
|
|
418
|
+
new_signal: dbus.func('void *dbus_message_new_signal(const char *path, const char *iface, const char *name)'),
|
|
419
|
+
set_path: dbus.func('int dbus_message_set_path(void *msg, const char *path)'),
|
|
420
|
+
unref: dbus.func('void dbus_message_unref(void *msg)'),
|
|
421
|
+
};
|
|
422
|
+
dbusCache = { koffi, f, DBusError, iter: () => koffi.alloc('uint8_t', 128) };
|
|
423
|
+
return dbusCache;
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
/** Opens the session bus, or explains why it cannot be opened. */
|
|
427
|
+
function openSessionBus() {
|
|
428
|
+
const { koffi, f, DBusError } = loadDbus();
|
|
429
|
+
const err = koffi.alloc(DBusError, 1);
|
|
430
|
+
f.error_init(err);
|
|
431
|
+
const conn = f.bus_get(0 /* DBUS_BUS_SESSION */, err);
|
|
432
|
+
if (!conn) {
|
|
433
|
+
if (!process.env.DBUS_SESSION_BUS_ADDRESS) {
|
|
434
|
+
throw new Error('no session bus (DBUS_SESSION_BUS_ADDRESS is not set)');
|
|
435
|
+
}
|
|
436
|
+
const decoded = f.error_is_set(err) ? koffi.decode(err, DBusError) : null;
|
|
437
|
+
const detail = decoded ? [decoded.name, decoded.message].filter(Boolean).join(': ') : 'unknown D-Bus error';
|
|
438
|
+
throw new Error(`session bus unavailable (${detail})`);
|
|
439
|
+
}
|
|
440
|
+
const err2 = koffi.alloc(DBusError, 1);
|
|
441
|
+
f.error_init(err2);
|
|
442
|
+
return { conn, err: err2, errorText: () => {
|
|
443
|
+
if (!f.error_is_set(err2)) return 'no error reported';
|
|
444
|
+
const decoded = koffi.decode(err2, DBusError);
|
|
445
|
+
return [decoded && decoded.name, decoded && decoded.message].filter(Boolean).join(': ') || 'unknown D-Bus error';
|
|
446
|
+
} };
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
/** Appends one string argument (portable form: pointer to pointer). */
|
|
450
|
+
function appendString(f, koffi, iter, value) {
|
|
451
|
+
const ref = koffi.alloc('char *', 1);
|
|
452
|
+
koffi.encode(ref, 'char *', value);
|
|
453
|
+
return f.iter_append(iter, 0x73 /* 's' */, ref);
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
/** Reads a string argument out of an iterator. */
|
|
457
|
+
function readString(f, koffi, iter) {
|
|
458
|
+
const ref = koffi.alloc('char *', 1);
|
|
459
|
+
f.iter_get(iter, ref);
|
|
460
|
+
return String(koffi.decode(ref, 'char *') || '');
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
/** Reads the first unsigned 32-bit argument out of an iterator. */
|
|
464
|
+
function readUint32(f, koffi, iter) {
|
|
465
|
+
const slot = koffi.alloc('uint32_t', 1);
|
|
466
|
+
f.iter_get(iter, slot);
|
|
467
|
+
return koffi.decode(slot, 'uint32_t');
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
// --- screenshots ---------------------------------------------------------------
|
|
471
|
+
|
|
472
|
+
/**
|
|
473
|
+
* Portable screenshot: org.freedesktop.portal.Screenshot (xdg-desktop-portal).
|
|
474
|
+
* Works on GNOME, KDE and wlroots compositors; the desktop asks the user to confirm the
|
|
475
|
+
* first time, exactly as it does for any other portal.
|
|
476
|
+
* The call returns a request handle, the file arrives with the Response signal.
|
|
477
|
+
*/
|
|
478
|
+
function portalScreenshot(dest, timeoutMs = 25000) {
|
|
479
|
+
const { koffi, f } = loadDbus();
|
|
480
|
+
const { conn, errorText } = openSessionBus();
|
|
481
|
+
|
|
482
|
+
// A D-Bus connection only receives broadcasts it has a match rule for: without this
|
|
483
|
+
// the Response signal never reaches us and the call looks like it hung.
|
|
484
|
+
f.add_match(conn, "type='signal',interface='org.freedesktop.portal.Request',member='Response'", null);
|
|
485
|
+
f.flush(conn);
|
|
486
|
+
|
|
487
|
+
const msg = f.new_call(
|
|
488
|
+
'org.freedesktop.portal.Desktop',
|
|
489
|
+
'/org/freedesktop/portal/desktop',
|
|
490
|
+
'org.freedesktop.portal.Screenshot',
|
|
491
|
+
'Screenshot'
|
|
492
|
+
);
|
|
493
|
+
if (!msg) throw new Error('cannot build the portal call');
|
|
494
|
+
const iter = loadDbus().iter();
|
|
495
|
+
f.iter_init_append(msg, iter);
|
|
496
|
+
if (!appendString(f, koffi, iter, '')) { // parent_window: none
|
|
497
|
+
f.unref(msg);
|
|
498
|
+
throw new Error('cannot append the parent window argument');
|
|
499
|
+
}
|
|
500
|
+
// a{sv} options: an empty dictionary is valid, the portal fills in its defaults
|
|
501
|
+
const dict = loadDbus().iter();
|
|
502
|
+
if (!f.open_container(iter, 0x61 /* 'a' */, '{sv}', dict)) {
|
|
503
|
+
f.unref(msg);
|
|
504
|
+
throw new Error('cannot build the options dictionary');
|
|
505
|
+
}
|
|
506
|
+
if (!f.close_container(iter, dict)) {
|
|
507
|
+
f.unref(msg);
|
|
508
|
+
throw new Error('cannot close the options dictionary');
|
|
509
|
+
}
|
|
510
|
+
|
|
511
|
+
const reply = f.send_block(conn, msg, timeoutMs, null);
|
|
512
|
+
f.unref(msg);
|
|
513
|
+
if (!reply) throw new Error('the portal did not answer');
|
|
514
|
+
const failure = f.get_error(reply);
|
|
515
|
+
if (failure) throw new Error(`portal unavailable (${String(failure)})`);
|
|
516
|
+
|
|
517
|
+
const replyIter = loadDbus().iter();
|
|
518
|
+
if (!f.iter_init(reply, replyIter)) throw new Error('the portal returned no request handle');
|
|
519
|
+
const handle = readString(f, koffi, replyIter);
|
|
520
|
+
if (!handle) throw new Error('the portal returned an empty request handle');
|
|
521
|
+
|
|
522
|
+
// wait for Response(u response, a{sv} results) on the request object
|
|
523
|
+
const deadline = Date.now() + timeoutMs;
|
|
524
|
+
while (Date.now() < deadline) {
|
|
525
|
+
f.dispatch(conn, 250);
|
|
526
|
+
const signal = f.pop(conn);
|
|
527
|
+
if (!signal) continue;
|
|
528
|
+
const path = String(f.get_path(signal) || '');
|
|
529
|
+
const isResponse = f.is_signal(signal, 'org.freedesktop.portal.Request', 'Response') && path === handle;
|
|
530
|
+
if (!isResponse) {
|
|
531
|
+
f.unref(signal);
|
|
532
|
+
continue;
|
|
533
|
+
}
|
|
534
|
+
const it = loadDbus().iter();
|
|
535
|
+
if (!f.iter_init(signal, it)) {
|
|
536
|
+
f.unref(signal);
|
|
537
|
+
throw new Error('empty portal response');
|
|
538
|
+
}
|
|
539
|
+
const code = readUint32(f, koffi, it); // 0 ok, 1 cancelled, 2 other
|
|
540
|
+
if (code !== 0) {
|
|
541
|
+
f.unref(signal);
|
|
542
|
+
throw new Error(code === 1 ? 'the user dismissed the screenshot dialog' : `the portal refused the screenshot (code ${code})`);
|
|
543
|
+
}
|
|
544
|
+
// second argument: a{sv}. The array iterates over dict entries ('{'), each of them
|
|
545
|
+
// holds the key string and then the variant that carries the value.
|
|
546
|
+
let uri = '';
|
|
547
|
+
if (f.iter_next(it)) {
|
|
548
|
+
const array = loadDbus().iter();
|
|
549
|
+
f.iter_recurse(it, array); // array -> first dict entry
|
|
550
|
+
// DBUS_TYPE_DICT_ENTRY is 'e' (0x65); '{' is only how it looks in a signature
|
|
551
|
+
while (f.iter_arg_type(array) === 0x65) {
|
|
552
|
+
const entry = loadDbus().iter();
|
|
553
|
+
f.iter_recurse(array, entry); // dict entry -> key
|
|
554
|
+
const key = readString(f, koffi, entry);
|
|
555
|
+
const variant = loadDbus().iter();
|
|
556
|
+
if (f.iter_next(entry)) {
|
|
557
|
+
f.iter_recurse(entry, variant); // variant -> value
|
|
558
|
+
if (key === 'uri') uri = readString(f, koffi, variant);
|
|
559
|
+
}
|
|
560
|
+
if (!f.iter_next(array)) break;
|
|
561
|
+
}
|
|
562
|
+
}
|
|
563
|
+
f.unref(signal);
|
|
564
|
+
if (!uri) throw new Error('the portal returned no file uri');
|
|
565
|
+
const source = uri.startsWith('file://') ? decodeURIComponent(uri.slice('file://'.length)) : uri;
|
|
566
|
+
fs.copyFileSync(source, dest);
|
|
567
|
+
return dest;
|
|
568
|
+
}
|
|
569
|
+
void errorText;
|
|
570
|
+
throw new Error('the portal never answered (no screenshot consent given?)');
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
/**
|
|
574
|
+
* GNOME Shell service: same result, no portal needed. Kept as the fallback because some
|
|
575
|
+
* NixOS/Flatpak setups ship the shell without the screenshot backend of the portal.
|
|
576
|
+
* Signature (bbs) -> (b success, s filename).
|
|
577
|
+
*/
|
|
578
|
+
function gnomeScreenshot(dest) {
|
|
579
|
+
const { koffi, f } = loadDbus();
|
|
580
|
+
const { conn, errorText } = openSessionBus();
|
|
581
|
+
const msg = f.new_call('org.gnome.Shell.Screenshot', '/org/gnome/Shell/Screenshot', 'org.gnome.Shell.Screenshot', 'Screenshot');
|
|
582
|
+
if (!msg) throw new Error('cannot build the D-Bus call');
|
|
583
|
+
const iter = loadDbus().iter();
|
|
584
|
+
f.iter_init_append(msg, iter);
|
|
585
|
+
const no = koffi.alloc('int', 1);
|
|
586
|
+
const noFlash = koffi.alloc('int', 1);
|
|
587
|
+
const ref = koffi.alloc('char *', 1);
|
|
588
|
+
koffi.encode(ref, 'char *', dest);
|
|
589
|
+
const built = f.iter_append(iter, 0x62, no) && f.iter_append(iter, 0x62, noFlash) && f.iter_append(iter, 0x73, ref);
|
|
590
|
+
if (!built) {
|
|
591
|
+
f.unref(msg);
|
|
592
|
+
throw new Error('cannot append the arguments of the screenshot call');
|
|
593
|
+
}
|
|
594
|
+
const reply = f.send_block(conn, msg, 20000, null);
|
|
595
|
+
f.unref(msg);
|
|
596
|
+
if (!reply) throw new Error(`screenshot call failed (${errorText()})`);
|
|
597
|
+
const name = f.get_error(reply);
|
|
598
|
+
if (name) throw new Error(`screenshot refused (${String(name)})`);
|
|
599
|
+
const replyIter = loadDbus().iter();
|
|
600
|
+
let ok = true;
|
|
601
|
+
if (f.iter_init(reply, replyIter) && f.get_type(reply) === 2 /* METHOD_RETURN */) {
|
|
602
|
+
const value = koffi.alloc('int', 1);
|
|
603
|
+
f.iter_get(replyIter, value);
|
|
604
|
+
ok = koffi.decode(value, 'int') !== 0;
|
|
605
|
+
}
|
|
606
|
+
if (!ok) throw new Error('the compositor reported it could not take the screenshot');
|
|
607
|
+
if (!fs.existsSync(dest)) throw new Error('the compositor answered OK but wrote no file');
|
|
608
|
+
return dest;
|
|
609
|
+
}
|
|
610
|
+
|
|
611
|
+
// ---------------------------------------------------------------- backend --
|
|
612
|
+
|
|
613
|
+
function x11Backend() {
|
|
614
|
+
// the X11 backend is still the only way to see XWayland windows and grab the root
|
|
615
|
+
try {
|
|
616
|
+
return require('./linux').backend;
|
|
617
|
+
} catch {
|
|
618
|
+
return null;
|
|
619
|
+
}
|
|
620
|
+
}
|
|
621
|
+
|
|
622
|
+
const wayland = {
|
|
623
|
+
name: 'wayland-uinput',
|
|
624
|
+
|
|
625
|
+
detect,
|
|
626
|
+
|
|
627
|
+
available() {
|
|
628
|
+
const session = detect();
|
|
629
|
+
if (!isLinux) return { ok: false, backend: 'wayland', error: 'Wayland support is Linux only' };
|
|
630
|
+
if (!session.wayland) {
|
|
631
|
+
return { ok: false, backend: 'wayland', error: 'not a Wayland session (XDG_SESSION_TYPE/WAYLAND_DISPLAY)' };
|
|
632
|
+
}
|
|
633
|
+
const access = uinputAccess();
|
|
634
|
+
if (!access.ok) return { ok: false, backend: 'wayland', error: access.error, session };
|
|
635
|
+
return {
|
|
636
|
+
ok: true,
|
|
637
|
+
backend: 'wayland',
|
|
638
|
+
detail: `uinput + ${session.compositor}${session.xwayland ? ' (XWayland for windows)' : ''}`,
|
|
639
|
+
session,
|
|
640
|
+
};
|
|
641
|
+
},
|
|
642
|
+
|
|
643
|
+
sysinfo() {
|
|
644
|
+
const session = detect();
|
|
645
|
+
return {
|
|
646
|
+
ok: true,
|
|
647
|
+
system: {
|
|
648
|
+
os_version: `${os.type()} ${os.release()}`,
|
|
649
|
+
session: 'wayland',
|
|
650
|
+
compositor: session.compositor,
|
|
651
|
+
xwayland: session.xwayland,
|
|
652
|
+
input: 'uinput',
|
|
653
|
+
screen_size: this.screenSize(),
|
|
654
|
+
},
|
|
655
|
+
};
|
|
656
|
+
},
|
|
657
|
+
|
|
658
|
+
screenSize() {
|
|
659
|
+
const x11 = x11Backend();
|
|
660
|
+
if (x11) {
|
|
661
|
+
try {
|
|
662
|
+
const size = x11.screenSize();
|
|
663
|
+
if (size && size.ok) return { ...size, source: 'xwayland' };
|
|
664
|
+
} catch {
|
|
665
|
+
/* no X display */
|
|
666
|
+
}
|
|
667
|
+
}
|
|
668
|
+
return { ok: false, error: 'screen size needs XWayland on Wayland (no compositor-neutral API)' };
|
|
669
|
+
},
|
|
670
|
+
|
|
671
|
+
windows() {
|
|
672
|
+
const x11 = x11Backend();
|
|
673
|
+
if (!x11) return { ok: true, windows: [], active_window: null, wayland: true, note: 'no XWayland: native Wayland windows cannot be listed' };
|
|
674
|
+
try {
|
|
675
|
+
const list = x11.windows();
|
|
676
|
+
if (!list.ok) return list;
|
|
677
|
+
return {
|
|
678
|
+
...list,
|
|
679
|
+
wayland: true,
|
|
680
|
+
note: 'XWayland windows only: Wayland hides the native ones from every client',
|
|
681
|
+
};
|
|
682
|
+
} catch (err) {
|
|
683
|
+
return { ok: true, windows: [], wayland: true, note: `XWayland unavailable (${err.message})` };
|
|
684
|
+
}
|
|
685
|
+
},
|
|
686
|
+
|
|
687
|
+
activeWindow() {
|
|
688
|
+
const x11 = x11Backend();
|
|
689
|
+
if (x11) {
|
|
690
|
+
try {
|
|
691
|
+
return x11.activeWindow();
|
|
692
|
+
} catch {
|
|
693
|
+
/* fall through */
|
|
694
|
+
}
|
|
695
|
+
}
|
|
696
|
+
return { ok: false, error: 'the focused window is not readable on Wayland' };
|
|
697
|
+
},
|
|
698
|
+
|
|
699
|
+
state(handle = null) {
|
|
700
|
+
const list = this.windows();
|
|
701
|
+
return {
|
|
702
|
+
ok: true,
|
|
703
|
+
windows: list.windows || [],
|
|
704
|
+
active_window: this.activeWindow(),
|
|
705
|
+
interactive_elements: [],
|
|
706
|
+
scrollable_elements: [],
|
|
707
|
+
interactive_elements_text: '',
|
|
708
|
+
scrollable_elements_text: '',
|
|
709
|
+
system: { os_version: `${os.type()} ${os.release()}`, session: 'wayland' },
|
|
710
|
+
metadata: { kind: 'wayland', xwayland: detect().xwayland, elements: 'unavailable on Wayland' },
|
|
711
|
+
handle,
|
|
712
|
+
};
|
|
713
|
+
},
|
|
714
|
+
|
|
715
|
+
moveMouse(x, y) {
|
|
716
|
+
// Wayland has no absolute pointer positioning without a compositor protocol: the
|
|
717
|
+
// honest answer is a relative move from where the pointer is now.
|
|
718
|
+
const pos = this.pointerPosition();
|
|
719
|
+
const dx = Number(x) - (pos.ok ? pos.x : 0);
|
|
720
|
+
const dy = Number(y) - (pos.ok ? pos.y : 0);
|
|
721
|
+
if (!pos.ok) return { ok: false, error: 'cannot read the pointer position on Wayland: use scroll/keys, or click with absolute X11 coordinates through XWayland' };
|
|
722
|
+
send([{ type: EV_REL, code: REL_X, value: dx }, { type: EV_REL, code: REL_Y, value: dy }]);
|
|
723
|
+
return { ok: true, x: Number(x), y: Number(y), relative: true };
|
|
724
|
+
},
|
|
725
|
+
|
|
726
|
+
pointerPosition() {
|
|
727
|
+
const x11 = x11Backend();
|
|
728
|
+
if (x11 && typeof x11.pointerPosition === 'function') {
|
|
729
|
+
try {
|
|
730
|
+
const pos = x11.pointerPosition();
|
|
731
|
+
if (pos && pos.ok) return pos;
|
|
732
|
+
} catch {
|
|
733
|
+
/* nothing */
|
|
734
|
+
}
|
|
735
|
+
}
|
|
736
|
+
return { ok: false, error: 'pointer position unavailable without XWayland' };
|
|
737
|
+
},
|
|
738
|
+
|
|
739
|
+
click(x, y, { button = 'left', count = 1 } = {}) {
|
|
740
|
+
if (x !== null && x !== undefined) {
|
|
741
|
+
const moved = this.moveMouse(x, y);
|
|
742
|
+
if (!moved.ok) return moved;
|
|
743
|
+
}
|
|
744
|
+
const code = button === 'right' ? BTN_RIGHT : button === 'middle' ? BTN_MIDDLE : BTN_LEFT;
|
|
745
|
+
const events = [];
|
|
746
|
+
for (let i = 0; i < Math.max(1, count); i += 1) {
|
|
747
|
+
events.push({ type: EV_KEY, code, value: 1 }, { type: EV_KEY, code, value: 0 });
|
|
748
|
+
}
|
|
749
|
+
send(events);
|
|
750
|
+
return { ok: true, x, y, button, count };
|
|
751
|
+
},
|
|
752
|
+
|
|
753
|
+
mouseButton(x, y, button, down) {
|
|
754
|
+
if (down && x !== null && x !== undefined) {
|
|
755
|
+
const moved = this.moveMouse(x, y);
|
|
756
|
+
if (!moved.ok) return moved;
|
|
757
|
+
}
|
|
758
|
+
const code = button === 'right' ? BTN_RIGHT : button === 'middle' ? BTN_MIDDLE : BTN_LEFT;
|
|
759
|
+
send([{ type: EV_KEY, code, value: down ? 1 : 0 }]);
|
|
760
|
+
return { ok: true, x, y, button, down };
|
|
761
|
+
},
|
|
762
|
+
|
|
763
|
+
drag(x, y, x2, y2) {
|
|
764
|
+
const start = this.mouseButton(x, y, 'left', true);
|
|
765
|
+
if (!start.ok) return start;
|
|
766
|
+
const moved = this.moveMouse(x2, y2);
|
|
767
|
+
this.mouseButton(x2, y2, 'left', false);
|
|
768
|
+
return { ok: moved.ok, from: [x, y], to: [x2, y2] };
|
|
769
|
+
},
|
|
770
|
+
|
|
771
|
+
scroll(amount, x = null, y = null) {
|
|
772
|
+
const notches = Math.trunc(Number(amount) || 0);
|
|
773
|
+
if (!notches) return { ok: true, amount: 0 };
|
|
774
|
+
const events = [];
|
|
775
|
+
for (let i = 0; i < Math.abs(notches); i += 1) events.push({ type: EV_REL, code: REL_WHEEL, value: notches > 0 ? 1 : -1 });
|
|
776
|
+
send(events);
|
|
777
|
+
if (x !== null && x !== undefined) this.moveMouse(x, y);
|
|
778
|
+
return { ok: true, amount: notches };
|
|
779
|
+
},
|
|
780
|
+
|
|
781
|
+
pressKeys(combo) {
|
|
782
|
+
const seq = keySequence(combo);
|
|
783
|
+
if (!seq.length) return { ok: false, error: `unknown key: ${combo}` };
|
|
784
|
+
send(seq.map((step) => ({ type: EV_KEY, code: step.code, value: step.down ? 1 : 0 })));
|
|
785
|
+
return { ok: true, keys: combo };
|
|
786
|
+
},
|
|
787
|
+
|
|
788
|
+
typeText(text, x = null, y = null) {
|
|
789
|
+
const value = String(text ?? '');
|
|
790
|
+
if (!value) return { ok: true, chars: 0, typed: 0 };
|
|
791
|
+
if (x !== null && x !== undefined) {
|
|
792
|
+
const moved = this.moveMouse(x, y);
|
|
793
|
+
if (!moved.ok) return moved;
|
|
794
|
+
}
|
|
795
|
+
const map = charMap();
|
|
796
|
+
const missing = [];
|
|
797
|
+
const events = [];
|
|
798
|
+
for (const ch of value) {
|
|
799
|
+
const hit = map.get(ch);
|
|
800
|
+
if (!hit) {
|
|
801
|
+
missing.push(ch);
|
|
802
|
+
continue;
|
|
803
|
+
}
|
|
804
|
+
if (hit.shift) events.push({ type: EV_KEY, code: KEYS.shift, value: 1 });
|
|
805
|
+
events.push({ type: EV_KEY, code: hit.code, value: 1 }, { type: EV_KEY, code: hit.code, value: 0 });
|
|
806
|
+
if (hit.shift) events.push({ type: EV_KEY, code: KEYS.shift, value: 0 });
|
|
807
|
+
}
|
|
808
|
+
if (missing.length) {
|
|
809
|
+
return {
|
|
810
|
+
ok: false,
|
|
811
|
+
error: `the active keymap cannot type ${missing.length} character(s): ${[...new Set(missing)].join(' ')}`,
|
|
812
|
+
unmapped: [...new Set(missing)],
|
|
813
|
+
layout: map.source,
|
|
814
|
+
};
|
|
815
|
+
}
|
|
816
|
+
send(events);
|
|
817
|
+
return { ok: true, chars: value.length, typed: value.length, layout: map.source };
|
|
818
|
+
},
|
|
819
|
+
|
|
820
|
+
screenshot(dest, { annotate = false } = {}) {
|
|
821
|
+
fs.mkdirSync(path.dirname(dest), { recursive: true });
|
|
822
|
+
const attempts = [];
|
|
823
|
+
try {
|
|
824
|
+
portalScreenshot(dest);
|
|
825
|
+
const size = pngSize(dest);
|
|
826
|
+
return { ok: true, file: dest, mime: 'image/png', method: 'xdg-desktop-portal', ...size, annotated: Boolean(annotate) };
|
|
827
|
+
} catch (err) {
|
|
828
|
+
attempts.push(`portal: ${err.message}`);
|
|
829
|
+
}
|
|
830
|
+
try {
|
|
831
|
+
gnomeScreenshot(dest);
|
|
832
|
+
const size = pngSize(dest);
|
|
833
|
+
return { ok: true, file: dest, mime: 'image/png', method: 'gnome-shell-dbus', ...size, annotated: Boolean(annotate) };
|
|
834
|
+
} catch (err) {
|
|
835
|
+
attempts.push(`gnome-shell: ${err.message}`);
|
|
836
|
+
}
|
|
837
|
+
const x11 = x11Backend();
|
|
838
|
+
if (x11) {
|
|
839
|
+
try {
|
|
840
|
+
const shot = x11.screenshot(dest, { annotate });
|
|
841
|
+
if (shot && shot.ok) return { ...shot, method: 'xwayland', partial: true };
|
|
842
|
+
attempts.push(`xwayland: ${(shot && shot.error) || 'failed'}`);
|
|
843
|
+
} catch (err) {
|
|
844
|
+
attempts.push(`xwayland: ${err.message}`);
|
|
845
|
+
}
|
|
846
|
+
}
|
|
847
|
+
return {
|
|
848
|
+
ok: false,
|
|
849
|
+
error: `screenshot on this Wayland compositor needs the GNOME Shell D-Bus service: ${attempts.join(' · ')}`,
|
|
850
|
+
attempts,
|
|
851
|
+
};
|
|
852
|
+
},
|
|
853
|
+
|
|
854
|
+
listApps() {
|
|
855
|
+
const dirs = String(process.env.XDG_DATA_DIRS || '/usr/share:/usr/local/share').split(':');
|
|
856
|
+
const apps = [];
|
|
857
|
+
const seen = new Set();
|
|
858
|
+
for (const dir of dirs.filter(Boolean)) {
|
|
859
|
+
const folder = path.join(dir, 'applications');
|
|
860
|
+
let files = [];
|
|
861
|
+
try {
|
|
862
|
+
files = fs.readdirSync(folder).filter((name) => name.endsWith('.desktop'));
|
|
863
|
+
} catch {
|
|
864
|
+
continue;
|
|
865
|
+
}
|
|
866
|
+
for (const file of files) {
|
|
867
|
+
try {
|
|
868
|
+
const text = fs.readFileSync(path.join(folder, file), 'utf8');
|
|
869
|
+
const name = /^Name=(.+)$/m.exec(text)?.[1];
|
|
870
|
+
const exec = /^Exec=(.+)$/m.exec(text)?.[1];
|
|
871
|
+
const hidden = /^(NoDisplay|Hidden)=true$/m.test(text);
|
|
872
|
+
if (!name || !exec || hidden) continue;
|
|
873
|
+
const id = file.replace(/\.desktop$/, '');
|
|
874
|
+
if (seen.has(id)) continue;
|
|
875
|
+
seen.add(id);
|
|
876
|
+
apps.push({ name, app_id: id, exec: exec.replace(/%[fFuU]/g, '').trim() });
|
|
877
|
+
} catch {
|
|
878
|
+
/* unreadable entry */
|
|
879
|
+
}
|
|
880
|
+
}
|
|
881
|
+
}
|
|
882
|
+
apps.sort((a, b) => a.name.localeCompare(b.name));
|
|
883
|
+
return { ok: true, apps: apps.slice(0, 200), wayland: true };
|
|
884
|
+
},
|
|
885
|
+
|
|
886
|
+
launchApp(name) {
|
|
887
|
+
const wanted = String(name || '').trim();
|
|
888
|
+
if (!wanted) return { ok: false, error: 'no app name' };
|
|
889
|
+
const { spawn } = require('node:child_process');
|
|
890
|
+
const known = this.listApps();
|
|
891
|
+
const hit = (known.apps || []).find((app) => app.name.toLowerCase() === wanted.toLowerCase() || app.app_id.toLowerCase() === wanted.toLowerCase());
|
|
892
|
+
const command = hit ? hit.exec : wanted;
|
|
893
|
+
try {
|
|
894
|
+
const child = spawn(command, { detached: true, stdio: 'ignore', shell: true, env: { ...process.env, DISPLAY: process.env.DISPLAY || '', WAYLAND_DISPLAY: process.env.WAYLAND_DISPLAY || '' } });
|
|
895
|
+
child.unref();
|
|
896
|
+
return { ok: true, app: hit ? hit.name : wanted, command };
|
|
897
|
+
} catch (err) {
|
|
898
|
+
return { ok: false, error: err.message };
|
|
899
|
+
}
|
|
900
|
+
},
|
|
901
|
+
|
|
902
|
+
focusWindow() {
|
|
903
|
+
return { ok: false, error: 'Wayland does not let a client focus another client\'s window' };
|
|
904
|
+
},
|
|
905
|
+
|
|
906
|
+
resizeWindow() {
|
|
907
|
+
return { ok: false, error: 'Wayland does not let a client resize another client\'s window' };
|
|
908
|
+
},
|
|
909
|
+
|
|
910
|
+
findWindow(name) {
|
|
911
|
+
const list = this.windows();
|
|
912
|
+
if (!list.ok) return list;
|
|
913
|
+
const { pickWindow } = require('../automation');
|
|
914
|
+
const hit = pickWindow(list.windows || [], name);
|
|
915
|
+
return hit ? { ok: true, window: hit, xwayland: true } : { ok: false, error: `window not found (XWayland only): ${name}` };
|
|
916
|
+
},
|
|
917
|
+
};
|
|
918
|
+
|
|
919
|
+
/** PNG width/height straight from the header, so the caller gets the numbers. */
|
|
920
|
+
function pngSize(file) {
|
|
921
|
+
try {
|
|
922
|
+
const head = Buffer.alloc(24);
|
|
923
|
+
const fd = fs.openSync(file, 'r');
|
|
924
|
+
fs.readSync(fd, head, 0, 24, 0);
|
|
925
|
+
fs.closeSync(fd);
|
|
926
|
+
if (head.readUInt32BE(0) !== 0x89504e47) return {};
|
|
927
|
+
return { width: head.readUInt32BE(16), height: head.readUInt32BE(20) };
|
|
928
|
+
} catch {
|
|
929
|
+
return {};
|
|
930
|
+
}
|
|
931
|
+
}
|
|
932
|
+
|
|
933
|
+
module.exports = {
|
|
934
|
+
backend: wayland,
|
|
935
|
+
detect,
|
|
936
|
+
uinputAccess,
|
|
937
|
+
uinputPath,
|
|
938
|
+
charMap,
|
|
939
|
+
usCharMap,
|
|
940
|
+
keySequence,
|
|
941
|
+
eventBuffer,
|
|
942
|
+
ensureDevice,
|
|
943
|
+
destroyDevice,
|
|
944
|
+
KEYS,
|
|
945
|
+
EVENT_SIZE,
|
|
946
|
+
pngSize,
|
|
947
|
+
};
|
package/src/automation.js
CHANGED
|
@@ -1000,13 +1000,28 @@ const automation = {
|
|
|
1000
1000
|
|
|
1001
1001
|
let nativeBackend = null;
|
|
1002
1002
|
let nativeKind = null;
|
|
1003
|
+
let nativeDegraded = null;
|
|
1003
1004
|
|
|
1004
1005
|
try {
|
|
1005
1006
|
if (isLinux) {
|
|
1006
1007
|
// Linux has no fallback any more: the FFI backend *is* the backend (the old one
|
|
1007
1008
|
// shelled out to xdotool, which is exactly what we do not want).
|
|
1008
|
-
|
|
1009
|
-
|
|
1009
|
+
//
|
|
1010
|
+
// On Wayland the X11 backend only ever reaches XWayland applications, so the
|
|
1011
|
+
// uinput backend takes over when it can open /dev/uinput. If it cannot, we stay on
|
|
1012
|
+
// X11 and say so instead of failing silently.
|
|
1013
|
+
const wayland = require('./automation/wayland');
|
|
1014
|
+
const session = wayland.detect();
|
|
1015
|
+
if (session.wayland && wayland.backend.available().ok) {
|
|
1016
|
+
nativeBackend = wayland.backend;
|
|
1017
|
+
nativeKind = 'wayland';
|
|
1018
|
+
} else {
|
|
1019
|
+
nativeBackend = require('./automation/linux').backend;
|
|
1020
|
+
nativeKind = 'x11';
|
|
1021
|
+
if (session.wayland) {
|
|
1022
|
+
nativeDegraded = wayland.backend.available().error || 'Wayland detected, uinput unavailable';
|
|
1023
|
+
}
|
|
1024
|
+
}
|
|
1010
1025
|
} else if (isWin) {
|
|
1011
1026
|
// The DLL ships in the package; PowerShell stays only as a safety net for the case
|
|
1012
1027
|
// where it cannot be loaded at all.
|
|
@@ -1040,6 +1055,7 @@ if (nativeBackend) {
|
|
|
1040
1055
|
kind: nativeKind,
|
|
1041
1056
|
name: nativeBackend.name,
|
|
1042
1057
|
dll: nativeKind === 'dll' ? require('./automation/win-dll').locate() : null,
|
|
1058
|
+
degraded: nativeDegraded,
|
|
1043
1059
|
},
|
|
1044
1060
|
available: () => nativeBackend.available(),
|
|
1045
1061
|
screenSize: () => nativeBackend.screenSize(),
|
|
@@ -188,6 +188,7 @@ Starts with `[ArenaCode mode: computer-agent]`. You act on the whole PC through
|
|
|
188
188
|
- Type exactly the text of the task: no extra spaces, line breaks or repeated characters.
|
|
189
189
|
- Before typing, click the field and confirm the cursor is in it. After a window changes, take a new screenshot before clicking. Do not reuse old coordinates.
|
|
190
190
|
- Coordinates are physical pixels, taken from the latest screenshot (`width` and `height` in its JSON).
|
|
191
|
+
- **Wayland** (`ui sysinfo` → `session: "wayland"`): `interactive_elements` is empty and `windows` only lists XWayland windows, so read the screen and click by coordinates from the screenshot. If a click with explicit `--x/--y` returns `ok: false` because the pointer position is unknown, do not retry it: reach the control with `ui keys` (tab, arrows, enter) or `ui control --action scroll`, and say in your reply that the desktop does not allow absolute positioning.
|
|
191
192
|
- Be efficient: group the predictable steps of the approved plan into short sequences, check at checkpoints rather than after every trivial step, do not repeat a completed action, and do not reopen what is already open.
|
|
192
193
|
|
|
193
194
|
### Safety
|
package/src/tui/app.js
CHANGED
|
@@ -305,6 +305,17 @@ class App {
|
|
|
305
305
|
this.invalidate();
|
|
306
306
|
return undefined;
|
|
307
307
|
}
|
|
308
|
+
// a left click on a card's affordance row ("more lines" / "collapse diff") toggles it
|
|
309
|
+
if (key.name === 'mouse' && key.button === 0 && !key.release) {
|
|
310
|
+
const hit = this.hitTest(key.y);
|
|
311
|
+
if (hit && hit.affordance) {
|
|
312
|
+
hit.item.expanded = hit.affordance === 'more';
|
|
313
|
+
this.invalidate();
|
|
314
|
+
this.scheduleSave();
|
|
315
|
+
return undefined;
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
|
|
308
319
|
// the wheel scrolls the transcript (three lines a notch, like every terminal app)
|
|
309
320
|
if (key.name === 'wheel') {
|
|
310
321
|
if (key.dir === 'up') st.scroll += 3;
|
|
@@ -333,6 +344,8 @@ class App {
|
|
|
333
344
|
return this.openHelp();
|
|
334
345
|
case 'l':
|
|
335
346
|
return this.openSessions();
|
|
347
|
+
case 'e':
|
|
348
|
+
return this.toggleLastDiff();
|
|
336
349
|
case 't': {
|
|
337
350
|
const names = themeNames();
|
|
338
351
|
const next = names[(names.indexOf(this.themeName) + 1) % names.length];
|
|
@@ -636,6 +649,7 @@ class App {
|
|
|
636
649
|
} },
|
|
637
650
|
{ title: 'Sessions (switch between sessions)', hint: 'ctrl+x l', slash: '/sessions', run: () => this.openSessions() },
|
|
638
651
|
{ title: 'Queue: tasks on the bridge', hint: '', slash: '/queue', run: () => this.openTasks() },
|
|
652
|
+
{ title: 'Expand the last diff', hint: 'ctrl+x e', run: () => this.toggleLastDiff() },
|
|
639
653
|
{ title: 'Toggle detail', hint: 'ctrl+x d', run: () => {
|
|
640
654
|
this.state.details = !this.state.details;
|
|
641
655
|
this.toast(`details ${this.state.details ? 'on' : 'off'}`);
|
|
@@ -891,6 +905,28 @@ class App {
|
|
|
891
905
|
return undefined;
|
|
892
906
|
}
|
|
893
907
|
|
|
908
|
+
/** Screen row (1-based, as the mouse reports it) -> the card drawn there, if any. */
|
|
909
|
+
hitTest(row) {
|
|
910
|
+
const view = this._chatView;
|
|
911
|
+
if (!view || !view.map) return null;
|
|
912
|
+
const index = view.start + (Number(row) - 1) - view.pad;
|
|
913
|
+
return view.map[index] || null;
|
|
914
|
+
}
|
|
915
|
+
|
|
916
|
+
/** `ctrl+x e`: expand (or close) the last diff in the transcript, for keyboards only. */
|
|
917
|
+
toggleLastDiff() {
|
|
918
|
+
const item = [...this.state.items].reverse().find((i) => i && i.diff);
|
|
919
|
+
if (!item) {
|
|
920
|
+
this.toast('no diff in this session');
|
|
921
|
+
return undefined;
|
|
922
|
+
}
|
|
923
|
+
item.expanded = !item.expanded;
|
|
924
|
+
this.toast(item.expanded ? `diff expanded (${item.diff.rows.length} rows)` : 'diff collapsed');
|
|
925
|
+
this.invalidate();
|
|
926
|
+
this.scheduleSave();
|
|
927
|
+
return undefined;
|
|
928
|
+
}
|
|
929
|
+
|
|
894
930
|
/** A list dialog draws group rows too: the selection index must skip them. */
|
|
895
931
|
sessionRowIndex(index) {
|
|
896
932
|
const at = Math.max(0, Math.min(this.sessions.length, index));
|
|
@@ -1611,6 +1647,7 @@ class App {
|
|
|
1611
1647
|
frame.push(footerFor());
|
|
1612
1648
|
if (prompt.cursor) cursor = { row: promptStart + prompt.cursor.row, col: prompt.cursor.col };
|
|
1613
1649
|
} else {
|
|
1650
|
+
const chatMap = [];
|
|
1614
1651
|
const chat = W2
|
|
1615
1652
|
.chatLines(t, st.items, {
|
|
1616
1653
|
width: W - 4,
|
|
@@ -1618,6 +1655,7 @@ class App {
|
|
|
1618
1655
|
agent: this.agentName(),
|
|
1619
1656
|
model: this.modelName(),
|
|
1620
1657
|
details: st.details ? 'full' : false,
|
|
1658
|
+
map: chatMap,
|
|
1621
1659
|
})
|
|
1622
1660
|
.map((line) => R.spaces(2) + line);
|
|
1623
1661
|
|
|
@@ -1625,7 +1663,11 @@ class App {
|
|
|
1625
1663
|
const maxScroll = Math.max(0, chat.length - bodyH);
|
|
1626
1664
|
st.scroll = Math.min(Math.max(0, st.scroll), maxScroll);
|
|
1627
1665
|
const start = Math.max(0, chat.length - bodyH - st.scroll);
|
|
1666
|
+
// a click comes in screen rows: remember how the visible slice maps onto `chat`
|
|
1667
|
+
this._chatView = { start, pad: 0, map: chatMap };
|
|
1628
1668
|
const visible = chat.slice(start, start + bodyH);
|
|
1669
|
+
const pad = Math.max(0, bodyH - visible.length);
|
|
1670
|
+
this._chatView.pad = pad;
|
|
1629
1671
|
while (visible.length < bodyH) visible.unshift('');
|
|
1630
1672
|
frame.push(...visible);
|
|
1631
1673
|
frame.push('');
|
|
@@ -1862,6 +1904,7 @@ function HELP_ROWS() {
|
|
|
1862
1904
|
['Session', 'ctrl+x n', 'new session'],
|
|
1863
1905
|
['Session', 'ctrl+x l', 'switch session'],
|
|
1864
1906
|
['Session', '/sessions', 'switch between sessions'],
|
|
1907
|
+
['Session', 'click / ctrl+x e', 'open or close a diff that was cut short'],
|
|
1865
1908
|
['Session', '/queue', 'tasks on the bridge'],
|
|
1866
1909
|
['Session', 'ctrl+x a', 'Ask and Proceed / Proceed'],
|
|
1867
1910
|
['Session', 'ctrl+x t', 'switch theme'],
|
package/src/tui/widgets.js
CHANGED
|
@@ -214,14 +214,22 @@ function diffLines(t, diff, { width = 80, max = 24 } = {}) {
|
|
|
214
214
|
out.push(`${line}${spaces(Math.max(0, width - R.vlen(head + body)))}${RESET}`);
|
|
215
215
|
}
|
|
216
216
|
if (rows.length > max) {
|
|
217
|
-
|
|
217
|
+
// the last row is the affordance: clicking it expands this very card (see chatLines' map)
|
|
218
|
+
out.push(paint(D.ctx.noBg, D.number, spaces(gutter) + `⋯ ${rows.length - max} more lines`, width));
|
|
219
|
+
out.affordance = 'more';
|
|
218
220
|
}
|
|
219
221
|
return out;
|
|
220
222
|
}
|
|
221
223
|
|
|
222
224
|
function chatLines(t, items, opts = {}) {
|
|
223
|
-
const { width, frame = 0, agent = 'Build', model = '', details = true } = opts;
|
|
225
|
+
const { width, frame = 0, agent = 'Build', model = '', details = true, map = null } = opts;
|
|
224
226
|
const out = [];
|
|
227
|
+
// `map` records, for every emitted line, which item it belongs to: the mouse needs it to
|
|
228
|
+
// know that the row under the pointer is that card's "more lines" affordance.
|
|
229
|
+
const mark = (from, item, extra = null) => {
|
|
230
|
+
if (!map) return;
|
|
231
|
+
for (let i = from; i < out.length; i += 1) map[i] = { item, ...(i === from ? extra : null) };
|
|
232
|
+
};
|
|
225
233
|
let prevKind = null;
|
|
226
234
|
// Blocks (a message, a card) get one empty line above and below; single lines
|
|
227
235
|
// (thoughts, notices) stay together, so nothing looks glued to its neighbour.
|
|
@@ -284,6 +292,7 @@ function chatLines(t, items, opts = {}) {
|
|
|
284
292
|
const meta = item.meta ? ` (${item.meta})` : '';
|
|
285
293
|
pushBlank();
|
|
286
294
|
if (item.diff && details && item.diff.rows && item.diff.rows.length) {
|
|
295
|
+
const from = out.length;
|
|
287
296
|
const bg = bgSeq(t.c.backgroundPanel);
|
|
288
297
|
const bar = `${fgSeq(t.c.background)}┃${RESET}`;
|
|
289
298
|
const inner = width - 3;
|
|
@@ -301,8 +310,19 @@ function chatLines(t, items, opts = {}) {
|
|
|
301
310
|
width - 1,
|
|
302
311
|
),
|
|
303
312
|
);
|
|
304
|
-
|
|
305
|
-
|
|
313
|
+
const diffRows = diffLines(t, item.diff, { width: inner, max: item.expanded ? Infinity : 20 });
|
|
314
|
+
for (const line of diffRows) out.push(bar + line + spaces(0));
|
|
315
|
+
if (diffRows.affordance === 'more') {
|
|
316
|
+
mark(from, item);
|
|
317
|
+
if (map) map[out.length - 1] = { item, affordance: 'more' };
|
|
318
|
+
} else if (item.expanded) {
|
|
319
|
+
// expanded: the card ends with a row that closes it again (also clickable)
|
|
320
|
+
const D = t.diff;
|
|
321
|
+
out.push(bar + `${bgSeq(D.ctx.noBg)}${fgSeq(D.number)}${spaces(3)}− collapse diff${spaces(Math.max(0, inner - 17))}${RESET}`);
|
|
322
|
+
mark(from, item);
|
|
323
|
+
if (map) map[out.length - 1] = { item, affordance: 'less' };
|
|
324
|
+
} else {
|
|
325
|
+
mark(from, item);
|
|
306
326
|
}
|
|
307
327
|
} else if (item.detail && details) {
|
|
308
328
|
// block tool: left border, panel background, command then output
|