@remcp/runtime 0.2.153 → 0.2.155

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,1791 +1,6 @@
1
- import path from 'node:path';
2
- import process from 'node:process';
3
- import { open } from 'node:fs/promises';
4
-
5
- import { image, multi, text } from '../util.mjs';
6
- import { capturePortalScreenshot, isWaylandSession } from '../screenshot-portal.mjs';
7
- import { hasWaylandRemoteDesktopGrant, portalPointerButton, portalPointerMotion, portalPointerMotionAbsolute, portalScroll, portalShortcut, portalTypeText, waylandPortalCandidate } from '../wayland-remote-desktop.mjs';
8
- import {
9
- clamp,
10
- commandExists,
11
- jsonResult,
12
- optionalString,
13
- readPrivateTempFile,
14
- removeTemp,
15
- requireEnum,
16
- runFile,
17
- spawnDetached,
18
- spawnDetachedWithInput,
19
- tempDir,
20
- unavailable,
21
- } from './common.mjs';
22
-
23
- function parseWindows(stdout) {
24
- const rows = [];
25
- for (const line of String(stdout).split(/\r?\n/)) {
26
- const match = line.match(/^(0x[0-9a-f]+)\s+\S+\s+(\d+)\s+(-?\d+)\s+(-?\d+)\s+(\d+)\s+(\d+)\s+(\S+)\s+\S+\s*(.*)$/i);
27
- if (!match) continue;
28
- rows.push({
29
- id: match[1],
30
- pid: Number(match[2]),
31
- x: Number(match[3]),
32
- y: Number(match[4]),
33
- width: Number(match[5]),
34
- height: Number(match[6]),
35
- app: match[7],
36
- title: match[8],
37
- });
38
- }
39
- return rows;
40
- }
41
-
42
- export function parseX11PixelBounds(stdout) {
43
- const x = Number(String(stdout).match(/Absolute upper-left X:\s*(-?\d+)/)?.[1]);
44
- const y = Number(String(stdout).match(/Absolute upper-left Y:\s*(-?\d+)/)?.[1]);
45
- const width = Number(String(stdout).match(/^\s*Width:\s*(\d+)/m)?.[1]);
46
- const height = Number(String(stdout).match(/^\s*Height:\s*(\d+)/m)?.[1]);
47
- if (![x,y,width,height].every(Number.isFinite) || width <= 0 || height <= 0) return null;
48
- return { x, y, width, height };
49
- }
50
-
51
- async function x11PixelBounds(id) {
52
- if (!commandExists('xwininfo')) return null;
53
- const { stdout } = await runFile('xwininfo', ['-id', String(id)], { label:'X11 window geometry', allowFailure:true, timeout:1000 });
54
- return parseX11PixelBounds(stdout);
55
- }
56
-
57
- export async function listWindows() {
58
- let wmRows = commandExists('wmctrl')
59
- ? parseWindows((await runFile('wmctrl', ['-lpGx'], { label: 'window inventory', allowFailure:true, timeout:3000 })).stdout)
60
- : [];
61
- if (!isWaylandSession()) {
62
- if (!wmRows.length && !commandExists('wmctrl')) unavailable('Window inventory', 'install wmctrl');
63
- return jsonResult(wmRows.map(row => ({
64
- ...row,
65
- wm_id: row.id,
66
- ui_id: null,
67
- backend: 'x11',
68
- actions: ['focus','minimize','maximize','restore','move','resize','move_resize','close'],
69
- })));
70
- }
71
-
72
- let uiRows = [];
73
- try {
74
- const snapshot = await uiSnapshot({ max_nodes: 2500, max_depth: 1 });
75
- const payload = JSON.parse(snapshot.content?.[0]?.text || '{}');
76
- const nodes = Array.isArray(payload) ? payload : payload.nodes || [];
77
- uiRows = nodes.filter(node => {
78
- const role = String(node.role || '').toLowerCase();
79
- return ['frame','window','dialog'].includes(role)
80
- && String(node.name || node.window || '').trim()
81
- && Number(node.width) > 0
82
- && Number(node.height) > 0
83
- && Number(node.x) > -1000000
84
- && Number(node.y) > -1000000;
85
- });
86
- } catch {}
87
-
88
- // AT-SPI enumeration can take several seconds on a busy desktop. Refresh the cheap
89
- // X11 inventory afterwards so an XWayland window that appeared while AT-SPI was
90
- // walking the tree is merged with its semantic window instead of being misclassified
91
- // as native Wayland for the rest of this call.
92
- if (commandExists('wmctrl')) {
93
- const refreshedWmRows = parseWindows((await runFile('wmctrl', ['-lpGx'], { label:'window inventory refresh', allowFailure:true, timeout:3000 })).stdout);
94
- if (refreshedWmRows.length) wmRows = refreshedWmRows;
95
- }
96
-
97
- const pixelBounds = new Map();
98
- if (commandExists('xwininfo')) {
99
- // wmctrl geometry on Mutter/XWayland is decoration-shifted and cannot be used for pixel crops.
100
- // xwininfo reports the actual client-area coordinates in the compositor screenshot space.
101
- for (const row of wmRows) {
102
- try {
103
- const bounds = await x11PixelBounds(row.id);
104
- if (bounds) pixelBounds.set(row.id, bounds);
105
- } catch {}
106
- }
107
- }
108
-
109
- const usedUi = new Set();
110
- const rows = wmRows.map(row => {
111
- const matchIndex = uiRows.findIndex((node, index) => {
112
- if (usedUi.has(index)) return false;
113
- if (Number(node.pid) > 0 && Number(row.pid) > 0 && Number(node.pid) !== Number(row.pid)) return false;
114
- const nodeTitle = String(node.name || node.window || '').trim().toLowerCase();
115
- const rowTitle = String(row.title || '').trim().toLowerCase();
116
- return nodeTitle && rowTitle && (nodeTitle === rowTitle || nodeTitle.includes(rowTitle) || rowTitle.includes(nodeTitle));
117
- });
118
- const node = matchIndex >= 0 ? uiRows[matchIndex] : null;
119
- if (matchIndex >= 0) usedUi.add(matchIndex);
120
- const xBounds = pixelBounds.get(row.id) || null;
121
- return {
122
- ...row,
123
- ...(xBounds || {}),
124
- pid: Number(row.pid) > 0 ? Number(row.pid) : (Number(node?.pid) > 0 ? Number(node.pid) : null),
125
- wm_bounds: xBounds ? { x:row.x, y:row.y, width:row.width, height:row.height } : null,
126
- geometry_source: xBounds ? 'xwininfo' : 'wmctrl',
127
- wm_id: row.id,
128
- ui_id: node?.id || null,
129
- backend: node ? 'xwayland+atspi' : 'xwayland',
130
- actions: ['focus','minimize','maximize','restore','move','resize','move_resize','close'],
131
- action_backends: isWaylandSession()
132
- ? { focus:'xdg-desktop-portal', minimize:'wmctrl', maximize:'wmctrl', restore:'wmctrl', move:'wmctrl', resize:'wmctrl', move_resize:'wmctrl', close:'wmctrl' }
133
- : { focus:'wmctrl', minimize:'wmctrl', maximize:'wmctrl', restore:'wmctrl', move:'wmctrl', resize:'wmctrl', move_resize:'wmctrl', close:'wmctrl' },
134
- focus_requires_portal_permission: isWaylandSession(),
135
- };
136
- });
137
-
138
- for (let index = 0; index < uiRows.length; index += 1) {
139
- if (usedUi.has(index)) continue;
140
- const node = uiRows[index];
141
- rows.push({
142
- id: node.id,
143
- wm_id: null,
144
- ui_id: node.id,
145
- pid: Number(node.pid) || null,
146
- x: Number(node.x),
147
- y: Number(node.y),
148
- width: Number(node.width),
149
- height: Number(node.height),
150
- app: node.app || null,
151
- title: node.name || node.window || '',
152
- backend: 'wayland-atspi',
153
- actions: ['focus','minimize','maximize','restore','move','resize','move_resize','close'],
154
- action_backend: 'xdg-desktop-portal',
155
- requires_portal_permission: true,
156
- });
157
- }
158
-
159
- if (!rows.length) unavailable('Window inventory', 'wmctrl and AT-SPI exposed no top-level windows');
160
- return jsonResult(rows);
161
- }
162
-
163
- function windowSelector(args = {}) {
164
- return {
165
- id: optionalString(args.id || args.window_id || args.windowId),
166
- pid: Number.isInteger(Number(args.pid)) ? Number(args.pid) : null,
167
- app: optionalString(args.app),
168
- title: optionalString(args.title || args.window_title || args.windowTitle),
169
- };
170
- }
171
-
172
- function windowMatchesSelector(item, selector) {
173
- return (!selector.id || item.id === selector.id || item.wm_id === selector.id || item.ui_id === selector.id)
174
- && (selector.pid == null || Number(item.pid) === selector.pid)
175
- && (!selector.app || String(item.app || '').toLowerCase().includes(selector.app.toLowerCase()))
176
- && (!selector.title || String(item.title || '').toLowerCase().includes(selector.title.toLowerCase()));
177
- }
178
-
179
- async function fastX11WindowMatch(args) {
180
- if (!commandExists('wmctrl')) return null;
181
- const selector = windowSelector(args);
182
- const rows = parseWindows((await runFile('wmctrl', ['-lpGx'], {
183
- label:'window match',
184
- allowFailure:true,
185
- timeout:2000,
186
- })).stdout);
187
- const row = rows.find(item => windowMatchesSelector({ ...item, wm_id:item.id, ui_id:null }, selector));
188
- if (!row) return null;
189
- return {
190
- ...row,
191
- wm_id:row.id,
192
- ui_id:null,
193
- backend:isWaylandSession() ? 'xwayland' : 'x11',
194
- };
195
- }
196
-
197
- async function windowMatch(args) {
198
- // Most desktop actions target an X11/XWayland window. Resolve that cheap native id
199
- // first instead of rebuilding the entire AT-SPI inventory before every action.
200
- // Native Wayland windows have no wmctrl row and fall through to the semantic inventory.
201
- const fast = await fastX11WindowMatch(args);
202
- if (fast) return fast;
203
-
204
- const rows = JSON.parse((await listWindows()).content[0].text);
205
- const selector = windowSelector(args);
206
- const row = rows.find(item => windowMatchesSelector(item, selector));
207
- if (!row) throw new Error('No matching window found');
208
- return row;
209
- }
210
-
211
- async function focusedAccessibilityContext() {
212
- if (!commandExists('python3')) return null;
213
- const script = String.raw`import gi,json
214
- gi.require_version("Atspi","2.0")
215
- from gi.repository import Atspi
216
- desktop=Atspi.get_desktop(0)
217
- queue=[];out=[];seen=0
218
- for i in range(desktop.get_child_count()):
219
- app=desktop.get_child_at_index(i)
220
- try:
221
- app_name=app.get_name() or ""
222
- pid=int(app.get_process_id())
223
- queue.append((app,app_name,pid,""))
224
- except Exception: pass
225
- while queue and seen<12000:
226
- node,app_name,pid,window_name=queue.pop(0);seen+=1
227
- try:
228
- role=node.get_role_name() or ""
229
- name=node.get_name() or ""
230
- if role.lower() in ("frame","window","dialog") and name: window_name=name
231
- if node.get_state_set().contains(Atspi.StateType.FOCUSED) and pid>0:
232
- out.append({"pid":pid,"app":app_name,"window":window_name,"role":role,"name":name})
233
- for i in range(node.get_child_count()): queue.append((node.get_child_at_index(i),app_name,pid,window_name))
234
- except Exception: pass
235
- print(json.dumps(out))`;
236
- const result = await runFile('python3', ['-c', script], { label:'focused accessibility context', allowFailure:true, timeout:2500 });
237
- let rows = [];
238
- try { rows = JSON.parse(result.stdout || '[]'); } catch {}
239
- if (!Array.isArray(rows) || !rows.length) return null;
240
- const score = row => {
241
- const app = String(row.app || '').toLowerCase();
242
- const windowName = String(row.window || '').trim();
243
- const role = String(row.role || '').toLowerCase();
244
- let value = windowName ? 100 : 0;
245
- if (app && app !== 'gnome-shell') value += 40;
246
- if (row.name) value += 15;
247
- if (!['window','filler','panel'].includes(role)) value += 15;
248
- if (/desktop icons/i.test(windowName)) value -= 80;
249
- if (app === 'gnome-shell' && !windowName) value -= 120;
250
- return value;
251
- };
252
- return rows.sort((a,b) => score(b)-score(a))[0] || null;
253
- }
254
-
255
- async function focusedAccessibilityPid() {
256
- return Number((await focusedAccessibilityContext())?.pid) || null;
257
- }
258
-
259
- async function targetAccessibilityFocus(row) {
260
- const pid = Number(row?.pid);
261
- if (!Number.isInteger(pid) || pid <= 0) return { focused:null, exact:null };
262
- try {
263
- const result = await uiSnapshot({ pid, max_nodes: 2200, max_depth: 24 });
264
- const payload = JSON.parse(result.content?.[0]?.text || '{}');
265
- const nodes = Array.isArray(payload) ? payload : payload.nodes || [];
266
- const topLevelActive = nodes.filter(node =>
267
- node.active === true
268
- && /frame|window|dialog/i.test(String(node.role || ''))
269
- && String(node.window || node.name || '').trim()
270
- );
271
- const active = topLevelActive[0] || null;
272
- const wanted = String(row.title || '').trim().toLowerCase();
273
- const exact = !wanted ? active : topLevelActive.find(node => {
274
- const current = String(node.window || node.name || '').trim().toLowerCase();
275
- return current && (wanted === current || wanted.includes(current) || current.includes(wanted));
276
- }) || null;
277
- return { active, exact };
278
- } catch {
279
- return { active:null, exact:null };
280
- }
281
- }
282
-
283
- async function inputWindowId(args = {}) {
284
- const explicit = Boolean(args.window_id || args.windowId || args.id || args.pid != null || args.app || args.title || args.window_title || args.windowTitle);
285
- if (explicit) return (await windowMatch(args)).wm_id || null;
286
- if (!isWaylandSession()) return null;
287
- const pid = await focusedAccessibilityPid();
288
- if (!pid) return null;
289
- try { return (await windowMatch({ pid })).wm_id || null; } catch { return null; }
290
- }
291
-
292
- async function focusInputWindow(windowId) {
293
- if (!windowId || !commandExists('wmctrl')) return;
294
- const result = await runFile('wmctrl', ['-ia', windowId], { label:'input window focus', allowFailure:true, timeout:2000 });
295
- if (result.code === 0) await new Promise(resolve => setTimeout(resolve, 40));
296
- }
297
-
298
- function explicitInputWindowSelector(args = {}) {
299
- const dedicatedWindowId = optionalString(args.window_id || args.windowId);
300
- const dedicatedTitle = optionalString(args.title);
301
- const hasSemanticElementTarget = Boolean(
302
- optionalString(args.id)
303
- || optionalString(args.name)
304
- || optionalString(args.role)
305
- || optionalString(args.automation_id || args.automationId)
306
- );
307
- if (!dedicatedWindowId && !dedicatedTitle && hasSemanticElementTarget) return null;
308
- const pid = Number.isInteger(Number(args.pid)) ? Number(args.pid) : null;
309
- const app = optionalString(args.app);
310
- const scopedTitle = optionalString(args.window_title || args.windowTitle);
311
- if (!dedicatedWindowId && !dedicatedTitle && pid == null && !app && !scopedTitle) return null;
312
- return {
313
- ...(dedicatedWindowId ? { id:dedicatedWindowId } : {}),
314
- ...(pid != null ? { pid } : {}),
315
- ...(app ? { app } : {}),
316
- ...((dedicatedTitle || scopedTitle) ? { title:dedicatedTitle || scopedTitle } : {}),
317
- };
318
- }
319
-
320
- async function focusExplicitWaylandInputTarget(args = {}) {
321
- const selector = explicitInputWindowSelector(args);
322
- if (!selector) return null;
323
- const row = await windowMatch(selector);
324
- await windowAction({ action:'focus', id:row.id, backend:inputBackend(args) });
325
- return row;
326
- }
327
-
328
- function nativeWaylandGeometryMatches(row, expected = {}, tolerance = 24) {
329
- for (const key of ['x','y','width','height']) {
330
- if (expected[key] == null) continue;
331
- const actual = Number(row?.[key]);
332
- const wanted = Number(expected[key]);
333
- if (!Number.isFinite(actual) || !Number.isFinite(wanted) || Math.abs(actual - wanted) > tolerance) return false;
334
- }
335
- return true;
336
- }
337
-
338
- async function verifyNativeWaylandGeometry(row, expected, label) {
339
- let latest = row;
340
- for (let attempt = 0; attempt < 6; attempt += 1) {
341
- if (attempt > 0) await new Promise(resolve => setTimeout(resolve, 120));
342
- try {
343
- latest = await windowMatch({ id:row.id });
344
- if (nativeWaylandGeometryMatches(latest, expected)) return latest;
345
- } catch {}
346
- }
347
- const actual = ['x','y','width','height']
348
- .map(key => `${key}=${Number.isFinite(Number(latest?.[key])) ? Number(latest[key]) : 'unknown'}`)
349
- .join(' ');
350
- const wanted = Object.entries(expected)
351
- .map(([key,value]) => `${key}=${value}`)
352
- .join(' ');
353
- throw new Error(`Could not verify native Wayland window ${label} for ${row.title || row.app || row.id}: wanted ${wanted}; observed ${actual}`);
354
- }
355
-
356
- async function nativeWaylandAccessibilityAction(row, actionNames) {
357
- const pid = Number(row?.pid);
358
- if (!Number.isInteger(pid) || pid <= 0 || !Array.isArray(actionNames) || !actionNames.length) {
359
- return { applied:false, available:[] };
360
- }
361
- const script = pyAtSpiPrelude() + `
362
- want_pid=${pid};want_window=${JSON.stringify(String(row.title || ''))};wanted_actions=${JSON.stringify(actionNames)}
363
- target=None
364
- for root,app_name,pid,active in q:
365
- if pid!=want_pid: continue
366
- try:
367
- for child in root:
368
- try:
369
- role=(child.getRoleName() or "").lower();name=child.name or ""
370
- if role not in ("frame","window","dialog"): continue
371
- if want_window and want_window.strip().lower()!=name.strip().lower(): continue
372
- target=child;break
373
- except: pass
374
- except: pass
375
- if target is not None: break
376
- if target is None:
377
- print(json.dumps({"applied":False,"available":[],"reason":"target window not found"}))
378
- else:
379
- try:
380
- actions=target.queryAction();available=[];chosen=-1;chosen_name=""
381
- for i in range(actions.nActions):
382
- try: action_name=actions.getName(i) or ""
383
- except: action_name=""
384
- available.append(action_name)
385
- if chosen<0 and action_name in wanted_actions:
386
- chosen=i;chosen_name=action_name
387
- if chosen<0:
388
- print(json.dumps({"applied":False,"available":available,"reason":"requested accessibility action unavailable"}))
389
- else:
390
- applied=actions.doAction(chosen)
391
- if applied is False: raise Exception("accessibility window action returned false")
392
- print(json.dumps({"applied":True,"action":chosen_name,"available":available}))
393
- except Exception as exc:
394
- print(json.dumps({"applied":False,"available":[],"reason":str(exc)}))
395
- `;
396
- const result = await runFile('python3', ['-c', script], { label:'AT-SPI window action', timeout:2000, allowFailure:true });
397
- try {
398
- const payload = JSON.parse(result.stdout || '{}');
399
- return payload && typeof payload === 'object' ? payload : { applied:false, available:[] };
400
- } catch {
401
- return { applied:false, available:[], reason:String(result.stderr || result.stdout || 'AT-SPI window action failed').trim() };
402
- }
403
- }
404
-
405
- async function verifyNativeWaylandWindowClosed(row, timeoutMs = 2500) {
406
- const deadline = Date.now() + timeoutMs;
407
- while (Date.now() < deadline) {
408
- try { await windowMatch({ id:row.id }); }
409
- catch { return; }
410
- await new Promise(resolve => setTimeout(resolve, 100));
411
- }
412
- throw new Error(`Could not verify native Wayland window close for ${row.title || row.app || row.id}`);
413
- }
414
-
415
- async function focusNativeWaylandForAction(row, backend) {
416
- await windowAction({ action:'focus', id:row.id, backend });
417
- const focusState = await targetAccessibilityFocus(row);
418
- if (!focusState.exact) throw new Error(`Could not verify native Wayland focus for ${row.title || row.app || row.id}`);
419
- }
420
-
421
- async function nativeWaylandWindowAction(row, action, args) {
422
- const backend = inputBackend(args);
423
- if (backend === 'x11') unavailable('Native Wayland window action', 'use backend=portal for compositor-managed windows');
424
- if (!waylandPortalCandidate()) unavailable('Native Wayland window action', 'no XDG RemoteDesktop portal is available');
425
- if (backend === 'auto' && !hasWaylandRemoteDesktopGrant()) portalPermissionHint('Native Wayland window action');
426
-
427
- const portalBackend = backend === 'portal' ? 'portal' : 'auto';
428
- const timeoutMs = backend === 'portal' ? 120_000 : 2500;
429
- if (action === 'close') {
430
- const semantic = await nativeWaylandAccessibilityAction(row, ['window.close']);
431
- if (semantic.applied) {
432
- await verifyNativeWaylandWindowClosed(row);
433
- return jsonResult({ action, backend:'atspi-window-action', id:row.id, accessibility_action:semantic.action });
434
- }
435
- // Some GTK apps disconnect from AT-SPI while handling window.close, before the action
436
- // call can return a success payload. Trust the observed target state, not that race.
437
- try {
438
- await verifyNativeWaylandWindowClosed(row, 400);
439
- return jsonResult({ action, backend:'atspi-window-action', id:row.id, accessibility_action:'window.close', verified_after_disconnect:true });
440
- } catch {}
441
- await focusNativeWaylandForAction(row, portalBackend);
442
- await portalShortcut('ALT+F4', { timeoutMs });
443
- await verifyNativeWaylandWindowClosed(row);
444
- return jsonResult({ action, backend:'xdg-desktop-portal', id:row.id });
445
- }
446
- if (action === 'minimize') {
447
- await focusNativeWaylandForAction(row, portalBackend);
448
- await portalShortcut('ALT+F9', { timeoutMs });
449
- }
450
- else if (action === 'maximize' || action === 'restore') {
451
- let maximized = false;
452
- try {
453
- const displays = JSON.parse((await displayInventory()).content?.[0]?.text || '[]');
454
- const cx = Number(row.x) + Number(row.width) / 2;
455
- const cy = Number(row.y) + Number(row.height) / 2;
456
- const display = displays.find(item =>
457
- Number.isFinite(cx) && Number.isFinite(cy)
458
- && cx >= Number(item.x) && cx < Number(item.x) + Number(item.width)
459
- && cy >= Number(item.y) && cy < Number(item.y) + Number(item.height)
460
- ) || displays.find(item => item.primary) || displays[0];
461
- if (display) {
462
- const tolerance = 24;
463
- const widthClose = Math.abs(Number(row.width) - Number(display.width)) <= tolerance;
464
- const heightClose = Number(row.height) >= Number(display.height) - 100;
465
- const xClose = Math.abs(Number(row.x) - Number(display.x)) <= tolerance;
466
- const yClose = Number(row.y) >= Number(display.y) && Number(row.y) - Number(display.y) <= 100;
467
- maximized = widthClose && heightClose && xClose && yClose;
468
- }
469
- } catch {}
470
- if ((action === 'maximize' && !maximized) || (action === 'restore' && maximized)) {
471
- await focusNativeWaylandForAction(row, portalBackend);
472
- await portalShortcut('ALT+F10', { timeoutMs });
473
- }
474
- } else {
475
- await focusNativeWaylandForAction(row, portalBackend);
476
- let currentX = Number(row.x), currentY = Number(row.y), currentWidth = Number(row.width), currentHeight = Number(row.height);
477
- if (![currentX,currentY,currentWidth,currentHeight].every(Number.isFinite)) unavailable('Native Wayland window geometry', 'accessibility bounds are unavailable');
478
-
479
- if (action === 'move' || action === 'move_resize') {
480
- const x = Number(args.x), y = Number(args.y);
481
- if (!Number.isFinite(x) || !Number.isFinite(y)) throw new Error('x and y are required for native Wayland window move');
482
- const titleOffset = Math.min(18, Math.max(10, currentHeight / 20));
483
- await dragDrop({
484
- from_x:currentX + currentWidth / 2,
485
- from_y:currentY + titleOffset,
486
- to_x:x + currentWidth / 2,
487
- to_y:y + titleOffset,
488
- backend:portalBackend,
489
- hold_ms:80,
490
- duration_ms:120,
491
- });
492
- const verified = await verifyNativeWaylandGeometry(row, { x, y }, 'move');
493
- currentX = Number(verified.x);
494
- currentY = Number(verified.y);
495
- currentWidth = Number(verified.width);
496
- currentHeight = Number(verified.height);
497
- }
498
-
499
- if (action === 'resize' || action === 'move_resize') {
500
- const width = Number(args.width), height = Number(args.height);
501
- if (!Number.isFinite(width) || !Number.isFinite(height) || width <= 0 || height <= 0) throw new Error('width and height must be positive for native Wayland window resize');
502
- await dragDrop({
503
- from_x:currentX + currentWidth - 4,
504
- from_y:currentY + currentHeight - 4,
505
- to_x:currentX + width - 4,
506
- to_y:currentY + height - 4,
507
- backend:portalBackend,
508
- hold_ms:80,
509
- duration_ms:120,
510
- });
511
- const verified = await verifyNativeWaylandGeometry(row, { width, height }, 'resize');
512
- currentX = Number(verified.x);
513
- currentY = Number(verified.y);
514
- currentWidth = Number(verified.width);
515
- currentHeight = Number(verified.height);
516
- }
517
- return jsonResult({
518
- action,
519
- backend:'xdg-desktop-portal',
520
- id:row.id,
521
- x:currentX,
522
- y:currentY,
523
- width:currentWidth,
524
- height:currentHeight,
525
- });
526
- }
527
- return jsonResult({ action, backend:'xdg-desktop-portal', id:row.id });
528
- }
529
-
530
- export async function windowAction(args) {
531
- const action = requireEnum(args.action, 'action', ['focus','minimize','maximize','restore','move','resize','move_resize','close']);
532
- const row = await windowMatch(args);
533
- const id = row.wm_id || row.id;
534
- const nativeWayland = isWaylandSession() && !row.wm_id && !/^0x/i.test(String(id || ''));
535
- if (action === 'focus') {
536
- if (isWaylandSession()) {
537
- const backend = inputBackend(args);
538
- if (row.ui_id && backend !== 'portal') {
539
- try {
540
- await uiAction({ action:'focus', id:row.ui_id });
541
- await new Promise(resolve => setTimeout(resolve, 80));
542
- const focusState = await targetAccessibilityFocus(row);
543
- if (focusState.exact) return text('Window action focus completed via verified AT-SPI focus.');
544
- } catch {}
545
- }
546
- // XWayland windows have a real X11 wm_id even inside a Wayland session. Prefer
547
- // direct activation for those windows, verify the result through accessibility,
548
- // and only fall back to compositor-level cycling if direct activation did not take.
549
- // This avoids dozens of Alt+Esc/AT-SPI round trips on busy desktops.
550
- if (row.wm_id && backend !== 'portal' && commandExists('wmctrl')) {
551
- const activated = await runFile('wmctrl', ['-ia', row.wm_id], { label:'XWayland window focus', allowFailure:true, timeout:2000 });
552
- if (activated.code === 0) {
553
- if (commandExists('xdotool')) {
554
- await runFile('xdotool', ['windowactivate','--sync',row.wm_id], { label:'XWayland window focus sync', allowFailure:true, timeout:3000 });
555
- }
556
- await new Promise(resolve => setTimeout(resolve, 80));
557
- const focusState = await targetAccessibilityFocus(row);
558
- if (focusState.exact) return text('Window action focus completed via verified XWayland activation.');
559
- }
560
- }
561
- if (backend === 'x11') unavailable('Wayland window focus', row.wm_id
562
- ? 'direct XWayland activation did not produce verifiable focus'
563
- : 'the target is native Wayland and has no X11 window id; use backend=portal');
564
- if (!waylandPortalCandidate()) unavailable('Wayland window focus', 'no XDG RemoteDesktop portal is available');
565
- if (backend === 'auto' && !hasWaylandRemoteDesktopGrant()) portalPermissionHint('Wayland window focus');
566
- const portalOptions = { timeoutMs: backend === 'portal' ? 120_000 : 2500 };
567
- let focusState = await targetAccessibilityFocus(row);
568
- if (focusState.exact) return text('Window action focus already matched the target window.');
569
-
570
- const rows = JSON.parse((await listWindows()).content?.[0]?.text || '[]');
571
- const windowCycles = Math.min(60, Math.max(8, rows.length + 4));
572
- for (let index = 0; index < windowCycles; index += 1) {
573
- await portalShortcut('ALT+ESC', portalOptions);
574
- await new Promise(resolve => setTimeout(resolve, 140));
575
- focusState = await targetAccessibilityFocus(row);
576
- if (focusState.exact) return text('Window action focus completed via verified Wayland Alt+Esc.');
577
- }
578
-
579
- const appCycles = Math.min(12, Math.max(3, new Set(rows.map(item => Number(item.pid) || String(item.app || ''))).size + 1));
580
- for (let index = 0; index < appCycles; index += 1) {
581
- await portalShortcut('ALT+TAB', portalOptions);
582
- await new Promise(resolve => setTimeout(resolve, 160));
583
- focusState = await targetAccessibilityFocus(row);
584
- if (focusState.exact) return text('Window action focus completed via verified Wayland Alt+Tab fallback.');
585
- if (focusState.active) {
586
- const sameAppWindows = rows.filter(item => Number(item.pid) === Number(row.pid)).length;
587
- for (let windowIndex = 0; windowIndex < Math.min(8, Math.max(2, sameAppWindows + 1)); windowIndex += 1) {
588
- await portalShortcut('ALT+`', portalOptions);
589
- await new Promise(resolve => setTimeout(resolve, 120));
590
- focusState = await targetAccessibilityFocus(row);
591
- if (focusState.exact) return text('Window action focus completed via verified Wayland same-app fallback.');
592
- }
593
- }
594
- }
595
- throw new Error(`Could not verify focus for ${row.title || row.app || row.id} after bounded Wayland window cycling`);
596
- }
597
- if (!commandExists('wmctrl')) unavailable('Window focus', 'install wmctrl');
598
- await runFile('wmctrl', ['-ia', id], { label: 'window focus' });
599
- if (commandExists('xdotool')) {
600
- const activated = await runFile('xdotool', ['windowactivate','--sync',id], { label:'window focus sync', allowFailure:true, timeout:3000 });
601
- if (activated.code !== 0) await new Promise(resolve => setTimeout(resolve, 80));
602
- }
603
- }
604
- else if (nativeWayland) return nativeWaylandWindowAction(row, action, args);
605
- else if (!commandExists('wmctrl')) unavailable('Window action', 'install wmctrl');
606
- else if (action === 'close') await runFile('wmctrl', ['-ic', id], { label: 'window close' });
607
- else if (action === 'minimize') await runFile('wmctrl', ['-ir', id, '-b', 'add,hidden'], { label: 'window minimize' });
608
- else if (action === 'maximize') await runFile('wmctrl', ['-ir', id, '-b', 'add,maximized_vert,maximized_horz'], { label: 'window maximize' });
609
- else if (action === 'restore') await runFile('wmctrl', ['-ir', id, '-b', 'remove,hidden,maximized_vert,maximized_horz'], { label: 'window restore' });
610
- else {
611
- const x = Number(args.x), y = Number(args.y), width = Number(args.width), height = Number(args.height);
612
- const geometry = `0,${Number.isFinite(x) ? Math.trunc(x) : -1},${Number.isFinite(y) ? Math.trunc(y) : -1},${Number.isFinite(width) ? Math.trunc(width) : -1},${Number.isFinite(height) ? Math.trunc(height) : -1}`;
613
- await runFile('wmctrl', ['-ir', id, '-e', geometry], { label: 'window geometry' });
614
- }
615
- return text(`Window action ${action} completed.`);
616
- }
617
-
618
- function pyAtSpiPrelude() {
619
- return `import json,sys,math
620
- try:
621
- import pyatspi
622
- except Exception:
623
- try:
624
- import gi
625
- gi.require_version("Atspi","2.0")
626
- from gi.repository import Atspi
627
- class _Component:
628
- def __init__(self,a): self.a=a
629
- def getExtents(self,coords): return self.a.get_extents(coords)
630
- def grabFocus(self): return self.a.grab_focus()
631
- class _Value:
632
- def __init__(self,a): self.a=a
633
- @property
634
- def currentValue(self): return self.a.get_current_value()
635
- @currentValue.setter
636
- def currentValue(self,v): self.a.set_current_value(float(v))
637
- class _Action:
638
- def __init__(self,a): self.a=a
639
- @property
640
- def nActions(self): return self.a.get_n_actions()
641
- def getName(self,i): return self.a.get_action_name(i)
642
- def doAction(self,i): return self.a.do_action(i)
643
- class _Editable:
644
- def __init__(self,a): self.a=a
645
- def setTextContents(self,v): return self.a.set_text_contents(v)
646
- class _Accessible:
647
- def __init__(self,a): self.a=a
648
- @property
649
- def name(self): return self.a.get_name()
650
- @property
651
- def childCount(self): return self.a.get_child_count()
652
- def getState(self): return self.a.get_state_set()
653
- def getRoleName(self): return self.a.get_role_name()
654
- def getChildAtIndex(self,i): return _Accessible(self.a.get_child_at_index(i))
655
- def queryComponent(self): return _Component(self.a)
656
- def queryValue(self):
657
- iface=self.a.get_value_iface()
658
- if iface is None: raise Exception("Value interface unavailable")
659
- return _Value(self.a)
660
- def queryAction(self): return _Action(self.a)
661
- def queryEditableText(self): return _Editable(self.a)
662
- def getProcessId(self):
663
- try: return int(self.a.get_process_id())
664
- except: return 0
665
- def __iter__(self):
666
- for i in range(self.childCount): yield self.getChildAtIndex(i)
667
- class _Registry:
668
- @staticmethod
669
- def getDesktop(i): return _Accessible(Atspi.get_desktop(i))
670
- class _Compat:
671
- Registry=_Registry
672
- STATE_ACTIVE=Atspi.StateType.ACTIVE
673
- STATE_SHOWING=Atspi.StateType.SHOWING
674
- STATE_FOCUSED=Atspi.StateType.FOCUSED
675
- DESKTOP_COORDS=Atspi.CoordType.SCREEN
676
- pyatspi=_Compat()
677
- except Exception as exc:
678
- print(json.dumps({"error":"AT-SPI unavailable: %s"%exc}));sys.exit(3)
679
- desktop=pyatspi.Registry.getDesktop(0)
680
- def _proc(e):
681
- for attr in ("getProcessId","get_process_id"):
682
- try:
683
- return int(getattr(e,attr)())
684
- except: pass
685
- try:
686
- app=e.getApplication()
687
- for attr in ("getProcessId","get_process_id"):
688
- try: return int(getattr(app,attr)())
689
- except: pass
690
- except: pass
691
- return 0
692
- q=[]
693
- for app in desktop:
694
- try:
695
- app_name=app.name or ""
696
- pid=_proc(app)
697
- active=False
698
- try:
699
- active=bool(app.getState().contains(pyatspi.STATE_ACTIVE))
700
- if not active:
701
- for child_i in range(min(app.childCount,64)):
702
- try:
703
- child=app.getChildAtIndex(child_i)
704
- if child.getState().contains(pyatspi.STATE_ACTIVE):
705
- active=True;break
706
- except: pass
707
- except: pass
708
- if app.childCount>0 or pid>0:q.append((app,app_name,pid,active))
709
- except:pass
710
- `;
711
- }
712
-
713
- export async function uiSnapshot(args = {}) {
714
- if (!commandExists('python3')) unavailable('Linux accessibility', 'python3 with python3-pyatspi is required');
715
- const maxNodes = clamp(args.max_nodes, 500, 1, 5000);
716
- const maxDepth = clamp(args.max_depth, 8, 1, 32);
717
- const wantedApp = JSON.stringify(optionalString(args.app) || '');
718
- const wantedPid = Number.isInteger(Number(args.pid)) ? Number(args.pid) : 0;
719
- const wantedWindow = JSON.stringify(optionalString(args.window_title || args.windowTitle) || '');
720
- const activeOnly = args.active_only === true;
721
- const script = pyAtSpiPrelude() + `
722
- want_app=${wantedApp};want_pid=${wantedPid};want_window=${wantedWindow};want_active=${activeOnly ? 'True' : 'False'}
723
- roots=[]
724
- for root,app_name,pid,active in q:
725
- if want_pid and pid!=want_pid: continue
726
- if want_app and want_app.lower() not in app_name.lower(): continue
727
- if want_active and not active: continue
728
- roots.append((root,app_name,pid,active))
729
- out=[];queue=[];seen=0
730
- for root_index,(e,app_name,pid,active) in enumerate(roots):
731
- if want_window:
732
- top_match=False
733
- try:
734
- for child in e:
735
- try:
736
- child_name=child.name or ""
737
- child_role=child.getRoleName().lower()
738
- if child_role in ("frame","dialog","window") and want_window.lower() in child_name.lower():
739
- top_match=True;break
740
- except: pass
741
- except: pass
742
- if not top_match: continue
743
- root_path="r" if pid else "a%d"%root_index
744
- queue.append((e,0,app_name,pid,active,"",root_path))
745
- while queue and len(out)<${maxNodes}:
746
- e,d,app_name,pid,active,window_name,node_path=queue.pop(0);current_index=seen;seen+=1
747
- try: role=e.getRoleName()
748
- except: role=""
749
- try: name=e.name or ""
750
- except: name=""
751
- if role.lower() in ("frame","dialog","window") and name: window_name=name
752
- try:
753
- ext=e.queryComponent().getExtents(pyatspi.DESKTOP_COORDS);box={"x":ext.x,"y":ext.y,"width":ext.width,"height":ext.height}
754
- except: box={"x":0,"y":0,"width":0,"height":0}
755
- try:
756
- value=e.queryValue().currentValue
757
- if isinstance(value,float) and not math.isfinite(value): value=""
758
- except: value=""
759
- focused=False;node_active=active
760
- try:
761
- states=e.getState();focused=bool(states.contains(pyatspi.STATE_FOCUSED));node_active=bool(node_active or states.contains(pyatspi.STATE_ACTIVE))
762
- except: pass
763
- if not want_window or want_window.lower() in window_name.lower():
764
- out.append({"id":"linux:%d:%s"%(pid,node_path),"index":current_index,"depth":d,"pid":pid,"app":app_name,"active":node_active,"focused":focused,"window":window_name,"role":role,"name":name,"value":value,**box})
765
- if d<${maxDepth}:
766
- try:
767
- for i in range(e.childCount):queue.append((e.getChildAtIndex(i),d+1,app_name,pid,active,window_name,node_path+"."+str(i)))
768
- except:pass
769
- print(json.dumps(out))`;
770
- const result = await runFile('python3', ['-c', script], { label: 'AT-SPI snapshot', timeout: 30_000, allowFailure: true });
771
- let parsed = null;
772
- try { parsed = JSON.parse(result.stdout || ''); } catch {}
773
- if (Array.isArray(parsed)) return jsonResult({ platform: 'linux', count: parsed.length, nodes: parsed });
774
- if (parsed?.error) unavailable('Linux accessibility', parsed.error);
775
- if (result.code !== 0) unavailable('Linux accessibility', (result.stderr || result.stdout).trim());
776
- return jsonResult({ platform: 'linux', count: 0, nodes: [] });
777
- }
778
-
779
- export async function uiAction(args = {}) {
780
- const action = requireEnum(args.action, 'action', ['click','invoke','focus','set_value','select','toggle','expand','collapse','scroll_into_view','set_range_value','add_to_selection','remove_from_selection']);
781
- if (!commandExists('python3')) unavailable('Linux accessibility action', 'python3 with python3-pyatspi is required');
782
- const id = optionalString(args.id);
783
- const idParts = id?.split(':') || [];
784
- const legacyIndex = idParts.length === 2 && idParts[0] === 'linux' && /^\d+$/.test(idParts[1]) ? Number(idParts[1]) : -1;
785
- const stablePid = idParts.length >= 3 && idParts[0] === 'linux' && /^\d+$/.test(idParts[1]) ? Number(idParts[1]) : 0;
786
- const stablePath = idParts.length >= 3 && idParts[0] === 'linux' ? idParts.slice(2).join(':') : '';
787
- const wantedPid = Number.isInteger(Number(args.pid)) ? Number(args.pid) : stablePid;
788
- const wantedApp = JSON.stringify(optionalString(args.app) || '');
789
- const wantedWindow = JSON.stringify(optionalString(args.window_title || args.windowTitle) || '');
790
- const wantedName = JSON.stringify(optionalString(args.name) || '');
791
- const wantedRole = JSON.stringify(optionalString(args.role) || '');
792
- const value = JSON.stringify(String(args.value ?? ''));
793
- const script = pyAtSpiPrelude() + `
794
- want_index=${legacyIndex};want_pid=${wantedPid};want_path=${JSON.stringify(stablePath)}
795
- want_app=${wantedApp};want_window=${wantedWindow};want_name=${wantedName};want_role=${wantedRole};action=${JSON.stringify(action)};value=${value}
796
- queue=[];seen=0;target=None;target_meta={}
797
- for root_index,(root,app_name,pid,active) in enumerate(q):
798
- if want_pid and pid!=want_pid: continue
799
- if want_app and want_app.lower() not in app_name.lower(): continue
800
- root_path="r" if pid else "a%d"%root_index
801
- queue.append((root,app_name,pid,"",root_path))
802
- while queue:
803
- e,app_name,pid,window_name,node_path=queue.pop(0);current_index=seen;seen+=1
804
- try: role=e.getRoleName();name=e.name or ""
805
- except: role="";name=""
806
- if role.lower() in ("frame","dialog","window") and name: window_name=name
807
- id_match=(not want_path or (pid==want_pid and node_path==want_path)) and (want_index<0 or current_index==want_index)
808
- selector_match=(not want_name or want_name.lower() in name.lower()) and (not want_role or want_role.lower() in role.lower()) and (not want_window or want_window.lower() in window_name.lower())
809
- if id_match and selector_match:
810
- target=e;target_meta={"pid":pid,"app":app_name,"window":window_name,"path":node_path};break
811
- try:
812
- for i in range(e.childCount):queue.append((e.getChildAtIndex(i),app_name,pid,window_name,node_path+"."+str(i)))
813
- except:pass
814
- if target is None:raise Exception("UI element not found")
815
- pointer_fallback=None
816
- if action in ("click","invoke"):
817
- try:
818
- applied=target.queryAction().doAction(0)
819
- if applied is False: raise Exception("accessibility action returned false")
820
- except Exception as action_error:
821
- try:
822
- extents=target.queryComponent().getExtents(pyatspi.DESKTOP_COORDS)
823
- def _rect_value(rect,name,index):
824
- try: return int(getattr(rect,name))
825
- except:
826
- try: return int(rect[index])
827
- except: return 0
828
- px=_rect_value(extents,"x",0);py=_rect_value(extents,"y",1)
829
- pw=_rect_value(extents,"width",2);ph=_rect_value(extents,"height",3)
830
- if pw<=0 or ph<=0: raise Exception("target has no usable screen bounds")
831
- pointer_fallback={"x":px+pw//2,"y":py+ph//2,"width":pw,"height":ph,"reason":str(action_error)}
832
- except Exception as bounds_error:
833
- raise Exception("accessibility action failed: %s; coordinate fallback unavailable: %s"%(action_error,bounds_error))
834
- elif action in ("select","toggle"):
835
- applied=target.queryAction().doAction(0)
836
- if applied is False: raise Exception(action+" accessibility action returned false")
837
- elif action=="focus":
838
- already_focused=False
839
- try: already_focused=bool(target.getState().contains(pyatspi.STATE_FOCUSED))
840
- except: pass
841
- if not already_focused:
842
- try:
843
- if target.queryComponent().grabFocus() is False: raise Exception("focus accessibility action returned false")
844
- except Exception as focus_error:
845
- try:
846
- if not target.getState().contains(pyatspi.STATE_FOCUSED): raise focus_error
847
- except Exception:
848
- raise focus_error
849
- elif action=="set_value":
850
- try:
851
- applied=target.queryEditableText().setTextContents(value)
852
- if applied is False: raise Exception("editable text write returned false")
853
- except Exception as editable_error:
854
- try: target.queryValue().currentValue=value
855
- except Exception as value_error: raise Exception("set_value failed: editable=%s; value=%s"%(editable_error,value_error))
856
- elif action=="set_range_value":
857
- target.queryValue().currentValue=float(value)
858
- elif action in ("expand","collapse"):
859
- actions=target.queryAction();chosen=-1
860
- for i in range(actions.nActions):
861
- try:
862
- action_name=(actions.getName(i) or "").lower()
863
- if action in action_name: chosen=i;break
864
- except: pass
865
- if chosen<0: raise Exception(action+" accessibility action unavailable")
866
- actions.doAction(chosen)
867
- elif action=="scroll_into_view":
868
- component=target.queryComponent()
869
- try: component.scrollTo(pyatspi.SCROLL_ANYWHERE)
870
- except:
871
- if not component.grabFocus(): raise Exception("scroll-to-view unavailable")
872
- elif action in ("add_to_selection","remove_from_selection"):
873
- parent=target.parent
874
- if parent is None: raise Exception("selection parent unavailable")
875
- selection=parent.querySelection();idx=target.getIndexInParent()
876
- if action=="add_to_selection":
877
- if not selection.selectChild(idx): raise Exception("could not add item to selection")
878
- else:
879
- if not selection.deselectChild(idx): raise Exception("could not remove item from selection")
880
- out={"action":action,"name":target.name or "","role":target.getRoleName(),**target_meta}
881
- if pointer_fallback is not None: out["pointer_fallback"]=pointer_fallback
882
- print(json.dumps(out))`;
883
- const result = await runFile('python3', ['-c', script], { label: 'AT-SPI action', timeout: 30_000, allowFailure: true });
884
- let payload = null;
885
- try { payload = JSON.parse(result.stdout || ''); } catch {}
886
- if (payload?.pointer_fallback) {
887
- const fallback = payload.pointer_fallback;
888
- const x = Number(fallback.x), y = Number(fallback.y);
889
- if (!Number.isFinite(x) || !Number.isFinite(y)) unavailable('Linux accessibility action', 'coordinate fallback returned invalid screen bounds');
890
- await pointer({ action:'click', x, y, ...(args.backend ? { backend:args.backend } : {}) });
891
- return jsonResult({
892
- action,
893
- id:id || null,
894
- name:payload.name || '',
895
- role:payload.role || '',
896
- backend:'pointer_fallback',
897
- x,
898
- y,
899
- fallback_reason:String(fallback.reason || 'accessibility action unavailable').slice(0, 500),
900
- });
901
- }
902
- if (payload && !payload.error) return text(JSON.stringify(payload));
903
- if (payload?.error) unavailable('Linux accessibility action', payload.error);
904
- if (result.code !== 0) {
905
- const detail = String(result.stderr || result.stdout || 'AT-SPI action failed').trim().split(/\r?\n/).filter(Boolean).at(-1) || 'AT-SPI action failed';
906
- unavailable('Linux accessibility action', detail);
907
- }
908
- return text(result.stdout.trim());
909
- }
910
-
911
- async function confirmClipboard(expected, backend, timeoutMs = 1500) {
912
- const deadline = Date.now() + timeoutMs;
913
- while (Date.now() < deadline) {
914
- const result = backend === 'wl-copy'
915
- ? await runFile('wl-paste', ['--no-newline'], { label: 'clipboard verify', allowFailure: true, timeout: 500 })
916
- : await runFile('xclip', ['-selection','clipboard','-o'], { label: 'clipboard verify', allowFailure: true, timeout: 500 });
917
- if (result.code === 0 && result.stdout === expected) return;
918
- await new Promise(resolve => setTimeout(resolve, 25));
919
- }
920
- throw new Error(`${backend} did not publish the requested clipboard text within ${timeoutMs} ms`);
921
- }
922
-
923
- export async function clipboard(args = {}) {
924
- const action = requireEnum(args.action, 'action', ['read','write','clear']);
925
- if (action !== 'read') {
926
- const value = action === 'clear' ? '' : String(args.text ?? args.value ?? '');
927
- const failures = [];
928
- if (commandExists('wl-copy')) {
929
- try {
930
- if (action === 'clear') await runFile('wl-copy', ['--clear'], { label: 'clipboard clear', timeout: 3000 });
931
- else {
932
- await spawnDetachedWithInput('wl-copy', [], value, { label: 'clipboard write', timeout: 3000 });
933
- if (commandExists('wl-paste')) await confirmClipboard(value, 'wl-copy');
934
- }
935
- return jsonResult({ action, length: value.length, backend: 'wl-copy' });
936
- } catch (error) { failures.push(`wl-copy: ${error instanceof Error ? error.message : String(error)}`); }
937
- }
938
- if (commandExists('xclip')) {
939
- try {
940
- await spawnDetachedWithInput('xclip', ['-selection','clipboard','-loops','0'], value, { label: 'clipboard write', timeout: 3000 });
941
- await confirmClipboard(value, 'xclip');
942
- return jsonResult({ action, length: value.length, backend: 'xclip' });
943
- } catch (error) { failures.push(`xclip: ${error instanceof Error ? error.message : String(error)}`); }
944
- }
945
- unavailable('Clipboard write', `${failures.join('; ') || 'install wl-clipboard or xclip'}`);
946
- }
947
- const failures = [];
948
- if (commandExists('wl-paste')) {
949
- const result = await runFile('wl-paste', ['--no-newline'], { label: 'clipboard read', allowFailure: true, timeout: 3000 });
950
- if (result.code === 0) return text(result.stdout);
951
- failures.push(`wl-paste: ${(result.stderr || result.stdout || `exit ${result.code}`).trim()}`);
952
- }
953
- if (commandExists('xclip')) {
954
- const result = await runFile('xclip', ['-selection','clipboard','-o'], { label: 'clipboard read', allowFailure: true, timeout: 3000 });
955
- if (result.code === 0) return text(result.stdout);
956
- failures.push(`xclip: ${(result.stderr || result.stdout || `exit ${result.code}`).trim()}`);
957
- }
958
- unavailable('Clipboard read', `${failures.join('; ') || 'install wl-clipboard or xclip'}`);
959
- }
960
-
961
- export async function typeTextWindowTarget(value, args = {}) {
962
- const row = await windowMatch(args);
963
- if (!row?.pid) return null;
964
- const snapshot = await uiSnapshot({
965
- pid:row.pid,
966
- window_title:row.title || undefined,
967
- max_nodes:2200,
968
- max_depth:24,
969
- });
970
- let payload = null;
971
- try { payload = JSON.parse(snapshot.content?.[0]?.text || '{}'); } catch {}
972
- const nodes = Array.isArray(payload) ? payload : payload?.nodes || [];
973
- const editableRoles = /(^|\b)(text|entry|textbox|edit|text field|search box|password text)(\b|$)/i;
974
- const candidates = nodes.filter(node => node?.id && editableRoles.test(String(node.role || '')));
975
- if (candidates.length !== 1) return null;
976
-
977
- const target = candidates[0];
978
- const result = await uiAction({ action:'set_value', id:target.id, value:String(value) });
979
- let action = result.content?.[0]?.text || '';
980
- try { action = JSON.parse(action); } catch {}
981
- return {
982
- id:target.id,
983
- pid:row.pid,
984
- app:row.app || target.app || '',
985
- window:row.title || target.window || '',
986
- role:target.role || '',
987
- name:target.name || '',
988
- action,
989
- };
990
- }
991
-
992
- export async function typeTextFocused(value) {
993
- if (!commandExists('python3')) return null;
994
- const script = String.raw`import gi,json,sys
995
- gi.require_version("Atspi","2.0")
996
- from gi.repository import Atspi
997
- value=sys.argv[1]
998
- desktop=Atspi.get_desktop(0)
999
- active=[];showing=[]
1000
- for i in range(desktop.get_child_count()):
1001
- app=desktop.get_child_at_index(i)
1002
- try:
1003
- states=app.get_state_set()
1004
- if states.contains(Atspi.StateType.ACTIVE): active.append(app);continue
1005
- visible=states.contains(Atspi.StateType.SHOWING)
1006
- if not visible:
1007
- for j in range(app.get_child_count()):
1008
- child=app.get_child_at_index(j);child_states=child.get_state_set()
1009
- if child_states.contains(Atspi.StateType.ACTIVE):
1010
- active.append(app);visible=False;break
1011
- if child_states.contains(Atspi.StateType.SHOWING):visible=True
1012
- if visible:showing.append(app)
1013
- except Exception: pass
1014
- queue=list(active or showing)
1015
- seen=0;target=None
1016
- while queue and seen<12000:
1017
- a=queue.pop(0);seen+=1
1018
- try:
1019
- states=a.get_state_set()
1020
- if states.contains(Atspi.StateType.FOCUSED) and states.contains(Atspi.StateType.EDITABLE) and a.get_editable_text_iface() is not None:
1021
- target=a;break
1022
- for i in range(a.get_child_count()): queue.append(a.get_child_at_index(i))
1023
- except Exception: pass
1024
- if target is None:
1025
- print(json.dumps({"ok":False,"reason":"no focused editable accessibility element"}));sys.exit(3)
1026
- try:
1027
- pos=max(0,int(target.get_caret_offset()))
1028
- selections=target.get_text_selections() or []
1029
- if selections:
1030
- selection=selections[0]
1031
- if selection.start_object==target and selection.end_object==target and selection.end_offset>selection.start_offset:
1032
- if not target.delete_text(selection.start_offset,selection.end_offset): raise RuntimeError("could not replace current selection")
1033
- pos=int(selection.start_offset)
1034
- ok=bool(target.insert_text(pos,value,len(value)))
1035
- print(json.dumps({"ok":ok,"role":target.get_role_name(),"name":target.get_name() or "","pid":target.get_process_id(),"inserted":len(value)}))
1036
- sys.exit(0 if ok else 4)
1037
- except Exception as exc:
1038
- print(json.dumps({"ok":False,"reason":str(exc)}));sys.exit(5)`;
1039
- const result = await runFile('python3', ['-c', script, String(value)], {
1040
- label: 'AT-SPI focused text input',
1041
- allowFailure: true,
1042
- timeout: 5000,
1043
- });
1044
- if (result.code !== 0) return null;
1045
- try {
1046
- const payload = JSON.parse(result.stdout || '{}');
1047
- return payload.ok ? payload : null;
1048
- } catch { return null; }
1049
- }
1050
-
1051
- function normalizeShortcut(shortcut) {
1052
- return shortcut.toLowerCase().replace(/cmd|command|meta/g, 'super').replace(/control/g, 'ctrl');
1053
- }
1054
-
1055
- function inputBackend(args = {}) {
1056
- return requireEnum(args.backend || 'auto', 'backend', ['auto','x11','portal']);
1057
- }
1058
-
1059
- function portalPermissionHint(feature) {
1060
- unavailable(feature, 'one-time XDG RemoteDesktop permission has not been granted; retry with backend=portal to request it');
1061
- }
1062
-
1063
- export async function keyboard(args = {}) {
1064
- const shortcut = optionalString(args.shortcut) || (Array.isArray(args.keys) ? args.keys.join('+') : optionalString(args.key));
1065
- if (!shortcut) throw new Error('shortcut or key is required');
1066
- const backend = inputBackend(args);
1067
- const wtypeArgs = () => {
1068
- const tokens = normalizeShortcut(shortcut).split('+').filter(Boolean);
1069
- const modifiers = tokens.slice(0,-1).map(v => ({ctrl:'ctrl',alt:'alt',shift:'shift',super:'logo'}[v])).filter(Boolean);
1070
- const key = tokens.at(-1);
1071
- const argv = [];
1072
- for (const mod of modifiers) argv.push('-M', mod);
1073
- argv.push('-P', key, '-p', key);
1074
- for (const mod of [...modifiers].reverse()) argv.push('-m', mod);
1075
- return argv;
1076
- };
1077
- if (isWaylandSession()) {
1078
- await focusExplicitWaylandInputTarget(args);
1079
- if (backend !== 'portal' && commandExists('wtype')) {
1080
- const result = await runFile('wtype', wtypeArgs(), { label: 'keyboard', allowFailure: true, timeout: 5000 });
1081
- if (result.code === 0) return text(`Sent ${shortcut} via wtype.`);
1082
- }
1083
- if (backend === 'portal') {
1084
- if (!waylandPortalCandidate()) unavailable('Wayland keyboard input', 'no XDG RemoteDesktop portal is available');
1085
- await portalShortcut(shortcut, { timeoutMs: 120_000 });
1086
- return text(`Sent ${shortcut} via the Wayland Remote Desktop portal.`);
1087
- }
1088
- if (backend === 'auto' && hasWaylandRemoteDesktopGrant()) {
1089
- try {
1090
- await portalShortcut(shortcut, { timeoutMs: 2500 });
1091
- return text(`Sent ${shortcut} via the Wayland Remote Desktop portal.`);
1092
- } catch {}
1093
- }
1094
- if (backend === 'auto' && waylandPortalCandidate()) portalPermissionHint('Wayland keyboard input');
1095
- unavailable('Wayland keyboard input', backend === 'x11'
1096
- ? 'XTEST/xdotool keyboard events are not reliably delivered through GNOME Wayland; use backend=portal or semantic accessibility input'
1097
- : 'no working keyboard backend is available');
1098
- }
1099
- if (commandExists('xdotool')) {
1100
- const windowId = await inputWindowId(args);
1101
- if (windowId) await focusInputWindow(windowId);
1102
- await runFile('xdotool', ['key','--clearmodifiers',...(windowId ? ['--window',windowId] : []),normalizeShortcut(shortcut)], { label: 'keyboard' });
1103
- return text(`Sent ${shortcut}${windowId ? ` to ${windowId}` : ''}.`);
1104
- }
1105
- if (commandExists('wtype')) {
1106
- await runFile('wtype', wtypeArgs(), { label: 'keyboard' });
1107
- return text(`Sent ${shortcut}.`);
1108
- }
1109
- unavailable('Keyboard input', 'install xdotool or wtype');
1110
- }
1111
-
1112
- export async function typeTextKeys(value, args = {}) {
1113
- const delay = clamp(args.delay_ms, 1, 0, 1000);
1114
- const backend = inputBackend(args);
1115
- if (isWaylandSession()) {
1116
- await focusExplicitWaylandInputTarget(args);
1117
- if (backend !== 'portal' && commandExists('wtype')) {
1118
- const result = await runFile('wtype', ['-d', String(delay), '--', String(value)], { label: 'type text', timeout: 30_000, allowFailure: true });
1119
- if (result.code === 0) return { ...result, backend: 'wtype' };
1120
- }
1121
- if (backend === 'portal') {
1122
- if (!waylandPortalCandidate()) unavailable('Wayland text input', 'no XDG RemoteDesktop portal is available');
1123
- await portalTypeText(String(value), { delayMs: delay, timeoutMs: 120_000 });
1124
- return { stdout: '', stderr: '', code: 0, backend: 'xdg-desktop-portal' };
1125
- }
1126
- if (backend === 'auto' && hasWaylandRemoteDesktopGrant()) {
1127
- try {
1128
- await portalTypeText(String(value), { delayMs: delay, timeoutMs: 2500 });
1129
- return { stdout: '', stderr: '', code: 0, backend: 'xdg-desktop-portal' };
1130
- } catch {}
1131
- }
1132
- if (backend === 'auto' && waylandPortalCandidate()) portalPermissionHint('Wayland text input');
1133
- unavailable('Wayland text input', backend === 'x11'
1134
- ? 'XTEST/xdotool text input is not reliably delivered through GNOME Wayland; use semantic accessibility typing or backend=portal'
1135
- : 'no working text input backend is available');
1136
- }
1137
- if (commandExists('xdotool')) {
1138
- const windowId = await inputWindowId(args);
1139
- if (windowId) await focusInputWindow(windowId);
1140
- const result = await runFile('xdotool', ['type','--clearmodifiers','--delay',String(delay),...(windowId ? ['--window',windowId] : []),String(value)], { label: 'type text', timeout: 30_000 });
1141
- return { ...result, backend: windowId ? 'xdotool-targeted' : 'xdotool' };
1142
- }
1143
- if (commandExists('wtype')) return runFile('wtype', ['-d', String(delay), '--', String(value)], { label: 'type text', timeout: 30_000 });
1144
- unavailable('Text input', 'install wtype or xdotool');
1145
- }
1146
-
1147
- function buttonNumber(button) {
1148
- return String(button || 'left').toLowerCase() === 'right' ? '3' : String(button || 'left').toLowerCase() === 'middle' ? '2' : '1';
1149
- }
1150
-
1151
- let portalPointerState = null;
1152
-
1153
- async function portalDesktopAnchor() {
1154
- const rows = JSON.parse((await displayInventory()).content?.[0]?.text || '[]');
1155
- const displays = Array.isArray(rows) ? rows.filter(row =>
1156
- Number.isFinite(Number(row.x)) && Number.isFinite(Number(row.y))
1157
- && Number.isFinite(Number(row.width)) && Number.isFinite(Number(row.height))
1158
- ) : [];
1159
- if (!displays.length) return { x:0, y:0 };
1160
- const left = Math.min(...displays.map(row => Number(row.x)));
1161
- const top = Math.min(...displays.map(row => Number(row.y)));
1162
- return { x:left, y:top };
1163
- }
1164
-
1165
- async function calibratePortalPointer(options = {}) {
1166
- const motionOptions = { ...options };
1167
- delete motionOptions.recalibrate;
1168
- await portalPointerMotion(-100000, -100000, motionOptions);
1169
- portalPointerState = await portalDesktopAnchor();
1170
- }
1171
-
1172
- async function portalMoveTo(x, y, options = {}) {
1173
- const targetX = Math.trunc(Number(x));
1174
- const targetY = Math.trunc(Number(y));
1175
- if (!Number.isFinite(targetX) || !Number.isFinite(targetY)) throw new Error('x and y must be finite coordinates');
1176
- const motionOptions = { ...options };
1177
- delete motionOptions.recalibrate;
1178
- try {
1179
- await portalPointerMotionAbsolute(targetX, targetY, motionOptions);
1180
- portalPointerState = { x:targetX, y:targetY };
1181
- return;
1182
- } catch (error) {
1183
- if (!/requires the EIS Remote Desktop backend/i.test(String(error?.message || error))) throw error;
1184
- }
1185
- // Legacy portal fallback: relative motion has no absolute position feedback,
1186
- // so recalibrate on every move instead of trusting cached coordinates.
1187
- await calibratePortalPointer(motionOptions);
1188
- await portalPointerMotion(targetX - portalPointerState.x, targetY - portalPointerState.y, motionOptions);
1189
- portalPointerState = { x:targetX, y:targetY };
1190
- }
1191
-
1192
- export async function pointer(args = {}) {
1193
- const action = requireEnum(args.action, 'action', ['move','click','double_click','right_click','down','up']);
1194
- const x = Number(args.x), y = Number(args.y);
1195
- const buttonName = action === 'right_click' ? 'right' : String(args.button || 'left').toLowerCase();
1196
- const backend = inputBackend(args);
1197
-
1198
- const runXdotool = async () => {
1199
- if (!commandExists('xdotool')) return false;
1200
- const argv = [];
1201
- if (Number.isFinite(x) && Number.isFinite(y)) argv.push('mousemove', String(Math.trunc(x)), String(Math.trunc(y)));
1202
- else if (action === 'move') throw new Error('x and y are required for pointer move');
1203
- const button = buttonNumber(buttonName);
1204
- if (action === 'click' || action === 'right_click') argv.push('click', button);
1205
- else if (action === 'double_click') argv.push('click','--repeat','2','--delay','70',button);
1206
- else if (action === 'down') argv.push('mousedown', button);
1207
- else if (action === 'up') argv.push('mouseup', button);
1208
- await runFile('xdotool', argv, { label: 'pointer' });
1209
- return true;
1210
- };
1211
-
1212
- const runPortal = async timeoutMs => {
1213
- const portalOptions = { timeoutMs };
1214
- if (Number.isFinite(x) && Number.isFinite(y)) await portalMoveTo(x, y, portalOptions);
1215
- else if (action === 'move') throw new Error('x and y are required for pointer move');
1216
- const click = async () => {
1217
- await portalPointerButton(buttonName, true, portalOptions);
1218
- // A zero-duration press/release can be accepted by the portal transport but
1219
- // coalesced before the target toolkit observes a real click. Keep a short,
1220
- // human-scale hold so successful return means a deliverable button gesture.
1221
- await new Promise(resolve => setTimeout(resolve, 60));
1222
- await portalPointerButton(buttonName, false, portalOptions);
1223
- await new Promise(resolve => setTimeout(resolve, 40));
1224
- };
1225
- if (action === 'click' || action === 'right_click') await click();
1226
- else if (action === 'double_click') {
1227
- await click();
1228
- await new Promise(resolve => setTimeout(resolve, 70));
1229
- await click();
1230
- } else if (action === 'down') await portalPointerButton(buttonName, true, portalOptions);
1231
- else if (action === 'up') {
1232
- await portalPointerButton(buttonName, false, portalOptions);
1233
- await new Promise(resolve => setTimeout(resolve, 40));
1234
- }
1235
- };
1236
-
1237
- if (isWaylandSession()) {
1238
- if (backend === 'portal') {
1239
- if (!waylandPortalCandidate()) unavailable('Wayland pointer control', 'no XDG RemoteDesktop portal is available');
1240
- await runPortal(120_000);
1241
- return text(`Pointer ${action} completed via the Wayland Remote Desktop portal.`);
1242
- }
1243
- if (backend === 'auto' && hasWaylandRemoteDesktopGrant()) {
1244
- try {
1245
- await runPortal(2500);
1246
- return text(`Pointer ${action} completed via the Wayland Remote Desktop portal.`);
1247
- } catch {}
1248
- }
1249
- if (backend !== 'portal' && await runXdotool()) return text(`Pointer ${action} completed via XWayland xdotool.`);
1250
- if (backend === 'auto' && waylandPortalCandidate()) portalPermissionHint('Wayland pointer control');
1251
- unavailable('Wayland pointer control', backend === 'x11' ? 'xdotool is unavailable' : 'no working pointer backend is available');
1252
- }
1253
-
1254
- if (await runXdotool()) return text(`Pointer ${action} completed.`);
1255
- unavailable('Pointer control', 'install xdotool');
1256
- }
1257
-
1258
- export async function dragDrop(args = {}) {
1259
- const backend = inputBackend(args);
1260
- const runXdotool = async () => {
1261
- if (!commandExists('xdotool')) return false;
1262
- const button = buttonNumber(args.button);
1263
- await runFile('xdotool', [
1264
- 'mousemove',String(Math.trunc(args.from_x)),String(Math.trunc(args.from_y)),
1265
- 'mousedown',button,
1266
- 'mousemove','--sync',String(Math.trunc(args.to_x)),String(Math.trunc(args.to_y)),
1267
- 'mouseup',button,
1268
- ], { label: 'drag and drop' });
1269
- return true;
1270
- };
1271
- const runPortal = async timeoutMs => {
1272
- const button = String(args.button || 'left').toLowerCase();
1273
- const portalOptions = { timeoutMs };
1274
- const fromX = Number(args.from_x), fromY = Number(args.from_y);
1275
- const toX = Number(args.to_x), toY = Number(args.to_y);
1276
- await portalMoveTo(fromX, fromY, portalOptions);
1277
- // Give the compositor/toolkit one frame to observe the source position before
1278
- // button-down. A transport ACK only proves that the portal accepted the event.
1279
- await new Promise(resolve => setTimeout(resolve, 60));
1280
- await portalPointerButton(button, true, portalOptions);
1281
- await new Promise(resolve => setTimeout(resolve, clamp(args.hold_ms, 120, 60, 5000)));
1282
-
1283
- // Keep the pressed pointer on a single absolute-motion transition. Repeated
1284
- // absolute moves can cross EIS region/device contexts and lose the logical
1285
- // pressed state. The EIS helper converts the desktop target into region-local
1286
- // coordinates, so one target move is sufficient and preserves buttons=1.
1287
- await portalMoveTo(toX, toY, portalOptions);
1288
- await new Promise(resolve => setTimeout(resolve, clamp(args.duration_ms, 160, 80, 5000)));
1289
- // Reassert the final target immediately before release. On Mutter/XWayland this
1290
- // mirrors a real pointer-up gesture and ensures the release is delivered as a
1291
- // pointerup/mouseup at the dragged target instead of only clearing button state.
1292
- await portalMoveTo(toX, toY, portalOptions);
1293
- await new Promise(resolve => setTimeout(resolve, 40));
1294
- await portalPointerButton(button, false, portalOptions);
1295
- await new Promise(resolve => setTimeout(resolve, 80));
1296
- };
1297
- if (isWaylandSession()) {
1298
- if (backend === 'portal') {
1299
- if (!waylandPortalCandidate()) unavailable('Wayland drag and drop', 'no XDG RemoteDesktop portal is available');
1300
- await runPortal(120_000);
1301
- return jsonResult({ from:[args.from_x,args.from_y], to:[args.to_x,args.to_y], backend:'xdg-desktop-portal' });
1302
- }
1303
- if (backend === 'auto' && hasWaylandRemoteDesktopGrant()) {
1304
- try {
1305
- await runPortal(2500);
1306
- return jsonResult({ from:[args.from_x,args.from_y], to:[args.to_x,args.to_y], backend:'xdg-desktop-portal' });
1307
- } catch {}
1308
- }
1309
- if (backend === 'x11') {
1310
- unavailable('Wayland drag and drop', 'XTEST/xdotool drag events are not reliably delivered through GNOME Wayland; use backend=portal');
1311
- }
1312
- if (backend === 'auto' && waylandPortalCandidate()) portalPermissionHint('Wayland drag and drop');
1313
- unavailable('Wayland drag and drop', 'no working consent-backed drag backend is available');
1314
- }
1315
- if (await runXdotool()) return jsonResult({ from:[args.from_x,args.from_y], to:[args.to_x,args.to_y], backend:'xdotool' });
1316
- unavailable('Drag and drop', 'install xdotool');
1317
- }
1318
-
1319
- export async function scroll(args = {}) {
1320
- const dx = Number(args.delta_x || 0), dy = Number(args.delta_y ?? args.delta ?? 0);
1321
- const backend = inputBackend(args);
1322
- const runXdotool = async () => {
1323
- if (!commandExists('xdotool')) return false;
1324
- if (dy) {
1325
- const button = dy < 0 ? '4' : '5';
1326
- const repeats = Math.max(1, Math.min(50, Math.ceil(Math.abs(dy) / 120)));
1327
- await runFile('xdotool', ['click','--repeat',String(repeats),button], { label: 'vertical scroll' });
1328
- }
1329
- if (dx) {
1330
- const button = dx < 0 ? '6' : '7';
1331
- const repeats = Math.max(1, Math.min(50, Math.ceil(Math.abs(dx) / 120)));
1332
- await runFile('xdotool', ['click','--repeat',String(repeats),button], { label: 'horizontal scroll' });
1333
- }
1334
- return true;
1335
- };
1336
- if (isWaylandSession()) {
1337
- if (backend === 'portal') {
1338
- if (!waylandPortalCandidate()) unavailable('Wayland scroll', 'no XDG RemoteDesktop portal is available');
1339
- await portalScroll(dx, dy, { timeoutMs: 120_000 });
1340
- return jsonResult({ delta_x: dx, delta_y: dy, backend:'xdg-desktop-portal' });
1341
- }
1342
- if (backend === 'auto' && hasWaylandRemoteDesktopGrant()) {
1343
- try {
1344
- await portalScroll(dx, dy, { timeoutMs: 2500 });
1345
- return jsonResult({ delta_x: dx, delta_y: dy, backend:'xdg-desktop-portal' });
1346
- } catch {}
1347
- }
1348
- if (backend !== 'portal' && await runXdotool()) return jsonResult({ delta_x: dx, delta_y: dy, backend:'xwayland-xdotool' });
1349
- if (backend === 'auto' && waylandPortalCandidate()) portalPermissionHint('Wayland scroll');
1350
- unavailable('Wayland scroll', backend === 'x11' ? 'xdotool is unavailable' : 'no working scroll backend is available');
1351
- }
1352
- if (await runXdotool()) return jsonResult({ delta_x: dx, delta_y: dy, backend:'xdotool' });
1353
- unavailable('Scroll', 'install xdotool');
1354
- }
1355
-
1356
- export async function cursorPosition() {
1357
- if (isWaylandSession()) {
1358
- unavailable('Cursor position', 'native Wayland does not expose an authoritative global cursor position; XWayland xdotool coordinates may be stale');
1359
- }
1360
- if (!commandExists('xdotool')) unavailable('Cursor position', 'install xdotool');
1361
- const result = await runFile('xdotool', ['getmouselocation','--shell'], { label:'cursor position', allowFailure:true, timeout:2000 });
1362
- const x = Number(result.stdout.match(/^X=(-?\d+)$/m)?.[1]);
1363
- const y = Number(result.stdout.match(/^Y=(-?\d+)$/m)?.[1]);
1364
- const screen = Number(result.stdout.match(/^SCREEN=(-?\d+)$/m)?.[1]);
1365
- if (result.code !== 0 || !Number.isFinite(x) || !Number.isFinite(y)) unavailable('Cursor position', (result.stderr || result.stdout || 'xdotool failed').trim());
1366
- return jsonResult({ x, y, screen: Number.isFinite(screen) ? screen : null });
1367
- }
1368
-
1369
- function unwrapDbus(value) {
1370
- if (value && typeof value === 'object' && Object.hasOwn(value, 'value')) return unwrapDbus(value.value);
1371
- if (Array.isArray(value)) return value.map(unwrapDbus);
1372
- if (value && typeof value === 'object') return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, unwrapDbus(item)]));
1373
- return value;
1374
- }
1375
-
1376
- async function mutterDisplayInventory() {
1377
- if (!isWaylandSession()) return null;
1378
- let dbus;
1379
- try { dbus = await import('@jellybrick/dbus-next'); } catch { return null; }
1380
- const bus = dbus.sessionBus();
1381
- try {
1382
- const object = await bus.getProxyObject('org.gnome.Mutter.DisplayConfig', '/org/gnome/Mutter/DisplayConfig');
1383
- const iface = object.getInterface('org.gnome.Mutter.DisplayConfig');
1384
- const state = unwrapDbus(await iface.GetCurrentState());
1385
- if (!Array.isArray(state) || state.length < 3) return null;
1386
- const monitors = Array.isArray(state[1]) ? state[1] : [];
1387
- const logical = Array.isArray(state[2]) ? state[2] : [];
1388
- const monitorByKey = new Map();
1389
- for (const monitor of monitors) {
1390
- const spec = Array.isArray(monitor?.[0]) ? monitor[0] : [];
1391
- const modes = Array.isArray(monitor?.[1]) ? monitor[1] : [];
1392
- const props = monitor?.[2] && typeof monitor[2] === 'object' ? monitor[2] : {};
1393
- const current = modes.find(mode => mode?.[6]?.['is-current'] === true)
1394
- || modes.find(mode => mode?.[6]?.['is-preferred'] === true)
1395
- || modes[0];
1396
- monitorByKey.set(JSON.stringify(spec), {
1397
- connector: String(spec[0] || ''),
1398
- vendor: String(spec[1] || ''),
1399
- product: String(spec[2] || ''),
1400
- serial: String(spec[3] || ''),
1401
- display_name: String(props['display-name'] || spec[0] || 'Display'),
1402
- builtin: Boolean(props['is-builtin']),
1403
- mode: current ? {
1404
- width: Number(current[1]) || null,
1405
- height: Number(current[2]) || null,
1406
- refresh_hz: Number(current[3]) || null,
1407
- } : null,
1408
- });
1409
- }
1410
- const rows = [];
1411
- for (const item of logical) {
1412
- if (!Array.isArray(item) || item.length < 6) continue;
1413
- const x = Number(item[0]), y = Number(item[1]), scale = Number(item[2]) || 1;
1414
- const transform = Number(item[3]) || 0;
1415
- const primary = Boolean(item[4]);
1416
- const specs = Array.isArray(item[5]) ? item[5] : [];
1417
- for (const spec of specs) {
1418
- const details = monitorByKey.get(JSON.stringify(spec)) || {
1419
- connector: String(spec?.[0] || ''),
1420
- vendor: String(spec?.[1] || ''),
1421
- product: String(spec?.[2] || ''),
1422
- serial: String(spec?.[3] || ''),
1423
- display_name: String(spec?.[0] || 'Display'),
1424
- mode: null,
1425
- };
1426
- const pixelWidth = Number(details.mode?.width) || null;
1427
- const pixelHeight = Number(details.mode?.height) || null;
1428
- const rotated = [1,3,5,7].includes(transform);
1429
- rows.push({
1430
- name: details.connector || details.display_name,
1431
- display_name: details.display_name,
1432
- primary,
1433
- x: Number.isFinite(x) ? x : null,
1434
- y: Number.isFinite(y) ? y : null,
1435
- width: pixelWidth == null ? null : Math.round((rotated ? pixelHeight : pixelWidth) / scale),
1436
- height: pixelHeight == null ? null : Math.round((rotated ? pixelWidth : pixelHeight) / scale),
1437
- pixel_width: pixelWidth,
1438
- pixel_height: pixelHeight,
1439
- scale,
1440
- refresh_hz: details.mode?.refresh_hz ?? null,
1441
- transform,
1442
- builtin: Boolean(details.builtin),
1443
- vendor: details.vendor || null,
1444
- product: details.product || null,
1445
- serial: details.serial || null,
1446
- backend: 'mutter',
1447
- });
1448
- }
1449
- }
1450
- return rows.length ? rows : null;
1451
- } catch {
1452
- return null;
1453
- } finally {
1454
- try { bus.disconnect(); } catch {}
1455
- }
1456
- }
1457
-
1458
- export function parseWlrRandrOutput(stdout) {
1459
- const rows = [];
1460
- let current = null;
1461
-
1462
- const finish = () => {
1463
- if (!current || current.enabled === false) return;
1464
- const mode = current.currentMode || current.firstMode;
1465
- if (!mode) return;
1466
- const scale = Number.isFinite(current.scale) && current.scale > 0 ? current.scale : 1;
1467
- const rotated = /(?:^|\D)(?:90|270)(?:\D|$)/.test(current.transform || '');
1468
- const pixelWidth = mode.width;
1469
- const pixelHeight = mode.height;
1470
- rows.push({
1471
- name:current.name,
1472
- display_name:current.description || current.model || current.name,
1473
- primary:false,
1474
- x:Number.isFinite(current.x) ? current.x : null,
1475
- y:Number.isFinite(current.y) ? current.y : null,
1476
- width:Math.round((rotated ? pixelHeight : pixelWidth) / scale),
1477
- height:Math.round((rotated ? pixelWidth : pixelHeight) / scale),
1478
- pixel_width:pixelWidth,
1479
- pixel_height:pixelHeight,
1480
- scale,
1481
- refresh_hz:mode.refresh_hz,
1482
- transform:current.transform || 'normal',
1483
- builtin:/^(?:eDP|LVDS|DSI)-/i.test(current.name),
1484
- vendor:current.make || null,
1485
- product:current.model || null,
1486
- serial:current.serial || null,
1487
- backend:'wlr-randr',
1488
- });
1489
- };
1490
-
1491
- for (const rawLine of String(stdout || '').split(/\r?\n/)) {
1492
- if (rawLine && !/^\s/.test(rawLine)) {
1493
- finish();
1494
- const header = rawLine.match(/^(\S+)(?:\s+"([^"]*)")?\s*$/);
1495
- current = header ? {
1496
- name:header[1],
1497
- description:header[2] || '',
1498
- enabled:true,
1499
- scale:1,
1500
- transform:'normal',
1501
- currentMode:null,
1502
- firstMode:null,
1503
- } : null;
1504
- continue;
1505
- }
1506
- if (!current) continue;
1507
- const line = rawLine.trim();
1508
- let match;
1509
- if ((match = line.match(/^Enabled:\s*(yes|no)$/i))) current.enabled = match[1].toLowerCase() === 'yes';
1510
- else if ((match = line.match(/^Position:\s*(-?\d+)\s*,\s*(-?\d+)$/i))) {
1511
- current.x = Number(match[1]);
1512
- current.y = Number(match[2]);
1513
- } else if ((match = line.match(/^Scale:\s*(\d+(?:\.\d+)?)$/i))) current.scale = Number(match[1]);
1514
- else if ((match = line.match(/^Transform:\s*(.+)$/i))) current.transform = match[1].trim();
1515
- else if ((match = line.match(/^Make:\s*(.*)$/i))) current.make = match[1].trim();
1516
- else if ((match = line.match(/^Model:\s*(.*)$/i))) current.model = match[1].trim();
1517
- else if ((match = line.match(/^Serial:\s*(.*)$/i))) current.serial = match[1].trim();
1518
- else if ((match = line.match(/^(\d+)x(\d+)\s+px,\s*([\d.]+)\s+Hz(?:\s+\(([^)]*)\))?/i))) {
1519
- const mode = { width:Number(match[1]), height:Number(match[2]), refresh_hz:Number(match[3]) };
1520
- current.firstMode ||= mode;
1521
- if (/\bcurrent\b/i.test(match[4] || '')) current.currentMode = mode;
1522
- }
1523
- }
1524
- finish();
1525
- return rows;
1526
- }
1527
-
1528
- export function parseXrandrDisplayOutput(stdout, scale = 1) {
1529
- const rows = [];
1530
- for (const line of String(stdout || '').split(/\r?\n/)) {
1531
- const match = line.match(/^(\S+) connected( primary)?(?: (\d+)x(\d+)\+(-?\d+)\+(-?\d+))?/);
1532
- if (match) rows.push({
1533
- name:match[1],
1534
- display_name:match[1],
1535
- primary:Boolean(match[2]),
1536
- width:match[3]?Number(match[3]):null,
1537
- height:match[4]?Number(match[4]):null,
1538
- pixel_width:match[3]?Number(match[3]):null,
1539
- pixel_height:match[4]?Number(match[4]):null,
1540
- x:match[5]?Number(match[5]):null,
1541
- y:match[6]?Number(match[6]):null,
1542
- scale,
1543
- backend:'xrandr',
1544
- });
1545
- }
1546
- return rows;
1547
- }
1548
-
1549
- async function xrandrDisplayScale({ commandExistsFn = commandExists, runFileFn = runFile, env = process.env } = {}) {
1550
- let scale = Number(env.GDK_SCALE || env.QT_SCALE_FACTOR || 0);
1551
- if (!Number.isFinite(scale) || scale <= 0) scale = 0;
1552
- if (!scale && commandExistsFn('gsettings')) {
1553
- const probe = await runFileFn('gsettings', ['get','org.gnome.desktop.interface','scaling-factor'], { label:'display scale', allowFailure:true, timeout:1500 });
1554
- const parsed = Number(String(probe.stdout || '').match(/(\d+(?:\.\d+)?)/)?.[1]);
1555
- if (Number.isFinite(parsed) && parsed > 0) scale = parsed;
1556
- }
1557
- return scale || 1;
1558
- }
1559
-
1560
- export async function resolveLinuxDisplayInventory({
1561
- wayland = isWaylandSession(),
1562
- mutterProbe = mutterDisplayInventory,
1563
- commandExistsFn = commandExists,
1564
- runFileFn = runFile,
1565
- env = process.env,
1566
- } = {}) {
1567
- if (wayland) {
1568
- const mutter = await mutterProbe().catch(() => null);
1569
- if (Array.isArray(mutter) && mutter.length) return mutter;
1570
-
1571
- if (commandExistsFn('wlr-randr')) {
1572
- const result = await runFileFn('wlr-randr', [], { label:'display inventory', allowFailure:true, timeout:2500, maxBuffer:4*1024*1024 });
1573
- if (result.code === 0) {
1574
- const rows = parseWlrRandrOutput(result.stdout);
1575
- if (rows.length) return rows;
1576
- }
1577
- }
1578
- }
1579
-
1580
- if (commandExistsFn('xrandr')) {
1581
- const result = await runFileFn('xrandr', ['--query'], { label:'display inventory', allowFailure:true, timeout:2500, maxBuffer:4*1024*1024 });
1582
- if (result.code === 0) {
1583
- const scale = await xrandrDisplayScale({ commandExistsFn, runFileFn, env });
1584
- const rows = parseXrandrDisplayOutput(result.stdout, scale);
1585
- if (rows.length) return rows;
1586
- }
1587
- }
1588
- return null;
1589
- }
1590
-
1591
- let displayCapabilityCache = { value:false, expiresAt:0, pending:null };
1592
-
1593
- export async function displayInventoryAvailable(options) {
1594
- if (options && Object.keys(options).length) {
1595
- return Boolean((await resolveLinuxDisplayInventory(options))?.length);
1596
- }
1597
- const now = Date.now();
1598
- if (displayCapabilityCache.expiresAt > now) return displayCapabilityCache.value;
1599
- if (displayCapabilityCache.pending) return displayCapabilityCache.pending;
1600
- const pending = resolveLinuxDisplayInventory()
1601
- .then(rows => {
1602
- const value = Boolean(rows?.length);
1603
- displayCapabilityCache = { value, expiresAt:Date.now() + 5000, pending:null };
1604
- return value;
1605
- })
1606
- .catch(() => {
1607
- displayCapabilityCache = { value:false, expiresAt:Date.now() + 5000, pending:null };
1608
- return false;
1609
- });
1610
- displayCapabilityCache.pending = pending;
1611
- return pending;
1612
- }
1613
-
1614
- export async function displayInventory() {
1615
- const rows = await resolveLinuxDisplayInventory();
1616
- if (!rows?.length) unavailable('Display inventory', 'GNOME Mutter DisplayConfig, wlr-randr or xrandr is required');
1617
- return jsonResult(rows);
1618
- }
1619
-
1620
- async function pngDimensions(file) {
1621
- const handle = await open(file, 'r');
1622
- try {
1623
- const buffer = Buffer.alloc(24);
1624
- const { bytesRead } = await handle.read(buffer, 0, buffer.length, 0);
1625
- if (bytesRead < buffer.length || buffer.subarray(0, 8).toString('hex') !== '89504e470d0a1a0a') return null;
1626
- const width = buffer.readUInt32BE(16);
1627
- const height = buffer.readUInt32BE(20);
1628
- if (!width || !height) return null;
1629
- return { width, height };
1630
- } finally {
1631
- await handle.close();
1632
- }
1633
- }
1634
-
1635
- export function regionScreenshotBackends({
1636
- wayland = isWaylandSession(),
1637
- grim = commandExists('grim'),
1638
- gnomeScreenshot = commandExists('gnome-screenshot'),
1639
- ffmpeg = commandExists('ffmpeg'),
1640
- imagemagick = commandExists('import'),
1641
- portalCandidate = waylandPortalCandidate(),
1642
- } = {}) {
1643
- const backends = [];
1644
- if (wayland && grim) backends.push('grim');
1645
- if (wayland) {
1646
- if (gnomeScreenshot && ffmpeg) backends.push('gnome-screenshot');
1647
- if (ffmpeg && portalCandidate) backends.push('portal');
1648
- } else if (imagemagick) {
1649
- backends.push('imagemagick');
1650
- }
1651
- return backends;
1652
- }
1653
-
1654
- export function portalCropGeometry(rect, displays = [], bitmap = null) {
1655
- const requested = {
1656
- x: Math.trunc(Number(rect?.x)),
1657
- y: Math.trunc(Number(rect?.y)),
1658
- width: Math.trunc(Number(rect?.width)),
1659
- height: Math.trunc(Number(rect?.height)),
1660
- };
1661
- if (![requested.x,requested.y,requested.width,requested.height].every(Number.isFinite)
1662
- || requested.width <= 0 || requested.height <= 0) {
1663
- throw new Error('portal crop requires a finite positive rectangle');
1664
- }
1665
- const visibleDisplays = Array.isArray(displays) ? displays.filter(item => {
1666
- const x = Number(item?.x), y = Number(item?.y), width = Number(item?.width), height = Number(item?.height);
1667
- return [x,y,width,height].every(Number.isFinite) && width > 0 && height > 0;
1668
- }) : [];
1669
- const bitmapWidth = Math.trunc(Number(bitmap?.width));
1670
- const bitmapHeight = Math.trunc(Number(bitmap?.height));
1671
- const hasBitmap = Number.isFinite(bitmapWidth) && bitmapWidth > 0 && Number.isFinite(bitmapHeight) && bitmapHeight > 0;
1672
-
1673
- let minX = 0, minY = 0;
1674
- let maxX = hasBitmap ? bitmapWidth : requested.x + requested.width;
1675
- let maxY = hasBitmap ? bitmapHeight : requested.y + requested.height;
1676
- if (visibleDisplays.length) {
1677
- minX = Math.min(...visibleDisplays.map(item => Number(item.x)));
1678
- minY = Math.min(...visibleDisplays.map(item => Number(item.y)));
1679
- maxX = Math.max(...visibleDisplays.map(item => Number(item.x) + Number(item.width)));
1680
- maxY = Math.max(...visibleDisplays.map(item => Number(item.y) + Number(item.height)));
1681
- }
1682
- const left = Math.max(requested.x, minX);
1683
- const top = Math.max(requested.y, minY);
1684
- const right = Math.min(requested.x + requested.width, maxX);
1685
- const bottom = Math.min(requested.y + requested.height, maxY);
1686
- if (right <= left || bottom <= top) throw new Error('screenshot region is outside visible desktop bounds');
1687
-
1688
- const virtualWidth = maxX - minX;
1689
- const virtualHeight = maxY - minY;
1690
- const scaleX = hasBitmap && virtualWidth > 0 ? bitmapWidth / virtualWidth : 1;
1691
- const scaleY = hasBitmap && virtualHeight > 0 ? bitmapHeight / virtualHeight : 1;
1692
- const x = Math.max(0, Math.floor((left - minX) * scaleX));
1693
- const y = Math.max(0, Math.floor((top - minY) * scaleY));
1694
- const sourceRight = hasBitmap
1695
- ? Math.min(bitmapWidth, Math.ceil((right - minX) * scaleX))
1696
- : Math.ceil((right - minX) * scaleX);
1697
- const sourceBottom = hasBitmap
1698
- ? Math.min(bitmapHeight, Math.ceil((bottom - minY) * scaleY))
1699
- : Math.ceil((bottom - minY) * scaleY);
1700
- const width = sourceRight - x;
1701
- const height = sourceBottom - y;
1702
- if (width <= 0 || height <= 0) throw new Error('screenshot region has no visible portal pixels');
1703
-
1704
- return {
1705
- x, y, width, height,
1706
- visible: { x:left, y:top, width:right-left, height:bottom-top },
1707
- clipped: left !== requested.x || top !== requested.y
1708
- || right !== requested.x + requested.width || bottom !== requested.y + requested.height,
1709
- };
1710
- }
1711
-
1712
- export async function screenshotRegion(args = {}) {
1713
- const x=Math.trunc(Number(args.x)), y=Math.trunc(Number(args.y)), width=Math.trunc(Number(args.width)), height=Math.trunc(Number(args.height));
1714
- if (![x,y,width,height].every(Number.isFinite) || width<=0 || height<=0) throw new Error('x, y, width and height are required; width/height must be positive');
1715
- if (width > 8192 || height > 8192 || width * height > 16_777_216) throw new Error('screenshot region is too large; width/height must be <= 8192 and area <= 16 megapixels');
1716
- const dir = await tempDir('remcp-region-');
1717
- const target = path.join(dir, 'region.png');
1718
- const full = path.join(dir, 'full.png');
1719
- let captured = { x, y, width, height };
1720
- let clipped = false;
1721
- let backend = '';
1722
- try {
1723
- const cropFullDesktop = async () => {
1724
- const inventory = await displayInventory().catch(() => null);
1725
- const displays = Array.isArray(inventory?.structuredContent?.data) ? inventory.structuredContent.data : [];
1726
- const bitmap = await pngDimensions(full);
1727
- const crop = portalCropGeometry({ x, y, width, height }, displays, bitmap);
1728
- captured = crop.visible;
1729
- clipped = crop.clipped;
1730
- await runFile('ffmpeg', ['-hide_banner','-loglevel','error','-y','-i',full,'-vf',`crop=${crop.width}:${crop.height}:${crop.x}:${crop.y}`,'-frames:v','1',target], { label:'crop desktop screenshot', timeout:30_000, maxBuffer:4*1024*1024 });
1731
- };
1732
-
1733
- const candidates = regionScreenshotBackends();
1734
- if (!candidates.length) {
1735
- unavailable('Region screenshots', isWaylandSession()
1736
- ? 'install grim, or gnome-screenshot/ffmpeg, or provide the XDG Desktop Portal plus ffmpeg'
1737
- : 'install ImageMagick import');
1738
- }
1739
-
1740
- let lastError = null;
1741
- for (const candidate of candidates) {
1742
- try {
1743
- if (candidate === 'grim') {
1744
- await runFile('grim', ['-g',`${x},${y} ${width}x${height}`,target], { label:'screenshot region' });
1745
- } else if (candidate === 'gnome-screenshot') {
1746
- await runFile('gnome-screenshot', ['-f',full], { label:'GNOME screenshot', timeout:10_000, maxBuffer:1024*1024 });
1747
- await cropFullDesktop();
1748
- } else if (candidate === 'portal') {
1749
- await capturePortalScreenshot(full);
1750
- await cropFullDesktop();
1751
- } else if (candidate === 'imagemagick') {
1752
- await runFile('import', ['-window','root','-crop',`${width}x${height}+${x}+${y}`,target], { label:'screenshot region' });
1753
- }
1754
- backend = candidate;
1755
- lastError = null;
1756
- break;
1757
- } catch (error) {
1758
- if (error?.code === 'PORTAL_CANCELLED') throw error;
1759
- lastError = error;
1760
- }
1761
- }
1762
- if (lastError) throw lastError;
1763
-
1764
- const {data} = await readPrivateTempFile(target, 4 * 1024 * 1024);
1765
- const baseSummary = clipped
1766
- ? `Captured visible ${captured.width}x${captured.height} at ${captured.x},${captured.y}; clipped from requested ${width}x${height} at ${x},${y}.`
1767
- : `Captured ${width}x${height} at ${x},${y}.`;
1768
- return multi([{type:'text',text:`${baseSummary} Backend: ${backend}.`}, image(data.toString('base64'),'image/png')]);
1769
- } finally { await removeTemp(dir); }
1770
- }
1771
-
1772
- export async function notification(args = {}) {
1773
- if (!commandExists('notify-send')) unavailable('Desktop notifications', 'install notify-send');
1774
- await runFile('notify-send', [optionalString(args.title)||'ReMCP', String(args.message ?? args.body ?? '')], { label:'notification' });
1775
- return text('Notification sent.');
1776
- }
1777
-
1778
- export async function launchApp(app, argv = [], options = {}) {
1779
- const pid = spawnDetached(app, argv, { cwd:options.cwd || undefined });
1780
- return `Launched ${app} (pid ${pid})${options.cwd ? ` in ${options.cwd}` : ''}.`;
1781
- }
1782
-
1783
- export async function openPath(target) {
1784
- if (!commandExists('xdg-open')) unavailable('Open path', 'xdg-open is required');
1785
- spawnDetached('xdg-open', [target]);
1786
- }
1787
-
1788
- export async function revealPath(target) {
1789
- if (!commandExists('xdg-open')) unavailable('Reveal path', 'xdg-open is required');
1790
- spawnDetached('xdg-open', [path.dirname(target)]);
1791
- }
1
+ // ReMCP Linux Desktop backend: stable public exports split by capability.
2
+ export { parseX11PixelBounds, listWindows } from './desktop-linux-windows.mjs';
3
+ export { windowAction, uiSnapshot, uiAction } from './desktop-linux-ui.mjs';
4
+ export { clipboard, typeTextWindowTarget, typeTextFocused, keyboard, typeTextKeys, pointer, dragDrop, scroll, cursorPosition } from './desktop-linux-input.mjs';
5
+ export { parseWlrRandrOutput, parseXrandrDisplayOutput, resolveLinuxDisplayInventory, displayInventoryAvailable, displayInventory, regionScreenshotBackends, portalCropGeometry } from './desktop-linux-displays.mjs';
6
+ export { screenshotRegion, notification, launchApp, openPath, revealPath } from './desktop-linux-screen.mjs';