@octavus/agent 1.0.22 → 1.0.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/computer-mcp/dist/index.js +17 -17
- package/dist/computer-mcp/linux/linux-driver.py +832 -291
- package/dist/index.js +51 -51
- package/package.json +1 -1
|
@@ -13,22 +13,27 @@ Dependencies (apt):
|
|
|
13
13
|
|
|
14
14
|
Usage mirrors the macOS driver exactly:
|
|
15
15
|
linux-driver.py observe --screenshot /tmp/shot.png [--pid PID]
|
|
16
|
-
linux-driver.py click <x> <y>
|
|
16
|
+
linux-driver.py click <x> <y> [--hold-seconds N]
|
|
17
17
|
linux-driver.py double-click <x> <y>
|
|
18
18
|
linux-driver.py right-click <x> <y>
|
|
19
19
|
linux-driver.py type <text>
|
|
20
20
|
linux-driver.py key <key-spec>
|
|
21
21
|
linux-driver.py scroll <direction> [steps] [--at x y]
|
|
22
|
-
linux-driver.py drag <from_x> <from_y> <to_x> <to_y> [--steps N] [--path x1,y1 ...]
|
|
22
|
+
linux-driver.py drag <from_x> <from_y> <to_x> <to_y> [--steps N] [--path x1,y1 ...] [--hold-seconds N]
|
|
23
23
|
linux-driver.py screenshot <path>
|
|
24
24
|
linux-driver.py screenshot-grid <path> --region x y w h --grid <2|3>
|
|
25
25
|
linux-driver.py activate <app-name>
|
|
26
26
|
"""
|
|
27
27
|
|
|
28
|
+
import ctypes
|
|
29
|
+
import ctypes.util
|
|
30
|
+
import functools
|
|
28
31
|
import glob
|
|
29
32
|
import json
|
|
30
33
|
import os
|
|
31
34
|
import re
|
|
35
|
+
import shlex
|
|
36
|
+
import signal
|
|
32
37
|
import subprocess
|
|
33
38
|
import sys
|
|
34
39
|
import time
|
|
@@ -38,8 +43,8 @@ import shutil
|
|
|
38
43
|
# Output helpers (match Swift driver JSON contract)
|
|
39
44
|
# ---------------------------------------------------------------------------
|
|
40
45
|
|
|
41
|
-
def print_success(path=None):
|
|
42
|
-
resp = {"success": True}
|
|
46
|
+
def print_success(path=None, **fields):
|
|
47
|
+
resp = {"success": True, **fields}
|
|
43
48
|
if path is not None:
|
|
44
49
|
resp["path"] = path
|
|
45
50
|
print(json.dumps(resp))
|
|
@@ -99,6 +104,10 @@ CONTAINER_ROLES = {
|
|
|
99
104
|
|
|
100
105
|
ACTIONABLE_ACTIONS = {"click", "press", "activate", "toggle", "jump"}
|
|
101
106
|
|
|
107
|
+
# Roles that are never a control, whatever action their toolkit advertises:
|
|
108
|
+
# LibreOffice exposes a "click" on every toolbar separator.
|
|
109
|
+
NON_ACTIONABLE_ROLES = {"separator"}
|
|
110
|
+
|
|
102
111
|
|
|
103
112
|
def _try_import_atspi():
|
|
104
113
|
"""Import AT-SPI2 via GObject introspection and connect to the a11y bus."""
|
|
@@ -132,6 +141,15 @@ def _overlaps_screen(x, y, w, h, screen_w, screen_h):
|
|
|
132
141
|
return x < screen_w and y < screen_h and x + w > 0 and y + h > 0
|
|
133
142
|
|
|
134
143
|
|
|
144
|
+
# The most children of one node the walk descends into. A spreadsheet reports its
|
|
145
|
+
# whole sheet as the table's children (Calc: every cell of 1024 x 1048576), which
|
|
146
|
+
# no walk could visit within a driver command - and its first rows would be no
|
|
147
|
+
# guide to what is on screen anyway. A node with more children than this is a
|
|
148
|
+
# data grid, left to grid coordinates; a UI container that is not one has orders
|
|
149
|
+
# of magnitude fewer.
|
|
150
|
+
_MAX_CHILDREN_WALKED = 2048
|
|
151
|
+
|
|
152
|
+
|
|
135
153
|
def _walk_atspi(
|
|
136
154
|
node,
|
|
137
155
|
elements,
|
|
@@ -161,6 +179,11 @@ def _walk_atspi(
|
|
|
161
179
|
except Exception:
|
|
162
180
|
pass
|
|
163
181
|
|
|
182
|
+
try:
|
|
183
|
+
role_name = node.get_role_name().lower()
|
|
184
|
+
except Exception:
|
|
185
|
+
role_name = ""
|
|
186
|
+
|
|
164
187
|
rect = None
|
|
165
188
|
try:
|
|
166
189
|
comp = node.get_component_iface()
|
|
@@ -178,6 +201,11 @@ def _walk_atspi(
|
|
|
178
201
|
)
|
|
179
202
|
except Exception:
|
|
180
203
|
rect = None
|
|
204
|
+
# GTK3 answers for an accessible without geometry of its own (a GtkNotebook
|
|
205
|
+
# page) with -1s: that is "no geometry", like having no Component interface at
|
|
206
|
+
# all, not a box that is off the screen.
|
|
207
|
+
if rect is not None and (rect[2] < 0 or rect[3] < 0):
|
|
208
|
+
rect = None
|
|
181
209
|
|
|
182
210
|
on_screen = (
|
|
183
211
|
rect is not None
|
|
@@ -197,11 +225,10 @@ def _walk_atspi(
|
|
|
197
225
|
# surfaces hidden-but-positioned elements. Off-screen and zero-area nodes fail
|
|
198
226
|
# the on_screen test regardless.
|
|
199
227
|
if rect is not None and on_screen and (trust_geometry or visible):
|
|
200
|
-
|
|
201
|
-
role_name
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
if role_name and (role_name in CLICKABLE_ROLES or _has_actionable(node, Atspi)):
|
|
228
|
+
if role_name and (
|
|
229
|
+
role_name in CLICKABLE_ROLES
|
|
230
|
+
or (role_name not in NON_ACTIONABLE_ROLES and _has_actionable(node, Atspi))
|
|
231
|
+
):
|
|
205
232
|
x, y, w, h = rect
|
|
206
233
|
area = w * h
|
|
207
234
|
screen_area = screen_w * screen_h
|
|
@@ -242,15 +269,22 @@ def _walk_atspi(
|
|
|
242
269
|
# Descend into children unless this node is provably off-screen. Keep walking
|
|
243
270
|
# through nodes with no geometry (structural containers such as the
|
|
244
271
|
# `application` node, which reports VISIBLE=False and no extents) and through
|
|
245
|
-
# on-screen containers that merely lack SHOWING
|
|
246
|
-
#
|
|
247
|
-
#
|
|
248
|
-
#
|
|
249
|
-
|
|
272
|
+
# on-screen containers that merely lack SHOWING; prune a node whose real box
|
|
273
|
+
# is off-screen or zero-area. For Chromium that prune also requires the node
|
|
274
|
+
# not to be SHOWING: web layouts routinely hang visible, absolutely positioned
|
|
275
|
+
# content under a zero-size wrapper, and off-viewport content is what bounds
|
|
276
|
+
# the walk over a large page. A native toolkit clips children to their
|
|
277
|
+
# parent's allocation, so a zero-area or off-screen native container shows
|
|
278
|
+
# nothing whatever its SHOWING flag says - LibreOffice keeps a SHOWING,
|
|
279
|
+
# zero-height copy of its menu bar whose items would otherwise be labeled on
|
|
280
|
+
# top of the toolbar drawn below it.
|
|
281
|
+
if rect is not None and not on_screen and (trust_geometry or not showing):
|
|
250
282
|
return
|
|
251
283
|
|
|
252
284
|
try:
|
|
253
285
|
n_children = node.get_child_count()
|
|
286
|
+
if n_children > _MAX_CHILDREN_WALKED:
|
|
287
|
+
return
|
|
254
288
|
for i in range(n_children):
|
|
255
289
|
child = node.get_child_at_index(i)
|
|
256
290
|
if child is not None:
|
|
@@ -328,33 +362,41 @@ def _dedup_elements(elements, screen_w, screen_h):
|
|
|
328
362
|
return deduped
|
|
329
363
|
|
|
330
364
|
|
|
331
|
-
def
|
|
365
|
+
def _xdotool_output(args, timeout=5):
|
|
366
|
+
"""stdout of an xdotool query, or None when it fails (no active window, a
|
|
367
|
+
window gone between two lookups, no WM support)."""
|
|
332
368
|
try:
|
|
333
|
-
|
|
334
|
-
["xdotool",
|
|
369
|
+
return subprocess.check_output(
|
|
370
|
+
["xdotool", *args], text=True, timeout=timeout, stderr=subprocess.DEVNULL
|
|
335
371
|
).strip()
|
|
336
|
-
pid = subprocess.check_output(
|
|
337
|
-
["xdotool", "getwindowpid", wid], text=True, timeout=5
|
|
338
|
-
).strip()
|
|
339
|
-
return int(pid)
|
|
340
372
|
except Exception:
|
|
341
373
|
return None
|
|
342
374
|
|
|
343
375
|
|
|
344
|
-
def
|
|
345
|
-
""
|
|
376
|
+
def _get_active_window_id():
|
|
377
|
+
out = _xdotool_output(["getactivewindow"])
|
|
378
|
+
try:
|
|
379
|
+
return int(out) if out else None
|
|
380
|
+
except ValueError:
|
|
381
|
+
return None
|
|
346
382
|
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
"""
|
|
383
|
+
|
|
384
|
+
def _window_pid(window_id):
|
|
385
|
+
out = _xdotool_output(["getwindowpid", str(window_id)])
|
|
351
386
|
try:
|
|
352
|
-
out
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
387
|
+
return int(out) if out else None
|
|
388
|
+
except ValueError:
|
|
389
|
+
return None
|
|
390
|
+
|
|
391
|
+
|
|
392
|
+
def _window_title(window_id):
|
|
393
|
+
return _xdotool_output(["getwindowname", str(window_id)]) or ""
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
def _window_geometry(window_id):
|
|
397
|
+
"""Screen rect of a window as {id, x, y, width, height}, or None when it is gone."""
|
|
398
|
+
out = _xdotool_output(["getwindowgeometry", "--shell", str(window_id)])
|
|
399
|
+
if not out:
|
|
358
400
|
return None
|
|
359
401
|
|
|
360
402
|
vals = {}
|
|
@@ -374,8 +416,58 @@ def _get_active_window_geometry():
|
|
|
374
416
|
return None
|
|
375
417
|
|
|
376
418
|
|
|
377
|
-
|
|
378
|
-
|
|
419
|
+
# Window types that sit in the WM's client list without ever being "in front":
|
|
420
|
+
# the desktop (GNOME's desktop-icons window spans the screen at the bottom of the
|
|
421
|
+
# stack) and panels/docks.
|
|
422
|
+
_BACKDROP_WINDOW_TYPES = ("_NET_WM_WINDOW_TYPE_DESKTOP", "_NET_WM_WINDOW_TYPE_DOCK")
|
|
423
|
+
|
|
424
|
+
|
|
425
|
+
def _is_backdrop_window(window_id):
|
|
426
|
+
"""Whether the window is the desktop, a dock, or minimized - never "in front"."""
|
|
427
|
+
with _X11() as x11:
|
|
428
|
+
types = x11.window_property(window_id, "_NET_WM_WINDOW_TYPE", max_items=8) or []
|
|
429
|
+
state = x11.window_property(window_id, "_NET_WM_STATE", max_items=32) or []
|
|
430
|
+
backdrop = {x11.atom(name) for name in _BACKDROP_WINDOW_TYPES}
|
|
431
|
+
return bool(backdrop & set(types)) or x11.atom("_NET_WM_STATE_HIDDEN") in state
|
|
432
|
+
|
|
433
|
+
|
|
434
|
+
def _foreground_window_id():
|
|
435
|
+
"""The window in front of the user: the top of the WM's stacking order.
|
|
436
|
+
|
|
437
|
+
Usually that is also the focused window. It is not when the window manager
|
|
438
|
+
stacks a window on top but withholds focus from it: mutter's focus-stealing
|
|
439
|
+
prevention does exactly that to a modal dialog another process opens on
|
|
440
|
+
behalf of the focused app (the portal file chooser Chrome's "Open" / "Save as"
|
|
441
|
+
bring up) after synthetic keystrokes, so the dialog covers the browser while
|
|
442
|
+
keystrokes keep landing in the browser. The topmost window is what the agent
|
|
443
|
+
sees and will click, so it is the one labeled - and it is given focus first so
|
|
444
|
+
typing reaches it too. Falls back to the focused window where the WM
|
|
445
|
+
publishes no stacking order, and is None when nothing but the desktop (or a
|
|
446
|
+
dock) is up - the desktop is never "the app in front".
|
|
447
|
+
"""
|
|
448
|
+
active = _get_active_window_id()
|
|
449
|
+
for window_id in reversed(_managed_windows_stacked()):
|
|
450
|
+
if _is_backdrop_window(window_id):
|
|
451
|
+
continue
|
|
452
|
+
if window_id != active:
|
|
453
|
+
_focus_window(window_id)
|
|
454
|
+
return window_id
|
|
455
|
+
if active is not None and not _is_backdrop_window(active):
|
|
456
|
+
return active
|
|
457
|
+
return None
|
|
458
|
+
|
|
459
|
+
|
|
460
|
+
def _process_name(pid):
|
|
461
|
+
"""The process's command name from /proc, or "" when unreadable."""
|
|
462
|
+
try:
|
|
463
|
+
with open(f"/proc/{pid}/comm", "rb") as f:
|
|
464
|
+
return f.read().decode("utf-8", "ignore").strip()
|
|
465
|
+
except Exception:
|
|
466
|
+
return ""
|
|
467
|
+
|
|
468
|
+
|
|
469
|
+
def _pid_is_chromium(pid):
|
|
470
|
+
"""Whether the PID is a Chrome/Chromium process.
|
|
379
471
|
|
|
380
472
|
Read from /proc rather than the window class: the cloud xdotool build has no
|
|
381
473
|
getwindowclassname and xprop is not installed, but the browser owns the
|
|
@@ -394,31 +486,118 @@ def _active_window_is_chromium(pid):
|
|
|
394
486
|
return False
|
|
395
487
|
|
|
396
488
|
|
|
397
|
-
def
|
|
398
|
-
|
|
489
|
+
def _pid_is_libreoffice(pid):
|
|
490
|
+
return pid is not None and _process_name(pid).startswith("soffice")
|
|
399
491
|
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
492
|
+
|
|
493
|
+
# How long LibreOffice gets to re-read its frame geometry after the pointer nudge
|
|
494
|
+
# below, before its extents are trusted.
|
|
495
|
+
_LIBREOFFICE_GEOMETRY_SETTLE = 0.3
|
|
496
|
+
|
|
497
|
+
|
|
498
|
+
def _refresh_libreoffice_geometry(window):
|
|
499
|
+
"""Make LibreOffice recompute where its widgets are before they are read.
|
|
500
|
+
|
|
501
|
+
LibreOffice (gtk3 VCL) caches its frame geometry and only re-reads it on a
|
|
502
|
+
pointer event: after the window opens and the window manager places and
|
|
503
|
+
decorates it, every toolbar, the document and the status bar keep reporting
|
|
504
|
+
AT-SPI extents one title-bar height above where they are drawn, until the
|
|
505
|
+
pointer first moves inside the window. A labeled toolbar button would then be
|
|
506
|
+
clicked on the menu bar above it. One motion event over the window's center
|
|
507
|
+
is enough; the pointer has no meaning to the agent between actions.
|
|
406
508
|
"""
|
|
407
|
-
|
|
408
|
-
|
|
509
|
+
center = (window["x"] + window["width"] // 2, window["y"] + window["height"] // 2)
|
|
510
|
+
if _pointer_location() == center:
|
|
511
|
+
center = (center[0] + 1, center[1])
|
|
512
|
+
_move_pointer(*center)
|
|
513
|
+
time.sleep(_LIBREOFFICE_GEOMETRY_SETTLE)
|
|
514
|
+
|
|
515
|
+
|
|
516
|
+
def _foreground_window():
|
|
517
|
+
"""What is in front: {id, title, pid, x, y, width, height, chromium}, or None.
|
|
518
|
+
|
|
519
|
+
Everything `observe` labels is scoped to this window - one desktop, what you
|
|
520
|
+
see is what you get - so it is also what the result reports back, letting the
|
|
521
|
+
agent (and anyone reading the trace) see which window the numbers belong to.
|
|
522
|
+
"""
|
|
523
|
+
window_id = _foreground_window_id()
|
|
524
|
+
window = _window_geometry(window_id) if window_id is not None else None
|
|
525
|
+
if window is None:
|
|
526
|
+
return None
|
|
527
|
+
pid = _window_pid(window["id"])
|
|
528
|
+
return {
|
|
529
|
+
**window,
|
|
530
|
+
"title": _window_title(window["id"]),
|
|
531
|
+
"pid": pid,
|
|
532
|
+
"chromium": _pid_is_chromium(pid),
|
|
533
|
+
}
|
|
534
|
+
|
|
535
|
+
|
|
536
|
+
def _apps_for_pid(desktop, target_pid):
|
|
537
|
+
"""The AT-SPI application node(s) owned by ``target_pid`` - strictly by PID.
|
|
538
|
+
|
|
539
|
+
A None target_pid (foreground PID unavailable) matches no app, so labeling never
|
|
540
|
+
falls back to walking the whole desktop, which would overlay the always-open
|
|
541
|
+
browser's labels onto whatever native app is in front.
|
|
542
|
+
"""
|
|
543
|
+
apps = []
|
|
544
|
+
if target_pid is None:
|
|
545
|
+
return apps
|
|
546
|
+
try:
|
|
547
|
+
n_apps = desktop.get_child_count()
|
|
548
|
+
except Exception:
|
|
549
|
+
return apps
|
|
409
550
|
for i in range(n_apps):
|
|
410
|
-
app = desktop.get_child_at_index(i)
|
|
411
|
-
if app is None:
|
|
412
|
-
continue
|
|
413
551
|
try:
|
|
414
|
-
|
|
415
|
-
|
|
552
|
+
app = desktop.get_child_at_index(i)
|
|
553
|
+
if app is not None and app.get_process_id() == target_pid:
|
|
554
|
+
apps.append(app)
|
|
416
555
|
except Exception:
|
|
417
556
|
continue
|
|
557
|
+
return apps
|
|
558
|
+
|
|
559
|
+
|
|
560
|
+
def _collect_elements(apps, window, Atspi, screen_w, screen_h, trust_geometry=False):
|
|
561
|
+
"""Collect the actionable elements of the foreground window from its app's tree.
|
|
562
|
+
|
|
563
|
+
Only the toplevel accessible that corresponds to the focused X11 window is
|
|
564
|
+
walked: an app's other windows (a second Files window behind the first, a
|
|
565
|
+
browser popup, a document behind a modal dialog) are not what the agent sees
|
|
566
|
+
and would otherwise be labeled on top of what is - the same control twice at
|
|
567
|
+
two positions. An app whose toplevel cannot be matched to the focused window
|
|
568
|
+
(nameless frames, a toolkit that never marks one ACTIVE) is walked whole, so a
|
|
569
|
+
scoping miss degrades to extra labels rather than none. `trust_geometry` is
|
|
570
|
+
forwarded to the walk so a native foreground app's on-screen controls are
|
|
571
|
+
included from geometry even when its toolkit leaves the AT-SPI VISIBLE/SHOWING
|
|
572
|
+
flags unset (GTK4).
|
|
573
|
+
"""
|
|
574
|
+
elements = []
|
|
575
|
+
for app in apps:
|
|
576
|
+
frame = _focused_frame(app, window, Atspi) if window is not None else None
|
|
418
577
|
if _uses_window_relative_extents(app):
|
|
419
|
-
|
|
578
|
+
# Window-relative extents can only be placed for the focused window,
|
|
579
|
+
# whose on-screen origin is known; without a matched frame there is
|
|
580
|
+
# nothing to place them against.
|
|
581
|
+
if frame is not None:
|
|
582
|
+
left, top = _client_side_decoration_offset(window["id"])
|
|
583
|
+
_walk_atspi(
|
|
584
|
+
frame,
|
|
585
|
+
elements,
|
|
586
|
+
Atspi,
|
|
587
|
+
screen_w,
|
|
588
|
+
screen_h,
|
|
589
|
+
trust_geometry=trust_geometry,
|
|
590
|
+
window_origin=(window["x"] + left, window["y"] + top),
|
|
591
|
+
)
|
|
420
592
|
else:
|
|
421
|
-
_walk_atspi(
|
|
593
|
+
_walk_atspi(
|
|
594
|
+
frame if frame is not None else app,
|
|
595
|
+
elements,
|
|
596
|
+
Atspi,
|
|
597
|
+
screen_w,
|
|
598
|
+
screen_h,
|
|
599
|
+
trust_geometry=trust_geometry,
|
|
600
|
+
)
|
|
422
601
|
return elements
|
|
423
602
|
|
|
424
603
|
|
|
@@ -440,33 +619,14 @@ def _uses_window_relative_extents(app):
|
|
|
440
619
|
return toolkit == "gtk" and major >= 4
|
|
441
620
|
|
|
442
621
|
|
|
443
|
-
def
|
|
444
|
-
"""
|
|
622
|
+
def _focused_frame(app, window, Atspi):
|
|
623
|
+
"""The app's toplevel accessible for the focused X11 window, or None.
|
|
445
624
|
|
|
446
|
-
|
|
447
|
-
|
|
625
|
+
Matched by the ACTIVE state when the toolkit sets it, else by title - a GTK
|
|
626
|
+
dialog's accessible carries its window title even when it is never marked
|
|
627
|
+
ACTIVE. An app with a single toplevel needs no matching: there is nothing
|
|
628
|
+
else it could be.
|
|
448
629
|
"""
|
|
449
|
-
window = _get_active_window_geometry()
|
|
450
|
-
if window is None:
|
|
451
|
-
return
|
|
452
|
-
frame = _focused_frame(app, window["id"], Atspi)
|
|
453
|
-
if frame is None:
|
|
454
|
-
return
|
|
455
|
-
left, top = _client_side_decoration_offset(window["id"])
|
|
456
|
-
_walk_atspi(
|
|
457
|
-
frame,
|
|
458
|
-
elements,
|
|
459
|
-
Atspi,
|
|
460
|
-
screen_w,
|
|
461
|
-
screen_h,
|
|
462
|
-
trust_geometry=trust_geometry,
|
|
463
|
-
window_origin=(window["x"] + left, window["y"] + top),
|
|
464
|
-
)
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
def _focused_frame(app, window_id, Atspi):
|
|
468
|
-
"""The app's toplevel accessible for the focused window: the one AT-SPI marks
|
|
469
|
-
ACTIVE, else the one titled like the focused X11 window."""
|
|
470
630
|
frames = []
|
|
471
631
|
try:
|
|
472
632
|
for i in range(app.get_child_count()):
|
|
@@ -475,6 +635,8 @@ def _focused_frame(app, window_id, Atspi):
|
|
|
475
635
|
frames.append(frame)
|
|
476
636
|
except Exception:
|
|
477
637
|
return None
|
|
638
|
+
if len(frames) == 1:
|
|
639
|
+
return frames[0]
|
|
478
640
|
|
|
479
641
|
for frame in frames:
|
|
480
642
|
try:
|
|
@@ -483,12 +645,7 @@ def _focused_frame(app, window_id, Atspi):
|
|
|
483
645
|
except Exception:
|
|
484
646
|
continue
|
|
485
647
|
|
|
486
|
-
|
|
487
|
-
title = subprocess.check_output(
|
|
488
|
-
["xdotool", "getwindowname", str(window_id)], text=True, timeout=5
|
|
489
|
-
).strip()
|
|
490
|
-
except Exception:
|
|
491
|
-
return None
|
|
648
|
+
title = window.get("title") or ""
|
|
492
649
|
for frame in frames:
|
|
493
650
|
try:
|
|
494
651
|
if title and frame.get_name() == title:
|
|
@@ -498,27 +655,32 @@ def _focused_frame(app, window_id, Atspi):
|
|
|
498
655
|
return None
|
|
499
656
|
|
|
500
657
|
|
|
501
|
-
|
|
658
|
+
# Xlib's default error handler exits the process. A window can be destroyed
|
|
659
|
+
# between the xdotool lookup and a property read (BadWindow), which must degrade
|
|
660
|
+
# to "no property" rather than take the driver down mid-observe. Installed once
|
|
661
|
+
# when libX11 is loaded; the callback has to stay referenced for as long as the
|
|
662
|
+
# library is in use, so it lives at module level.
|
|
663
|
+
_IGNORE_X_ERRORS = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p)(
|
|
664
|
+
lambda _display, _error: 0
|
|
665
|
+
)
|
|
502
666
|
|
|
503
667
|
|
|
504
|
-
|
|
505
|
-
|
|
668
|
+
@functools.lru_cache(maxsize=None)
|
|
669
|
+
def _load_xlib():
|
|
670
|
+
"""libX11 with the prototypes the driver uses, loaded once; None when unavailable.
|
|
506
671
|
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
measures window-relative extents from the visible frame, not the X11 window's
|
|
510
|
-
corner. Read through libX11 directly because xprop is not installed on the
|
|
511
|
-
cloud image. (0, 0) for a window without client-side decorations.
|
|
672
|
+
Window properties are read through libX11 directly because xprop is not
|
|
673
|
+
installed on the cloud image.
|
|
512
674
|
"""
|
|
513
675
|
try:
|
|
514
|
-
import ctypes
|
|
515
|
-
import ctypes.util
|
|
516
|
-
|
|
517
676
|
xlib = ctypes.cdll.LoadLibrary(ctypes.util.find_library("X11") or "libX11.so.6")
|
|
518
677
|
except Exception:
|
|
519
|
-
return
|
|
678
|
+
return None
|
|
520
679
|
xlib.XOpenDisplay.argtypes = [ctypes.c_char_p]
|
|
521
680
|
xlib.XOpenDisplay.restype = ctypes.c_void_p
|
|
681
|
+
xlib.XCloseDisplay.argtypes = [ctypes.c_void_p]
|
|
682
|
+
xlib.XDefaultRootWindow.argtypes = [ctypes.c_void_p]
|
|
683
|
+
xlib.XDefaultRootWindow.restype = ctypes.c_ulong
|
|
522
684
|
xlib.XInternAtom.argtypes = [ctypes.c_void_p, ctypes.c_char_p, ctypes.c_int]
|
|
523
685
|
xlib.XInternAtom.restype = ctypes.c_ulong
|
|
524
686
|
xlib.XGetWindowProperty.argtypes = [
|
|
@@ -536,38 +698,56 @@ def _client_side_decoration_offset(window_id):
|
|
|
536
698
|
ctypes.POINTER(ctypes.c_void_p),
|
|
537
699
|
]
|
|
538
700
|
xlib.XFree.argtypes = [ctypes.c_void_p]
|
|
539
|
-
xlib.XCloseDisplay.argtypes = [ctypes.c_void_p]
|
|
540
701
|
xlib.XSetErrorHandler.restype = ctypes.c_void_p
|
|
702
|
+
xlib.XSetErrorHandler(_IGNORE_X_ERRORS)
|
|
703
|
+
return xlib
|
|
541
704
|
|
|
542
|
-
# Xlib's default error handler exits the process. The focused window can be
|
|
543
|
-
# destroyed between the xdotool lookup and this read (BadWindow), which must
|
|
544
|
-
# degrade to no offset rather than take the driver down mid-observe. The
|
|
545
|
-
# callback has to stay referenced while the display is open.
|
|
546
|
-
ignore_x_errors = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p)(
|
|
547
|
-
lambda _display, _error: 0
|
|
548
|
-
)
|
|
549
|
-
xlib.XSetErrorHandler(ignore_x_errors)
|
|
550
705
|
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
706
|
+
class _X11:
|
|
707
|
+
"""One connection to the X display for a batch of reads.
|
|
708
|
+
|
|
709
|
+
Opened on entry and closed on exit. Every read answers with a neutral value
|
|
710
|
+
(0 for an atom, None for a property) instead of raising, so a window destroyed
|
|
711
|
+
between the xdotool lookup and the read - or a box without libX11 or a display
|
|
712
|
+
at all - degrades to "no property".
|
|
713
|
+
"""
|
|
714
|
+
|
|
715
|
+
def __enter__(self):
|
|
716
|
+
self._xlib = _load_xlib()
|
|
717
|
+
self._display = self._xlib.XOpenDisplay(None) if self._xlib else None
|
|
718
|
+
return self
|
|
719
|
+
|
|
720
|
+
def __exit__(self, *_exc):
|
|
721
|
+
if self._display:
|
|
722
|
+
self._xlib.XCloseDisplay(self._display)
|
|
723
|
+
|
|
724
|
+
def atom(self, name):
|
|
725
|
+
"""The atom for ``name``, or 0 when it was never interned on this server."""
|
|
726
|
+
if not self._display:
|
|
727
|
+
return 0
|
|
728
|
+
return int(self._xlib.XInternAtom(self._display, name.encode(), True))
|
|
729
|
+
|
|
730
|
+
def window_property(self, window_id, name, max_items=1024):
|
|
731
|
+
"""A format-32 property of ``window_id`` (the root window when None) as a
|
|
732
|
+
list of ints, or None when the window or the property does not exist."""
|
|
733
|
+
atom = self.atom(name)
|
|
556
734
|
if not atom:
|
|
557
|
-
return
|
|
735
|
+
return None
|
|
736
|
+
if window_id is None:
|
|
737
|
+
window_id = self._xlib.XDefaultRootWindow(self._display)
|
|
558
738
|
actual_type = ctypes.c_ulong()
|
|
559
739
|
actual_format = ctypes.c_int()
|
|
560
740
|
n_items = ctypes.c_ulong()
|
|
561
741
|
bytes_after = ctypes.c_ulong()
|
|
562
742
|
data = ctypes.c_void_p()
|
|
563
|
-
status =
|
|
564
|
-
|
|
743
|
+
status = self._xlib.XGetWindowProperty(
|
|
744
|
+
self._display,
|
|
565
745
|
window_id,
|
|
566
746
|
atom,
|
|
567
747
|
0,
|
|
568
|
-
|
|
748
|
+
max_items,
|
|
569
749
|
False,
|
|
570
|
-
|
|
750
|
+
0, # AnyPropertyType
|
|
571
751
|
ctypes.byref(actual_type),
|
|
572
752
|
ctypes.byref(actual_format),
|
|
573
753
|
ctypes.byref(n_items),
|
|
@@ -575,49 +755,69 @@ def _client_side_decoration_offset(window_id):
|
|
|
575
755
|
ctypes.byref(data),
|
|
576
756
|
)
|
|
577
757
|
if status != 0 or not data.value:
|
|
578
|
-
return
|
|
758
|
+
return None
|
|
579
759
|
try:
|
|
580
|
-
if actual_format.value != 32 or n_items.value
|
|
581
|
-
return
|
|
760
|
+
if actual_format.value != 32 or n_items.value == 0:
|
|
761
|
+
return None
|
|
582
762
|
# Xlib hands format-32 property items back as C longs.
|
|
583
|
-
|
|
584
|
-
return int(left), int(top)
|
|
763
|
+
return [int(v) for v in ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[: n_items.value]]
|
|
585
764
|
finally:
|
|
586
|
-
|
|
587
|
-
|
|
765
|
+
self._xlib.XFree(data)
|
|
766
|
+
|
|
767
|
+
|
|
768
|
+
def _x11_window_property(window_id, name, max_items=1024):
|
|
769
|
+
"""One format-32 property read on its own connection (see ``_X11``)."""
|
|
770
|
+
with _X11() as x11:
|
|
771
|
+
return x11.window_property(window_id, name, max_items)
|
|
772
|
+
|
|
773
|
+
|
|
774
|
+
def _managed_windows_stacked():
|
|
775
|
+
"""The window manager's client windows, bottom to top (`_NET_CLIENT_LIST_STACKING`).
|
|
776
|
+
|
|
777
|
+
These are the real, mapped toplevels a user sees - not the unmapped client-leader
|
|
778
|
+
and helper windows a toolkit also creates (which `xdotool search` returns too, and
|
|
779
|
+
which cannot be activated). Empty when the WM does not publish the list.
|
|
780
|
+
"""
|
|
781
|
+
return _x11_window_property(None, "_NET_CLIENT_LIST_STACKING") or []
|
|
782
|
+
|
|
783
|
+
|
|
784
|
+
def _client_side_decoration_offset(window_id):
|
|
785
|
+
"""(left, top) inset of a window's visible frame within its X11 window.
|
|
786
|
+
|
|
787
|
+
GTK draws client-side decorations - its drop shadow - inside the X11 window and
|
|
788
|
+
advertises their size in `_GTK_FRAME_EXTENTS` (left, right, top, bottom); GTK
|
|
789
|
+
measures window-relative extents from the visible frame, not the X11 window's
|
|
790
|
+
corner. (0, 0) for a window without client-side decorations.
|
|
791
|
+
"""
|
|
792
|
+
extents = _x11_window_property(window_id, "_GTK_FRAME_EXTENTS", max_items=4)
|
|
793
|
+
if not extents or len(extents) != 4:
|
|
588
794
|
return 0, 0
|
|
589
|
-
|
|
590
|
-
|
|
795
|
+
left, _right, top, _bottom = extents
|
|
796
|
+
return left, top
|
|
591
797
|
|
|
592
798
|
|
|
593
|
-
def
|
|
594
|
-
"""
|
|
799
|
+
def _chromium_apps(desktop):
|
|
800
|
+
"""The Chrome/Chromium AT-SPI app node(s), matched by name.
|
|
595
801
|
|
|
596
|
-
Recovery for when the foreground window is Chromium but
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
browser's labels.
|
|
802
|
+
Recovery for when the foreground window is Chromium but its active-window PID
|
|
803
|
+
disagrees with the AT-SPI app PID (a wrapper or zygote owning the window).
|
|
804
|
+
Scoping by name keeps the recovery on the browser alone, so a native app in
|
|
805
|
+
front is never overlaid with the always-open browser's labels.
|
|
601
806
|
"""
|
|
602
|
-
|
|
807
|
+
apps = []
|
|
603
808
|
try:
|
|
604
809
|
n_apps = desktop.get_child_count()
|
|
605
810
|
except Exception:
|
|
606
|
-
return
|
|
811
|
+
return apps
|
|
607
812
|
for i in range(n_apps):
|
|
608
813
|
try:
|
|
609
814
|
app = desktop.get_child_at_index(i)
|
|
815
|
+
name = (app.get_name() or "").lower() if app is not None else ""
|
|
610
816
|
except Exception:
|
|
611
817
|
continue
|
|
612
|
-
if app is None:
|
|
613
|
-
continue
|
|
614
|
-
try:
|
|
615
|
-
name = (app.get_name() or "").lower()
|
|
616
|
-
except Exception:
|
|
617
|
-
name = ""
|
|
618
818
|
if "chrome" in name or "chromium" in name:
|
|
619
|
-
|
|
620
|
-
return
|
|
819
|
+
apps.append(app)
|
|
820
|
+
return apps
|
|
621
821
|
|
|
622
822
|
|
|
623
823
|
def _within_rect(elements, rect):
|
|
@@ -634,7 +834,60 @@ def _within_rect(elements, rect):
|
|
|
634
834
|
return kept
|
|
635
835
|
|
|
636
836
|
|
|
837
|
+
# Retries for a Chromium tree that is still being built: Chromium exports its
|
|
838
|
+
# web content lazily, populating the tree on the first assistive-technology
|
|
839
|
+
# query, so the very first walk after a navigation can come back empty.
|
|
840
|
+
_CHROMIUM_EXPORT_RETRIES = 2
|
|
841
|
+
_CHROMIUM_EXPORT_RETRY_DELAY = 0.6
|
|
842
|
+
|
|
843
|
+
|
|
844
|
+
def _empty_result_diagnostic(window, apps_found, foreground_is_chromium):
|
|
845
|
+
"""Why `observe` found nothing, for the agent and for anyone reading the trace.
|
|
846
|
+
|
|
847
|
+
"Found 0 actionable elements" has several unrelated causes with different
|
|
848
|
+
remedies, and the trace cannot tell them apart without this: no application
|
|
849
|
+
window in front (only the desktop, or a window manager reporting nothing
|
|
850
|
+
active), a window without an owning process, an app that is not on the
|
|
851
|
+
accessibility bus at all (a toolkit without AT-SPI support, or a browser that
|
|
852
|
+
started with accessibility off and stays that way until restarted), or an app
|
|
853
|
+
on the bus whose focused window exports no actionable controls.
|
|
854
|
+
"""
|
|
855
|
+
if window is None:
|
|
856
|
+
return (
|
|
857
|
+
"No application window is in front: only the desktop is showing, or the "
|
|
858
|
+
"window manager reports nothing active."
|
|
859
|
+
)
|
|
860
|
+
title = window.get("title") or "(untitled)"
|
|
861
|
+
if window.get("pid") is None:
|
|
862
|
+
return (
|
|
863
|
+
f'The focused window "{title}" reports no owning process, '
|
|
864
|
+
"so its accessibility tree cannot be located."
|
|
865
|
+
)
|
|
866
|
+
if not apps_found:
|
|
867
|
+
if foreground_is_chromium:
|
|
868
|
+
return (
|
|
869
|
+
"The browser in front is not exporting an accessibility tree (it is not "
|
|
870
|
+
"registered on the accessibility bus), so none of its controls or page "
|
|
871
|
+
"content can be labeled until it is restarted."
|
|
872
|
+
)
|
|
873
|
+
process = _process_name(window["pid"]) or f"pid {window['pid']}"
|
|
874
|
+
return (
|
|
875
|
+
f'The app in front ("{title}", {process}) is not registered on the '
|
|
876
|
+
"accessibility bus: it exports no accessibility tree."
|
|
877
|
+
)
|
|
878
|
+
return (
|
|
879
|
+
f'The app in front ("{title}") is on the accessibility bus but exports no '
|
|
880
|
+
"actionable controls for its focused window."
|
|
881
|
+
)
|
|
882
|
+
|
|
883
|
+
|
|
637
884
|
def enumerate_elements(pid=None):
|
|
885
|
+
"""Label the foreground window's actionable elements.
|
|
886
|
+
|
|
887
|
+
Returns (elements, window, diagnostic): the numbered elements, the foreground
|
|
888
|
+
window descriptor (None when no application window is in front), and - only
|
|
889
|
+
when no element was found - a diagnostic naming the cause.
|
|
890
|
+
"""
|
|
638
891
|
Atspi = _try_import_atspi()
|
|
639
892
|
if Atspi is None:
|
|
640
893
|
exit_with_error(
|
|
@@ -642,34 +895,38 @@ def enumerate_elements(pid=None):
|
|
|
642
895
|
)
|
|
643
896
|
|
|
644
897
|
screen_w, screen_h = get_screen_size()
|
|
645
|
-
|
|
898
|
+
window = _foreground_window()
|
|
899
|
+
if pid is not None and window is not None:
|
|
900
|
+
window = {**window, "pid": pid, "chromium": _pid_is_chromium(pid)}
|
|
901
|
+
target_pid = pid or (window["pid"] if window is not None else None)
|
|
646
902
|
# A native foreground app (GTK4 etc.) leaves the AT-SPI VISIBLE/SHOWING flags
|
|
647
903
|
# unset on its on-screen descendants, so trust geometry for it; Chromium's web
|
|
648
904
|
# a11y flags are reliable, so keep requiring VISIBLE there.
|
|
649
|
-
foreground_is_chromium =
|
|
905
|
+
foreground_is_chromium = bool(window and window["chromium"])
|
|
906
|
+
if window is not None and _pid_is_libreoffice(target_pid):
|
|
907
|
+
_refresh_libreoffice_geometry(window)
|
|
650
908
|
|
|
651
909
|
desktop = Atspi.get_desktop(0)
|
|
910
|
+
apps = _apps_for_pid(desktop, target_pid)
|
|
652
911
|
elements = _collect_elements(
|
|
653
|
-
|
|
912
|
+
apps, window, Atspi, screen_w, screen_h, trust_geometry=not foreground_is_chromium
|
|
654
913
|
)
|
|
655
914
|
|
|
656
|
-
# The foreground app exported its own actionable tree - use it verbatim. It is
|
|
657
|
-
# already scoped to the single window the agent sees (one desktop, WYSIWYG).
|
|
658
915
|
if not elements and foreground_is_chromium:
|
|
659
|
-
#
|
|
660
|
-
# PID
|
|
661
|
-
#
|
|
662
|
-
#
|
|
663
|
-
#
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
for _ in range(2):
|
|
667
|
-
time.sleep(0.6)
|
|
916
|
+
# The browser's PID-scoped walk came back empty: either its tree is still
|
|
917
|
+
# being exported, or its window PID is not its AT-SPI PID. Retry briefly on
|
|
918
|
+
# the browser's own app node(s), clipped to the focused window. Never the
|
|
919
|
+
# whole desktop, which would overlay the always-open browser's labels on
|
|
920
|
+
# whatever native app is in front.
|
|
921
|
+
for _ in range(_CHROMIUM_EXPORT_RETRIES):
|
|
922
|
+
time.sleep(_CHROMIUM_EXPORT_RETRY_DELAY)
|
|
668
923
|
desktop = Atspi.get_desktop(0)
|
|
669
|
-
|
|
924
|
+
apps = apps or _chromium_apps(desktop)
|
|
925
|
+
elements = _within_rect(
|
|
926
|
+
_collect_elements(apps, window, Atspi, screen_w, screen_h), window
|
|
927
|
+
)
|
|
670
928
|
if elements:
|
|
671
929
|
break
|
|
672
|
-
elements = _within_rect(elements, active_rect)
|
|
673
930
|
|
|
674
931
|
# A native (non-Chromium) foreground app that exports nothing yields an empty
|
|
675
932
|
# result on purpose: the agent falls back to screenshot_grid coordinates
|
|
@@ -680,7 +937,10 @@ def enumerate_elements(pid=None):
|
|
|
680
937
|
for idx, el in enumerate(deduped):
|
|
681
938
|
el["number"] = idx + 1
|
|
682
939
|
|
|
683
|
-
|
|
940
|
+
diagnostic = None
|
|
941
|
+
if not deduped:
|
|
942
|
+
diagnostic = _empty_result_diagnostic(window, bool(apps), foreground_is_chromium)
|
|
943
|
+
return deduped, window, diagnostic
|
|
684
944
|
|
|
685
945
|
|
|
686
946
|
# ---------------------------------------------------------------------------
|
|
@@ -917,6 +1177,10 @@ def annotate_screenshot(screenshot_path, elements):
|
|
|
917
1177
|
# slightly-less-synchronous move instead of a hard failure.
|
|
918
1178
|
_MOUSEMOVE_SYNC_TIMEOUT = 3
|
|
919
1179
|
|
|
1180
|
+
# The longest a press-and-hold may keep the button down. A command's timeout
|
|
1181
|
+
# grows by the requested hold, so this bound is what actually limits it.
|
|
1182
|
+
_MAX_HOLD_SECONDS = 20
|
|
1183
|
+
|
|
920
1184
|
# Every keysym a modifier can be held under, left and right variants.
|
|
921
1185
|
_MODIFIER_KEYSYMS = [
|
|
922
1186
|
"Control_L", "Control_R", "Alt_L", "Alt_R", "Shift_L", "Shift_R", "Super_L", "Super_R",
|
|
@@ -946,30 +1210,76 @@ def release_modifiers(keysyms=_MODIFIER_KEYSYMS):
|
|
|
946
1210
|
except (FileNotFoundError, subprocess.TimeoutExpired):
|
|
947
1211
|
pass
|
|
948
1212
|
|
|
949
|
-
def
|
|
950
|
-
ix, iy
|
|
1213
|
+
def _move_pointer(ix, iy):
|
|
1214
|
+
"""Move the pointer to integer (ix, iy) with a bounded ``--sync`` wait.
|
|
1215
|
+
|
|
1216
|
+
Returns True when the X server confirmed the arrival. When the sync stalls
|
|
1217
|
+
(or errors) the move is re-issued without ``--sync``, which returns
|
|
1218
|
+
immediately but carries no arrival confirmation, and False is returned so
|
|
1219
|
+
the caller can give the async motion a longer settle.
|
|
1220
|
+
"""
|
|
951
1221
|
try:
|
|
952
1222
|
subprocess.run(
|
|
953
|
-
["xdotool", "mousemove", "--sync", ix, iy],
|
|
1223
|
+
["xdotool", "mousemove", "--sync", str(ix), str(iy)],
|
|
954
1224
|
capture_output=True,
|
|
955
1225
|
text=True,
|
|
956
1226
|
timeout=_MOUSEMOVE_SYNC_TIMEOUT,
|
|
957
1227
|
check=True,
|
|
958
1228
|
)
|
|
959
|
-
|
|
960
|
-
return
|
|
1229
|
+
return True
|
|
961
1230
|
except FileNotFoundError:
|
|
962
1231
|
exit_with_error("Required command not found: xdotool")
|
|
963
1232
|
except (subprocess.TimeoutExpired, subprocess.CalledProcessError):
|
|
964
1233
|
pass
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
# clicks, since a non-sync move carries no server arrival confirmation.
|
|
968
|
-
_run(["xdotool", "mousemove", ix, iy])
|
|
969
|
-
time.sleep(0.1)
|
|
1234
|
+
_run(["xdotool", "mousemove", str(ix), str(iy)])
|
|
1235
|
+
return False
|
|
970
1236
|
|
|
971
|
-
def
|
|
1237
|
+
def _pointer_location():
|
|
1238
|
+
"""The pointer's current integer (x, y), or None when it cannot be read."""
|
|
1239
|
+
try:
|
|
1240
|
+
out = subprocess.run(
|
|
1241
|
+
["xdotool", "getmouselocation", "--shell"],
|
|
1242
|
+
capture_output=True,
|
|
1243
|
+
text=True,
|
|
1244
|
+
timeout=_MOUSEMOVE_SYNC_TIMEOUT,
|
|
1245
|
+
check=True,
|
|
1246
|
+
).stdout
|
|
1247
|
+
except (FileNotFoundError, subprocess.TimeoutExpired, subprocess.CalledProcessError):
|
|
1248
|
+
return None
|
|
1249
|
+
values = dict(line.split("=", 1) for line in out.split() if "=" in line)
|
|
1250
|
+
try:
|
|
1251
|
+
return int(values["X"]), int(values["Y"])
|
|
1252
|
+
except (KeyError, ValueError):
|
|
1253
|
+
return None
|
|
1254
|
+
|
|
1255
|
+
def mouse_move(x, y):
|
|
1256
|
+
target = (int(x), int(y))
|
|
1257
|
+
# `mousemove --sync` waits for the pointer to leave its origin, so a move to
|
|
1258
|
+
# the pixel it is already on never confirms and would burn the whole sync
|
|
1259
|
+
# timeout (clicking the same element twice, a drag that starts where the last
|
|
1260
|
+
# click landed, a repeated scroll at one spot). Nothing to move: return.
|
|
1261
|
+
if _pointer_location() == target:
|
|
1262
|
+
return
|
|
1263
|
+
confirmed = _move_pointer(*target)
|
|
1264
|
+
time.sleep(0.025 if confirmed else 0.1)
|
|
1265
|
+
|
|
1266
|
+
def _hold_button(button, hold_seconds):
|
|
1267
|
+
"""Press ``button``, keep it down for ``hold_seconds``, then release it.
|
|
1268
|
+
|
|
1269
|
+
The release runs even when the hold is interrupted, so a press-and-hold
|
|
1270
|
+
can never leave the button logically down.
|
|
1271
|
+
"""
|
|
1272
|
+
_run(["xdotool", "mousedown", button])
|
|
1273
|
+
try:
|
|
1274
|
+
time.sleep(hold_seconds)
|
|
1275
|
+
finally:
|
|
1276
|
+
_run(["xdotool", "mouseup", button], check=False)
|
|
1277
|
+
|
|
1278
|
+
def mouse_click(x, y, hold_seconds=0.0):
|
|
972
1279
|
mouse_move(x, y)
|
|
1280
|
+
if hold_seconds > 0:
|
|
1281
|
+
_hold_button("1", hold_seconds)
|
|
1282
|
+
return
|
|
973
1283
|
_run(["xdotool", "click", "1"])
|
|
974
1284
|
|
|
975
1285
|
def mouse_double_click(x, y):
|
|
@@ -1047,12 +1357,20 @@ def _interpolate_points(from_x, from_y, to_x, to_y, steps):
|
|
|
1047
1357
|
return points
|
|
1048
1358
|
|
|
1049
1359
|
|
|
1050
|
-
def mouse_drag(from_x, from_y, to_x, to_y, steps=10, waypoints=None):
|
|
1360
|
+
def mouse_drag(from_x, from_y, to_x, to_y, steps=10, waypoints=None, hold_seconds=0.0):
|
|
1051
1361
|
"""Perform a smooth drag from one position to another.
|
|
1052
1362
|
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1363
|
+
Presses the button at the start - keeping it down for ``hold_seconds``
|
|
1364
|
+
first when given, the press-and-hold-then-slide gesture - then moves
|
|
1365
|
+
through the interpolated points with bounded ``--sync`` waits so Chrome
|
|
1366
|
+
recognizes the drag, and releases at the end. The 16ms sleep between
|
|
1367
|
+
moves matches 60fps frame timing.
|
|
1368
|
+
|
|
1369
|
+
A point that lands on the pixel the pointer is already on is not sent to
|
|
1370
|
+
xdotool: its ``mousemove --sync`` waits for the pointer to leave its
|
|
1371
|
+
origin, so a move to the current position blocks until the command
|
|
1372
|
+
timeout (a zero-length drag, a freehand loop whose points round to one
|
|
1373
|
+
pixel). The frame sleep still runs so the gesture keeps its timing.
|
|
1056
1374
|
|
|
1057
1375
|
When waypoints are provided the last waypoint is the destination, so
|
|
1058
1376
|
(to_x, to_y) is not appended to the segment list. The button is
|
|
@@ -1060,8 +1378,11 @@ def mouse_drag(from_x, from_y, to_x, to_y, steps=10, waypoints=None):
|
|
|
1060
1378
|
cursor stuck in the down state.
|
|
1061
1379
|
"""
|
|
1062
1380
|
mouse_move(from_x, from_y)
|
|
1381
|
+
last = (int(from_x), int(from_y))
|
|
1063
1382
|
_run(["xdotool", "mousedown", "1"])
|
|
1064
1383
|
try:
|
|
1384
|
+
if hold_seconds > 0:
|
|
1385
|
+
time.sleep(hold_seconds)
|
|
1065
1386
|
if waypoints:
|
|
1066
1387
|
segments = [(from_x, from_y)] + list(waypoints)
|
|
1067
1388
|
else:
|
|
@@ -1070,7 +1391,10 @@ def mouse_drag(from_x, from_y, to_x, to_y, steps=10, waypoints=None):
|
|
|
1070
1391
|
sx, sy = segments[seg_idx]
|
|
1071
1392
|
ex, ey = segments[seg_idx + 1]
|
|
1072
1393
|
for px, py in _interpolate_points(sx, sy, ex, ey, steps):
|
|
1073
|
-
|
|
1394
|
+
target = (int(px), int(py))
|
|
1395
|
+
if target != last:
|
|
1396
|
+
_move_pointer(*target)
|
|
1397
|
+
last = target
|
|
1074
1398
|
time.sleep(0.016)
|
|
1075
1399
|
finally:
|
|
1076
1400
|
_run(["xdotool", "mouseup", "1"], check=False)
|
|
@@ -1104,17 +1428,61 @@ _DESKTOP_DIRS = [
|
|
|
1104
1428
|
]
|
|
1105
1429
|
|
|
1106
1430
|
|
|
1107
|
-
|
|
1108
|
-
|
|
1431
|
+
# How long a freshly launched app gets to show a window before `activate` reports
|
|
1432
|
+
# it as launched but not visible yet. LibreOffice and Evince routinely need well
|
|
1433
|
+
# over the few seconds a snappy app takes on these computers. A driver command
|
|
1434
|
+
# has 30 s in total, so this leaves room for the lookups before the launch and
|
|
1435
|
+
# the settle after it.
|
|
1436
|
+
_LAUNCH_WINDOW_TIMEOUT = 20.0
|
|
1437
|
+
|
|
1438
|
+
# The %f/%U/... placeholders in a .desktop Exec= line (files/URLs to open, icon,
|
|
1439
|
+
# caption), none of which apply to a bare launch.
|
|
1440
|
+
_DESKTOP_FIELD_CODES = re.compile(r"\s+%[fFuUdDnNickvm]")
|
|
1109
1441
|
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1442
|
+
|
|
1443
|
+
def _desktop_entry_lines(content):
|
|
1444
|
+
"""Yield the lines of a .desktop file's ``[Desktop Entry]`` group only.
|
|
1445
|
+
|
|
1446
|
+
``[Desktop Action ...]`` groups carry their own Name= and Exec= (the
|
|
1447
|
+
LibreOffice Start Center's actions are named "Writer", "Calc", ...), so
|
|
1448
|
+
reading them would match a component's short name to the launcher's entry
|
|
1449
|
+
and open the Start Center for "Calc".
|
|
1450
|
+
"""
|
|
1451
|
+
in_entry = False
|
|
1452
|
+
for line in content.splitlines():
|
|
1453
|
+
if line.startswith("["):
|
|
1454
|
+
in_entry = line.strip() == "[Desktop Entry]"
|
|
1455
|
+
elif in_entry:
|
|
1456
|
+
yield line
|
|
1457
|
+
|
|
1458
|
+
|
|
1459
|
+
def _desktop_entry_bool(fields, key):
|
|
1460
|
+
return fields.get(key, "").strip().lower() == "true"
|
|
1461
|
+
|
|
1462
|
+
|
|
1463
|
+
def _resolve_from_desktop_files(name):
|
|
1464
|
+
"""Search .desktop files for an app matching *name*, best match first.
|
|
1465
|
+
|
|
1466
|
+
Matches the Name= and GenericName= fields of the main group
|
|
1467
|
+
(case-insensitive) and the desktop file basename (e.g. "chromium" matches
|
|
1468
|
+
"chromium-browser.desktop"). A Name= match ranks before a GenericName= match,
|
|
1469
|
+
which ranks before a basename match - "Text Editor" is the Name of GNOME's
|
|
1470
|
+
editor but only the GenericName of vim's entry, and "libreoffice" is a
|
|
1471
|
+
substring of every libreoffice-*.desktop basename but names the Start Center.
|
|
1472
|
+
Ties break on the desktop id, so the choice never depends on directory order.
|
|
1473
|
+
|
|
1474
|
+
Entries that cannot show a window of their own are left out: `Terminal=true`
|
|
1475
|
+
programs (vim, htop) run inside a terminal emulator the agent has to open
|
|
1476
|
+
first, and `Hidden`/`NoDisplay` entries are not launchers a user sees.
|
|
1477
|
+
|
|
1478
|
+
Returns a list of (argv, desktop_id, window_classes) launch candidates. argv is
|
|
1479
|
+
the Exec= command with its arguments kept and the field codes removed, so
|
|
1480
|
+
"LibreOffice Calc" launches `libreoffice --calc` - not the bare `libreoffice`,
|
|
1481
|
+
which opens the Start Center instead of the component the agent asked for.
|
|
1482
|
+
window_classes are the WM_CLASS patterns that identify the entry's windows.
|
|
1114
1483
|
"""
|
|
1115
1484
|
name_lower = name.lower()
|
|
1116
|
-
|
|
1117
|
-
desktop_ids = []
|
|
1485
|
+
ranked = []
|
|
1118
1486
|
|
|
1119
1487
|
for desktop_dir in _DESKTOP_DIRS:
|
|
1120
1488
|
for filepath in glob.glob(os.path.join(desktop_dir, "*.desktop")):
|
|
@@ -1124,32 +1492,69 @@ def _resolve_from_desktop_files(name):
|
|
|
1124
1492
|
except Exception:
|
|
1125
1493
|
continue
|
|
1126
1494
|
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
break
|
|
1495
|
+
fields = {}
|
|
1496
|
+
for line in _desktop_entry_lines(content):
|
|
1497
|
+
key, sep, value = line.partition("=")
|
|
1498
|
+
if sep and key.strip() not in fields:
|
|
1499
|
+
fields[key.strip()] = value.strip()
|
|
1500
|
+
if fields.get("Type", "Application") != "Application":
|
|
1501
|
+
continue
|
|
1502
|
+
if any(_desktop_entry_bool(fields, key) for key in ("Terminal", "Hidden", "NoDisplay")):
|
|
1503
|
+
continue
|
|
1137
1504
|
|
|
1138
|
-
|
|
1505
|
+
desktop_id = os.path.basename(filepath)
|
|
1506
|
+
basename = os.path.splitext(desktop_id)[0].lower()
|
|
1507
|
+
if fields.get("Name", "").lower() == name_lower:
|
|
1508
|
+
rank = 0
|
|
1509
|
+
elif fields.get("GenericName", "").lower() == name_lower:
|
|
1510
|
+
rank = 1
|
|
1511
|
+
elif name_lower in basename:
|
|
1512
|
+
rank = 2
|
|
1513
|
+
else:
|
|
1139
1514
|
continue
|
|
1140
1515
|
|
|
1141
|
-
|
|
1516
|
+
exec_cmd = _DESKTOP_FIELD_CODES.sub("", fields.get("Exec", ""))
|
|
1517
|
+
try:
|
|
1518
|
+
argv = shlex.split(exec_cmd) if exec_cmd else []
|
|
1519
|
+
except ValueError:
|
|
1520
|
+
argv = []
|
|
1521
|
+
classes = _window_classes_for_entry(argv, desktop_id, fields.get("StartupWMClass"))
|
|
1522
|
+
ranked.append((rank, desktop_id, argv, classes))
|
|
1523
|
+
|
|
1524
|
+
ranked.sort(key=lambda item: (item[0], item[1]))
|
|
1525
|
+
return [(argv, desktop_id, classes) for _rank, desktop_id, argv, classes in ranked]
|
|
1142
1526
|
|
|
1143
|
-
for line in content.splitlines():
|
|
1144
|
-
if line.startswith("Exec="):
|
|
1145
|
-
exec_cmd = line.split("=", 1)[1].strip()
|
|
1146
|
-
exec_cmd = re.sub(r"\s+%[fFuUdDnNickvm]", "", exec_cmd)
|
|
1147
|
-
binary = exec_cmd.split()[0]
|
|
1148
|
-
if binary not in binaries:
|
|
1149
|
-
binaries.append(binary)
|
|
1150
|
-
break
|
|
1151
1527
|
|
|
1152
|
-
|
|
1528
|
+
# Exec= commands that are wrappers around the real program, so their name says
|
|
1529
|
+
# nothing about the window's class.
|
|
1530
|
+
_LAUNCHER_WRAPPERS = {"env", "sh", "bash", "flatpak", "snap", "gtk-launch", "xdg-open"}
|
|
1531
|
+
|
|
1532
|
+
|
|
1533
|
+
def _window_classes_for_entry(argv, desktop_id, startup_wm_class=None):
|
|
1534
|
+
"""WM_CLASS patterns (case-insensitive regexes) that identify an entry's windows.
|
|
1535
|
+
|
|
1536
|
+
`StartupWMClass=` is the entry's own statement of its class and is used alone
|
|
1537
|
+
when present. LibreOffice's component entries carry it, and the guesses below
|
|
1538
|
+
would otherwise name the whole family too: `libreoffice`, the Exec= basename,
|
|
1539
|
+
is a substring of every component's class, so asking for Calc would focus an
|
|
1540
|
+
open Writer window instead of launching Calc. Without it, a .desktop id is the
|
|
1541
|
+
most specific signal; a reverse-DNS id (`org.gnome.Evince`) also contributes
|
|
1542
|
+
its last segment, since GTK3 apps set WM_CLASS from the program name; and the
|
|
1543
|
+
executable's basename covers apps whose class is their binary
|
|
1544
|
+
(`gnome-calculator`) - without a desktop entry it is all there is.
|
|
1545
|
+
"""
|
|
1546
|
+
if startup_wm_class:
|
|
1547
|
+
return [re.escape(startup_wm_class)]
|
|
1548
|
+
classes = []
|
|
1549
|
+
if desktop_id:
|
|
1550
|
+
base = os.path.splitext(desktop_id)[0]
|
|
1551
|
+
classes.append(base)
|
|
1552
|
+
if "." in base:
|
|
1553
|
+
classes.append(base.rsplit(".", 1)[1])
|
|
1554
|
+
if argv and os.path.basename(argv[0]) not in _LAUNCHER_WRAPPERS:
|
|
1555
|
+
classes.append(os.path.basename(argv[0]))
|
|
1556
|
+
seen = set()
|
|
1557
|
+
return [re.escape(c) for c in classes if c and not (c.lower() in seen or seen.add(c.lower()))]
|
|
1153
1558
|
|
|
1154
1559
|
|
|
1155
1560
|
# WM_CLASS regex fragments for apps whose window title does not match the app name
|
|
@@ -1165,100 +1570,202 @@ _WINDOW_CLASS_ALIASES = {
|
|
|
1165
1570
|
"firefox": ["firefox"],
|
|
1166
1571
|
}
|
|
1167
1572
|
|
|
1573
|
+
# How long the window manager gets to report a requested window as active before
|
|
1574
|
+
# the next activation method is tried. Mutter answers a pager-sourced
|
|
1575
|
+
# _NET_ACTIVE_WINDOW request within a frame or two; the bound only matters when a
|
|
1576
|
+
# request is refused outright.
|
|
1577
|
+
_ACTIVATION_CONFIRM_TIMEOUT = 1.0
|
|
1168
1578
|
|
|
1169
|
-
def activate_app(name):
|
|
1170
|
-
def find_windows(flag, query):
|
|
1171
|
-
try:
|
|
1172
|
-
r = subprocess.run(
|
|
1173
|
-
["xdotool", "search", flag, query],
|
|
1174
|
-
capture_output=True, text=True, timeout=5,
|
|
1175
|
-
)
|
|
1176
|
-
return [wid for wid in r.stdout.strip().split("\n") if wid]
|
|
1177
|
-
except Exception:
|
|
1178
|
-
return []
|
|
1179
|
-
|
|
1180
|
-
def activate_existing_window():
|
|
1181
|
-
# Match an already-open window by title (--name) and by window class
|
|
1182
|
-
# (--class). A Chrome `--app=<url>` window carries the page title, not the
|
|
1183
|
-
# app name, so a title-only search misses it; the class aliases below focus
|
|
1184
|
-
# the running browser window instead of launching a second, duplicate one.
|
|
1185
|
-
attempts = [("--name", name), ("--name", name.lower())]
|
|
1186
|
-
attempts += [("--class", cls) for cls in _WINDOW_CLASS_ALIASES.get(name.lower(), [])]
|
|
1187
|
-
for flag, query in attempts:
|
|
1188
|
-
wids = find_windows(flag, query)
|
|
1189
|
-
if wids:
|
|
1190
|
-
# No --sync: `windowactivate --sync` blocks until the WM confirms
|
|
1191
|
-
# activation, which never arrives for a borderless/maximized
|
|
1192
|
-
# window under fluxbox/xfwm and hangs until the subprocess
|
|
1193
|
-
# timeout. A plain activate plus a short settle is enough; treat a
|
|
1194
|
-
# non-zero exit as best-effort so it can't abort labeling.
|
|
1195
|
-
_run(["xdotool", "windowactivate", wids[0]], check=False)
|
|
1196
|
-
time.sleep(0.2)
|
|
1197
|
-
return True
|
|
1198
|
-
return False
|
|
1199
1579
|
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
while time.time() < deadline:
|
|
1203
|
-
if activate_existing_window():
|
|
1204
|
-
return True
|
|
1205
|
-
time.sleep(0.2)
|
|
1206
|
-
return False
|
|
1580
|
+
def _search_windows(flag, pattern):
|
|
1581
|
+
"""Visible X11 windows matching an xdotool search (`--class` / `--name`).
|
|
1207
1582
|
|
|
1208
|
-
|
|
1209
|
-
|
|
1583
|
+
`--onlyvisible` leaves out the unmapped client-leader and helper windows a
|
|
1584
|
+
toolkit creates alongside its real toplevel: they share its class and title,
|
|
1585
|
+
sort first, and cannot be activated (the WM has never managed them).
|
|
1586
|
+
"""
|
|
1587
|
+
try:
|
|
1588
|
+
r = subprocess.run(
|
|
1589
|
+
["xdotool", "search", "--onlyvisible", flag, pattern],
|
|
1590
|
+
capture_output=True, text=True, timeout=5,
|
|
1591
|
+
)
|
|
1592
|
+
return [int(wid) for wid in r.stdout.split() if wid.isdigit()]
|
|
1593
|
+
except Exception:
|
|
1594
|
+
return []
|
|
1210
1595
|
|
|
1211
|
-
desktop_binaries, desktop_ids = _resolve_from_desktop_files(name)
|
|
1212
1596
|
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
if direct not in candidates:
|
|
1216
|
-
candidates.append(direct)
|
|
1217
|
-
if name != direct and name not in candidates:
|
|
1218
|
-
candidates.append(name)
|
|
1597
|
+
def _find_app_window(name_patterns, class_patterns):
|
|
1598
|
+
"""The topmost window of the app, matched by WM_CLASS first, then by title.
|
|
1219
1599
|
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1600
|
+
Class is the reliable identity of an app's windows: a title only sometimes
|
|
1601
|
+
carries the app's name (Files shows "Home", a document viewer shows the
|
|
1602
|
+
document's title), and a title search can hit a browser tab that happens to
|
|
1603
|
+
mention the app. Among several windows the one highest in the WM's stacking
|
|
1604
|
+
order is the one the user last worked in. Falls back to xdotool's own order
|
|
1605
|
+
when the WM publishes no stacking list.
|
|
1606
|
+
"""
|
|
1607
|
+
stacked = _managed_windows_stacked()
|
|
1608
|
+
attempts = [("--class", c) for c in class_patterns] + [("--name", n) for n in name_patterns]
|
|
1609
|
+
for flag, pattern in attempts:
|
|
1610
|
+
wids = _search_windows(flag, pattern)
|
|
1611
|
+
if not wids:
|
|
1612
|
+
continue
|
|
1613
|
+
if stacked:
|
|
1614
|
+
matched = set(wids)
|
|
1615
|
+
managed = [wid for wid in stacked if wid in matched]
|
|
1616
|
+
if managed:
|
|
1617
|
+
return managed[-1]
|
|
1224
1618
|
continue
|
|
1619
|
+
return wids[-1]
|
|
1620
|
+
return None
|
|
1225
1621
|
|
|
1226
|
-
launch_attempted = True
|
|
1227
|
-
try:
|
|
1228
|
-
subprocess.Popen(
|
|
1229
|
-
[executable],
|
|
1230
|
-
stdout=subprocess.DEVNULL,
|
|
1231
|
-
stderr=subprocess.DEVNULL,
|
|
1232
|
-
)
|
|
1233
|
-
if wait_for_window():
|
|
1234
|
-
return
|
|
1235
|
-
except Exception:
|
|
1236
|
-
pass
|
|
1237
1622
|
|
|
1238
|
-
|
|
1623
|
+
def _focus_window(window_id):
|
|
1624
|
+
"""Make ``window_id`` the active window and confirm the WM agreed.
|
|
1625
|
+
|
|
1626
|
+
A plain `xdotool windowactivate` (a pager-sourced _NET_ACTIVE_WINDOW request)
|
|
1627
|
+
is what mutter and the other EWMH managers honor. Its `--sync` is not used: it
|
|
1628
|
+
waits without a bound of its own for the WM to report the window active, so a
|
|
1629
|
+
refused request would hang until the subprocess timeout. Confirmation is read
|
|
1630
|
+
back from the WM under a bound instead, and a refused request falls through to
|
|
1631
|
+
a raise + focus and then to wmctrl. Returns whether the window ended up active.
|
|
1632
|
+
"""
|
|
1633
|
+
methods = [
|
|
1634
|
+
[["xdotool", "windowactivate", str(window_id)]],
|
|
1635
|
+
[["xdotool", "windowraise", str(window_id)], ["xdotool", "windowfocus", str(window_id)]],
|
|
1636
|
+
]
|
|
1637
|
+
if shutil.which("wmctrl"):
|
|
1638
|
+
methods.append([["wmctrl", "-ia", str(window_id)]])
|
|
1639
|
+
for commands in methods:
|
|
1640
|
+
for command in commands:
|
|
1641
|
+
try:
|
|
1642
|
+
subprocess.run(command, capture_output=True, text=True, timeout=5, check=False)
|
|
1643
|
+
except (FileNotFoundError, subprocess.TimeoutExpired):
|
|
1644
|
+
continue
|
|
1645
|
+
deadline = time.time() + _ACTIVATION_CONFIRM_TIMEOUT
|
|
1646
|
+
while time.time() < deadline:
|
|
1647
|
+
active = _get_active_window_id()
|
|
1648
|
+
# A WM that publishes no active window cannot confirm or deny; the
|
|
1649
|
+
# request was delivered, which is all there is to go on.
|
|
1650
|
+
if active is None or active == window_id:
|
|
1651
|
+
return True
|
|
1652
|
+
time.sleep(0.1)
|
|
1653
|
+
return False
|
|
1654
|
+
|
|
1655
|
+
|
|
1656
|
+
def activate_app(name):
|
|
1657
|
+
"""Focus the app's window, launching the app first when it has none.
|
|
1658
|
+
|
|
1659
|
+
Returns True when a window of the app is in front, False when the app was
|
|
1660
|
+
launched but has not shown a window within the launch timeout - it is still
|
|
1661
|
+
starting, so the caller should wait rather than launch it again. Raises
|
|
1662
|
+
RuntimeError when nothing launchable matches the name, when the launched
|
|
1663
|
+
process exits with a failure before showing a window, or when the app has a
|
|
1664
|
+
window the window manager refuses to bring to the front.
|
|
1665
|
+
"""
|
|
1666
|
+
candidates = _resolve_from_desktop_files(name)
|
|
1667
|
+
name_patterns = [re.escape(name)]
|
|
1668
|
+
class_patterns = list(_WINDOW_CLASS_ALIASES.get(name.lower(), []))
|
|
1669
|
+
for _argv, _desktop_id, classes in candidates:
|
|
1670
|
+
class_patterns += [c for c in classes if c not in class_patterns]
|
|
1671
|
+
|
|
1672
|
+
def focus_or_raise(window_id):
|
|
1673
|
+
if _focus_window(window_id):
|
|
1674
|
+
return True
|
|
1675
|
+
title = _window_title(window_id) or "(untitled)"
|
|
1676
|
+
raise RuntimeError(
|
|
1677
|
+
f"'{name}' has a window (\"{title}\") but the window manager would not "
|
|
1678
|
+
"bring it to the front."
|
|
1679
|
+
)
|
|
1680
|
+
|
|
1681
|
+
# An app that is already running is focused, never launched again: a second
|
|
1682
|
+
# launch of a single-instance app opens a second window (or just re-presents
|
|
1683
|
+
# the first), and the agent then labels two of the same window.
|
|
1684
|
+
existing = _find_app_window(name_patterns, class_patterns)
|
|
1685
|
+
if existing is not None:
|
|
1686
|
+
return focus_or_raise(existing)
|
|
1687
|
+
|
|
1688
|
+
known = {argv[0] for argv, _, _ in candidates if argv}
|
|
1689
|
+
for direct in (name.lower(), name):
|
|
1690
|
+
if direct not in known:
|
|
1691
|
+
candidates.append(([direct], None, _window_classes_for_entry([direct], None)))
|
|
1692
|
+
known.add(direct)
|
|
1693
|
+
|
|
1694
|
+
# Launch once: the first candidate whose executable exists. Each candidate
|
|
1695
|
+
# names the same app, so launching more than one would only stack windows
|
|
1696
|
+
# and make the agent wait for every one in turn.
|
|
1697
|
+
launch = None
|
|
1698
|
+
for argv, desktop_id, classes in candidates:
|
|
1699
|
+
if not argv:
|
|
1700
|
+
continue
|
|
1701
|
+
executable = shutil.which(argv[0])
|
|
1702
|
+
if executable is not None:
|
|
1703
|
+
launch = ([executable, *argv[1:]], desktop_id, classes)
|
|
1704
|
+
break
|
|
1705
|
+
|
|
1706
|
+
# A desktop entry whose Exec= cannot be resolved directly (D-Bus activated
|
|
1707
|
+
# apps, launcher wrappers off PATH) still launches through gtk-launch.
|
|
1708
|
+
if launch is None:
|
|
1239
1709
|
gtk_launch = shutil.which("gtk-launch")
|
|
1240
|
-
if
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
except Exception:
|
|
1251
|
-
pass
|
|
1710
|
+
with_entry = [(desktop_id, classes) for _, desktop_id, classes in candidates if desktop_id]
|
|
1711
|
+
if gtk_launch and with_entry:
|
|
1712
|
+
desktop_id, classes = with_entry[0]
|
|
1713
|
+
launch = ([gtk_launch, desktop_id], desktop_id, classes)
|
|
1714
|
+
|
|
1715
|
+
if launch is None:
|
|
1716
|
+
raise RuntimeError(
|
|
1717
|
+
f"Could not find an application named '{name}': no open window, "
|
|
1718
|
+
f".desktop entry, or executable matches it."
|
|
1719
|
+
)
|
|
1252
1720
|
|
|
1721
|
+
argv, _desktop_id, launch_classes = launch
|
|
1722
|
+
windows_before = set(_managed_windows_stacked())
|
|
1253
1723
|
try:
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1724
|
+
# Its own session with no inherited stdio: the app must outlive this
|
|
1725
|
+
# driver process (one process per command) and never block on, or be
|
|
1726
|
+
# killed with, the driver's pipes and process group.
|
|
1727
|
+
proc = subprocess.Popen(
|
|
1728
|
+
argv,
|
|
1729
|
+
stdin=subprocess.DEVNULL,
|
|
1730
|
+
stdout=subprocess.DEVNULL,
|
|
1731
|
+
stderr=subprocess.DEVNULL,
|
|
1732
|
+
start_new_session=True,
|
|
1733
|
+
)
|
|
1734
|
+
except OSError as e:
|
|
1735
|
+
raise RuntimeError(f"Could not launch '{name}' ({argv[0]}): {e}") from e
|
|
1736
|
+
|
|
1737
|
+
def launched_window():
|
|
1738
|
+
# The app's window by class (its launcher wrapper may exec a differently
|
|
1739
|
+
# named binary - LibreOffice's `soffice` - and a single-instance app may
|
|
1740
|
+
# hand the launch to its running process), else whatever managed window
|
|
1741
|
+
# appeared since the launch: that is the app's, whatever class it chose.
|
|
1742
|
+
window_id = _find_app_window(name_patterns, launch_classes)
|
|
1743
|
+
if window_id is not None:
|
|
1744
|
+
return window_id
|
|
1745
|
+
appeared = [wid for wid in _managed_windows_stacked() if wid not in windows_before]
|
|
1746
|
+
return appeared[-1] if appeared else None
|
|
1747
|
+
|
|
1748
|
+
deadline = time.time() + _LAUNCH_WINDOW_TIMEOUT
|
|
1749
|
+
while time.time() < deadline:
|
|
1750
|
+
window_id = launched_window()
|
|
1751
|
+
# A window the WM will not activate yet is still settling (a splash, a
|
|
1752
|
+
# frame the app is about to replace with its real one): keep waiting for
|
|
1753
|
+
# one it will, rather than report the app as unfocusable.
|
|
1754
|
+
if window_id is not None and _focus_window(window_id):
|
|
1755
|
+
return True
|
|
1756
|
+
# A launcher that hands off to a running instance exits 0 with no
|
|
1757
|
+
# window of its own, so only a failure exit ends the wait early.
|
|
1758
|
+
code = proc.poll()
|
|
1759
|
+
if code is not None and code != 0:
|
|
1760
|
+
status = f"signal {-code}" if code < 0 else f"exit status {code}"
|
|
1761
|
+
raise RuntimeError(
|
|
1762
|
+
f"'{name}' crashed on launch ({status}) before showing a window."
|
|
1763
|
+
)
|
|
1764
|
+
time.sleep(0.2)
|
|
1765
|
+
window_id = launched_window()
|
|
1766
|
+
if window_id is not None:
|
|
1767
|
+
return focus_or_raise(window_id)
|
|
1768
|
+
return False
|
|
1262
1769
|
|
|
1263
1770
|
|
|
1264
1771
|
# ---------------------------------------------------------------------------
|
|
@@ -1273,6 +1780,32 @@ def activate_app(name):
|
|
|
1273
1780
|
_INPUT_COMMANDS = {"click", "double-click", "right-click", "type", "key", "scroll", "drag"}
|
|
1274
1781
|
|
|
1275
1782
|
|
|
1783
|
+
def _parse_hold_seconds(args):
|
|
1784
|
+
"""Read an optional ``--hold-seconds N`` out of a command's trailing args."""
|
|
1785
|
+
for i, arg in enumerate(args):
|
|
1786
|
+
if arg != "--hold-seconds":
|
|
1787
|
+
continue
|
|
1788
|
+
try:
|
|
1789
|
+
hold = float(args[i + 1])
|
|
1790
|
+
except (IndexError, ValueError):
|
|
1791
|
+
exit_with_error("--hold-seconds requires a number")
|
|
1792
|
+
if hold < 0 or hold > _MAX_HOLD_SECONDS:
|
|
1793
|
+
exit_with_error(f"--hold-seconds must be between 0 and {_MAX_HOLD_SECONDS}")
|
|
1794
|
+
return hold
|
|
1795
|
+
return 0.0
|
|
1796
|
+
|
|
1797
|
+
|
|
1798
|
+
def _exit_on_sigterm(signum, frame):
|
|
1799
|
+
"""Unwind on SIGTERM instead of dying in place.
|
|
1800
|
+
|
|
1801
|
+
A driver that outlives its command timeout is ended with SIGTERM. Python's
|
|
1802
|
+
default disposition terminates without unwinding, which would skip the
|
|
1803
|
+
``finally`` blocks that release a mouse button held by a drag or a
|
|
1804
|
+
press-and-hold; raising SystemExit runs them first.
|
|
1805
|
+
"""
|
|
1806
|
+
sys.exit(1)
|
|
1807
|
+
|
|
1808
|
+
|
|
1276
1809
|
def main():
|
|
1277
1810
|
args = sys.argv[1:]
|
|
1278
1811
|
if not args:
|
|
@@ -1280,6 +1813,7 @@ def main():
|
|
|
1280
1813
|
|
|
1281
1814
|
command = args[0]
|
|
1282
1815
|
if command in _INPUT_COMMANDS:
|
|
1816
|
+
signal.signal(signal.SIGTERM, _exit_on_sigterm)
|
|
1283
1817
|
release_modifiers()
|
|
1284
1818
|
|
|
1285
1819
|
if command == "observe":
|
|
@@ -1296,7 +1830,7 @@ def main():
|
|
|
1296
1830
|
else:
|
|
1297
1831
|
i += 1
|
|
1298
1832
|
|
|
1299
|
-
elements = enumerate_elements(pid=pid)
|
|
1833
|
+
elements, window, diagnostic = enumerate_elements(pid=pid)
|
|
1300
1834
|
capture_screenshot(screenshot_path)
|
|
1301
1835
|
annotate_screenshot(screenshot_path, elements)
|
|
1302
1836
|
|
|
@@ -1304,12 +1838,16 @@ def main():
|
|
|
1304
1838
|
"elements": elements,
|
|
1305
1839
|
"screenshotPath": screenshot_path,
|
|
1306
1840
|
}
|
|
1841
|
+
if window is not None:
|
|
1842
|
+
result["window"] = {"title": window["title"], "pid": window["pid"]}
|
|
1843
|
+
if diagnostic:
|
|
1844
|
+
result["diagnostic"] = diagnostic
|
|
1307
1845
|
print(json.dumps(result, indent=2))
|
|
1308
1846
|
|
|
1309
1847
|
elif command == "click":
|
|
1310
1848
|
if len(args) < 3:
|
|
1311
|
-
exit_with_error("Usage: linux-driver.py click <x> <y>")
|
|
1312
|
-
mouse_click(float(args[1]), float(args[2]))
|
|
1849
|
+
exit_with_error("Usage: linux-driver.py click <x> <y> [--hold-seconds N]")
|
|
1850
|
+
mouse_click(float(args[1]), float(args[2]), _parse_hold_seconds(args[3:]))
|
|
1313
1851
|
print_success()
|
|
1314
1852
|
|
|
1315
1853
|
elif command == "double-click":
|
|
@@ -1363,12 +1901,13 @@ def main():
|
|
|
1363
1901
|
if len(args) < 5:
|
|
1364
1902
|
exit_with_error(
|
|
1365
1903
|
"Usage: linux-driver.py drag <from_x> <from_y> <to_x> <to_y> "
|
|
1366
|
-
"[--steps N] [--path x1,y1 ...]"
|
|
1904
|
+
"[--steps N] [--path x1,y1 ...] [--hold-seconds N]"
|
|
1367
1905
|
)
|
|
1368
1906
|
from_x, from_y = float(args[1]), float(args[2])
|
|
1369
1907
|
to_x, to_y = float(args[3]), float(args[4])
|
|
1370
1908
|
steps = 10
|
|
1371
1909
|
waypoints = None
|
|
1910
|
+
hold_seconds = _parse_hold_seconds(args[5:])
|
|
1372
1911
|
di = 5
|
|
1373
1912
|
while di < len(args):
|
|
1374
1913
|
if args[di] == "--steps" and di + 1 < len(args):
|
|
@@ -1383,7 +1922,7 @@ def main():
|
|
|
1383
1922
|
di += 1
|
|
1384
1923
|
else:
|
|
1385
1924
|
di += 1
|
|
1386
|
-
mouse_drag(from_x, from_y, to_x, to_y, steps, waypoints)
|
|
1925
|
+
mouse_drag(from_x, from_y, to_x, to_y, steps, waypoints, hold_seconds)
|
|
1387
1926
|
print_success()
|
|
1388
1927
|
|
|
1389
1928
|
elif command == "screenshot":
|
|
@@ -1426,11 +1965,13 @@ def main():
|
|
|
1426
1965
|
exit_with_error("Usage: linux-driver.py activate <app-name>")
|
|
1427
1966
|
app_name = " ".join(args[1:])
|
|
1428
1967
|
try:
|
|
1429
|
-
activate_app(app_name)
|
|
1968
|
+
window_visible = activate_app(app_name)
|
|
1430
1969
|
except RuntimeError as e:
|
|
1431
1970
|
exit_with_error(str(e))
|
|
1432
1971
|
time.sleep(0.5)
|
|
1433
|
-
|
|
1972
|
+
# `windowVisible: false` reports an app that was launched but has not shown
|
|
1973
|
+
# a window yet, so the caller waits instead of launching it again.
|
|
1974
|
+
print_success(windowVisible=window_visible)
|
|
1434
1975
|
|
|
1435
1976
|
else:
|
|
1436
1977
|
exit_with_error(f"Unknown command: '{command}'")
|