@octavus/agent 1.0.24 → 1.0.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/computer-mcp/dist/index.js +14 -14
- package/dist/computer-mcp/linux/linux-driver.py +638 -250
- package/dist/index.js +40 -40
- package/package.json +1 -1
|
@@ -25,6 +25,9 @@ Usage mirrors the macOS driver exactly:
|
|
|
25
25
|
linux-driver.py activate <app-name>
|
|
26
26
|
"""
|
|
27
27
|
|
|
28
|
+
import ctypes
|
|
29
|
+
import ctypes.util
|
|
30
|
+
import functools
|
|
28
31
|
import glob
|
|
29
32
|
import json
|
|
30
33
|
import os
|
|
@@ -101,6 +104,10 @@ CONTAINER_ROLES = {
|
|
|
101
104
|
|
|
102
105
|
ACTIONABLE_ACTIONS = {"click", "press", "activate", "toggle", "jump"}
|
|
103
106
|
|
|
107
|
+
# Roles that are never a control, whatever action their toolkit advertises:
|
|
108
|
+
# LibreOffice exposes a "click" on every toolbar separator.
|
|
109
|
+
NON_ACTIONABLE_ROLES = {"separator"}
|
|
110
|
+
|
|
104
111
|
|
|
105
112
|
def _try_import_atspi():
|
|
106
113
|
"""Import AT-SPI2 via GObject introspection and connect to the a11y bus."""
|
|
@@ -134,6 +141,15 @@ def _overlaps_screen(x, y, w, h, screen_w, screen_h):
|
|
|
134
141
|
return x < screen_w and y < screen_h and x + w > 0 and y + h > 0
|
|
135
142
|
|
|
136
143
|
|
|
144
|
+
# The most children of one node the walk descends into. A spreadsheet reports its
|
|
145
|
+
# whole sheet as the table's children (Calc: every cell of 1024 x 1048576), which
|
|
146
|
+
# no walk could visit within a driver command - and its first rows would be no
|
|
147
|
+
# guide to what is on screen anyway. A node with more children than this is a
|
|
148
|
+
# data grid, left to grid coordinates; a UI container that is not one has orders
|
|
149
|
+
# of magnitude fewer.
|
|
150
|
+
_MAX_CHILDREN_WALKED = 2048
|
|
151
|
+
|
|
152
|
+
|
|
137
153
|
def _walk_atspi(
|
|
138
154
|
node,
|
|
139
155
|
elements,
|
|
@@ -163,6 +179,11 @@ def _walk_atspi(
|
|
|
163
179
|
except Exception:
|
|
164
180
|
pass
|
|
165
181
|
|
|
182
|
+
try:
|
|
183
|
+
role_name = node.get_role_name().lower()
|
|
184
|
+
except Exception:
|
|
185
|
+
role_name = ""
|
|
186
|
+
|
|
166
187
|
rect = None
|
|
167
188
|
try:
|
|
168
189
|
comp = node.get_component_iface()
|
|
@@ -180,6 +201,11 @@ def _walk_atspi(
|
|
|
180
201
|
)
|
|
181
202
|
except Exception:
|
|
182
203
|
rect = None
|
|
204
|
+
# GTK3 answers for an accessible without geometry of its own (a GtkNotebook
|
|
205
|
+
# page) with -1s: that is "no geometry", like having no Component interface at
|
|
206
|
+
# all, not a box that is off the screen.
|
|
207
|
+
if rect is not None and (rect[2] < 0 or rect[3] < 0):
|
|
208
|
+
rect = None
|
|
183
209
|
|
|
184
210
|
on_screen = (
|
|
185
211
|
rect is not None
|
|
@@ -199,11 +225,10 @@ def _walk_atspi(
|
|
|
199
225
|
# surfaces hidden-but-positioned elements. Off-screen and zero-area nodes fail
|
|
200
226
|
# the on_screen test regardless.
|
|
201
227
|
if rect is not None and on_screen and (trust_geometry or visible):
|
|
202
|
-
|
|
203
|
-
role_name
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
if role_name and (role_name in CLICKABLE_ROLES or _has_actionable(node, Atspi)):
|
|
228
|
+
if role_name and (
|
|
229
|
+
role_name in CLICKABLE_ROLES
|
|
230
|
+
or (role_name not in NON_ACTIONABLE_ROLES and _has_actionable(node, Atspi))
|
|
231
|
+
):
|
|
207
232
|
x, y, w, h = rect
|
|
208
233
|
area = w * h
|
|
209
234
|
screen_area = screen_w * screen_h
|
|
@@ -244,15 +269,22 @@ def _walk_atspi(
|
|
|
244
269
|
# Descend into children unless this node is provably off-screen. Keep walking
|
|
245
270
|
# through nodes with no geometry (structural containers such as the
|
|
246
271
|
# `application` node, which reports VISIBLE=False and no extents) and through
|
|
247
|
-
# on-screen containers that merely lack SHOWING
|
|
248
|
-
#
|
|
249
|
-
#
|
|
250
|
-
#
|
|
251
|
-
|
|
272
|
+
# on-screen containers that merely lack SHOWING; prune a node whose real box
|
|
273
|
+
# is off-screen or zero-area. For Chromium that prune also requires the node
|
|
274
|
+
# not to be SHOWING: web layouts routinely hang visible, absolutely positioned
|
|
275
|
+
# content under a zero-size wrapper, and off-viewport content is what bounds
|
|
276
|
+
# the walk over a large page. A native toolkit clips children to their
|
|
277
|
+
# parent's allocation, so a zero-area or off-screen native container shows
|
|
278
|
+
# nothing whatever its SHOWING flag says - LibreOffice keeps a SHOWING,
|
|
279
|
+
# zero-height copy of its menu bar whose items would otherwise be labeled on
|
|
280
|
+
# top of the toolbar drawn below it.
|
|
281
|
+
if rect is not None and not on_screen and (trust_geometry or not showing):
|
|
252
282
|
return
|
|
253
283
|
|
|
254
284
|
try:
|
|
255
285
|
n_children = node.get_child_count()
|
|
286
|
+
if n_children > _MAX_CHILDREN_WALKED:
|
|
287
|
+
return
|
|
256
288
|
for i in range(n_children):
|
|
257
289
|
child = node.get_child_at_index(i)
|
|
258
290
|
if child is not None:
|
|
@@ -330,33 +362,41 @@ def _dedup_elements(elements, screen_w, screen_h):
|
|
|
330
362
|
return deduped
|
|
331
363
|
|
|
332
364
|
|
|
333
|
-
def
|
|
365
|
+
def _xdotool_output(args, timeout=5):
|
|
366
|
+
"""stdout of an xdotool query, or None when it fails (no active window, a
|
|
367
|
+
window gone between two lookups, no WM support)."""
|
|
334
368
|
try:
|
|
335
|
-
|
|
336
|
-
["xdotool",
|
|
337
|
-
).strip()
|
|
338
|
-
pid = subprocess.check_output(
|
|
339
|
-
["xdotool", "getwindowpid", wid], text=True, timeout=5
|
|
369
|
+
return subprocess.check_output(
|
|
370
|
+
["xdotool", *args], text=True, timeout=timeout, stderr=subprocess.DEVNULL
|
|
340
371
|
).strip()
|
|
341
|
-
return int(pid)
|
|
342
372
|
except Exception:
|
|
343
373
|
return None
|
|
344
374
|
|
|
345
375
|
|
|
346
|
-
def
|
|
347
|
-
""
|
|
376
|
+
def _get_active_window_id():
|
|
377
|
+
out = _xdotool_output(["getactivewindow"])
|
|
378
|
+
try:
|
|
379
|
+
return int(out) if out else None
|
|
380
|
+
except ValueError:
|
|
381
|
+
return None
|
|
382
|
+
|
|
348
383
|
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
be clipped to, or placed within, the visible app.
|
|
352
|
-
"""
|
|
384
|
+
def _window_pid(window_id):
|
|
385
|
+
out = _xdotool_output(["getwindowpid", str(window_id)])
|
|
353
386
|
try:
|
|
354
|
-
out
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
387
|
+
return int(out) if out else None
|
|
388
|
+
except ValueError:
|
|
389
|
+
return None
|
|
390
|
+
|
|
391
|
+
|
|
392
|
+
def _window_title(window_id):
|
|
393
|
+
return _xdotool_output(["getwindowname", str(window_id)]) or ""
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
def _window_geometry(window_id):
|
|
397
|
+
"""Screen rect of a window as {id, x, y, width, height}, or None when it is gone."""
|
|
398
|
+
out = _xdotool_output(["getwindowgeometry", "--shell", str(window_id)])
|
|
399
|
+
if not out:
|
|
360
400
|
return None
|
|
361
401
|
|
|
362
402
|
vals = {}
|
|
@@ -376,8 +416,58 @@ def _get_active_window_geometry():
|
|
|
376
416
|
return None
|
|
377
417
|
|
|
378
418
|
|
|
379
|
-
|
|
380
|
-
|
|
419
|
+
# Window types that sit in the WM's client list without ever being "in front":
|
|
420
|
+
# the desktop (GNOME's desktop-icons window spans the screen at the bottom of the
|
|
421
|
+
# stack) and panels/docks.
|
|
422
|
+
_BACKDROP_WINDOW_TYPES = ("_NET_WM_WINDOW_TYPE_DESKTOP", "_NET_WM_WINDOW_TYPE_DOCK")
|
|
423
|
+
|
|
424
|
+
|
|
425
|
+
def _is_backdrop_window(window_id):
|
|
426
|
+
"""Whether the window is the desktop, a dock, or minimized - never "in front"."""
|
|
427
|
+
with _X11() as x11:
|
|
428
|
+
types = x11.window_property(window_id, "_NET_WM_WINDOW_TYPE", max_items=8) or []
|
|
429
|
+
state = x11.window_property(window_id, "_NET_WM_STATE", max_items=32) or []
|
|
430
|
+
backdrop = {x11.atom(name) for name in _BACKDROP_WINDOW_TYPES}
|
|
431
|
+
return bool(backdrop & set(types)) or x11.atom("_NET_WM_STATE_HIDDEN") in state
|
|
432
|
+
|
|
433
|
+
|
|
434
|
+
def _foreground_window_id():
|
|
435
|
+
"""The window in front of the user: the top of the WM's stacking order.
|
|
436
|
+
|
|
437
|
+
Usually that is also the focused window. It is not when the window manager
|
|
438
|
+
stacks a window on top but withholds focus from it: mutter's focus-stealing
|
|
439
|
+
prevention does exactly that to a modal dialog another process opens on
|
|
440
|
+
behalf of the focused app (the portal file chooser Chrome's "Open" / "Save as"
|
|
441
|
+
bring up) after synthetic keystrokes, so the dialog covers the browser while
|
|
442
|
+
keystrokes keep landing in the browser. The topmost window is what the agent
|
|
443
|
+
sees and will click, so it is the one labeled - and it is given focus first so
|
|
444
|
+
typing reaches it too. Falls back to the focused window where the WM
|
|
445
|
+
publishes no stacking order, and is None when nothing but the desktop (or a
|
|
446
|
+
dock) is up - the desktop is never "the app in front".
|
|
447
|
+
"""
|
|
448
|
+
active = _get_active_window_id()
|
|
449
|
+
for window_id in reversed(_managed_windows_stacked()):
|
|
450
|
+
if _is_backdrop_window(window_id):
|
|
451
|
+
continue
|
|
452
|
+
if window_id != active:
|
|
453
|
+
_focus_window(window_id)
|
|
454
|
+
return window_id
|
|
455
|
+
if active is not None and not _is_backdrop_window(active):
|
|
456
|
+
return active
|
|
457
|
+
return None
|
|
458
|
+
|
|
459
|
+
|
|
460
|
+
def _process_name(pid):
|
|
461
|
+
"""The process's command name from /proc, or "" when unreadable."""
|
|
462
|
+
try:
|
|
463
|
+
with open(f"/proc/{pid}/comm", "rb") as f:
|
|
464
|
+
return f.read().decode("utf-8", "ignore").strip()
|
|
465
|
+
except Exception:
|
|
466
|
+
return ""
|
|
467
|
+
|
|
468
|
+
|
|
469
|
+
def _pid_is_chromium(pid):
|
|
470
|
+
"""Whether the PID is a Chrome/Chromium process.
|
|
381
471
|
|
|
382
472
|
Read from /proc rather than the window class: the cloud xdotool build has no
|
|
383
473
|
getwindowclassname and xprop is not installed, but the browser owns the
|
|
@@ -396,31 +486,118 @@ def _active_window_is_chromium(pid):
|
|
|
396
486
|
return False
|
|
397
487
|
|
|
398
488
|
|
|
399
|
-
def
|
|
400
|
-
|
|
489
|
+
def _pid_is_libreoffice(pid):
|
|
490
|
+
return pid is not None and _process_name(pid).startswith("soffice")
|
|
491
|
+
|
|
492
|
+
|
|
493
|
+
# How long LibreOffice gets to re-read its frame geometry after the pointer nudge
|
|
494
|
+
# below, before its extents are trusted.
|
|
495
|
+
_LIBREOFFICE_GEOMETRY_SETTLE = 0.3
|
|
496
|
+
|
|
497
|
+
|
|
498
|
+
def _refresh_libreoffice_geometry(window):
|
|
499
|
+
"""Make LibreOffice recompute where its widgets are before they are read.
|
|
401
500
|
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
501
|
+
LibreOffice (gtk3 VCL) caches its frame geometry and only re-reads it on a
|
|
502
|
+
pointer event: after the window opens and the window manager places and
|
|
503
|
+
decorates it, every toolbar, the document and the status bar keep reporting
|
|
504
|
+
AT-SPI extents one title-bar height above where they are drawn, until the
|
|
505
|
+
pointer first moves inside the window. A labeled toolbar button would then be
|
|
506
|
+
clicked on the menu bar above it. One motion event over the window's center
|
|
507
|
+
is enough; the pointer has no meaning to the agent between actions.
|
|
408
508
|
"""
|
|
409
|
-
|
|
410
|
-
|
|
509
|
+
center = (window["x"] + window["width"] // 2, window["y"] + window["height"] // 2)
|
|
510
|
+
if _pointer_location() == center:
|
|
511
|
+
center = (center[0] + 1, center[1])
|
|
512
|
+
_move_pointer(*center)
|
|
513
|
+
time.sleep(_LIBREOFFICE_GEOMETRY_SETTLE)
|
|
514
|
+
|
|
515
|
+
|
|
516
|
+
def _foreground_window():
|
|
517
|
+
"""What is in front: {id, title, pid, x, y, width, height, chromium}, or None.
|
|
518
|
+
|
|
519
|
+
Everything `observe` labels is scoped to this window - one desktop, what you
|
|
520
|
+
see is what you get - so it is also what the result reports back, letting the
|
|
521
|
+
agent (and anyone reading the trace) see which window the numbers belong to.
|
|
522
|
+
"""
|
|
523
|
+
window_id = _foreground_window_id()
|
|
524
|
+
window = _window_geometry(window_id) if window_id is not None else None
|
|
525
|
+
if window is None:
|
|
526
|
+
return None
|
|
527
|
+
pid = _window_pid(window["id"])
|
|
528
|
+
return {
|
|
529
|
+
**window,
|
|
530
|
+
"title": _window_title(window["id"]),
|
|
531
|
+
"pid": pid,
|
|
532
|
+
"chromium": _pid_is_chromium(pid),
|
|
533
|
+
}
|
|
534
|
+
|
|
535
|
+
|
|
536
|
+
def _apps_for_pid(desktop, target_pid):
|
|
537
|
+
"""The AT-SPI application node(s) owned by ``target_pid`` - strictly by PID.
|
|
538
|
+
|
|
539
|
+
A None target_pid (foreground PID unavailable) matches no app, so labeling never
|
|
540
|
+
falls back to walking the whole desktop, which would overlay the always-open
|
|
541
|
+
browser's labels onto whatever native app is in front.
|
|
542
|
+
"""
|
|
543
|
+
apps = []
|
|
544
|
+
if target_pid is None:
|
|
545
|
+
return apps
|
|
546
|
+
try:
|
|
547
|
+
n_apps = desktop.get_child_count()
|
|
548
|
+
except Exception:
|
|
549
|
+
return apps
|
|
411
550
|
for i in range(n_apps):
|
|
412
|
-
app = desktop.get_child_at_index(i)
|
|
413
|
-
if app is None:
|
|
414
|
-
continue
|
|
415
551
|
try:
|
|
416
|
-
|
|
417
|
-
|
|
552
|
+
app = desktop.get_child_at_index(i)
|
|
553
|
+
if app is not None and app.get_process_id() == target_pid:
|
|
554
|
+
apps.append(app)
|
|
418
555
|
except Exception:
|
|
419
556
|
continue
|
|
557
|
+
return apps
|
|
558
|
+
|
|
559
|
+
|
|
560
|
+
def _collect_elements(apps, window, Atspi, screen_w, screen_h, trust_geometry=False):
|
|
561
|
+
"""Collect the actionable elements of the foreground window from its app's tree.
|
|
562
|
+
|
|
563
|
+
Only the toplevel accessible that corresponds to the focused X11 window is
|
|
564
|
+
walked: an app's other windows (a second Files window behind the first, a
|
|
565
|
+
browser popup, a document behind a modal dialog) are not what the agent sees
|
|
566
|
+
and would otherwise be labeled on top of what is - the same control twice at
|
|
567
|
+
two positions. An app whose toplevel cannot be matched to the focused window
|
|
568
|
+
(nameless frames, a toolkit that never marks one ACTIVE) is walked whole, so a
|
|
569
|
+
scoping miss degrades to extra labels rather than none. `trust_geometry` is
|
|
570
|
+
forwarded to the walk so a native foreground app's on-screen controls are
|
|
571
|
+
included from geometry even when its toolkit leaves the AT-SPI VISIBLE/SHOWING
|
|
572
|
+
flags unset (GTK4).
|
|
573
|
+
"""
|
|
574
|
+
elements = []
|
|
575
|
+
for app in apps:
|
|
576
|
+
frame = _focused_frame(app, window, Atspi) if window is not None else None
|
|
420
577
|
if _uses_window_relative_extents(app):
|
|
421
|
-
|
|
578
|
+
# Window-relative extents can only be placed for the focused window,
|
|
579
|
+
# whose on-screen origin is known; without a matched frame there is
|
|
580
|
+
# nothing to place them against.
|
|
581
|
+
if frame is not None:
|
|
582
|
+
left, top = _client_side_decoration_offset(window["id"])
|
|
583
|
+
_walk_atspi(
|
|
584
|
+
frame,
|
|
585
|
+
elements,
|
|
586
|
+
Atspi,
|
|
587
|
+
screen_w,
|
|
588
|
+
screen_h,
|
|
589
|
+
trust_geometry=trust_geometry,
|
|
590
|
+
window_origin=(window["x"] + left, window["y"] + top),
|
|
591
|
+
)
|
|
422
592
|
else:
|
|
423
|
-
_walk_atspi(
|
|
593
|
+
_walk_atspi(
|
|
594
|
+
frame if frame is not None else app,
|
|
595
|
+
elements,
|
|
596
|
+
Atspi,
|
|
597
|
+
screen_w,
|
|
598
|
+
screen_h,
|
|
599
|
+
trust_geometry=trust_geometry,
|
|
600
|
+
)
|
|
424
601
|
return elements
|
|
425
602
|
|
|
426
603
|
|
|
@@ -442,33 +619,14 @@ def _uses_window_relative_extents(app):
|
|
|
442
619
|
return toolkit == "gtk" and major >= 4
|
|
443
620
|
|
|
444
621
|
|
|
445
|
-
def
|
|
446
|
-
"""
|
|
622
|
+
def _focused_frame(app, window, Atspi):
|
|
623
|
+
"""The app's toplevel accessible for the focused X11 window, or None.
|
|
447
624
|
|
|
448
|
-
|
|
449
|
-
|
|
625
|
+
Matched by the ACTIVE state when the toolkit sets it, else by title - a GTK
|
|
626
|
+
dialog's accessible carries its window title even when it is never marked
|
|
627
|
+
ACTIVE. An app with a single toplevel needs no matching: there is nothing
|
|
628
|
+
else it could be.
|
|
450
629
|
"""
|
|
451
|
-
window = _get_active_window_geometry()
|
|
452
|
-
if window is None:
|
|
453
|
-
return
|
|
454
|
-
frame = _focused_frame(app, window["id"], Atspi)
|
|
455
|
-
if frame is None:
|
|
456
|
-
return
|
|
457
|
-
left, top = _client_side_decoration_offset(window["id"])
|
|
458
|
-
_walk_atspi(
|
|
459
|
-
frame,
|
|
460
|
-
elements,
|
|
461
|
-
Atspi,
|
|
462
|
-
screen_w,
|
|
463
|
-
screen_h,
|
|
464
|
-
trust_geometry=trust_geometry,
|
|
465
|
-
window_origin=(window["x"] + left, window["y"] + top),
|
|
466
|
-
)
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
def _focused_frame(app, window_id, Atspi):
|
|
470
|
-
"""The app's toplevel accessible for the focused window: the one AT-SPI marks
|
|
471
|
-
ACTIVE, else the one titled like the focused X11 window."""
|
|
472
630
|
frames = []
|
|
473
631
|
try:
|
|
474
632
|
for i in range(app.get_child_count()):
|
|
@@ -477,6 +635,8 @@ def _focused_frame(app, window_id, Atspi):
|
|
|
477
635
|
frames.append(frame)
|
|
478
636
|
except Exception:
|
|
479
637
|
return None
|
|
638
|
+
if len(frames) == 1:
|
|
639
|
+
return frames[0]
|
|
480
640
|
|
|
481
641
|
for frame in frames:
|
|
482
642
|
try:
|
|
@@ -485,12 +645,7 @@ def _focused_frame(app, window_id, Atspi):
|
|
|
485
645
|
except Exception:
|
|
486
646
|
continue
|
|
487
647
|
|
|
488
|
-
|
|
489
|
-
title = subprocess.check_output(
|
|
490
|
-
["xdotool", "getwindowname", str(window_id)], text=True, timeout=5
|
|
491
|
-
).strip()
|
|
492
|
-
except Exception:
|
|
493
|
-
return None
|
|
648
|
+
title = window.get("title") or ""
|
|
494
649
|
for frame in frames:
|
|
495
650
|
try:
|
|
496
651
|
if title and frame.get_name() == title:
|
|
@@ -500,27 +655,32 @@ def _focused_frame(app, window_id, Atspi):
|
|
|
500
655
|
return None
|
|
501
656
|
|
|
502
657
|
|
|
503
|
-
|
|
658
|
+
# Xlib's default error handler exits the process. A window can be destroyed
|
|
659
|
+
# between the xdotool lookup and a property read (BadWindow), which must degrade
|
|
660
|
+
# to "no property" rather than take the driver down mid-observe. Installed once
|
|
661
|
+
# when libX11 is loaded; the callback has to stay referenced for as long as the
|
|
662
|
+
# library is in use, so it lives at module level.
|
|
663
|
+
_IGNORE_X_ERRORS = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p)(
|
|
664
|
+
lambda _display, _error: 0
|
|
665
|
+
)
|
|
504
666
|
|
|
505
667
|
|
|
506
|
-
|
|
507
|
-
|
|
668
|
+
@functools.lru_cache(maxsize=None)
|
|
669
|
+
def _load_xlib():
|
|
670
|
+
"""libX11 with the prototypes the driver uses, loaded once; None when unavailable.
|
|
508
671
|
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
measures window-relative extents from the visible frame, not the X11 window's
|
|
512
|
-
corner. Read through libX11 directly because xprop is not installed on the
|
|
513
|
-
cloud image. (0, 0) for a window without client-side decorations.
|
|
672
|
+
Window properties are read through libX11 directly because xprop is not
|
|
673
|
+
installed on the cloud image.
|
|
514
674
|
"""
|
|
515
675
|
try:
|
|
516
|
-
import ctypes
|
|
517
|
-
import ctypes.util
|
|
518
|
-
|
|
519
676
|
xlib = ctypes.cdll.LoadLibrary(ctypes.util.find_library("X11") or "libX11.so.6")
|
|
520
677
|
except Exception:
|
|
521
|
-
return
|
|
678
|
+
return None
|
|
522
679
|
xlib.XOpenDisplay.argtypes = [ctypes.c_char_p]
|
|
523
680
|
xlib.XOpenDisplay.restype = ctypes.c_void_p
|
|
681
|
+
xlib.XCloseDisplay.argtypes = [ctypes.c_void_p]
|
|
682
|
+
xlib.XDefaultRootWindow.argtypes = [ctypes.c_void_p]
|
|
683
|
+
xlib.XDefaultRootWindow.restype = ctypes.c_ulong
|
|
524
684
|
xlib.XInternAtom.argtypes = [ctypes.c_void_p, ctypes.c_char_p, ctypes.c_int]
|
|
525
685
|
xlib.XInternAtom.restype = ctypes.c_ulong
|
|
526
686
|
xlib.XGetWindowProperty.argtypes = [
|
|
@@ -538,38 +698,56 @@ def _client_side_decoration_offset(window_id):
|
|
|
538
698
|
ctypes.POINTER(ctypes.c_void_p),
|
|
539
699
|
]
|
|
540
700
|
xlib.XFree.argtypes = [ctypes.c_void_p]
|
|
541
|
-
xlib.XCloseDisplay.argtypes = [ctypes.c_void_p]
|
|
542
701
|
xlib.XSetErrorHandler.restype = ctypes.c_void_p
|
|
702
|
+
xlib.XSetErrorHandler(_IGNORE_X_ERRORS)
|
|
703
|
+
return xlib
|
|
543
704
|
|
|
544
|
-
# Xlib's default error handler exits the process. The focused window can be
|
|
545
|
-
# destroyed between the xdotool lookup and this read (BadWindow), which must
|
|
546
|
-
# degrade to no offset rather than take the driver down mid-observe. The
|
|
547
|
-
# callback has to stay referenced while the display is open.
|
|
548
|
-
ignore_x_errors = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p)(
|
|
549
|
-
lambda _display, _error: 0
|
|
550
|
-
)
|
|
551
|
-
xlib.XSetErrorHandler(ignore_x_errors)
|
|
552
705
|
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
706
|
+
class _X11:
|
|
707
|
+
"""One connection to the X display for a batch of reads.
|
|
708
|
+
|
|
709
|
+
Opened on entry and closed on exit. Every read answers with a neutral value
|
|
710
|
+
(0 for an atom, None for a property) instead of raising, so a window destroyed
|
|
711
|
+
between the xdotool lookup and the read - or a box without libX11 or a display
|
|
712
|
+
at all - degrades to "no property".
|
|
713
|
+
"""
|
|
714
|
+
|
|
715
|
+
def __enter__(self):
|
|
716
|
+
self._xlib = _load_xlib()
|
|
717
|
+
self._display = self._xlib.XOpenDisplay(None) if self._xlib else None
|
|
718
|
+
return self
|
|
719
|
+
|
|
720
|
+
def __exit__(self, *_exc):
|
|
721
|
+
if self._display:
|
|
722
|
+
self._xlib.XCloseDisplay(self._display)
|
|
723
|
+
|
|
724
|
+
def atom(self, name):
|
|
725
|
+
"""The atom for ``name``, or 0 when it was never interned on this server."""
|
|
726
|
+
if not self._display:
|
|
727
|
+
return 0
|
|
728
|
+
return int(self._xlib.XInternAtom(self._display, name.encode(), True))
|
|
729
|
+
|
|
730
|
+
def window_property(self, window_id, name, max_items=1024):
|
|
731
|
+
"""A format-32 property of ``window_id`` (the root window when None) as a
|
|
732
|
+
list of ints, or None when the window or the property does not exist."""
|
|
733
|
+
atom = self.atom(name)
|
|
558
734
|
if not atom:
|
|
559
|
-
return
|
|
735
|
+
return None
|
|
736
|
+
if window_id is None:
|
|
737
|
+
window_id = self._xlib.XDefaultRootWindow(self._display)
|
|
560
738
|
actual_type = ctypes.c_ulong()
|
|
561
739
|
actual_format = ctypes.c_int()
|
|
562
740
|
n_items = ctypes.c_ulong()
|
|
563
741
|
bytes_after = ctypes.c_ulong()
|
|
564
742
|
data = ctypes.c_void_p()
|
|
565
|
-
status =
|
|
566
|
-
|
|
743
|
+
status = self._xlib.XGetWindowProperty(
|
|
744
|
+
self._display,
|
|
567
745
|
window_id,
|
|
568
746
|
atom,
|
|
569
747
|
0,
|
|
570
|
-
|
|
748
|
+
max_items,
|
|
571
749
|
False,
|
|
572
|
-
|
|
750
|
+
0, # AnyPropertyType
|
|
573
751
|
ctypes.byref(actual_type),
|
|
574
752
|
ctypes.byref(actual_format),
|
|
575
753
|
ctypes.byref(n_items),
|
|
@@ -577,49 +755,69 @@ def _client_side_decoration_offset(window_id):
|
|
|
577
755
|
ctypes.byref(data),
|
|
578
756
|
)
|
|
579
757
|
if status != 0 or not data.value:
|
|
580
|
-
return
|
|
758
|
+
return None
|
|
581
759
|
try:
|
|
582
|
-
if actual_format.value != 32 or n_items.value
|
|
583
|
-
return
|
|
760
|
+
if actual_format.value != 32 or n_items.value == 0:
|
|
761
|
+
return None
|
|
584
762
|
# Xlib hands format-32 property items back as C longs.
|
|
585
|
-
|
|
586
|
-
return int(left), int(top)
|
|
763
|
+
return [int(v) for v in ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[: n_items.value]]
|
|
587
764
|
finally:
|
|
588
|
-
|
|
589
|
-
|
|
765
|
+
self._xlib.XFree(data)
|
|
766
|
+
|
|
767
|
+
|
|
768
|
+
def _x11_window_property(window_id, name, max_items=1024):
|
|
769
|
+
"""One format-32 property read on its own connection (see ``_X11``)."""
|
|
770
|
+
with _X11() as x11:
|
|
771
|
+
return x11.window_property(window_id, name, max_items)
|
|
772
|
+
|
|
773
|
+
|
|
774
|
+
def _managed_windows_stacked():
|
|
775
|
+
"""The window manager's client windows, bottom to top (`_NET_CLIENT_LIST_STACKING`).
|
|
776
|
+
|
|
777
|
+
These are the real, mapped toplevels a user sees - not the unmapped client-leader
|
|
778
|
+
and helper windows a toolkit also creates (which `xdotool search` returns too, and
|
|
779
|
+
which cannot be activated). Empty when the WM does not publish the list.
|
|
780
|
+
"""
|
|
781
|
+
return _x11_window_property(None, "_NET_CLIENT_LIST_STACKING") or []
|
|
782
|
+
|
|
783
|
+
|
|
784
|
+
def _client_side_decoration_offset(window_id):
|
|
785
|
+
"""(left, top) inset of a window's visible frame within its X11 window.
|
|
786
|
+
|
|
787
|
+
GTK draws client-side decorations - its drop shadow - inside the X11 window and
|
|
788
|
+
advertises their size in `_GTK_FRAME_EXTENTS` (left, right, top, bottom); GTK
|
|
789
|
+
measures window-relative extents from the visible frame, not the X11 window's
|
|
790
|
+
corner. (0, 0) for a window without client-side decorations.
|
|
791
|
+
"""
|
|
792
|
+
extents = _x11_window_property(window_id, "_GTK_FRAME_EXTENTS", max_items=4)
|
|
793
|
+
if not extents or len(extents) != 4:
|
|
590
794
|
return 0, 0
|
|
591
|
-
|
|
592
|
-
|
|
795
|
+
left, _right, top, _bottom = extents
|
|
796
|
+
return left, top
|
|
593
797
|
|
|
594
798
|
|
|
595
|
-
def
|
|
596
|
-
"""
|
|
799
|
+
def _chromium_apps(desktop):
|
|
800
|
+
"""The Chrome/Chromium AT-SPI app node(s), matched by name.
|
|
597
801
|
|
|
598
|
-
Recovery for when the foreground window is Chromium but
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
browser's labels.
|
|
802
|
+
Recovery for when the foreground window is Chromium but its active-window PID
|
|
803
|
+
disagrees with the AT-SPI app PID (a wrapper or zygote owning the window).
|
|
804
|
+
Scoping by name keeps the recovery on the browser alone, so a native app in
|
|
805
|
+
front is never overlaid with the always-open browser's labels.
|
|
603
806
|
"""
|
|
604
|
-
|
|
807
|
+
apps = []
|
|
605
808
|
try:
|
|
606
809
|
n_apps = desktop.get_child_count()
|
|
607
810
|
except Exception:
|
|
608
|
-
return
|
|
811
|
+
return apps
|
|
609
812
|
for i in range(n_apps):
|
|
610
813
|
try:
|
|
611
814
|
app = desktop.get_child_at_index(i)
|
|
815
|
+
name = (app.get_name() or "").lower() if app is not None else ""
|
|
612
816
|
except Exception:
|
|
613
817
|
continue
|
|
614
|
-
if app is None:
|
|
615
|
-
continue
|
|
616
|
-
try:
|
|
617
|
-
name = (app.get_name() or "").lower()
|
|
618
|
-
except Exception:
|
|
619
|
-
name = ""
|
|
620
818
|
if "chrome" in name or "chromium" in name:
|
|
621
|
-
|
|
622
|
-
return
|
|
819
|
+
apps.append(app)
|
|
820
|
+
return apps
|
|
623
821
|
|
|
624
822
|
|
|
625
823
|
def _within_rect(elements, rect):
|
|
@@ -636,7 +834,60 @@ def _within_rect(elements, rect):
|
|
|
636
834
|
return kept
|
|
637
835
|
|
|
638
836
|
|
|
837
|
+
# Retries for a Chromium tree that is still being built: Chromium exports its
|
|
838
|
+
# web content lazily, populating the tree on the first assistive-technology
|
|
839
|
+
# query, so the very first walk after a navigation can come back empty.
|
|
840
|
+
_CHROMIUM_EXPORT_RETRIES = 2
|
|
841
|
+
_CHROMIUM_EXPORT_RETRY_DELAY = 0.6
|
|
842
|
+
|
|
843
|
+
|
|
844
|
+
def _empty_result_diagnostic(window, apps_found, foreground_is_chromium):
|
|
845
|
+
"""Why `observe` found nothing, for the agent and for anyone reading the trace.
|
|
846
|
+
|
|
847
|
+
"Found 0 actionable elements" has several unrelated causes with different
|
|
848
|
+
remedies, and the trace cannot tell them apart without this: no application
|
|
849
|
+
window in front (only the desktop, or a window manager reporting nothing
|
|
850
|
+
active), a window without an owning process, an app that is not on the
|
|
851
|
+
accessibility bus at all (a toolkit without AT-SPI support, or a browser that
|
|
852
|
+
started with accessibility off and stays that way until restarted), or an app
|
|
853
|
+
on the bus whose focused window exports no actionable controls.
|
|
854
|
+
"""
|
|
855
|
+
if window is None:
|
|
856
|
+
return (
|
|
857
|
+
"No application window is in front: only the desktop is showing, or the "
|
|
858
|
+
"window manager reports nothing active."
|
|
859
|
+
)
|
|
860
|
+
title = window.get("title") or "(untitled)"
|
|
861
|
+
if window.get("pid") is None:
|
|
862
|
+
return (
|
|
863
|
+
f'The focused window "{title}" reports no owning process, '
|
|
864
|
+
"so its accessibility tree cannot be located."
|
|
865
|
+
)
|
|
866
|
+
if not apps_found:
|
|
867
|
+
if foreground_is_chromium:
|
|
868
|
+
return (
|
|
869
|
+
"The browser in front is not exporting an accessibility tree (it is not "
|
|
870
|
+
"registered on the accessibility bus), so none of its controls or page "
|
|
871
|
+
"content can be labeled until it is restarted."
|
|
872
|
+
)
|
|
873
|
+
process = _process_name(window["pid"]) or f"pid {window['pid']}"
|
|
874
|
+
return (
|
|
875
|
+
f'The app in front ("{title}", {process}) is not registered on the '
|
|
876
|
+
"accessibility bus: it exports no accessibility tree."
|
|
877
|
+
)
|
|
878
|
+
return (
|
|
879
|
+
f'The app in front ("{title}") is on the accessibility bus but exports no '
|
|
880
|
+
"actionable controls for its focused window."
|
|
881
|
+
)
|
|
882
|
+
|
|
883
|
+
|
|
639
884
|
def enumerate_elements(pid=None):
|
|
885
|
+
"""Label the foreground window's actionable elements.
|
|
886
|
+
|
|
887
|
+
Returns (elements, window, diagnostic): the numbered elements, the foreground
|
|
888
|
+
window descriptor (None when no application window is in front), and - only
|
|
889
|
+
when no element was found - a diagnostic naming the cause.
|
|
890
|
+
"""
|
|
640
891
|
Atspi = _try_import_atspi()
|
|
641
892
|
if Atspi is None:
|
|
642
893
|
exit_with_error(
|
|
@@ -644,34 +895,38 @@ def enumerate_elements(pid=None):
|
|
|
644
895
|
)
|
|
645
896
|
|
|
646
897
|
screen_w, screen_h = get_screen_size()
|
|
647
|
-
|
|
898
|
+
window = _foreground_window()
|
|
899
|
+
if pid is not None and window is not None:
|
|
900
|
+
window = {**window, "pid": pid, "chromium": _pid_is_chromium(pid)}
|
|
901
|
+
target_pid = pid or (window["pid"] if window is not None else None)
|
|
648
902
|
# A native foreground app (GTK4 etc.) leaves the AT-SPI VISIBLE/SHOWING flags
|
|
649
903
|
# unset on its on-screen descendants, so trust geometry for it; Chromium's web
|
|
650
904
|
# a11y flags are reliable, so keep requiring VISIBLE there.
|
|
651
|
-
foreground_is_chromium =
|
|
905
|
+
foreground_is_chromium = bool(window and window["chromium"])
|
|
906
|
+
if window is not None and _pid_is_libreoffice(target_pid):
|
|
907
|
+
_refresh_libreoffice_geometry(window)
|
|
652
908
|
|
|
653
909
|
desktop = Atspi.get_desktop(0)
|
|
910
|
+
apps = _apps_for_pid(desktop, target_pid)
|
|
654
911
|
elements = _collect_elements(
|
|
655
|
-
|
|
912
|
+
apps, window, Atspi, screen_w, screen_h, trust_geometry=not foreground_is_chromium
|
|
656
913
|
)
|
|
657
914
|
|
|
658
|
-
# The foreground app exported its own actionable tree - use it verbatim. It is
|
|
659
|
-
# already scoped to the single window the agent sees (one desktop, WYSIWYG).
|
|
660
915
|
if not elements and foreground_is_chromium:
|
|
661
|
-
#
|
|
662
|
-
# PID
|
|
663
|
-
#
|
|
664
|
-
#
|
|
665
|
-
#
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
for _ in range(2):
|
|
669
|
-
time.sleep(0.6)
|
|
916
|
+
# The browser's PID-scoped walk came back empty: either its tree is still
|
|
917
|
+
# being exported, or its window PID is not its AT-SPI PID. Retry briefly on
|
|
918
|
+
# the browser's own app node(s), clipped to the focused window. Never the
|
|
919
|
+
# whole desktop, which would overlay the always-open browser's labels on
|
|
920
|
+
# whatever native app is in front.
|
|
921
|
+
for _ in range(_CHROMIUM_EXPORT_RETRIES):
|
|
922
|
+
time.sleep(_CHROMIUM_EXPORT_RETRY_DELAY)
|
|
670
923
|
desktop = Atspi.get_desktop(0)
|
|
671
|
-
|
|
924
|
+
apps = apps or _chromium_apps(desktop)
|
|
925
|
+
elements = _within_rect(
|
|
926
|
+
_collect_elements(apps, window, Atspi, screen_w, screen_h), window
|
|
927
|
+
)
|
|
672
928
|
if elements:
|
|
673
929
|
break
|
|
674
|
-
elements = _within_rect(elements, active_rect)
|
|
675
930
|
|
|
676
931
|
# A native (non-Chromium) foreground app that exports nothing yields an empty
|
|
677
932
|
# result on purpose: the agent falls back to screenshot_grid coordinates
|
|
@@ -682,7 +937,10 @@ def enumerate_elements(pid=None):
|
|
|
682
937
|
for idx, el in enumerate(deduped):
|
|
683
938
|
el["number"] = idx + 1
|
|
684
939
|
|
|
685
|
-
|
|
940
|
+
diagnostic = None
|
|
941
|
+
if not deduped:
|
|
942
|
+
diagnostic = _empty_result_diagnostic(window, bool(apps), foreground_is_chromium)
|
|
943
|
+
return deduped, window, diagnostic
|
|
686
944
|
|
|
687
945
|
|
|
688
946
|
# ---------------------------------------------------------------------------
|
|
@@ -1198,23 +1456,33 @@ def _desktop_entry_lines(content):
|
|
|
1198
1456
|
yield line
|
|
1199
1457
|
|
|
1200
1458
|
|
|
1459
|
+
def _desktop_entry_bool(fields, key):
|
|
1460
|
+
return fields.get(key, "").strip().lower() == "true"
|
|
1461
|
+
|
|
1462
|
+
|
|
1201
1463
|
def _resolve_from_desktop_files(name):
|
|
1202
|
-
"""Search .desktop files for an app matching *name
|
|
1464
|
+
"""Search .desktop files for an app matching *name*, best match first.
|
|
1203
1465
|
|
|
1204
1466
|
Matches the Name= and GenericName= fields of the main group
|
|
1205
1467
|
(case-insensitive) and the desktop file basename (e.g. "chromium" matches
|
|
1206
|
-
"chromium-browser.desktop").
|
|
1207
|
-
basename
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1468
|
+
"chromium-browser.desktop"). A Name= match ranks before a GenericName= match,
|
|
1469
|
+
which ranks before a basename match - "Text Editor" is the Name of GNOME's
|
|
1470
|
+
editor but only the GenericName of vim's entry, and "libreoffice" is a
|
|
1471
|
+
substring of every libreoffice-*.desktop basename but names the Start Center.
|
|
1472
|
+
Ties break on the desktop id, so the choice never depends on directory order.
|
|
1473
|
+
|
|
1474
|
+
Entries that cannot show a window of their own are left out: `Terminal=true`
|
|
1475
|
+
programs (vim, htop) run inside a terminal emulator the agent has to open
|
|
1476
|
+
first, and `Hidden`/`NoDisplay` entries are not launchers a user sees.
|
|
1477
|
+
|
|
1478
|
+
Returns a list of (argv, desktop_id, window_classes) launch candidates. argv is
|
|
1479
|
+
the Exec= command with its arguments kept and the field codes removed, so
|
|
1480
|
+
"LibreOffice Calc" launches `libreoffice --calc` - not the bare `libreoffice`,
|
|
1481
|
+
which opens the Start Center instead of the component the agent asked for.
|
|
1482
|
+
window_classes are the WM_CLASS patterns that identify the entry's windows.
|
|
1214
1483
|
"""
|
|
1215
1484
|
name_lower = name.lower()
|
|
1216
|
-
|
|
1217
|
-
partial = []
|
|
1485
|
+
ranked = []
|
|
1218
1486
|
|
|
1219
1487
|
for desktop_dir in _DESKTOP_DIRS:
|
|
1220
1488
|
for filepath in glob.glob(os.path.join(desktop_dir, "*.desktop")):
|
|
@@ -1224,48 +1492,69 @@ def _resolve_from_desktop_files(name):
|
|
|
1224
1492
|
except Exception:
|
|
1225
1493
|
continue
|
|
1226
1494
|
|
|
1227
|
-
|
|
1228
|
-
exec_cmd = None
|
|
1495
|
+
fields = {}
|
|
1229
1496
|
for line in _desktop_entry_lines(content):
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1497
|
+
key, sep, value = line.partition("=")
|
|
1498
|
+
if sep and key.strip() not in fields:
|
|
1499
|
+
fields[key.strip()] = value.strip()
|
|
1500
|
+
if fields.get("Type", "Application") != "Application":
|
|
1501
|
+
continue
|
|
1502
|
+
if any(_desktop_entry_bool(fields, key) for key in ("Terminal", "Hidden", "NoDisplay")):
|
|
1503
|
+
continue
|
|
1234
1504
|
|
|
1235
1505
|
desktop_id = os.path.basename(filepath)
|
|
1236
1506
|
basename = os.path.splitext(desktop_id)[0].lower()
|
|
1237
|
-
if
|
|
1238
|
-
|
|
1507
|
+
if fields.get("Name", "").lower() == name_lower:
|
|
1508
|
+
rank = 0
|
|
1509
|
+
elif fields.get("GenericName", "").lower() == name_lower:
|
|
1510
|
+
rank = 1
|
|
1239
1511
|
elif name_lower in basename:
|
|
1240
|
-
|
|
1512
|
+
rank = 2
|
|
1241
1513
|
else:
|
|
1242
1514
|
continue
|
|
1243
1515
|
|
|
1516
|
+
exec_cmd = _DESKTOP_FIELD_CODES.sub("", fields.get("Exec", ""))
|
|
1244
1517
|
try:
|
|
1245
1518
|
argv = shlex.split(exec_cmd) if exec_cmd else []
|
|
1246
1519
|
except ValueError:
|
|
1247
1520
|
argv = []
|
|
1248
|
-
|
|
1521
|
+
classes = _window_classes_for_entry(argv, desktop_id, fields.get("StartupWMClass"))
|
|
1522
|
+
ranked.append((rank, desktop_id, argv, classes))
|
|
1249
1523
|
|
|
1250
|
-
|
|
1524
|
+
ranked.sort(key=lambda item: (item[0], item[1]))
|
|
1525
|
+
return [(argv, desktop_id, classes) for _rank, desktop_id, argv, classes in ranked]
|
|
1251
1526
|
|
|
1252
1527
|
|
|
1253
|
-
|
|
1254
|
-
|
|
1528
|
+
# Exec= commands that are wrappers around the real program, so their name says
|
|
1529
|
+
# nothing about the window's class.
|
|
1530
|
+
_LAUNCHER_WRAPPERS = {"env", "sh", "bash", "flatpak", "snap", "gtk-launch", "xdg-open"}
|
|
1255
1531
|
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1532
|
+
|
|
1533
|
+
def _window_classes_for_entry(argv, desktop_id, startup_wm_class=None):
|
|
1534
|
+
"""WM_CLASS patterns (case-insensitive regexes) that identify an entry's windows.
|
|
1535
|
+
|
|
1536
|
+
`StartupWMClass=` is the entry's own statement of its class and is used alone
|
|
1537
|
+
when present. LibreOffice's component entries carry it, and the guesses below
|
|
1538
|
+
would otherwise name the whole family too: `libreoffice`, the Exec= basename,
|
|
1539
|
+
is a substring of every component's class, so asking for Calc would focus an
|
|
1540
|
+
open Writer window instead of launching Calc. Without it, a .desktop id is the
|
|
1541
|
+
most specific signal; a reverse-DNS id (`org.gnome.Evince`) also contributes
|
|
1542
|
+
its last segment, since GTK3 apps set WM_CLASS from the program name; and the
|
|
1543
|
+
executable's basename covers apps whose class is their binary
|
|
1544
|
+
(`gnome-calculator`) - without a desktop entry it is all there is.
|
|
1261
1545
|
"""
|
|
1546
|
+
if startup_wm_class:
|
|
1547
|
+
return [re.escape(startup_wm_class)]
|
|
1548
|
+
classes = []
|
|
1262
1549
|
if desktop_id:
|
|
1263
1550
|
base = os.path.splitext(desktop_id)[0]
|
|
1264
|
-
classes
|
|
1551
|
+
classes.append(base)
|
|
1265
1552
|
if "." in base:
|
|
1266
1553
|
classes.append(base.rsplit(".", 1)[1])
|
|
1267
|
-
|
|
1268
|
-
|
|
1554
|
+
if argv and os.path.basename(argv[0]) not in _LAUNCHER_WRAPPERS:
|
|
1555
|
+
classes.append(os.path.basename(argv[0]))
|
|
1556
|
+
seen = set()
|
|
1557
|
+
return [re.escape(c) for c in classes if c and not (c.lower() in seen or seen.add(c.lower()))]
|
|
1269
1558
|
|
|
1270
1559
|
|
|
1271
1560
|
# WM_CLASS regex fragments for apps whose window title does not match the app name
|
|
@@ -1281,94 +1570,147 @@ _WINDOW_CLASS_ALIASES = {
|
|
|
1281
1570
|
"firefox": ["firefox"],
|
|
1282
1571
|
}
|
|
1283
1572
|
|
|
1573
|
+
# How long the window manager gets to report a requested window as active before
|
|
1574
|
+
# the next activation method is tried. Mutter answers a pager-sourced
|
|
1575
|
+
# _NET_ACTIVE_WINDOW request within a frame or two; the bound only matters when a
|
|
1576
|
+
# request is refused outright.
|
|
1577
|
+
_ACTIVATION_CONFIRM_TIMEOUT = 1.0
|
|
1284
1578
|
|
|
1285
|
-
def activate_app(name):
|
|
1286
|
-
"""Focus the app's window, launching the app first when it has none.
|
|
1287
1579
|
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1580
|
+
def _search_windows(flag, pattern):
|
|
1581
|
+
"""Visible X11 windows matching an xdotool search (`--class` / `--name`).
|
|
1582
|
+
|
|
1583
|
+
`--onlyvisible` leaves out the unmapped client-leader and helper windows a
|
|
1584
|
+
toolkit creates alongside its real toplevel: they share its class and title,
|
|
1585
|
+
sort first, and cannot be activated (the WM has never managed them).
|
|
1293
1586
|
"""
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
return
|
|
1325
|
-
|
|
1587
|
+
try:
|
|
1588
|
+
r = subprocess.run(
|
|
1589
|
+
["xdotool", "search", "--onlyvisible", flag, pattern],
|
|
1590
|
+
capture_output=True, text=True, timeout=5,
|
|
1591
|
+
)
|
|
1592
|
+
return [int(wid) for wid in r.stdout.split() if wid.isdigit()]
|
|
1593
|
+
except Exception:
|
|
1594
|
+
return []
|
|
1595
|
+
|
|
1596
|
+
|
|
1597
|
+
def _find_app_window(name_patterns, class_patterns):
|
|
1598
|
+
"""The topmost window of the app, matched by WM_CLASS first, then by title.
|
|
1599
|
+
|
|
1600
|
+
Class is the reliable identity of an app's windows: a title only sometimes
|
|
1601
|
+
carries the app's name (Files shows "Home", a document viewer shows the
|
|
1602
|
+
document's title), and a title search can hit a browser tab that happens to
|
|
1603
|
+
mention the app. Among several windows the one highest in the WM's stacking
|
|
1604
|
+
order is the one the user last worked in. Falls back to xdotool's own order
|
|
1605
|
+
when the WM publishes no stacking list.
|
|
1606
|
+
"""
|
|
1607
|
+
stacked = _managed_windows_stacked()
|
|
1608
|
+
attempts = [("--class", c) for c in class_patterns] + [("--name", n) for n in name_patterns]
|
|
1609
|
+
for flag, pattern in attempts:
|
|
1610
|
+
wids = _search_windows(flag, pattern)
|
|
1611
|
+
if not wids:
|
|
1612
|
+
continue
|
|
1613
|
+
if stacked:
|
|
1614
|
+
matched = set(wids)
|
|
1615
|
+
managed = [wid for wid in stacked if wid in matched]
|
|
1616
|
+
if managed:
|
|
1617
|
+
return managed[-1]
|
|
1618
|
+
continue
|
|
1619
|
+
return wids[-1]
|
|
1620
|
+
return None
|
|
1621
|
+
|
|
1622
|
+
|
|
1623
|
+
def _focus_window(window_id):
|
|
1624
|
+
"""Make ``window_id`` the active window and confirm the WM agreed.
|
|
1326
1625
|
|
|
1327
|
-
|
|
1328
|
-
|
|
1626
|
+
A plain `xdotool windowactivate` (a pager-sourced _NET_ACTIVE_WINDOW request)
|
|
1627
|
+
is what mutter and the other EWMH managers honor. Its `--sync` is not used: it
|
|
1628
|
+
waits without a bound of its own for the WM to report the window active, so a
|
|
1629
|
+
refused request would hang until the subprocess timeout. Confirmation is read
|
|
1630
|
+
back from the WM under a bound instead, and a refused request falls through to
|
|
1631
|
+
a raise + focus and then to wmctrl. Returns whether the window ended up active.
|
|
1632
|
+
"""
|
|
1633
|
+
methods = [
|
|
1634
|
+
[["xdotool", "windowactivate", str(window_id)]],
|
|
1635
|
+
[["xdotool", "windowraise", str(window_id)], ["xdotool", "windowfocus", str(window_id)]],
|
|
1636
|
+
]
|
|
1637
|
+
if shutil.which("wmctrl"):
|
|
1638
|
+
methods.append([["wmctrl", "-ia", str(window_id)]])
|
|
1639
|
+
for commands in methods:
|
|
1640
|
+
for command in commands:
|
|
1641
|
+
try:
|
|
1642
|
+
subprocess.run(command, capture_output=True, text=True, timeout=5, check=False)
|
|
1643
|
+
except (FileNotFoundError, subprocess.TimeoutExpired):
|
|
1644
|
+
continue
|
|
1645
|
+
deadline = time.time() + _ACTIVATION_CONFIRM_TIMEOUT
|
|
1329
1646
|
while time.time() < deadline:
|
|
1330
|
-
|
|
1647
|
+
active = _get_active_window_id()
|
|
1648
|
+
# A WM that publishes no active window cannot confirm or deny; the
|
|
1649
|
+
# request was delivered, which is all there is to go on.
|
|
1650
|
+
if active is None or active == window_id:
|
|
1331
1651
|
return True
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
code = proc.poll()
|
|
1335
|
-
if code is not None and code != 0:
|
|
1336
|
-
status = f"signal {-code}" if code < 0 else f"exit status {code}"
|
|
1337
|
-
raise RuntimeError(
|
|
1338
|
-
f"'{name}' crashed on launch ({status}) before showing a window."
|
|
1339
|
-
)
|
|
1340
|
-
time.sleep(0.2)
|
|
1341
|
-
return False
|
|
1652
|
+
time.sleep(0.1)
|
|
1653
|
+
return False
|
|
1342
1654
|
|
|
1343
|
-
if activate_existing_window():
|
|
1344
|
-
return True
|
|
1345
1655
|
|
|
1656
|
+
def activate_app(name):
|
|
1657
|
+
"""Focus the app's window, launching the app first when it has none.
|
|
1658
|
+
|
|
1659
|
+
Returns True when a window of the app is in front, False when the app was
|
|
1660
|
+
launched but has not shown a window within the launch timeout - it is still
|
|
1661
|
+
starting, so the caller should wait rather than launch it again. Raises
|
|
1662
|
+
RuntimeError when nothing launchable matches the name, when the launched
|
|
1663
|
+
process exits with a failure before showing a window, or when the app has a
|
|
1664
|
+
window the window manager refuses to bring to the front.
|
|
1665
|
+
"""
|
|
1346
1666
|
candidates = _resolve_from_desktop_files(name)
|
|
1347
|
-
|
|
1667
|
+
name_patterns = [re.escape(name)]
|
|
1668
|
+
class_patterns = list(_WINDOW_CLASS_ALIASES.get(name.lower(), []))
|
|
1669
|
+
for _argv, _desktop_id, classes in candidates:
|
|
1670
|
+
class_patterns += [c for c in classes if c not in class_patterns]
|
|
1671
|
+
|
|
1672
|
+
def focus_or_raise(window_id):
|
|
1673
|
+
if _focus_window(window_id):
|
|
1674
|
+
return True
|
|
1675
|
+
title = _window_title(window_id) or "(untitled)"
|
|
1676
|
+
raise RuntimeError(
|
|
1677
|
+
f"'{name}' has a window (\"{title}\") but the window manager would not "
|
|
1678
|
+
"bring it to the front."
|
|
1679
|
+
)
|
|
1680
|
+
|
|
1681
|
+
# An app that is already running is focused, never launched again: a second
|
|
1682
|
+
# launch of a single-instance app opens a second window (or just re-presents
|
|
1683
|
+
# the first), and the agent then labels two of the same window.
|
|
1684
|
+
existing = _find_app_window(name_patterns, class_patterns)
|
|
1685
|
+
if existing is not None:
|
|
1686
|
+
return focus_or_raise(existing)
|
|
1687
|
+
|
|
1688
|
+
known = {argv[0] for argv, _, _ in candidates if argv}
|
|
1348
1689
|
for direct in (name.lower(), name):
|
|
1349
1690
|
if direct not in known:
|
|
1350
|
-
candidates.append(([direct], None))
|
|
1691
|
+
candidates.append(([direct], None, _window_classes_for_entry([direct], None)))
|
|
1351
1692
|
known.add(direct)
|
|
1352
1693
|
|
|
1353
1694
|
# Launch once: the first candidate whose executable exists. Each candidate
|
|
1354
1695
|
# names the same app, so launching more than one would only stack windows
|
|
1355
1696
|
# and make the agent wait for every one in turn.
|
|
1356
1697
|
launch = None
|
|
1357
|
-
for argv, desktop_id in candidates:
|
|
1698
|
+
for argv, desktop_id, classes in candidates:
|
|
1358
1699
|
if not argv:
|
|
1359
1700
|
continue
|
|
1360
1701
|
executable = shutil.which(argv[0])
|
|
1361
1702
|
if executable is not None:
|
|
1362
|
-
launch = ([executable, *argv[1:]], desktop_id)
|
|
1703
|
+
launch = ([executable, *argv[1:]], desktop_id, classes)
|
|
1363
1704
|
break
|
|
1364
1705
|
|
|
1365
1706
|
# A desktop entry whose Exec= cannot be resolved directly (D-Bus activated
|
|
1366
1707
|
# apps, launcher wrappers off PATH) still launches through gtk-launch.
|
|
1367
1708
|
if launch is None:
|
|
1368
1709
|
gtk_launch = shutil.which("gtk-launch")
|
|
1369
|
-
|
|
1370
|
-
if gtk_launch and
|
|
1371
|
-
|
|
1710
|
+
with_entry = [(desktop_id, classes) for _, desktop_id, classes in candidates if desktop_id]
|
|
1711
|
+
if gtk_launch and with_entry:
|
|
1712
|
+
desktop_id, classes = with_entry[0]
|
|
1713
|
+
launch = ([gtk_launch, desktop_id], desktop_id, classes)
|
|
1372
1714
|
|
|
1373
1715
|
if launch is None:
|
|
1374
1716
|
raise RuntimeError(
|
|
@@ -1376,12 +1718,54 @@ def activate_app(name):
|
|
|
1376
1718
|
f".desktop entry, or executable matches it."
|
|
1377
1719
|
)
|
|
1378
1720
|
|
|
1379
|
-
argv,
|
|
1721
|
+
argv, _desktop_id, launch_classes = launch
|
|
1722
|
+
windows_before = set(_managed_windows_stacked())
|
|
1380
1723
|
try:
|
|
1381
|
-
|
|
1724
|
+
# Its own session with no inherited stdio: the app must outlive this
|
|
1725
|
+
# driver process (one process per command) and never block on, or be
|
|
1726
|
+
# killed with, the driver's pipes and process group.
|
|
1727
|
+
proc = subprocess.Popen(
|
|
1728
|
+
argv,
|
|
1729
|
+
stdin=subprocess.DEVNULL,
|
|
1730
|
+
stdout=subprocess.DEVNULL,
|
|
1731
|
+
stderr=subprocess.DEVNULL,
|
|
1732
|
+
start_new_session=True,
|
|
1733
|
+
)
|
|
1382
1734
|
except OSError as e:
|
|
1383
1735
|
raise RuntimeError(f"Could not launch '{name}' ({argv[0]}): {e}") from e
|
|
1384
|
-
|
|
1736
|
+
|
|
1737
|
+
def launched_window():
|
|
1738
|
+
# The app's window by class (its launcher wrapper may exec a differently
|
|
1739
|
+
# named binary - LibreOffice's `soffice` - and a single-instance app may
|
|
1740
|
+
# hand the launch to its running process), else whatever managed window
|
|
1741
|
+
# appeared since the launch: that is the app's, whatever class it chose.
|
|
1742
|
+
window_id = _find_app_window(name_patterns, launch_classes)
|
|
1743
|
+
if window_id is not None:
|
|
1744
|
+
return window_id
|
|
1745
|
+
appeared = [wid for wid in _managed_windows_stacked() if wid not in windows_before]
|
|
1746
|
+
return appeared[-1] if appeared else None
|
|
1747
|
+
|
|
1748
|
+
deadline = time.time() + _LAUNCH_WINDOW_TIMEOUT
|
|
1749
|
+
while time.time() < deadline:
|
|
1750
|
+
window_id = launched_window()
|
|
1751
|
+
# A window the WM will not activate yet is still settling (a splash, a
|
|
1752
|
+
# frame the app is about to replace with its real one): keep waiting for
|
|
1753
|
+
# one it will, rather than report the app as unfocusable.
|
|
1754
|
+
if window_id is not None and _focus_window(window_id):
|
|
1755
|
+
return True
|
|
1756
|
+
# A launcher that hands off to a running instance exits 0 with no
|
|
1757
|
+
# window of its own, so only a failure exit ends the wait early.
|
|
1758
|
+
code = proc.poll()
|
|
1759
|
+
if code is not None and code != 0:
|
|
1760
|
+
status = f"signal {-code}" if code < 0 else f"exit status {code}"
|
|
1761
|
+
raise RuntimeError(
|
|
1762
|
+
f"'{name}' crashed on launch ({status}) before showing a window."
|
|
1763
|
+
)
|
|
1764
|
+
time.sleep(0.2)
|
|
1765
|
+
window_id = launched_window()
|
|
1766
|
+
if window_id is not None:
|
|
1767
|
+
return focus_or_raise(window_id)
|
|
1768
|
+
return False
|
|
1385
1769
|
|
|
1386
1770
|
|
|
1387
1771
|
# ---------------------------------------------------------------------------
|
|
@@ -1446,7 +1830,7 @@ def main():
|
|
|
1446
1830
|
else:
|
|
1447
1831
|
i += 1
|
|
1448
1832
|
|
|
1449
|
-
elements = enumerate_elements(pid=pid)
|
|
1833
|
+
elements, window, diagnostic = enumerate_elements(pid=pid)
|
|
1450
1834
|
capture_screenshot(screenshot_path)
|
|
1451
1835
|
annotate_screenshot(screenshot_path, elements)
|
|
1452
1836
|
|
|
@@ -1454,6 +1838,10 @@ def main():
|
|
|
1454
1838
|
"elements": elements,
|
|
1455
1839
|
"screenshotPath": screenshot_path,
|
|
1456
1840
|
}
|
|
1841
|
+
if window is not None:
|
|
1842
|
+
result["window"] = {"title": window["title"], "pid": window["pid"]}
|
|
1843
|
+
if diagnostic:
|
|
1844
|
+
result["diagnostic"] = diagnostic
|
|
1457
1845
|
print(json.dumps(result, indent=2))
|
|
1458
1846
|
|
|
1459
1847
|
elif command == "click":
|