@octavus/agent 1.0.24 → 1.0.25

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -25,6 +25,9 @@ Usage mirrors the macOS driver exactly:
25
25
  linux-driver.py activate <app-name>
26
26
  """
27
27
 
28
+ import ctypes
29
+ import ctypes.util
30
+ import functools
28
31
  import glob
29
32
  import json
30
33
  import os
@@ -101,6 +104,10 @@ CONTAINER_ROLES = {
101
104
 
102
105
  ACTIONABLE_ACTIONS = {"click", "press", "activate", "toggle", "jump"}
103
106
 
107
+ # Roles that are never a control, whatever action their toolkit advertises:
108
+ # LibreOffice exposes a "click" on every toolbar separator.
109
+ NON_ACTIONABLE_ROLES = {"separator"}
110
+
104
111
 
105
112
  def _try_import_atspi():
106
113
  """Import AT-SPI2 via GObject introspection and connect to the a11y bus."""
@@ -134,6 +141,15 @@ def _overlaps_screen(x, y, w, h, screen_w, screen_h):
134
141
  return x < screen_w and y < screen_h and x + w > 0 and y + h > 0
135
142
 
136
143
 
144
+ # The most children of one node the walk descends into. A spreadsheet reports its
145
+ # whole sheet as the table's children (Calc: every cell of 1024 x 1048576), which
146
+ # no walk could visit within a driver command - and its first rows would be no
147
+ # guide to what is on screen anyway. A node with more children than this is a
148
+ # data grid, left to grid coordinates; a UI container that is not one has orders
149
+ # of magnitude fewer.
150
+ _MAX_CHILDREN_WALKED = 2048
151
+
152
+
137
153
  def _walk_atspi(
138
154
  node,
139
155
  elements,
@@ -163,6 +179,11 @@ def _walk_atspi(
163
179
  except Exception:
164
180
  pass
165
181
 
182
+ try:
183
+ role_name = node.get_role_name().lower()
184
+ except Exception:
185
+ role_name = ""
186
+
166
187
  rect = None
167
188
  try:
168
189
  comp = node.get_component_iface()
@@ -180,6 +201,11 @@ def _walk_atspi(
180
201
  )
181
202
  except Exception:
182
203
  rect = None
204
+ # GTK3 answers for an accessible without geometry of its own (a GtkNotebook
205
+ # page) with -1s: that is "no geometry", like having no Component interface at
206
+ # all, not a box that is off the screen.
207
+ if rect is not None and (rect[2] < 0 or rect[3] < 0):
208
+ rect = None
183
209
 
184
210
  on_screen = (
185
211
  rect is not None
@@ -199,11 +225,10 @@ def _walk_atspi(
199
225
  # surfaces hidden-but-positioned elements. Off-screen and zero-area nodes fail
200
226
  # the on_screen test regardless.
201
227
  if rect is not None and on_screen and (trust_geometry or visible):
202
- try:
203
- role_name = node.get_role_name().lower()
204
- except Exception:
205
- role_name = ""
206
- if role_name and (role_name in CLICKABLE_ROLES or _has_actionable(node, Atspi)):
228
+ if role_name and (
229
+ role_name in CLICKABLE_ROLES
230
+ or (role_name not in NON_ACTIONABLE_ROLES and _has_actionable(node, Atspi))
231
+ ):
207
232
  x, y, w, h = rect
208
233
  area = w * h
209
234
  screen_area = screen_w * screen_h
@@ -244,15 +269,22 @@ def _walk_atspi(
244
269
  # Descend into children unless this node is provably off-screen. Keep walking
245
270
  # through nodes with no geometry (structural containers such as the
246
271
  # `application` node, which reports VISIBLE=False and no extents) and through
247
- # on-screen containers that merely lack SHOWING - prune only a node that has a
248
- # real box that is off-screen or zero-area and is also not SHOWING. That
249
- # bounds the walk over large web trees (off-viewport content) without severing
250
- # a native app's subtree at a SHOWING-less container.
251
- if rect is not None and not on_screen and not showing:
272
+ # on-screen containers that merely lack SHOWING; prune a node whose real box
273
+ # is off-screen or zero-area. For Chromium that prune also requires the node
274
+ # not to be SHOWING: web layouts routinely hang visible, absolutely positioned
275
+ # content under a zero-size wrapper, and off-viewport content is what bounds
276
+ # the walk over a large page. A native toolkit clips children to their
277
+ # parent's allocation, so a zero-area or off-screen native container shows
278
+ # nothing whatever its SHOWING flag says - LibreOffice keeps a SHOWING,
279
+ # zero-height copy of its menu bar whose items would otherwise be labeled on
280
+ # top of the toolbar drawn below it.
281
+ if rect is not None and not on_screen and (trust_geometry or not showing):
252
282
  return
253
283
 
254
284
  try:
255
285
  n_children = node.get_child_count()
286
+ if n_children > _MAX_CHILDREN_WALKED:
287
+ return
256
288
  for i in range(n_children):
257
289
  child = node.get_child_at_index(i)
258
290
  if child is not None:
@@ -330,33 +362,41 @@ def _dedup_elements(elements, screen_w, screen_h):
330
362
  return deduped
331
363
 
332
364
 
333
- def _get_active_window_pid():
365
+ def _xdotool_output(args, timeout=5):
366
+ """stdout of an xdotool query, or None when it fails (no active window, a
367
+ window gone between two lookups, no WM support)."""
334
368
  try:
335
- wid = subprocess.check_output(
336
- ["xdotool", "getactivewindow"], text=True, timeout=5
337
- ).strip()
338
- pid = subprocess.check_output(
339
- ["xdotool", "getwindowpid", wid], text=True, timeout=5
369
+ return subprocess.check_output(
370
+ ["xdotool", *args], text=True, timeout=timeout, stderr=subprocess.DEVNULL
340
371
  ).strip()
341
- return int(pid)
342
372
  except Exception:
343
373
  return None
344
374
 
345
375
 
346
- def _get_active_window_geometry():
347
- """Screen rect of the focused window as {id, x, y, width, height}, or None.
376
+ def _get_active_window_id():
377
+ out = _xdotool_output(["getactivewindow"])
378
+ try:
379
+ return int(out) if out else None
380
+ except ValueError:
381
+ return None
382
+
348
383
 
349
- The focused window is the one the agent is actually looking at (one desktop,
350
- what-you-see-is-what-you-get), so it is the reference for anything that must
351
- be clipped to, or placed within, the visible app.
352
- """
384
+ def _window_pid(window_id):
385
+ out = _xdotool_output(["getwindowpid", str(window_id)])
353
386
  try:
354
- out = subprocess.check_output(
355
- ["xdotool", "getactivewindow", "getwindowgeometry", "--shell"],
356
- text=True,
357
- timeout=5,
358
- )
359
- except Exception:
387
+ return int(out) if out else None
388
+ except ValueError:
389
+ return None
390
+
391
+
392
+ def _window_title(window_id):
393
+ return _xdotool_output(["getwindowname", str(window_id)]) or ""
394
+
395
+
396
+ def _window_geometry(window_id):
397
+ """Screen rect of a window as {id, x, y, width, height}, or None when it is gone."""
398
+ out = _xdotool_output(["getwindowgeometry", "--shell", str(window_id)])
399
+ if not out:
360
400
  return None
361
401
 
362
402
  vals = {}
@@ -376,8 +416,58 @@ def _get_active_window_geometry():
376
416
  return None
377
417
 
378
418
 
379
- def _active_window_is_chromium(pid):
380
- """Whether the foreground PID is a Chrome/Chromium process.
419
+ # Window types that sit in the WM's client list without ever being "in front":
420
+ # the desktop (GNOME's desktop-icons window spans the screen at the bottom of the
421
+ # stack) and panels/docks.
422
+ _BACKDROP_WINDOW_TYPES = ("_NET_WM_WINDOW_TYPE_DESKTOP", "_NET_WM_WINDOW_TYPE_DOCK")
423
+
424
+
425
+ def _is_backdrop_window(window_id):
426
+ """Whether the window is the desktop, a dock, or minimized - never "in front"."""
427
+ with _X11() as x11:
428
+ types = x11.window_property(window_id, "_NET_WM_WINDOW_TYPE", max_items=8) or []
429
+ state = x11.window_property(window_id, "_NET_WM_STATE", max_items=32) or []
430
+ backdrop = {x11.atom(name) for name in _BACKDROP_WINDOW_TYPES}
431
+ return bool(backdrop & set(types)) or x11.atom("_NET_WM_STATE_HIDDEN") in state
432
+
433
+
434
+ def _foreground_window_id():
435
+ """The window in front of the user: the top of the WM's stacking order.
436
+
437
+ Usually that is also the focused window. It is not when the window manager
438
+ stacks a window on top but withholds focus from it: mutter's focus-stealing
439
+ prevention does exactly that to a modal dialog another process opens on
440
+ behalf of the focused app (the portal file chooser Chrome's "Open" / "Save as"
441
+ bring up) after synthetic keystrokes, so the dialog covers the browser while
442
+ keystrokes keep landing in the browser. The topmost window is what the agent
443
+ sees and will click, so it is the one labeled - and it is given focus first so
444
+ typing reaches it too. Falls back to the focused window where the WM
445
+ publishes no stacking order, and is None when nothing but the desktop (or a
446
+ dock) is up - the desktop is never "the app in front".
447
+ """
448
+ active = _get_active_window_id()
449
+ for window_id in reversed(_managed_windows_stacked()):
450
+ if _is_backdrop_window(window_id):
451
+ continue
452
+ if window_id != active:
453
+ _focus_window(window_id)
454
+ return window_id
455
+ if active is not None and not _is_backdrop_window(active):
456
+ return active
457
+ return None
458
+
459
+
460
+ def _process_name(pid):
461
+ """The process's command name from /proc, or "" when unreadable."""
462
+ try:
463
+ with open(f"/proc/{pid}/comm", "rb") as f:
464
+ return f.read().decode("utf-8", "ignore").strip()
465
+ except Exception:
466
+ return ""
467
+
468
+
469
+ def _pid_is_chromium(pid):
470
+ """Whether the PID is a Chrome/Chromium process.
381
471
 
382
472
  Read from /proc rather than the window class: the cloud xdotool build has no
383
473
  getwindowclassname and xprop is not installed, but the browser owns the
@@ -396,31 +486,118 @@ def _active_window_is_chromium(pid):
396
486
  return False
397
487
 
398
488
 
399
- def _collect_elements(desktop, target_pid, Atspi, screen_w, screen_h, trust_geometry=False):
400
- """Collect actionable elements from the AT-SPI app(s) matching target_pid.
489
+ def _pid_is_libreoffice(pid):
490
+ return pid is not None and _process_name(pid).startswith("soffice")
491
+
492
+
493
+ # How long LibreOffice gets to re-read its frame geometry after the pointer nudge
494
+ # below, before its extents are trusted.
495
+ _LIBREOFFICE_GEOMETRY_SETTLE = 0.3
496
+
497
+
498
+ def _refresh_libreoffice_geometry(window):
499
+ """Make LibreOffice recompute where its widgets are before they are read.
401
500
 
402
- Strictly PID-scoped: a None target_pid (foreground PID unavailable) matches no
403
- app and returns empty, so labeling never falls back to walking the whole
404
- desktop, which would overlay the always-open browser's labels onto whatever
405
- native app is in front. `trust_geometry` is forwarded to the walk so a native
406
- foreground app's on-screen controls are included from geometry even when its
407
- toolkit leaves the AT-SPI VISIBLE/SHOWING flags unset (GTK4).
501
+ LibreOffice (gtk3 VCL) caches its frame geometry and only re-reads it on a
502
+ pointer event: after the window opens and the window manager places and
503
+ decorates it, every toolbar, the document and the status bar keep reporting
504
+ AT-SPI extents one title-bar height above where they are drawn, until the
505
+ pointer first moves inside the window. A labeled toolbar button would then be
506
+ clicked on the menu bar above it. One motion event over the window's center
507
+ is enough; the pointer has no meaning to the agent between actions.
408
508
  """
409
- elements = []
410
- n_apps = desktop.get_child_count()
509
+ center = (window["x"] + window["width"] // 2, window["y"] + window["height"] // 2)
510
+ if _pointer_location() == center:
511
+ center = (center[0] + 1, center[1])
512
+ _move_pointer(*center)
513
+ time.sleep(_LIBREOFFICE_GEOMETRY_SETTLE)
514
+
515
+
516
+ def _foreground_window():
517
+ """What is in front: {id, title, pid, x, y, width, height, chromium}, or None.
518
+
519
+ Everything `observe` labels is scoped to this window - one desktop, what you
520
+ see is what you get - so it is also what the result reports back, letting the
521
+ agent (and anyone reading the trace) see which window the numbers belong to.
522
+ """
523
+ window_id = _foreground_window_id()
524
+ window = _window_geometry(window_id) if window_id is not None else None
525
+ if window is None:
526
+ return None
527
+ pid = _window_pid(window["id"])
528
+ return {
529
+ **window,
530
+ "title": _window_title(window["id"]),
531
+ "pid": pid,
532
+ "chromium": _pid_is_chromium(pid),
533
+ }
534
+
535
+
536
+ def _apps_for_pid(desktop, target_pid):
537
+ """The AT-SPI application node(s) owned by ``target_pid`` - strictly by PID.
538
+
539
+ A None target_pid (foreground PID unavailable) matches no app, so labeling never
540
+ falls back to walking the whole desktop, which would overlay the always-open
541
+ browser's labels onto whatever native app is in front.
542
+ """
543
+ apps = []
544
+ if target_pid is None:
545
+ return apps
546
+ try:
547
+ n_apps = desktop.get_child_count()
548
+ except Exception:
549
+ return apps
411
550
  for i in range(n_apps):
412
- app = desktop.get_child_at_index(i)
413
- if app is None:
414
- continue
415
551
  try:
416
- if app.get_process_id() != target_pid:
417
- continue
552
+ app = desktop.get_child_at_index(i)
553
+ if app is not None and app.get_process_id() == target_pid:
554
+ apps.append(app)
418
555
  except Exception:
419
556
  continue
557
+ return apps
558
+
559
+
560
+ def _collect_elements(apps, window, Atspi, screen_w, screen_h, trust_geometry=False):
561
+ """Collect the actionable elements of the foreground window from its app's tree.
562
+
563
+ Only the toplevel accessible that corresponds to the focused X11 window is
564
+ walked: an app's other windows (a second Files window behind the first, a
565
+ browser popup, a document behind a modal dialog) are not what the agent sees
566
+ and would otherwise be labeled on top of what is - the same control twice at
567
+ two positions. An app whose toplevel cannot be matched to the focused window
568
+ (nameless frames, a toolkit that never marks one ACTIVE) is walked whole, so a
569
+ scoping miss degrades to extra labels rather than none. `trust_geometry` is
570
+ forwarded to the walk so a native foreground app's on-screen controls are
571
+ included from geometry even when its toolkit leaves the AT-SPI VISIBLE/SHOWING
572
+ flags unset (GTK4).
573
+ """
574
+ elements = []
575
+ for app in apps:
576
+ frame = _focused_frame(app, window, Atspi) if window is not None else None
420
577
  if _uses_window_relative_extents(app):
421
- _walk_focused_window(app, elements, Atspi, screen_w, screen_h, trust_geometry)
578
+ # Window-relative extents can only be placed for the focused window,
579
+ # whose on-screen origin is known; without a matched frame there is
580
+ # nothing to place them against.
581
+ if frame is not None:
582
+ left, top = _client_side_decoration_offset(window["id"])
583
+ _walk_atspi(
584
+ frame,
585
+ elements,
586
+ Atspi,
587
+ screen_w,
588
+ screen_h,
589
+ trust_geometry=trust_geometry,
590
+ window_origin=(window["x"] + left, window["y"] + top),
591
+ )
422
592
  else:
423
- _walk_atspi(app, elements, Atspi, screen_w, screen_h, trust_geometry=trust_geometry)
593
+ _walk_atspi(
594
+ frame if frame is not None else app,
595
+ elements,
596
+ Atspi,
597
+ screen_w,
598
+ screen_h,
599
+ trust_geometry=trust_geometry,
600
+ )
424
601
  return elements
425
602
 
426
603
 
@@ -442,33 +619,14 @@ def _uses_window_relative_extents(app):
442
619
  return toolkit == "gtk" and major >= 4
443
620
 
444
621
 
445
- def _walk_focused_window(app, elements, Atspi, screen_w, screen_h, trust_geometry):
446
- """Walk the app's focused window, placing its window-relative extents on screen.
622
+ def _focused_frame(app, window, Atspi):
623
+ """The app's toplevel accessible for the focused X11 window, or None.
447
624
 
448
- Only the focused window is walked: the app's other windows have their own
449
- origins, which the agent cannot see past the window in front anyway.
625
+ Matched by the ACTIVE state when the toolkit sets it, else by title - a GTK
626
+ dialog's accessible carries its window title even when it is never marked
627
+ ACTIVE. An app with a single toplevel needs no matching: there is nothing
628
+ else it could be.
450
629
  """
451
- window = _get_active_window_geometry()
452
- if window is None:
453
- return
454
- frame = _focused_frame(app, window["id"], Atspi)
455
- if frame is None:
456
- return
457
- left, top = _client_side_decoration_offset(window["id"])
458
- _walk_atspi(
459
- frame,
460
- elements,
461
- Atspi,
462
- screen_w,
463
- screen_h,
464
- trust_geometry=trust_geometry,
465
- window_origin=(window["x"] + left, window["y"] + top),
466
- )
467
-
468
-
469
- def _focused_frame(app, window_id, Atspi):
470
- """The app's toplevel accessible for the focused window: the one AT-SPI marks
471
- ACTIVE, else the one titled like the focused X11 window."""
472
630
  frames = []
473
631
  try:
474
632
  for i in range(app.get_child_count()):
@@ -477,6 +635,8 @@ def _focused_frame(app, window_id, Atspi):
477
635
  frames.append(frame)
478
636
  except Exception:
479
637
  return None
638
+ if len(frames) == 1:
639
+ return frames[0]
480
640
 
481
641
  for frame in frames:
482
642
  try:
@@ -485,12 +645,7 @@ def _focused_frame(app, window_id, Atspi):
485
645
  except Exception:
486
646
  continue
487
647
 
488
- try:
489
- title = subprocess.check_output(
490
- ["xdotool", "getwindowname", str(window_id)], text=True, timeout=5
491
- ).strip()
492
- except Exception:
493
- return None
648
+ title = window.get("title") or ""
494
649
  for frame in frames:
495
650
  try:
496
651
  if title and frame.get_name() == title:
@@ -500,27 +655,32 @@ def _focused_frame(app, window_id, Atspi):
500
655
  return None
501
656
 
502
657
 
503
- _XA_CARDINAL = 6
658
+ # Xlib's default error handler exits the process. A window can be destroyed
659
+ # between the xdotool lookup and a property read (BadWindow), which must degrade
660
+ # to "no property" rather than take the driver down mid-observe. Installed once
661
+ # when libX11 is loaded; the callback has to stay referenced for as long as the
662
+ # library is in use, so it lives at module level.
663
+ _IGNORE_X_ERRORS = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p)(
664
+ lambda _display, _error: 0
665
+ )
504
666
 
505
667
 
506
- def _client_side_decoration_offset(window_id):
507
- """(left, top) inset of a window's visible frame within its X11 window.
668
+ @functools.lru_cache(maxsize=None)
669
+ def _load_xlib():
670
+ """libX11 with the prototypes the driver uses, loaded once; None when unavailable.
508
671
 
509
- GTK draws client-side decorations - its drop shadow - inside the X11 window and
510
- advertises their size in `_GTK_FRAME_EXTENTS` (left, right, top, bottom); GTK
511
- measures window-relative extents from the visible frame, not the X11 window's
512
- corner. Read through libX11 directly because xprop is not installed on the
513
- cloud image. (0, 0) for a window without client-side decorations.
672
+ Window properties are read through libX11 directly because xprop is not
673
+ installed on the cloud image.
514
674
  """
515
675
  try:
516
- import ctypes
517
- import ctypes.util
518
-
519
676
  xlib = ctypes.cdll.LoadLibrary(ctypes.util.find_library("X11") or "libX11.so.6")
520
677
  except Exception:
521
- return 0, 0
678
+ return None
522
679
  xlib.XOpenDisplay.argtypes = [ctypes.c_char_p]
523
680
  xlib.XOpenDisplay.restype = ctypes.c_void_p
681
+ xlib.XCloseDisplay.argtypes = [ctypes.c_void_p]
682
+ xlib.XDefaultRootWindow.argtypes = [ctypes.c_void_p]
683
+ xlib.XDefaultRootWindow.restype = ctypes.c_ulong
524
684
  xlib.XInternAtom.argtypes = [ctypes.c_void_p, ctypes.c_char_p, ctypes.c_int]
525
685
  xlib.XInternAtom.restype = ctypes.c_ulong
526
686
  xlib.XGetWindowProperty.argtypes = [
@@ -538,38 +698,56 @@ def _client_side_decoration_offset(window_id):
538
698
  ctypes.POINTER(ctypes.c_void_p),
539
699
  ]
540
700
  xlib.XFree.argtypes = [ctypes.c_void_p]
541
- xlib.XCloseDisplay.argtypes = [ctypes.c_void_p]
542
701
  xlib.XSetErrorHandler.restype = ctypes.c_void_p
702
+ xlib.XSetErrorHandler(_IGNORE_X_ERRORS)
703
+ return xlib
543
704
 
544
- # Xlib's default error handler exits the process. The focused window can be
545
- # destroyed between the xdotool lookup and this read (BadWindow), which must
546
- # degrade to no offset rather than take the driver down mid-observe. The
547
- # callback has to stay referenced while the display is open.
548
- ignore_x_errors = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p)(
549
- lambda _display, _error: 0
550
- )
551
- xlib.XSetErrorHandler(ignore_x_errors)
552
705
 
553
- display = xlib.XOpenDisplay(None)
554
- if not display:
555
- return 0, 0
556
- try:
557
- atom = xlib.XInternAtom(display, b"_GTK_FRAME_EXTENTS", True)
706
+ class _X11:
707
+ """One connection to the X display for a batch of reads.
708
+
709
+ Opened on entry and closed on exit. Every read answers with a neutral value
710
+ (0 for an atom, None for a property) instead of raising, so a window destroyed
711
+ between the xdotool lookup and the read - or a box without libX11 or a display
712
+ at all - degrades to "no property".
713
+ """
714
+
715
+ def __enter__(self):
716
+ self._xlib = _load_xlib()
717
+ self._display = self._xlib.XOpenDisplay(None) if self._xlib else None
718
+ return self
719
+
720
+ def __exit__(self, *_exc):
721
+ if self._display:
722
+ self._xlib.XCloseDisplay(self._display)
723
+
724
+ def atom(self, name):
725
+ """The atom for ``name``, or 0 when it was never interned on this server."""
726
+ if not self._display:
727
+ return 0
728
+ return int(self._xlib.XInternAtom(self._display, name.encode(), True))
729
+
730
+ def window_property(self, window_id, name, max_items=1024):
731
+ """A format-32 property of ``window_id`` (the root window when None) as a
732
+ list of ints, or None when the window or the property does not exist."""
733
+ atom = self.atom(name)
558
734
  if not atom:
559
- return 0, 0
735
+ return None
736
+ if window_id is None:
737
+ window_id = self._xlib.XDefaultRootWindow(self._display)
560
738
  actual_type = ctypes.c_ulong()
561
739
  actual_format = ctypes.c_int()
562
740
  n_items = ctypes.c_ulong()
563
741
  bytes_after = ctypes.c_ulong()
564
742
  data = ctypes.c_void_p()
565
- status = xlib.XGetWindowProperty(
566
- display,
743
+ status = self._xlib.XGetWindowProperty(
744
+ self._display,
567
745
  window_id,
568
746
  atom,
569
747
  0,
570
- 4,
748
+ max_items,
571
749
  False,
572
- _XA_CARDINAL,
750
+ 0, # AnyPropertyType
573
751
  ctypes.byref(actual_type),
574
752
  ctypes.byref(actual_format),
575
753
  ctypes.byref(n_items),
@@ -577,49 +755,69 @@ def _client_side_decoration_offset(window_id):
577
755
  ctypes.byref(data),
578
756
  )
579
757
  if status != 0 or not data.value:
580
- return 0, 0
758
+ return None
581
759
  try:
582
- if actual_format.value != 32 or n_items.value != 4:
583
- return 0, 0
760
+ if actual_format.value != 32 or n_items.value == 0:
761
+ return None
584
762
  # Xlib hands format-32 property items back as C longs.
585
- left, _right, top, _bottom = ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[:4]
586
- return int(left), int(top)
763
+ return [int(v) for v in ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[: n_items.value]]
587
764
  finally:
588
- xlib.XFree(data)
589
- except Exception:
765
+ self._xlib.XFree(data)
766
+
767
+
768
+ def _x11_window_property(window_id, name, max_items=1024):
769
+ """One format-32 property read on its own connection (see ``_X11``)."""
770
+ with _X11() as x11:
771
+ return x11.window_property(window_id, name, max_items)
772
+
773
+
774
+ def _managed_windows_stacked():
775
+ """The window manager's client windows, bottom to top (`_NET_CLIENT_LIST_STACKING`).
776
+
777
+ These are the real, mapped toplevels a user sees - not the unmapped client-leader
778
+ and helper windows a toolkit also creates (which `xdotool search` returns too, and
779
+ which cannot be activated). Empty when the WM does not publish the list.
780
+ """
781
+ return _x11_window_property(None, "_NET_CLIENT_LIST_STACKING") or []
782
+
783
+
784
+ def _client_side_decoration_offset(window_id):
785
+ """(left, top) inset of a window's visible frame within its X11 window.
786
+
787
+ GTK draws client-side decorations - its drop shadow - inside the X11 window and
788
+ advertises their size in `_GTK_FRAME_EXTENTS` (left, right, top, bottom); GTK
789
+ measures window-relative extents from the visible frame, not the X11 window's
790
+ corner. (0, 0) for a window without client-side decorations.
791
+ """
792
+ extents = _x11_window_property(window_id, "_GTK_FRAME_EXTENTS", max_items=4)
793
+ if not extents or len(extents) != 4:
590
794
  return 0, 0
591
- finally:
592
- xlib.XCloseDisplay(display)
795
+ left, _right, top, _bottom = extents
796
+ return left, top
593
797
 
594
798
 
595
- def _collect_chromium(desktop, Atspi, screen_w, screen_h):
596
- """Walk only the Chrome/Chromium AT-SPI app node(s), matched by name.
799
+ def _chromium_apps(desktop):
800
+ """The Chrome/Chromium AT-SPI app node(s), matched by name.
597
801
 
598
- Recovery for when the foreground window is Chromium but the PID-scoped walk
599
- came back empty (a lazily-exported page tree, or an active-window PID that
600
- disagrees with the AT-SPI app PID). Scoping by name keeps the recovery on the
601
- browser alone, so a native app in front is never overlaid with the always-open
602
- browser's labels.
802
+ Recovery for when the foreground window is Chromium but its active-window PID
803
+ disagrees with the AT-SPI app PID (a wrapper or zygote owning the window).
804
+ Scoping by name keeps the recovery on the browser alone, so a native app in
805
+ front is never overlaid with the always-open browser's labels.
603
806
  """
604
- elements = []
807
+ apps = []
605
808
  try:
606
809
  n_apps = desktop.get_child_count()
607
810
  except Exception:
608
- return elements
811
+ return apps
609
812
  for i in range(n_apps):
610
813
  try:
611
814
  app = desktop.get_child_at_index(i)
815
+ name = (app.get_name() or "").lower() if app is not None else ""
612
816
  except Exception:
613
817
  continue
614
- if app is None:
615
- continue
616
- try:
617
- name = (app.get_name() or "").lower()
618
- except Exception:
619
- name = ""
620
818
  if "chrome" in name or "chromium" in name:
621
- _walk_atspi(app, elements, Atspi, screen_w, screen_h)
622
- return elements
819
+ apps.append(app)
820
+ return apps
623
821
 
624
822
 
625
823
  def _within_rect(elements, rect):
@@ -636,7 +834,60 @@ def _within_rect(elements, rect):
636
834
  return kept
637
835
 
638
836
 
837
+ # Retries for a Chromium tree that is still being built: Chromium exports its
838
+ # web content lazily, populating the tree on the first assistive-technology
839
+ # query, so the very first walk after a navigation can come back empty.
840
+ _CHROMIUM_EXPORT_RETRIES = 2
841
+ _CHROMIUM_EXPORT_RETRY_DELAY = 0.6
842
+
843
+
844
+ def _empty_result_diagnostic(window, apps_found, foreground_is_chromium):
845
+ """Why `observe` found nothing, for the agent and for anyone reading the trace.
846
+
847
+ "Found 0 actionable elements" has several unrelated causes with different
848
+ remedies, and the trace cannot tell them apart without this: no application
849
+ window in front (only the desktop, or a window manager reporting nothing
850
+ active), a window without an owning process, an app that is not on the
851
+ accessibility bus at all (a toolkit without AT-SPI support, or a browser that
852
+ started with accessibility off and stays that way until restarted), or an app
853
+ on the bus whose focused window exports no actionable controls.
854
+ """
855
+ if window is None:
856
+ return (
857
+ "No application window is in front: only the desktop is showing, or the "
858
+ "window manager reports nothing active."
859
+ )
860
+ title = window.get("title") or "(untitled)"
861
+ if window.get("pid") is None:
862
+ return (
863
+ f'The focused window "{title}" reports no owning process, '
864
+ "so its accessibility tree cannot be located."
865
+ )
866
+ if not apps_found:
867
+ if foreground_is_chromium:
868
+ return (
869
+ "The browser in front is not exporting an accessibility tree (it is not "
870
+ "registered on the accessibility bus), so none of its controls or page "
871
+ "content can be labeled until it is restarted."
872
+ )
873
+ process = _process_name(window["pid"]) or f"pid {window['pid']}"
874
+ return (
875
+ f'The app in front ("{title}", {process}) is not registered on the '
876
+ "accessibility bus: it exports no accessibility tree."
877
+ )
878
+ return (
879
+ f'The app in front ("{title}") is on the accessibility bus but exports no '
880
+ "actionable controls for its focused window."
881
+ )
882
+
883
+
639
884
  def enumerate_elements(pid=None):
885
+ """Label the foreground window's actionable elements.
886
+
887
+ Returns (elements, window, diagnostic): the numbered elements, the foreground
888
+ window descriptor (None when no application window is in front), and - only
889
+ when no element was found - a diagnostic naming the cause.
890
+ """
640
891
  Atspi = _try_import_atspi()
641
892
  if Atspi is None:
642
893
  exit_with_error(
@@ -644,34 +895,38 @@ def enumerate_elements(pid=None):
644
895
  )
645
896
 
646
897
  screen_w, screen_h = get_screen_size()
647
- target_pid = pid or _get_active_window_pid()
898
+ window = _foreground_window()
899
+ if pid is not None and window is not None:
900
+ window = {**window, "pid": pid, "chromium": _pid_is_chromium(pid)}
901
+ target_pid = pid or (window["pid"] if window is not None else None)
648
902
  # A native foreground app (GTK4 etc.) leaves the AT-SPI VISIBLE/SHOWING flags
649
903
  # unset on its on-screen descendants, so trust geometry for it; Chromium's web
650
904
  # a11y flags are reliable, so keep requiring VISIBLE there.
651
- foreground_is_chromium = _active_window_is_chromium(target_pid)
905
+ foreground_is_chromium = bool(window and window["chromium"])
906
+ if window is not None and _pid_is_libreoffice(target_pid):
907
+ _refresh_libreoffice_geometry(window)
652
908
 
653
909
  desktop = Atspi.get_desktop(0)
910
+ apps = _apps_for_pid(desktop, target_pid)
654
911
  elements = _collect_elements(
655
- desktop, target_pid, Atspi, screen_w, screen_h, trust_geometry=not foreground_is_chromium
912
+ apps, window, Atspi, screen_w, screen_h, trust_geometry=not foreground_is_chromium
656
913
  )
657
914
 
658
- # The foreground app exported its own actionable tree - use it verbatim. It is
659
- # already scoped to the single window the agent sees (one desktop, WYSIWYG).
660
915
  if not elements and foreground_is_chromium:
661
- # Chromium publishes its accessibility tree lazily, and its active-window
662
- # PID can disagree with the AT-SPI app PID, so the PID-scoped walk above
663
- # can miss it. Recover by walking only the Chrome app node(s), retrying
664
- # briefly for the lazy export, then clip to the focused window. We never
665
- # fall back to the whole desktop, which would overlay the always-open
666
- # browser's labels on top of whatever native app is in front.
667
- active_rect = _get_active_window_geometry()
668
- for _ in range(2):
669
- time.sleep(0.6)
916
+ # The browser's PID-scoped walk came back empty: either its tree is still
917
+ # being exported, or its window PID is not its AT-SPI PID. Retry briefly on
918
+ # the browser's own app node(s), clipped to the focused window. Never the
919
+ # whole desktop, which would overlay the always-open browser's labels on
920
+ # whatever native app is in front.
921
+ for _ in range(_CHROMIUM_EXPORT_RETRIES):
922
+ time.sleep(_CHROMIUM_EXPORT_RETRY_DELAY)
670
923
  desktop = Atspi.get_desktop(0)
671
- elements = _collect_chromium(desktop, Atspi, screen_w, screen_h)
924
+ apps = apps or _chromium_apps(desktop)
925
+ elements = _within_rect(
926
+ _collect_elements(apps, window, Atspi, screen_w, screen_h), window
927
+ )
672
928
  if elements:
673
929
  break
674
- elements = _within_rect(elements, active_rect)
675
930
 
676
931
  # A native (non-Chromium) foreground app that exports nothing yields an empty
677
932
  # result on purpose: the agent falls back to screenshot_grid coordinates
@@ -682,7 +937,10 @@ def enumerate_elements(pid=None):
682
937
  for idx, el in enumerate(deduped):
683
938
  el["number"] = idx + 1
684
939
 
685
- return deduped
940
+ diagnostic = None
941
+ if not deduped:
942
+ diagnostic = _empty_result_diagnostic(window, bool(apps), foreground_is_chromium)
943
+ return deduped, window, diagnostic
686
944
 
687
945
 
688
946
  # ---------------------------------------------------------------------------
@@ -1198,23 +1456,33 @@ def _desktop_entry_lines(content):
1198
1456
  yield line
1199
1457
 
1200
1458
 
1459
+ def _desktop_entry_bool(fields, key):
1460
+ return fields.get(key, "").strip().lower() == "true"
1461
+
1462
+
1201
1463
  def _resolve_from_desktop_files(name):
1202
- """Search .desktop files for an app matching *name*.
1464
+ """Search .desktop files for an app matching *name*, best match first.
1203
1465
 
1204
1466
  Matches the Name= and GenericName= fields of the main group
1205
1467
  (case-insensitive) and the desktop file basename (e.g. "chromium" matches
1206
- "chromium-browser.desktop"). Exact Name/GenericName matches order before
1207
- basename matches: "libreoffice" is a substring of every libreoffice-*.desktop
1208
- basename but names the Start Center.
1209
-
1210
- Returns a list of (argv, desktop_id) launch candidates. argv is the Exec=
1211
- command with its arguments kept and the field codes removed, so "LibreOffice
1212
- Calc" launches `libreoffice --calc` - not the bare `libreoffice`, which opens
1213
- the Start Center instead of the component the agent asked for.
1468
+ "chromium-browser.desktop"). A Name= match ranks before a GenericName= match,
1469
+ which ranks before a basename match - "Text Editor" is the Name of GNOME's
1470
+ editor but only the GenericName of vim's entry, and "libreoffice" is a
1471
+ substring of every libreoffice-*.desktop basename but names the Start Center.
1472
+ Ties break on the desktop id, so the choice never depends on directory order.
1473
+
1474
+ Entries that cannot show a window of their own are left out: `Terminal=true`
1475
+ programs (vim, htop) run inside a terminal emulator the agent has to open
1476
+ first, and `Hidden`/`NoDisplay` entries are not launchers a user sees.
1477
+
1478
+ Returns a list of (argv, desktop_id, window_classes) launch candidates. argv is
1479
+ the Exec= command with its arguments kept and the field codes removed, so
1480
+ "LibreOffice Calc" launches `libreoffice --calc` - not the bare `libreoffice`,
1481
+ which opens the Start Center instead of the component the agent asked for.
1482
+ window_classes are the WM_CLASS patterns that identify the entry's windows.
1214
1483
  """
1215
1484
  name_lower = name.lower()
1216
- exact = []
1217
- partial = []
1485
+ ranked = []
1218
1486
 
1219
1487
  for desktop_dir in _DESKTOP_DIRS:
1220
1488
  for filepath in glob.glob(os.path.join(desktop_dir, "*.desktop")):
@@ -1224,48 +1492,69 @@ def _resolve_from_desktop_files(name):
1224
1492
  except Exception:
1225
1493
  continue
1226
1494
 
1227
- names = set()
1228
- exec_cmd = None
1495
+ fields = {}
1229
1496
  for line in _desktop_entry_lines(content):
1230
- if line.startswith("Name=") or line.startswith("GenericName="):
1231
- names.add(line.split("=", 1)[1].strip().lower())
1232
- elif line.startswith("Exec=") and exec_cmd is None:
1233
- exec_cmd = _DESKTOP_FIELD_CODES.sub("", line.split("=", 1)[1].strip())
1497
+ key, sep, value = line.partition("=")
1498
+ if sep and key.strip() not in fields:
1499
+ fields[key.strip()] = value.strip()
1500
+ if fields.get("Type", "Application") != "Application":
1501
+ continue
1502
+ if any(_desktop_entry_bool(fields, key) for key in ("Terminal", "Hidden", "NoDisplay")):
1503
+ continue
1234
1504
 
1235
1505
  desktop_id = os.path.basename(filepath)
1236
1506
  basename = os.path.splitext(desktop_id)[0].lower()
1237
- if name_lower in names:
1238
- bucket = exact
1507
+ if fields.get("Name", "").lower() == name_lower:
1508
+ rank = 0
1509
+ elif fields.get("GenericName", "").lower() == name_lower:
1510
+ rank = 1
1239
1511
  elif name_lower in basename:
1240
- bucket = partial
1512
+ rank = 2
1241
1513
  else:
1242
1514
  continue
1243
1515
 
1516
+ exec_cmd = _DESKTOP_FIELD_CODES.sub("", fields.get("Exec", ""))
1244
1517
  try:
1245
1518
  argv = shlex.split(exec_cmd) if exec_cmd else []
1246
1519
  except ValueError:
1247
1520
  argv = []
1248
- bucket.append((argv, desktop_id))
1521
+ classes = _window_classes_for_entry(argv, desktop_id, fields.get("StartupWMClass"))
1522
+ ranked.append((rank, desktop_id, argv, classes))
1249
1523
 
1250
- return exact + partial
1524
+ ranked.sort(key=lambda item: (item[0], item[1]))
1525
+ return [(argv, desktop_id, classes) for _rank, desktop_id, argv, classes in ranked]
1251
1526
 
1252
1527
 
1253
- def _window_classes_for_launch(argv, desktop_id):
1254
- """WM_CLASS regexes that identify the window of a launch we just made.
1528
+ # Exec= commands that are wrappers around the real program, so their name says
1529
+ # nothing about the window's class.
1530
+ _LAUNCHER_WRAPPERS = {"env", "sh", "bash", "flatpak", "snap", "gtk-launch", "xdg-open"}
1255
1531
 
1256
- A .desktop id is the most specific signal (`libreoffice-calc` names the Calc
1257
- window, never the Start Center that a bare `libreoffice` class would also
1258
- match); a reverse-DNS id (`org.gnome.Evince`) also contributes its last
1259
- segment, since GTK3 apps set WM_CLASS from the program name. Without a
1260
- desktop entry, the executable's basename is all there is.
1532
+
1533
+ def _window_classes_for_entry(argv, desktop_id, startup_wm_class=None):
1534
+ """WM_CLASS patterns (case-insensitive regexes) that identify an entry's windows.
1535
+
1536
+ `StartupWMClass=` is the entry's own statement of its class and is used alone
1537
+ when present. LibreOffice's component entries carry it, and the guesses below
1538
+ would otherwise name the whole family too: `libreoffice`, the Exec= basename,
1539
+ is a substring of every component's class, so asking for Calc would focus an
1540
+ open Writer window instead of launching Calc. Without it, a .desktop id is the
1541
+ most specific signal; a reverse-DNS id (`org.gnome.Evince`) also contributes
1542
+ its last segment, since GTK3 apps set WM_CLASS from the program name; and the
1543
+ executable's basename covers apps whose class is their binary
1544
+ (`gnome-calculator`) - without a desktop entry it is all there is.
1261
1545
  """
1546
+ if startup_wm_class:
1547
+ return [re.escape(startup_wm_class)]
1548
+ classes = []
1262
1549
  if desktop_id:
1263
1550
  base = os.path.splitext(desktop_id)[0]
1264
- classes = [base]
1551
+ classes.append(base)
1265
1552
  if "." in base:
1266
1553
  classes.append(base.rsplit(".", 1)[1])
1267
- return classes
1268
- return [os.path.basename(argv[0])]
1554
+ if argv and os.path.basename(argv[0]) not in _LAUNCHER_WRAPPERS:
1555
+ classes.append(os.path.basename(argv[0]))
1556
+ seen = set()
1557
+ return [re.escape(c) for c in classes if c and not (c.lower() in seen or seen.add(c.lower()))]
1269
1558
 
1270
1559
 
1271
1560
  # WM_CLASS regex fragments for apps whose window title does not match the app name
@@ -1281,94 +1570,147 @@ _WINDOW_CLASS_ALIASES = {
1281
1570
  "firefox": ["firefox"],
1282
1571
  }
1283
1572
 
1573
+ # How long the window manager gets to report a requested window as active before
1574
+ # the next activation method is tried. Mutter answers a pager-sourced
1575
+ # _NET_ACTIVE_WINDOW request within a frame or two; the bound only matters when a
1576
+ # request is refused outright.
1577
+ _ACTIVATION_CONFIRM_TIMEOUT = 1.0
1284
1578
 
1285
- def activate_app(name):
1286
- """Focus the app's window, launching the app first when it has none.
1287
1579
 
1288
- Returns True when a window of the app is focused, False when the app was
1289
- launched but has not shown a window within the launch timeout - it is
1290
- still starting, so the caller should wait rather than launch it again.
1291
- Raises RuntimeError when nothing launchable matches the name, or when the
1292
- launched process exits with a failure before showing a window.
1580
+ def _search_windows(flag, pattern):
1581
+ """Visible X11 windows matching an xdotool search (`--class` / `--name`).
1582
+
1583
+ `--onlyvisible` leaves out the unmapped client-leader and helper windows a
1584
+ toolkit creates alongside its real toplevel: they share its class and title,
1585
+ sort first, and cannot be activated (the WM has never managed them).
1293
1586
  """
1294
- def find_windows(flag, query):
1295
- try:
1296
- r = subprocess.run(
1297
- ["xdotool", "search", flag, query],
1298
- capture_output=True, text=True, timeout=5,
1299
- )
1300
- return [wid for wid in r.stdout.strip().split("\n") if wid]
1301
- except Exception:
1302
- return []
1303
-
1304
- def activate_existing_window(launch_classes=()):
1305
- # Match an already-open window by title (--name) and by window class
1306
- # (--class). A Chrome `--app=<url>` window carries the page title, not the
1307
- # app name, so a title-only search misses it; the class aliases below focus
1308
- # the running browser window instead of launching a second, duplicate one.
1309
- # After a launch, the classes derived from what was launched find windows
1310
- # whose title never carries the app name (Evince shows the document's).
1311
- attempts = [("--name", name), ("--name", name.lower())]
1312
- attempts += [("--class", cls) for cls in _WINDOW_CLASS_ALIASES.get(name.lower(), [])]
1313
- attempts += [("--class", cls) for cls in launch_classes]
1314
- for flag, query in attempts:
1315
- wids = find_windows(flag, query)
1316
- if wids:
1317
- # No --sync: `windowactivate --sync` blocks until the WM confirms
1318
- # activation, which never arrives for a borderless/maximized
1319
- # window under fluxbox/xfwm and hangs until the subprocess
1320
- # timeout. A plain activate plus a short settle is enough; treat a
1321
- # non-zero exit as best-effort so it can't abort labeling.
1322
- _run(["xdotool", "windowactivate", wids[0]], check=False)
1323
- time.sleep(0.2)
1324
- return True
1325
- return False
1587
+ try:
1588
+ r = subprocess.run(
1589
+ ["xdotool", "search", "--onlyvisible", flag, pattern],
1590
+ capture_output=True, text=True, timeout=5,
1591
+ )
1592
+ return [int(wid) for wid in r.stdout.split() if wid.isdigit()]
1593
+ except Exception:
1594
+ return []
1595
+
1596
+
1597
+ def _find_app_window(name_patterns, class_patterns):
1598
+ """The topmost window of the app, matched by WM_CLASS first, then by title.
1599
+
1600
+ Class is the reliable identity of an app's windows: a title only sometimes
1601
+ carries the app's name (Files shows "Home", a document viewer shows the
1602
+ document's title), and a title search can hit a browser tab that happens to
1603
+ mention the app. Among several windows the one highest in the WM's stacking
1604
+ order is the one the user last worked in. Falls back to xdotool's own order
1605
+ when the WM publishes no stacking list.
1606
+ """
1607
+ stacked = _managed_windows_stacked()
1608
+ attempts = [("--class", c) for c in class_patterns] + [("--name", n) for n in name_patterns]
1609
+ for flag, pattern in attempts:
1610
+ wids = _search_windows(flag, pattern)
1611
+ if not wids:
1612
+ continue
1613
+ if stacked:
1614
+ matched = set(wids)
1615
+ managed = [wid for wid in stacked if wid in matched]
1616
+ if managed:
1617
+ return managed[-1]
1618
+ continue
1619
+ return wids[-1]
1620
+ return None
1621
+
1622
+
1623
+ def _focus_window(window_id):
1624
+ """Make ``window_id`` the active window and confirm the WM agreed.
1326
1625
 
1327
- def wait_for_window(proc, launch_classes):
1328
- deadline = time.time() + _LAUNCH_WINDOW_TIMEOUT
1626
+ A plain `xdotool windowactivate` (a pager-sourced _NET_ACTIVE_WINDOW request)
1627
+ is what mutter and the other EWMH managers honor. Its `--sync` is not used: it
1628
+ waits without a bound of its own for the WM to report the window active, so a
1629
+ refused request would hang until the subprocess timeout. Confirmation is read
1630
+ back from the WM under a bound instead, and a refused request falls through to
1631
+ a raise + focus and then to wmctrl. Returns whether the window ended up active.
1632
+ """
1633
+ methods = [
1634
+ [["xdotool", "windowactivate", str(window_id)]],
1635
+ [["xdotool", "windowraise", str(window_id)], ["xdotool", "windowfocus", str(window_id)]],
1636
+ ]
1637
+ if shutil.which("wmctrl"):
1638
+ methods.append([["wmctrl", "-ia", str(window_id)]])
1639
+ for commands in methods:
1640
+ for command in commands:
1641
+ try:
1642
+ subprocess.run(command, capture_output=True, text=True, timeout=5, check=False)
1643
+ except (FileNotFoundError, subprocess.TimeoutExpired):
1644
+ continue
1645
+ deadline = time.time() + _ACTIVATION_CONFIRM_TIMEOUT
1329
1646
  while time.time() < deadline:
1330
- if activate_existing_window(launch_classes):
1647
+ active = _get_active_window_id()
1648
+ # A WM that publishes no active window cannot confirm or deny; the
1649
+ # request was delivered, which is all there is to go on.
1650
+ if active is None or active == window_id:
1331
1651
  return True
1332
- # A launcher that hands off to a running instance exits 0 with no
1333
- # window of its own, so only a failure exit ends the wait early.
1334
- code = proc.poll()
1335
- if code is not None and code != 0:
1336
- status = f"signal {-code}" if code < 0 else f"exit status {code}"
1337
- raise RuntimeError(
1338
- f"'{name}' crashed on launch ({status}) before showing a window."
1339
- )
1340
- time.sleep(0.2)
1341
- return False
1652
+ time.sleep(0.1)
1653
+ return False
1342
1654
 
1343
- if activate_existing_window():
1344
- return True
1345
1655
 
1656
+ def activate_app(name):
1657
+ """Focus the app's window, launching the app first when it has none.
1658
+
1659
+ Returns True when a window of the app is in front, False when the app was
1660
+ launched but has not shown a window within the launch timeout - it is still
1661
+ starting, so the caller should wait rather than launch it again. Raises
1662
+ RuntimeError when nothing launchable matches the name, when the launched
1663
+ process exits with a failure before showing a window, or when the app has a
1664
+ window the window manager refuses to bring to the front.
1665
+ """
1346
1666
  candidates = _resolve_from_desktop_files(name)
1347
- known = {argv[0] for argv, _ in candidates if argv}
1667
+ name_patterns = [re.escape(name)]
1668
+ class_patterns = list(_WINDOW_CLASS_ALIASES.get(name.lower(), []))
1669
+ for _argv, _desktop_id, classes in candidates:
1670
+ class_patterns += [c for c in classes if c not in class_patterns]
1671
+
1672
+ def focus_or_raise(window_id):
1673
+ if _focus_window(window_id):
1674
+ return True
1675
+ title = _window_title(window_id) or "(untitled)"
1676
+ raise RuntimeError(
1677
+ f"'{name}' has a window (\"{title}\") but the window manager would not "
1678
+ "bring it to the front."
1679
+ )
1680
+
1681
+ # An app that is already running is focused, never launched again: a second
1682
+ # launch of a single-instance app opens a second window (or just re-presents
1683
+ # the first), and the agent then labels two of the same window.
1684
+ existing = _find_app_window(name_patterns, class_patterns)
1685
+ if existing is not None:
1686
+ return focus_or_raise(existing)
1687
+
1688
+ known = {argv[0] for argv, _, _ in candidates if argv}
1348
1689
  for direct in (name.lower(), name):
1349
1690
  if direct not in known:
1350
- candidates.append(([direct], None))
1691
+ candidates.append(([direct], None, _window_classes_for_entry([direct], None)))
1351
1692
  known.add(direct)
1352
1693
 
1353
1694
  # Launch once: the first candidate whose executable exists. Each candidate
1354
1695
  # names the same app, so launching more than one would only stack windows
1355
1696
  # and make the agent wait for every one in turn.
1356
1697
  launch = None
1357
- for argv, desktop_id in candidates:
1698
+ for argv, desktop_id, classes in candidates:
1358
1699
  if not argv:
1359
1700
  continue
1360
1701
  executable = shutil.which(argv[0])
1361
1702
  if executable is not None:
1362
- launch = ([executable, *argv[1:]], desktop_id)
1703
+ launch = ([executable, *argv[1:]], desktop_id, classes)
1363
1704
  break
1364
1705
 
1365
1706
  # A desktop entry whose Exec= cannot be resolved directly (D-Bus activated
1366
1707
  # apps, launcher wrappers off PATH) still launches through gtk-launch.
1367
1708
  if launch is None:
1368
1709
  gtk_launch = shutil.which("gtk-launch")
1369
- desktop_ids = [desktop_id for _, desktop_id in candidates if desktop_id]
1370
- if gtk_launch and desktop_ids:
1371
- launch = ([gtk_launch, desktop_ids[0]], desktop_ids[0])
1710
+ with_entry = [(desktop_id, classes) for _, desktop_id, classes in candidates if desktop_id]
1711
+ if gtk_launch and with_entry:
1712
+ desktop_id, classes = with_entry[0]
1713
+ launch = ([gtk_launch, desktop_id], desktop_id, classes)
1372
1714
 
1373
1715
  if launch is None:
1374
1716
  raise RuntimeError(
@@ -1376,12 +1718,54 @@ def activate_app(name):
1376
1718
  f".desktop entry, or executable matches it."
1377
1719
  )
1378
1720
 
1379
- argv, desktop_id = launch
1721
+ argv, _desktop_id, launch_classes = launch
1722
+ windows_before = set(_managed_windows_stacked())
1380
1723
  try:
1381
- proc = subprocess.Popen(argv, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
1724
+ # Its own session with no inherited stdio: the app must outlive this
1725
+ # driver process (one process per command) and never block on, or be
1726
+ # killed with, the driver's pipes and process group.
1727
+ proc = subprocess.Popen(
1728
+ argv,
1729
+ stdin=subprocess.DEVNULL,
1730
+ stdout=subprocess.DEVNULL,
1731
+ stderr=subprocess.DEVNULL,
1732
+ start_new_session=True,
1733
+ )
1382
1734
  except OSError as e:
1383
1735
  raise RuntimeError(f"Could not launch '{name}' ({argv[0]}): {e}") from e
1384
- return wait_for_window(proc, _window_classes_for_launch(argv, desktop_id))
1736
+
1737
+ def launched_window():
1738
+ # The app's window by class (its launcher wrapper may exec a differently
1739
+ # named binary - LibreOffice's `soffice` - and a single-instance app may
1740
+ # hand the launch to its running process), else whatever managed window
1741
+ # appeared since the launch: that is the app's, whatever class it chose.
1742
+ window_id = _find_app_window(name_patterns, launch_classes)
1743
+ if window_id is not None:
1744
+ return window_id
1745
+ appeared = [wid for wid in _managed_windows_stacked() if wid not in windows_before]
1746
+ return appeared[-1] if appeared else None
1747
+
1748
+ deadline = time.time() + _LAUNCH_WINDOW_TIMEOUT
1749
+ while time.time() < deadline:
1750
+ window_id = launched_window()
1751
+ # A window the WM will not activate yet is still settling (a splash, a
1752
+ # frame the app is about to replace with its real one): keep waiting for
1753
+ # one it will, rather than report the app as unfocusable.
1754
+ if window_id is not None and _focus_window(window_id):
1755
+ return True
1756
+ # A launcher that hands off to a running instance exits 0 with no
1757
+ # window of its own, so only a failure exit ends the wait early.
1758
+ code = proc.poll()
1759
+ if code is not None and code != 0:
1760
+ status = f"signal {-code}" if code < 0 else f"exit status {code}"
1761
+ raise RuntimeError(
1762
+ f"'{name}' crashed on launch ({status}) before showing a window."
1763
+ )
1764
+ time.sleep(0.2)
1765
+ window_id = launched_window()
1766
+ if window_id is not None:
1767
+ return focus_or_raise(window_id)
1768
+ return False
1385
1769
 
1386
1770
 
1387
1771
  # ---------------------------------------------------------------------------
@@ -1446,7 +1830,7 @@ def main():
1446
1830
  else:
1447
1831
  i += 1
1448
1832
 
1449
- elements = enumerate_elements(pid=pid)
1833
+ elements, window, diagnostic = enumerate_elements(pid=pid)
1450
1834
  capture_screenshot(screenshot_path)
1451
1835
  annotate_screenshot(screenshot_path, elements)
1452
1836
 
@@ -1454,6 +1838,10 @@ def main():
1454
1838
  "elements": elements,
1455
1839
  "screenshotPath": screenshot_path,
1456
1840
  }
1841
+ if window is not None:
1842
+ result["window"] = {"title": window["title"], "pid": window["pid"]}
1843
+ if diagnostic:
1844
+ result["diagnostic"] = diagnostic
1457
1845
  print(json.dumps(result, indent=2))
1458
1846
 
1459
1847
  elif command == "click":