@octavus/agent 1.0.20 → 1.0.24

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -13,13 +13,13 @@ Dependencies (apt):
13
13
 
14
14
  Usage mirrors the macOS driver exactly:
15
15
  linux-driver.py observe --screenshot /tmp/shot.png [--pid PID]
16
- linux-driver.py click <x> <y>
16
+ linux-driver.py click <x> <y> [--hold-seconds N]
17
17
  linux-driver.py double-click <x> <y>
18
18
  linux-driver.py right-click <x> <y>
19
19
  linux-driver.py type <text>
20
20
  linux-driver.py key <key-spec>
21
21
  linux-driver.py scroll <direction> [steps] [--at x y]
22
- linux-driver.py drag <from_x> <from_y> <to_x> <to_y> [--steps N] [--path x1,y1 ...]
22
+ linux-driver.py drag <from_x> <from_y> <to_x> <to_y> [--steps N] [--path x1,y1 ...] [--hold-seconds N]
23
23
  linux-driver.py screenshot <path>
24
24
  linux-driver.py screenshot-grid <path> --region x y w h --grid <2|3>
25
25
  linux-driver.py activate <app-name>
@@ -29,6 +29,8 @@ import glob
29
29
  import json
30
30
  import os
31
31
  import re
32
+ import shlex
33
+ import signal
32
34
  import subprocess
33
35
  import sys
34
36
  import time
@@ -38,8 +40,8 @@ import shutil
38
40
  # Output helpers (match Swift driver JSON contract)
39
41
  # ---------------------------------------------------------------------------
40
42
 
41
- def print_success(path=None):
42
- resp = {"success": True}
43
+ def print_success(path=None, **fields):
44
+ resp = {"success": True, **fields}
43
45
  if path is not None:
44
46
  resp["path"] = path
45
47
  print(json.dumps(resp))
@@ -133,8 +135,22 @@ def _overlaps_screen(x, y, w, h, screen_w, screen_h):
133
135
 
134
136
 
135
137
  def _walk_atspi(
136
- node, elements, Atspi, screen_w, screen_h, depth=0, max_depth=20, trust_geometry=False
138
+ node,
139
+ elements,
140
+ Atspi,
141
+ screen_w,
142
+ screen_h,
143
+ depth=0,
144
+ max_depth=20,
145
+ trust_geometry=False,
146
+ window_origin=None,
137
147
  ):
148
+ """Collect the on-screen actionable elements under ``node``.
149
+
150
+ ``window_origin`` is set for a toolkit whose extents are window-relative
151
+ (see ``_uses_window_relative_extents``): extents are then read in WINDOW
152
+ coordinates and offset by the window's on-screen client-area origin.
153
+ """
138
154
  if depth > max_depth:
139
155
  return
140
156
 
@@ -151,8 +167,17 @@ def _walk_atspi(
151
167
  try:
152
168
  comp = node.get_component_iface()
153
169
  if comp is not None:
154
- extents = comp.get_extents(Atspi.CoordType.SCREEN)
155
- rect = (extents.x, extents.y, extents.width, extents.height)
170
+ if window_origin is None:
171
+ extents = comp.get_extents(Atspi.CoordType.SCREEN)
172
+ rect = (extents.x, extents.y, extents.width, extents.height)
173
+ else:
174
+ extents = comp.get_extents(Atspi.CoordType.WINDOW)
175
+ rect = (
176
+ extents.x + window_origin[0],
177
+ extents.y + window_origin[1],
178
+ extents.width,
179
+ extents.height,
180
+ )
156
181
  except Exception:
157
182
  rect = None
158
183
 
@@ -232,7 +257,15 @@ def _walk_atspi(
232
257
  child = node.get_child_at_index(i)
233
258
  if child is not None:
234
259
  _walk_atspi(
235
- child, elements, Atspi, screen_w, screen_h, depth + 1, max_depth, trust_geometry
260
+ child,
261
+ elements,
262
+ Atspi,
263
+ screen_w,
264
+ screen_h,
265
+ depth + 1,
266
+ max_depth,
267
+ trust_geometry,
268
+ window_origin,
236
269
  )
237
270
  except Exception:
238
271
  pass
@@ -311,10 +344,11 @@ def _get_active_window_pid():
311
344
 
312
345
 
313
346
  def _get_active_window_geometry():
314
- """Screen rect of the focused window as {x, y, width, height}, or None.
347
+ """Screen rect of the focused window as {id, x, y, width, height}, or None.
315
348
 
316
- Used to clip a recovered element set to the window the agent is actually
317
- looking at (one desktop, what-you-see-is-what-you-get).
349
+ The focused window is the one the agent is actually looking at (one desktop,
350
+ what-you-see-is-what-you-get), so it is the reference for anything that must
351
+ be clipped to, or placed within, the visible app.
318
352
  """
319
353
  try:
320
354
  out = subprocess.check_output(
@@ -332,6 +366,7 @@ def _get_active_window_geometry():
332
366
  vals[key.strip()] = value.strip()
333
367
  try:
334
368
  return {
369
+ "id": int(vals["WINDOW"]),
335
370
  "x": int(vals["X"]),
336
371
  "y": int(vals["Y"]),
337
372
  "width": int(vals["WIDTH"]),
@@ -382,10 +417,181 @@ def _collect_elements(desktop, target_pid, Atspi, screen_w, screen_h, trust_geom
382
417
  continue
383
418
  except Exception:
384
419
  continue
385
- _walk_atspi(app, elements, Atspi, screen_w, screen_h, trust_geometry=trust_geometry)
420
+ if _uses_window_relative_extents(app):
421
+ _walk_focused_window(app, elements, Atspi, screen_w, screen_h, trust_geometry)
422
+ else:
423
+ _walk_atspi(app, elements, Atspi, screen_w, screen_h, trust_geometry=trust_geometry)
386
424
  return elements
387
425
 
388
426
 
427
+ def _uses_window_relative_extents(app):
428
+ """Whether the app's AT-SPI extents are relative to its window, not the screen.
429
+
430
+ GTK 4 has no global coordinates: 4.6 answers a SCREEN extents query with
431
+ window-relative values ("Screen coordinates not supported") and later releases
432
+ report every SCREEN origin as 0,0. Taken as screen coordinates, every label
433
+ lands near the screen's top-left corner (GNOME's Activities button and dock)
434
+ instead of on its control. GTK 3 (through the ATK bridge), Chromium, and Qt
435
+ report true screen coordinates.
436
+ """
437
+ try:
438
+ toolkit = (app.get_toolkit_name() or "").lower()
439
+ major = int((app.get_toolkit_version() or "0").split(".")[0])
440
+ except Exception:
441
+ return False
442
+ return toolkit == "gtk" and major >= 4
443
+
444
+
445
+ def _walk_focused_window(app, elements, Atspi, screen_w, screen_h, trust_geometry):
446
+ """Walk the app's focused window, placing its window-relative extents on screen.
447
+
448
+ Only the focused window is walked: the app's other windows have their own
449
+ origins, which the agent cannot see past the window in front anyway.
450
+ """
451
+ window = _get_active_window_geometry()
452
+ if window is None:
453
+ return
454
+ frame = _focused_frame(app, window["id"], Atspi)
455
+ if frame is None:
456
+ return
457
+ left, top = _client_side_decoration_offset(window["id"])
458
+ _walk_atspi(
459
+ frame,
460
+ elements,
461
+ Atspi,
462
+ screen_w,
463
+ screen_h,
464
+ trust_geometry=trust_geometry,
465
+ window_origin=(window["x"] + left, window["y"] + top),
466
+ )
467
+
468
+
469
+ def _focused_frame(app, window_id, Atspi):
470
+ """The app's toplevel accessible for the focused window: the one AT-SPI marks
471
+ ACTIVE, else the one titled like the focused X11 window."""
472
+ frames = []
473
+ try:
474
+ for i in range(app.get_child_count()):
475
+ frame = app.get_child_at_index(i)
476
+ if frame is not None:
477
+ frames.append(frame)
478
+ except Exception:
479
+ return None
480
+
481
+ for frame in frames:
482
+ try:
483
+ if frame.get_state_set().contains(Atspi.StateType.ACTIVE):
484
+ return frame
485
+ except Exception:
486
+ continue
487
+
488
+ try:
489
+ title = subprocess.check_output(
490
+ ["xdotool", "getwindowname", str(window_id)], text=True, timeout=5
491
+ ).strip()
492
+ except Exception:
493
+ return None
494
+ for frame in frames:
495
+ try:
496
+ if title and frame.get_name() == title:
497
+ return frame
498
+ except Exception:
499
+ continue
500
+ return None
501
+
502
+
503
+ _XA_CARDINAL = 6
504
+
505
+
506
+ def _client_side_decoration_offset(window_id):
507
+ """(left, top) inset of a window's visible frame within its X11 window.
508
+
509
+ GTK draws client-side decorations - its drop shadow - inside the X11 window and
510
+ advertises their size in `_GTK_FRAME_EXTENTS` (left, right, top, bottom); GTK
511
+ measures window-relative extents from the visible frame, not the X11 window's
512
+ corner. Read through libX11 directly because xprop is not installed on the
513
+ cloud image. (0, 0) for a window without client-side decorations.
514
+ """
515
+ try:
516
+ import ctypes
517
+ import ctypes.util
518
+
519
+ xlib = ctypes.cdll.LoadLibrary(ctypes.util.find_library("X11") or "libX11.so.6")
520
+ except Exception:
521
+ return 0, 0
522
+ xlib.XOpenDisplay.argtypes = [ctypes.c_char_p]
523
+ xlib.XOpenDisplay.restype = ctypes.c_void_p
524
+ xlib.XInternAtom.argtypes = [ctypes.c_void_p, ctypes.c_char_p, ctypes.c_int]
525
+ xlib.XInternAtom.restype = ctypes.c_ulong
526
+ xlib.XGetWindowProperty.argtypes = [
527
+ ctypes.c_void_p,
528
+ ctypes.c_ulong,
529
+ ctypes.c_ulong,
530
+ ctypes.c_long,
531
+ ctypes.c_long,
532
+ ctypes.c_int,
533
+ ctypes.c_ulong,
534
+ ctypes.POINTER(ctypes.c_ulong),
535
+ ctypes.POINTER(ctypes.c_int),
536
+ ctypes.POINTER(ctypes.c_ulong),
537
+ ctypes.POINTER(ctypes.c_ulong),
538
+ ctypes.POINTER(ctypes.c_void_p),
539
+ ]
540
+ xlib.XFree.argtypes = [ctypes.c_void_p]
541
+ xlib.XCloseDisplay.argtypes = [ctypes.c_void_p]
542
+ xlib.XSetErrorHandler.restype = ctypes.c_void_p
543
+
544
+ # Xlib's default error handler exits the process. The focused window can be
545
+ # destroyed between the xdotool lookup and this read (BadWindow), which must
546
+ # degrade to no offset rather than take the driver down mid-observe. The
547
+ # callback has to stay referenced while the display is open.
548
+ ignore_x_errors = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p)(
549
+ lambda _display, _error: 0
550
+ )
551
+ xlib.XSetErrorHandler(ignore_x_errors)
552
+
553
+ display = xlib.XOpenDisplay(None)
554
+ if not display:
555
+ return 0, 0
556
+ try:
557
+ atom = xlib.XInternAtom(display, b"_GTK_FRAME_EXTENTS", True)
558
+ if not atom:
559
+ return 0, 0
560
+ actual_type = ctypes.c_ulong()
561
+ actual_format = ctypes.c_int()
562
+ n_items = ctypes.c_ulong()
563
+ bytes_after = ctypes.c_ulong()
564
+ data = ctypes.c_void_p()
565
+ status = xlib.XGetWindowProperty(
566
+ display,
567
+ window_id,
568
+ atom,
569
+ 0,
570
+ 4,
571
+ False,
572
+ _XA_CARDINAL,
573
+ ctypes.byref(actual_type),
574
+ ctypes.byref(actual_format),
575
+ ctypes.byref(n_items),
576
+ ctypes.byref(bytes_after),
577
+ ctypes.byref(data),
578
+ )
579
+ if status != 0 or not data.value:
580
+ return 0, 0
581
+ try:
582
+ if actual_format.value != 32 or n_items.value != 4:
583
+ return 0, 0
584
+ # Xlib hands format-32 property items back as C longs.
585
+ left, _right, top, _bottom = ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[:4]
586
+ return int(left), int(top)
587
+ finally:
588
+ xlib.XFree(data)
589
+ except Exception:
590
+ return 0, 0
591
+ finally:
592
+ xlib.XCloseDisplay(display)
593
+
594
+
389
595
  def _collect_chromium(desktop, Atspi, screen_w, screen_h):
390
596
  """Walk only the Chrome/Chromium AT-SPI app node(s), matched by name.
391
597
 
@@ -713,30 +919,109 @@ def annotate_screenshot(screenshot_path, elements):
713
919
  # slightly-less-synchronous move instead of a hard failure.
714
920
  _MOUSEMOVE_SYNC_TIMEOUT = 3
715
921
 
716
- def mouse_move(x, y):
717
- ix, iy = str(int(x)), str(int(y))
922
+ # The longest a press-and-hold may keep the button down. A command's timeout
923
+ # grows by the requested hold, so this bound is what actually limits it.
924
+ _MAX_HOLD_SECONDS = 20
925
+
926
+ # Every keysym a modifier can be held under, left and right variants.
927
+ _MODIFIER_KEYSYMS = [
928
+ "Control_L", "Control_R", "Alt_L", "Alt_R", "Shift_L", "Shift_R", "Super_L", "Super_R",
929
+ ]
930
+ _MODIFIER_KEYUP_TIMEOUT = 3
931
+
932
+ def release_modifiers(keysyms=_MODIFIER_KEYSYMS):
933
+ """Release modifier keys the X server still counts as held.
934
+
935
+ The driver runs one process per action and has no hold-key primitive, so a
936
+ modifier that is logically down when an action starts is always stale - most
937
+ often a combo whose release never landed because the ``xdotool key`` that
938
+ pressed it was killed by the ``_run`` timeout under display lag. Left in
939
+ place, a stale Ctrl turns the next wheel scroll into browser zoom and the
940
+ next click into a modified click. ``--clearmodifiers`` does not cure it: it
941
+ restores whatever it found held once its own keystroke is done. Best-effort
942
+ and never fatal: a failed release must not fail the action itself.
943
+ """
944
+ try:
945
+ subprocess.run(
946
+ ["xdotool", "keyup", *keysyms],
947
+ capture_output=True,
948
+ text=True,
949
+ timeout=_MODIFIER_KEYUP_TIMEOUT,
950
+ check=False,
951
+ )
952
+ except (FileNotFoundError, subprocess.TimeoutExpired):
953
+ pass
954
+
955
+ def _move_pointer(ix, iy):
956
+ """Move the pointer to integer (ix, iy) with a bounded ``--sync`` wait.
957
+
958
+ Returns True when the X server confirmed the arrival. When the sync stalls
959
+ (or errors) the move is re-issued without ``--sync``, which returns
960
+ immediately but carries no arrival confirmation, and False is returned so
961
+ the caller can give the async motion a longer settle.
962
+ """
718
963
  try:
719
964
  subprocess.run(
720
- ["xdotool", "mousemove", "--sync", ix, iy],
965
+ ["xdotool", "mousemove", "--sync", str(ix), str(iy)],
721
966
  capture_output=True,
722
967
  text=True,
723
968
  timeout=_MOUSEMOVE_SYNC_TIMEOUT,
724
969
  check=True,
725
970
  )
726
- time.sleep(0.025)
727
- return
971
+ return True
728
972
  except FileNotFoundError:
729
973
  exit_with_error("Required command not found: xdotool")
730
974
  except (subprocess.TimeoutExpired, subprocess.CalledProcessError):
731
975
  pass
732
- # Sync confirmation stalled - re-issue without --sync (returns immediately)
733
- # and give the async motion a slightly longer settle before the caller
734
- # clicks, since a non-sync move carries no server arrival confirmation.
735
- _run(["xdotool", "mousemove", ix, iy])
736
- time.sleep(0.1)
976
+ _run(["xdotool", "mousemove", str(ix), str(iy)])
977
+ return False
737
978
 
738
- def mouse_click(x, y):
979
+ def _pointer_location():
980
+ """The pointer's current integer (x, y), or None when it cannot be read."""
981
+ try:
982
+ out = subprocess.run(
983
+ ["xdotool", "getmouselocation", "--shell"],
984
+ capture_output=True,
985
+ text=True,
986
+ timeout=_MOUSEMOVE_SYNC_TIMEOUT,
987
+ check=True,
988
+ ).stdout
989
+ except (FileNotFoundError, subprocess.TimeoutExpired, subprocess.CalledProcessError):
990
+ return None
991
+ values = dict(line.split("=", 1) for line in out.split() if "=" in line)
992
+ try:
993
+ return int(values["X"]), int(values["Y"])
994
+ except (KeyError, ValueError):
995
+ return None
996
+
997
+ def mouse_move(x, y):
998
+ target = (int(x), int(y))
999
+ # `mousemove --sync` waits for the pointer to leave its origin, so a move to
1000
+ # the pixel it is already on never confirms and would burn the whole sync
1001
+ # timeout (clicking the same element twice, a drag that starts where the last
1002
+ # click landed, a repeated scroll at one spot). Nothing to move: return.
1003
+ if _pointer_location() == target:
1004
+ return
1005
+ confirmed = _move_pointer(*target)
1006
+ time.sleep(0.025 if confirmed else 0.1)
1007
+
1008
+ def _hold_button(button, hold_seconds):
1009
+ """Press ``button``, keep it down for ``hold_seconds``, then release it.
1010
+
1011
+ The release runs even when the hold is interrupted, so a press-and-hold
1012
+ can never leave the button logically down.
1013
+ """
1014
+ _run(["xdotool", "mousedown", button])
1015
+ try:
1016
+ time.sleep(hold_seconds)
1017
+ finally:
1018
+ _run(["xdotool", "mouseup", button], check=False)
1019
+
1020
+ def mouse_click(x, y, hold_seconds=0.0):
739
1021
  mouse_move(x, y)
1022
+ if hold_seconds > 0:
1023
+ _hold_button("1", hold_seconds)
1024
+ return
740
1025
  _run(["xdotool", "click", "1"])
741
1026
 
742
1027
  def mouse_double_click(x, y):
@@ -768,6 +1053,14 @@ _MOD_MAP = {
768
1053
  "ctrl": "ctrl", "control": "ctrl",
769
1054
  }
770
1055
 
1056
+ # The keysyms a combo's modifier is pressed under, for the release guarantee.
1057
+ _MOD_KEYSYMS = {
1058
+ "super": ["Super_L", "Super_R"],
1059
+ "shift": ["Shift_L", "Shift_R"],
1060
+ "alt": ["Alt_L", "Alt_R"],
1061
+ "ctrl": ["Control_L", "Control_R"],
1062
+ }
1063
+
771
1064
  def press_key(key_spec):
772
1065
  parts = key_spec.lower().split("+")
773
1066
  mods = []
@@ -782,7 +1075,18 @@ def press_key(key_spec):
782
1075
  xdot_key = _KEY_MAP.get(key_name, key_name)
783
1076
 
784
1077
  combo = "+".join(mods + [xdot_key]) if mods else xdot_key
785
- _run(["xdotool", "key", "--clearmodifiers", combo])
1078
+ if not mods:
1079
+ _run(["xdotool", "key", "--clearmodifiers", combo])
1080
+ return
1081
+ # ``xdotool key`` presses the combo's modifiers, taps the key, then releases
1082
+ # them in order. Killed part-way (the ``_run`` timeout under display lag),
1083
+ # it leaves the modifiers logically held in the X server, and every later
1084
+ # wheel scroll and click arrives modified. The release runs on every exit,
1085
+ # including the SystemExit the timeout raises, so no combo leaves one behind.
1086
+ try:
1087
+ _run(["xdotool", "key", "--clearmodifiers", combo])
1088
+ finally:
1089
+ release_modifiers([keysym for mod in mods for keysym in _MOD_KEYSYMS[mod]])
786
1090
 
787
1091
  def _interpolate_points(from_x, from_y, to_x, to_y, steps):
788
1092
  """Generate linearly interpolated points between two coordinates."""
@@ -795,12 +1099,20 @@ def _interpolate_points(from_x, from_y, to_x, to_y, steps):
795
1099
  return points
796
1100
 
797
1101
 
798
- def mouse_drag(from_x, from_y, to_x, to_y, steps=10, waypoints=None):
1102
+ def mouse_drag(from_x, from_y, to_x, to_y, steps=10, waypoints=None, hold_seconds=0.0):
799
1103
  """Perform a smooth drag from one position to another.
800
1104
 
801
- Uses xdotool mousedown/mousemove/mouseup with --sync to ensure
802
- Chrome recognizes the drag gesture. The 16ms sleep between moves
803
- matches 60fps frame timing.
1105
+ Presses the button at the start - keeping it down for ``hold_seconds``
1106
+ first when given, the press-and-hold-then-slide gesture - then moves
1107
+ through the interpolated points with bounded ``--sync`` waits so Chrome
1108
+ recognizes the drag, and releases at the end. The 16ms sleep between
1109
+ moves matches 60fps frame timing.
1110
+
1111
+ A point that lands on the pixel the pointer is already on is not sent to
1112
+ xdotool: its ``mousemove --sync`` waits for the pointer to leave its
1113
+ origin, so a move to the current position blocks until the command
1114
+ timeout (a zero-length drag, a freehand loop whose points round to one
1115
+ pixel). The frame sleep still runs so the gesture keeps its timing.
804
1116
 
805
1117
  When waypoints are provided the last waypoint is the destination, so
806
1118
  (to_x, to_y) is not appended to the segment list. The button is
@@ -808,8 +1120,11 @@ def mouse_drag(from_x, from_y, to_x, to_y, steps=10, waypoints=None):
808
1120
  cursor stuck in the down state.
809
1121
  """
810
1122
  mouse_move(from_x, from_y)
1123
+ last = (int(from_x), int(from_y))
811
1124
  _run(["xdotool", "mousedown", "1"])
812
1125
  try:
1126
+ if hold_seconds > 0:
1127
+ time.sleep(hold_seconds)
813
1128
  if waypoints:
814
1129
  segments = [(from_x, from_y)] + list(waypoints)
815
1130
  else:
@@ -818,7 +1133,10 @@ def mouse_drag(from_x, from_y, to_x, to_y, steps=10, waypoints=None):
818
1133
  sx, sy = segments[seg_idx]
819
1134
  ex, ey = segments[seg_idx + 1]
820
1135
  for px, py in _interpolate_points(sx, sy, ex, ey, steps):
821
- _run(["xdotool", "mousemove", "--sync", str(int(px)), str(int(py))])
1136
+ target = (int(px), int(py))
1137
+ if target != last:
1138
+ _move_pointer(*target)
1139
+ last = target
822
1140
  time.sleep(0.016)
823
1141
  finally:
824
1142
  _run(["xdotool", "mouseup", "1"], check=False)
@@ -834,8 +1152,10 @@ def do_scroll(direction, steps=3, at_x=None, at_y=None):
834
1152
  time.sleep(0.05)
835
1153
 
836
1154
  button = "5" if direction == "down" else "4"
1155
+ # Chrome turns a wheel event with Ctrl held into zoom, and nine notches
1156
+ # reach its 25% floor; a scroll must never carry a modifier, held or stale.
837
1157
  for _ in range(steps):
838
- _run(["xdotool", "click", "--repeat", "3", button])
1158
+ _run(["xdotool", "click", "--clearmodifiers", "--repeat", "3", button])
839
1159
  time.sleep(0.016)
840
1160
 
841
1161
 
@@ -850,17 +1170,51 @@ _DESKTOP_DIRS = [
850
1170
  ]
851
1171
 
852
1172
 
1173
+ # How long a freshly launched app gets to show a window before `activate` reports
1174
+ # it as launched but not visible yet. LibreOffice and Evince routinely need well
1175
+ # over the few seconds a snappy app takes on these computers. A driver command
1176
+ # has 30 s in total, so this leaves room for the lookups before the launch and
1177
+ # the settle after it.
1178
+ _LAUNCH_WINDOW_TIMEOUT = 20.0
1179
+
1180
+ # The %f/%U/... placeholders in a .desktop Exec= line (files/URLs to open, icon,
1181
+ # caption), none of which apply to a bare launch.
1182
+ _DESKTOP_FIELD_CODES = re.compile(r"\s+%[fFuUdDnNickvm]")
1183
+
1184
+
1185
+ def _desktop_entry_lines(content):
1186
+ """Yield the lines of a .desktop file's ``[Desktop Entry]`` group only.
1187
+
1188
+ ``[Desktop Action ...]`` groups carry their own Name= and Exec= (the
1189
+ LibreOffice Start Center's actions are named "Writer", "Calc", ...), so
1190
+ reading them would match a component's short name to the launcher's entry
1191
+ and open the Start Center for "Calc".
1192
+ """
1193
+ in_entry = False
1194
+ for line in content.splitlines():
1195
+ if line.startswith("["):
1196
+ in_entry = line.strip() == "[Desktop Entry]"
1197
+ elif in_entry:
1198
+ yield line
1199
+
1200
+
853
1201
  def _resolve_from_desktop_files(name):
854
1202
  """Search .desktop files for an app matching *name*.
855
1203
 
856
- Checks the Name= and GenericName= fields (case-insensitive), plus the
857
- desktop file basename (e.g. "chromium" matches "chromium-browser.desktop").
858
- Returns (binaries, desktop_ids) - binary commands from Exec= lines and
859
- matching .desktop filenames for gtk-launch.
1204
+ Matches the Name= and GenericName= fields of the main group
1205
+ (case-insensitive) and the desktop file basename (e.g. "chromium" matches
1206
+ "chromium-browser.desktop"). Exact Name/GenericName matches order before
1207
+ basename matches: "libreoffice" is a substring of every libreoffice-*.desktop
1208
+ basename but names the Start Center.
1209
+
1210
+ Returns a list of (argv, desktop_id) launch candidates. argv is the Exec=
1211
+ command with its arguments kept and the field codes removed, so "LibreOffice
1212
+ Calc" launches `libreoffice --calc` - not the bare `libreoffice`, which opens
1213
+ the Start Center instead of the component the agent asked for.
860
1214
  """
861
1215
  name_lower = name.lower()
862
- binaries = []
863
- desktop_ids = []
1216
+ exact = []
1217
+ partial = []
864
1218
 
865
1219
  for desktop_dir in _DESKTOP_DIRS:
866
1220
  for filepath in glob.glob(os.path.join(desktop_dir, "*.desktop")):
@@ -870,32 +1224,48 @@ def _resolve_from_desktop_files(name):
870
1224
  except Exception:
871
1225
  continue
872
1226
 
873
- basename = os.path.splitext(os.path.basename(filepath))[0].lower()
874
- matched = name_lower in basename
1227
+ names = set()
1228
+ exec_cmd = None
1229
+ for line in _desktop_entry_lines(content):
1230
+ if line.startswith("Name=") or line.startswith("GenericName="):
1231
+ names.add(line.split("=", 1)[1].strip().lower())
1232
+ elif line.startswith("Exec=") and exec_cmd is None:
1233
+ exec_cmd = _DESKTOP_FIELD_CODES.sub("", line.split("=", 1)[1].strip())
1234
+
1235
+ desktop_id = os.path.basename(filepath)
1236
+ basename = os.path.splitext(desktop_id)[0].lower()
1237
+ if name_lower in names:
1238
+ bucket = exact
1239
+ elif name_lower in basename:
1240
+ bucket = partial
1241
+ else:
1242
+ continue
875
1243
 
876
- if not matched:
877
- for line in content.splitlines():
878
- if line.startswith("Name=") or line.startswith("GenericName="):
879
- value = line.split("=", 1)[1].strip().lower()
880
- if value == name_lower:
881
- matched = True
882
- break
1244
+ try:
1245
+ argv = shlex.split(exec_cmd) if exec_cmd else []
1246
+ except ValueError:
1247
+ argv = []
1248
+ bucket.append((argv, desktop_id))
883
1249
 
884
- if not matched:
885
- continue
1250
+ return exact + partial
886
1251
 
887
- desktop_ids.append(os.path.basename(filepath))
888
1252
 
889
- for line in content.splitlines():
890
- if line.startswith("Exec="):
891
- exec_cmd = line.split("=", 1)[1].strip()
892
- exec_cmd = re.sub(r"\s+%[fFuUdDnNickvm]", "", exec_cmd)
893
- binary = exec_cmd.split()[0]
894
- if binary not in binaries:
895
- binaries.append(binary)
896
- break
1253
+ def _window_classes_for_launch(argv, desktop_id):
1254
+ """WM_CLASS regexes that identify the window of a launch we just made.
897
1255
 
898
- return binaries, desktop_ids
1256
+ A .desktop id is the most specific signal (`libreoffice-calc` names the Calc
1257
+ window, never the Start Center that a bare `libreoffice` class would also
1258
+ match); a reverse-DNS id (`org.gnome.Evince`) also contributes its last
1259
+ segment, since GTK3 apps set WM_CLASS from the program name. Without a
1260
+ desktop entry, the executable's basename is all there is.
1261
+ """
1262
+ if desktop_id:
1263
+ base = os.path.splitext(desktop_id)[0]
1264
+ classes = [base]
1265
+ if "." in base:
1266
+ classes.append(base.rsplit(".", 1)[1])
1267
+ return classes
1268
+ return [os.path.basename(argv[0])]
899
1269
 
900
1270
 
901
1271
  # WM_CLASS regex fragments for apps whose window title does not match the app name
@@ -913,6 +1283,14 @@ _WINDOW_CLASS_ALIASES = {
913
1283
 
914
1284
 
915
1285
  def activate_app(name):
1286
+ """Focus the app's window, launching the app first when it has none.
1287
+
1288
+ Returns True when a window of the app is focused, False when the app was
1289
+ launched but has not shown a window within the launch timeout - it is
1290
+ still starting, so the caller should wait rather than launch it again.
1291
+ Raises RuntimeError when nothing launchable matches the name, or when the
1292
+ launched process exits with a failure before showing a window.
1293
+ """
916
1294
  def find_windows(flag, query):
917
1295
  try:
918
1296
  r = subprocess.run(
@@ -923,13 +1301,16 @@ def activate_app(name):
923
1301
  except Exception:
924
1302
  return []
925
1303
 
926
- def activate_existing_window():
1304
+ def activate_existing_window(launch_classes=()):
927
1305
  # Match an already-open window by title (--name) and by window class
928
1306
  # (--class). A Chrome `--app=<url>` window carries the page title, not the
929
1307
  # app name, so a title-only search misses it; the class aliases below focus
930
1308
  # the running browser window instead of launching a second, duplicate one.
1309
+ # After a launch, the classes derived from what was launched find windows
1310
+ # whose title never carries the app name (Evince shows the document's).
931
1311
  attempts = [("--name", name), ("--name", name.lower())]
932
1312
  attempts += [("--class", cls) for cls in _WINDOW_CLASS_ALIASES.get(name.lower(), [])]
1313
+ attempts += [("--class", cls) for cls in launch_classes]
933
1314
  for flag, query in attempts:
934
1315
  wids = find_windows(flag, query)
935
1316
  if wids:
@@ -943,80 +1324,113 @@ def activate_app(name):
943
1324
  return True
944
1325
  return False
945
1326
 
946
- def wait_for_window(timeout_s=4.0):
947
- deadline = time.time() + timeout_s
1327
+ def wait_for_window(proc, launch_classes):
1328
+ deadline = time.time() + _LAUNCH_WINDOW_TIMEOUT
948
1329
  while time.time() < deadline:
949
- if activate_existing_window():
1330
+ if activate_existing_window(launch_classes):
950
1331
  return True
1332
+ # A launcher that hands off to a running instance exits 0 with no
1333
+ # window of its own, so only a failure exit ends the wait early.
1334
+ code = proc.poll()
1335
+ if code is not None and code != 0:
1336
+ status = f"signal {-code}" if code < 0 else f"exit status {code}"
1337
+ raise RuntimeError(
1338
+ f"'{name}' crashed on launch ({status}) before showing a window."
1339
+ )
951
1340
  time.sleep(0.2)
952
1341
  return False
953
1342
 
954
1343
  if activate_existing_window():
955
- return
956
-
957
- desktop_binaries, desktop_ids = _resolve_from_desktop_files(name)
958
-
959
- candidates = list(desktop_binaries)
960
- direct = name.lower()
961
- if direct not in candidates:
962
- candidates.append(direct)
963
- if name != direct and name not in candidates:
964
- candidates.append(name)
965
-
966
- launch_attempted = False
967
- for command in candidates:
968
- executable = shutil.which(command)
969
- if executable is None:
1344
+ return True
1345
+
1346
+ candidates = _resolve_from_desktop_files(name)
1347
+ known = {argv[0] for argv, _ in candidates if argv}
1348
+ for direct in (name.lower(), name):
1349
+ if direct not in known:
1350
+ candidates.append(([direct], None))
1351
+ known.add(direct)
1352
+
1353
+ # Launch once: the first candidate whose executable exists. Each candidate
1354
+ # names the same app, so launching more than one would only stack windows
1355
+ # and make the agent wait for every one in turn.
1356
+ launch = None
1357
+ for argv, desktop_id in candidates:
1358
+ if not argv:
970
1359
  continue
971
-
972
- launch_attempted = True
973
- try:
974
- subprocess.Popen(
975
- [executable],
976
- stdout=subprocess.DEVNULL,
977
- stderr=subprocess.DEVNULL,
978
- )
979
- if wait_for_window():
980
- return
981
- except Exception:
982
- pass
983
-
984
- if not launch_attempted and desktop_ids:
1360
+ executable = shutil.which(argv[0])
1361
+ if executable is not None:
1362
+ launch = ([executable, *argv[1:]], desktop_id)
1363
+ break
1364
+
1365
+ # A desktop entry whose Exec= cannot be resolved directly (D-Bus activated
1366
+ # apps, launcher wrappers off PATH) still launches through gtk-launch.
1367
+ if launch is None:
985
1368
  gtk_launch = shutil.which("gtk-launch")
986
- if gtk_launch:
987
- for desktop_id in desktop_ids:
988
- try:
989
- subprocess.Popen(
990
- [gtk_launch, desktop_id],
991
- stdout=subprocess.DEVNULL,
992
- stderr=subprocess.DEVNULL,
993
- )
994
- if wait_for_window():
995
- return
996
- except Exception:
997
- pass
1369
+ desktop_ids = [desktop_id for _, desktop_id in candidates if desktop_id]
1370
+ if gtk_launch and desktop_ids:
1371
+ launch = ([gtk_launch, desktop_ids[0]], desktop_ids[0])
1372
+
1373
+ if launch is None:
1374
+ raise RuntimeError(
1375
+ f"Could not find an application named '{name}': no open window, "
1376
+ f".desktop entry, or executable matches it."
1377
+ )
998
1378
 
1379
+ argv, desktop_id = launch
999
1380
  try:
1000
- active_pid = _get_active_window_pid()
1001
- except Exception:
1002
- active_pid = None
1003
-
1004
- raise RuntimeError(
1005
- f"Could not open or focus application '{name}'. "
1006
- f"Active window pid remains {active_pid if active_pid is not None else 'unknown'}."
1007
- )
1381
+ proc = subprocess.Popen(argv, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
1382
+ except OSError as e:
1383
+ raise RuntimeError(f"Could not launch '{name}' ({argv[0]}): {e}") from e
1384
+ return wait_for_window(proc, _window_classes_for_launch(argv, desktop_id))
1008
1385
 
1009
1386
 
1010
1387
  # ---------------------------------------------------------------------------
1011
1388
  # Main CLI
1012
1389
  # ---------------------------------------------------------------------------
1013
1390
 
1391
+ # Commands that synthesize keyboard or mouse input. Each starts from a neutral
1392
+ # modifier state: the driver is one process per action with no hold-key
1393
+ # primitive, so any modifier still held when an action begins is stale by
1394
+ # construction (see ``release_modifiers``), and this is a headless machine with
1395
+ # no person at the keyboard whose keys could be released by mistake.
1396
+ _INPUT_COMMANDS = {"click", "double-click", "right-click", "type", "key", "scroll", "drag"}
1397
+
1398
+
1399
+ def _parse_hold_seconds(args):
1400
+ """Read an optional ``--hold-seconds N`` out of a command's trailing args."""
1401
+ for i, arg in enumerate(args):
1402
+ if arg != "--hold-seconds":
1403
+ continue
1404
+ try:
1405
+ hold = float(args[i + 1])
1406
+ except (IndexError, ValueError):
1407
+ exit_with_error("--hold-seconds requires a number")
1408
+ if hold < 0 or hold > _MAX_HOLD_SECONDS:
1409
+ exit_with_error(f"--hold-seconds must be between 0 and {_MAX_HOLD_SECONDS}")
1410
+ return hold
1411
+ return 0.0
1412
+
1413
+
1414
+ def _exit_on_sigterm(signum, frame):
1415
+ """Unwind on SIGTERM instead of dying in place.
1416
+
1417
+ A driver that outlives its command timeout is ended with SIGTERM. Python's
1418
+ default disposition terminates without unwinding, which would skip the
1419
+ ``finally`` blocks that release a mouse button held by a drag or a
1420
+ press-and-hold; raising SystemExit runs them first.
1421
+ """
1422
+ sys.exit(1)
1423
+
1424
+
1014
1425
  def main():
1015
1426
  args = sys.argv[1:]
1016
1427
  if not args:
1017
1428
  exit_with_error("Usage: linux-driver.py <command> [args...]")
1018
1429
 
1019
1430
  command = args[0]
1431
+ if command in _INPUT_COMMANDS:
1432
+ signal.signal(signal.SIGTERM, _exit_on_sigterm)
1433
+ release_modifiers()
1020
1434
 
1021
1435
  if command == "observe":
1022
1436
  pid = None
@@ -1044,8 +1458,8 @@ def main():
1044
1458
 
1045
1459
  elif command == "click":
1046
1460
  if len(args) < 3:
1047
- exit_with_error("Usage: linux-driver.py click <x> <y>")
1048
- mouse_click(float(args[1]), float(args[2]))
1461
+ exit_with_error("Usage: linux-driver.py click <x> <y> [--hold-seconds N]")
1462
+ mouse_click(float(args[1]), float(args[2]), _parse_hold_seconds(args[3:]))
1049
1463
  print_success()
1050
1464
 
1051
1465
  elif command == "double-click":
@@ -1099,12 +1513,13 @@ def main():
1099
1513
  if len(args) < 5:
1100
1514
  exit_with_error(
1101
1515
  "Usage: linux-driver.py drag <from_x> <from_y> <to_x> <to_y> "
1102
- "[--steps N] [--path x1,y1 ...]"
1516
+ "[--steps N] [--path x1,y1 ...] [--hold-seconds N]"
1103
1517
  )
1104
1518
  from_x, from_y = float(args[1]), float(args[2])
1105
1519
  to_x, to_y = float(args[3]), float(args[4])
1106
1520
  steps = 10
1107
1521
  waypoints = None
1522
+ hold_seconds = _parse_hold_seconds(args[5:])
1108
1523
  di = 5
1109
1524
  while di < len(args):
1110
1525
  if args[di] == "--steps" and di + 1 < len(args):
@@ -1119,7 +1534,7 @@ def main():
1119
1534
  di += 1
1120
1535
  else:
1121
1536
  di += 1
1122
- mouse_drag(from_x, from_y, to_x, to_y, steps, waypoints)
1537
+ mouse_drag(from_x, from_y, to_x, to_y, steps, waypoints, hold_seconds)
1123
1538
  print_success()
1124
1539
 
1125
1540
  elif command == "screenshot":
@@ -1162,11 +1577,13 @@ def main():
1162
1577
  exit_with_error("Usage: linux-driver.py activate <app-name>")
1163
1578
  app_name = " ".join(args[1:])
1164
1579
  try:
1165
- activate_app(app_name)
1580
+ window_visible = activate_app(app_name)
1166
1581
  except RuntimeError as e:
1167
1582
  exit_with_error(str(e))
1168
1583
  time.sleep(0.5)
1169
- print_success()
1584
+ # `windowVisible: false` reports an app that was launched but has not shown
1585
+ # a window yet, so the caller waits instead of launching it again.
1586
+ print_success(windowVisible=window_visible)
1170
1587
 
1171
1588
  else:
1172
1589
  exit_with_error(f"Unknown command: '{command}'")