@octavus/agent 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,966 @@
1
+ #!/usr/bin/env python3
2
+ """
3
+ Linux driver for octavus-computer-use MCP.
4
+
5
+ Implements the same CLI + JSON protocol as the macOS Swift driver,
6
+ using AT-SPI2 for accessibility tree, xdotool for input simulation,
7
+ scrot/xdpyinfo for screenshots, and Pillow for badge annotation.
8
+
9
+ Dependencies (apt):
10
+ python3-gi gir1.2-atspi-2.0 at-spi2-core xdotool scrot python3-pil
11
+ fonts-dejavu-core (a scalable bold TTF for grid/label overlays; see
12
+ _BOLD_FONT_CANDIDATES)
13
+
14
+ Usage mirrors the macOS driver exactly:
15
+ linux-driver.py observe --screenshot /tmp/shot.png [--pid PID]
16
+ linux-driver.py click <x> <y>
17
+ linux-driver.py double-click <x> <y>
18
+ linux-driver.py right-click <x> <y>
19
+ linux-driver.py type <text>
20
+ linux-driver.py key <key-spec>
21
+ linux-driver.py scroll <direction> [steps] [--at x y]
22
+ linux-driver.py drag <from_x> <from_y> <to_x> <to_y> [--steps N] [--path x1,y1 ...]
23
+ linux-driver.py screenshot <path>
24
+ linux-driver.py screenshot-grid <path> --region x y w h --grid <2|3>
25
+ linux-driver.py activate <app-name>
26
+ """
27
+
28
+ import glob
29
+ import json
30
+ import os
31
+ import re
32
+ import subprocess
33
+ import sys
34
+ import time
35
+ import shutil
36
+
37
+ # ---------------------------------------------------------------------------
38
+ # Output helpers (match Swift driver JSON contract)
39
+ # ---------------------------------------------------------------------------
40
+
41
+ def print_success(path=None):
42
+ resp = {"success": True}
43
+ if path is not None:
44
+ resp["path"] = path
45
+ print(json.dumps(resp))
46
+
47
+ def exit_with_error(message):
48
+ print(json.dumps({"error": message}))
49
+ sys.exit(1)
50
+
51
+ def _run(cmd, check=True, capture=True):
52
+ try:
53
+ r = subprocess.run(
54
+ cmd,
55
+ capture_output=capture,
56
+ text=True,
57
+ timeout=15,
58
+ check=check,
59
+ )
60
+ return r
61
+ except FileNotFoundError:
62
+ exit_with_error(f"Required command not found: {cmd[0]}")
63
+ except subprocess.CalledProcessError as e:
64
+ exit_with_error(f"Command failed: {' '.join(cmd)}\n{e.stderr or e.stdout or ''}")
65
+ except subprocess.TimeoutExpired:
66
+ exit_with_error(f"Command timed out: {' '.join(cmd)}")
67
+
68
+ # ---------------------------------------------------------------------------
69
+ # Screen geometry
70
+ # ---------------------------------------------------------------------------
71
+
72
+ def get_screen_size():
73
+ """Return (width, height) of the root window via xdotool."""
74
+ try:
75
+ out = subprocess.check_output(
76
+ ["xdotool", "getdisplaygeometry"], text=True, timeout=5
77
+ ).strip()
78
+ w, h = out.split()
79
+ return int(w), int(h)
80
+ except Exception:
81
+ return 1920, 1080
82
+
83
+ # ---------------------------------------------------------------------------
84
+ # AT-SPI2 accessibility tree
85
+ # ---------------------------------------------------------------------------
86
+
87
+ CLICKABLE_ROLES = {
88
+ "push button", "toggle button", "link", "menu item", "menu",
89
+ "check box", "check menu item", "radio button", "radio menu item",
90
+ "combo box", "text", "password text", "entry", "spin button",
91
+ "slider", "tab", "page tab", "tool bar button",
92
+ }
93
+
94
+ CONTAINER_ROLES = {
95
+ "frame", "dialog", "window", "scroll pane", "panel",
96
+ "filler", "application", "desktop frame", "root pane",
97
+ "layered pane", "split pane", "viewport",
98
+ }
99
+
100
+ ACTIONABLE_ACTIONS = {"click", "press", "activate", "toggle", "jump"}
101
+
102
+
103
+ def _try_import_atspi():
104
+ """Import AT-SPI2 via GObject introspection and connect to the a11y bus."""
105
+ try:
106
+ import gi
107
+ gi.require_version("Atspi", "2.0")
108
+ from gi.repository import Atspi
109
+ Atspi.init()
110
+ return Atspi
111
+ except Exception:
112
+ return None
113
+
114
+
115
+ def _has_actionable(node, Atspi):
116
+ try:
117
+ action = node.get_action_iface()
118
+ if action is None:
119
+ return False
120
+ n = action.get_n_actions()
121
+ for i in range(n):
122
+ name = action.get_action_name(i).lower()
123
+ if name in ACTIONABLE_ACTIONS:
124
+ return True
125
+ except Exception:
126
+ pass
127
+ return False
128
+
129
+
130
+ def _walk_atspi(node, elements, Atspi, screen_w, screen_h, depth=0, max_depth=20):
131
+ if depth > max_depth:
132
+ return
133
+ try:
134
+ state_set = node.get_state_set()
135
+ if not state_set.contains(Atspi.StateType.VISIBLE):
136
+ return
137
+ if not state_set.contains(Atspi.StateType.SHOWING):
138
+ return
139
+ except Exception:
140
+ pass
141
+
142
+ try:
143
+ comp = node.get_component_iface()
144
+ if comp is not None:
145
+ rect = comp.get_extents(Atspi.CoordType.SCREEN)
146
+ x, y, w, h = rect.x, rect.y, rect.width, rect.height
147
+ if w > 0 and h > 0:
148
+ role_name = node.get_role_name().lower()
149
+ is_click = role_name in CLICKABLE_ROLES or _has_actionable(node, Atspi)
150
+
151
+ if is_click:
152
+ area = w * h
153
+ screen_area = screen_w * screen_h
154
+ if role_name in CONTAINER_ROLES and area > screen_area * 0.25:
155
+ pass
156
+ elif area > screen_area * 0.5:
157
+ pass
158
+ else:
159
+ title = ""
160
+ try:
161
+ title = node.get_name() or ""
162
+ except Exception:
163
+ pass
164
+ if not title:
165
+ try:
166
+ title = node.get_description() or ""
167
+ except Exception:
168
+ pass
169
+ if not title:
170
+ try:
171
+ text_iface = node.get_text_iface()
172
+ if text_iface:
173
+ val = text_iface.get_text(0, min(80, text_iface.get_character_count()))
174
+ if val:
175
+ title = val
176
+ except Exception:
177
+ pass
178
+ if len(title) > 60:
179
+ title = title[:60] + "..."
180
+
181
+ state = _get_element_state(node, role_name, Atspi)
182
+
183
+ elements.append({
184
+ "role": role_name,
185
+ "title": title,
186
+ "state": state,
187
+ "x": x, "y": y,
188
+ "width": w, "height": h,
189
+ })
190
+ except Exception:
191
+ pass
192
+
193
+ try:
194
+ n_children = node.get_child_count()
195
+ for i in range(n_children):
196
+ child = node.get_child_at_index(i)
197
+ if child is not None:
198
+ _walk_atspi(child, elements, Atspi, screen_w, screen_h, depth + 1, max_depth)
199
+ except Exception:
200
+ pass
201
+
202
+
203
+ def _get_element_state(node, role_name, Atspi):
204
+ parts = []
205
+ try:
206
+ state_set = node.get_state_set()
207
+ if role_name in ("check box", "check menu item", "radio button", "radio menu item", "toggle button"):
208
+ if state_set.contains(Atspi.StateType.CHECKED):
209
+ parts.append("selected")
210
+ else:
211
+ parts.append("unselected")
212
+ if state_set.contains(Atspi.StateType.FOCUSED):
213
+ parts.append("focused")
214
+ if role_name in ("text", "password text", "entry", "spin button", "combo box"):
215
+ try:
216
+ text_iface = node.get_text_iface()
217
+ if text_iface:
218
+ val = text_iface.get_text(0, min(40, text_iface.get_character_count()))
219
+ if val:
220
+ if len(val) > 40:
221
+ val = val[:40] + "..."
222
+ parts.append(f'value="{val}"')
223
+ except Exception:
224
+ pass
225
+ except Exception:
226
+ pass
227
+ return ", ".join(parts)
228
+
229
+
230
+ def _dedup_elements(elements, screen_w, screen_h):
231
+ elements.sort(key=lambda e: e["width"] * e["height"])
232
+
233
+ deduped = []
234
+ for el in elements:
235
+ area = el["width"] * el["height"]
236
+ if area > screen_w * screen_h * 0.5:
237
+ continue
238
+
239
+ cx = el["x"] + el["width"] / 2
240
+ cy = el["y"] + el["height"] / 2
241
+ is_dup = False
242
+ for ex in deduped:
243
+ ex_cx = ex["x"] + ex["width"] / 2
244
+ ex_cy = ex["y"] + ex["height"] / 2
245
+ dist = ((cx - ex_cx) ** 2 + (cy - ex_cy) ** 2) ** 0.5
246
+ if dist < 12.0:
247
+ is_dup = True
248
+ break
249
+ if (ex["x"] >= el["x"] and ex["y"] >= el["y"]
250
+ and ex["x"] + ex["width"] <= el["x"] + el["width"]
251
+ and ex["y"] + ex["height"] <= el["y"] + el["height"]
252
+ and dist < 30.0):
253
+ is_dup = True
254
+ break
255
+ if not is_dup:
256
+ deduped.append(el)
257
+
258
+ deduped.sort(key=lambda e: (round(e["y"] / 15) * 15, e["x"]))
259
+ return deduped
260
+
261
+
262
+ def _get_active_window_pid():
263
+ try:
264
+ wid = subprocess.check_output(
265
+ ["xdotool", "getactivewindow"], text=True, timeout=5
266
+ ).strip()
267
+ pid = subprocess.check_output(
268
+ ["xdotool", "getwindowpid", wid], text=True, timeout=5
269
+ ).strip()
270
+ return int(pid)
271
+ except Exception:
272
+ return None
273
+
274
+
275
+ def _collect_elements(desktop, target_pid, Atspi, screen_w, screen_h):
276
+ elements = []
277
+ matched_target = False
278
+
279
+ n_apps = desktop.get_child_count()
280
+ for i in range(n_apps):
281
+ app = desktop.get_child_at_index(i)
282
+ if app is None:
283
+ continue
284
+
285
+ if target_pid is not None:
286
+ try:
287
+ app_pid = app.get_process_id()
288
+ if app_pid != target_pid:
289
+ continue
290
+ matched_target = True
291
+ except Exception:
292
+ continue
293
+
294
+ _walk_atspi(app, elements, Atspi, screen_w, screen_h)
295
+
296
+ return elements, matched_target
297
+
298
+
299
+ def enumerate_elements(pid=None):
300
+ Atspi = _try_import_atspi()
301
+ if Atspi is None:
302
+ exit_with_error(
303
+ "AT-SPI2 not available. Install: sudo apt install python3-gi gir1.2-atspi-2.0 at-spi2-core"
304
+ )
305
+
306
+ screen_w, screen_h = get_screen_size()
307
+ target_pid = pid or _get_active_window_pid()
308
+
309
+ desktop = Atspi.get_desktop(0)
310
+ elements, matched_target = _collect_elements(
311
+ desktop, target_pid, Atspi, screen_w, screen_h
312
+ )
313
+
314
+ # Linux/X11 accessibility can disagree with xdotool's active-window PID,
315
+ # especially for Chromium/Electron renderers. If the focused PID yields
316
+ # nothing, retry against the full AT-SPI desktop tree.
317
+ if target_pid is not None and (not matched_target or not elements):
318
+ elements, _ = _collect_elements(desktop, None, Atspi, screen_w, screen_h)
319
+
320
+ # Chromium publishes its accessibility tree to AT-SPI lazily, so a label fired
321
+ # right after navigation can beat that export and see an empty desktop. The
322
+ # a11y bus is already enabled by the runtime at startup, so wait briefly and
323
+ # re-walk the full tree once before giving up.
324
+ if not elements:
325
+ time.sleep(0.6)
326
+ desktop = Atspi.get_desktop(0)
327
+ elements, _ = _collect_elements(desktop, None, Atspi, screen_w, screen_h)
328
+
329
+ deduped = _dedup_elements(elements, screen_w, screen_h)
330
+
331
+ for idx, el in enumerate(deduped):
332
+ el["number"] = idx + 1
333
+
334
+ return deduped
335
+
336
+
337
+ # ---------------------------------------------------------------------------
338
+ # Screenshot capture
339
+ # ---------------------------------------------------------------------------
340
+
341
+ def capture_screenshot(output_path):
342
+ """Take a screenshot using scrot (X11)."""
343
+ _run(["scrot", "-o", output_path])
344
+
345
+
346
+ def crop_screenshot(screenshot_path, region):
347
+ """Crop a screenshot to the given region dict {x, y, width, height}."""
348
+ try:
349
+ from PIL import Image
350
+ except ImportError:
351
+ exit_with_error(
352
+ "Pillow not installed. Install: pip3 install Pillow (or sudo apt install python3-pil)"
353
+ )
354
+ img = Image.open(screenshot_path)
355
+ img_w, img_h = img.size
356
+ screen_w, screen_h = get_screen_size()
357
+ scale_x = img_w / screen_w
358
+ scale_y = img_h / screen_h
359
+
360
+ left = int(region["x"] * scale_x)
361
+ top = int(region["y"] * scale_y)
362
+ right = int((region["x"] + region["width"]) * scale_x)
363
+ bottom = int((region["y"] + region["height"]) * scale_y)
364
+
365
+ left = max(0, min(left, img_w))
366
+ top = max(0, min(top, img_h))
367
+ right = max(left, min(right, img_w))
368
+ bottom = max(top, min(bottom, img_h))
369
+
370
+ cropped = img.crop((left, top, right, bottom))
371
+ cropped.info = img.info
372
+ cropped.save(screenshot_path, "PNG")
373
+
374
+
375
+ # Bold sans-serif TrueType candidates, in preference order. DejaVu is the
376
+ # baseline on both the E2B image and the cloud AMI; Liberation and FreeSans are
377
+ # fallbacks so a substrate missing DejaVu still renders readable, scalable
378
+ # overlays instead of PIL's fixed-size bitmap default (which ignores the
379
+ # requested size and makes grid/label text unreadable).
380
+ _BOLD_FONT_CANDIDATES = (
381
+ "/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf",
382
+ "/usr/share/fonts/truetype/liberation/LiberationSans-Bold.ttf",
383
+ "/usr/share/fonts/truetype/freefont/FreeSansBold.ttf",
384
+ )
385
+
386
+
387
+ def load_bold_font(size):
388
+ """Load a scalable bold font at the requested size from a known path,
389
+ falling back to PIL's default only when no TrueType font is available."""
390
+ from PIL import ImageFont
391
+
392
+ for path in _BOLD_FONT_CANDIDATES:
393
+ try:
394
+ return ImageFont.truetype(path, size)
395
+ except Exception:
396
+ continue
397
+ return ImageFont.load_default()
398
+
399
+
400
+ def draw_grid_overlay(screenshot_path, grid_size):
401
+ """Draw a numbered grid overlay on an existing screenshot.
402
+ grid_size is 2 or 3 (for 2x2 or 3x3)."""
403
+ try:
404
+ from PIL import Image, ImageDraw
405
+ except ImportError:
406
+ exit_with_error(
407
+ "Pillow not installed. Install: pip3 install Pillow (or sudo apt install python3-pil)"
408
+ )
409
+
410
+ img = Image.open(screenshot_path).convert("RGBA")
411
+ overlay = Image.new("RGBA", img.size, (0, 0, 0, 0))
412
+ draw = ImageDraw.Draw(overlay, "RGBA")
413
+ img_w, img_h = img.size
414
+ cell_w = img_w / grid_size
415
+ cell_h = img_h / grid_size
416
+ cell_min = min(cell_w, cell_h)
417
+
418
+ line_width = max(1, min(int(cell_min / 40), 4))
419
+ for i in range(1, grid_size):
420
+ x = int(i * cell_w)
421
+ draw.line([(x, 0), (x, img_h)], fill=(230, 25, 25, 216), width=line_width)
422
+ for i in range(1, grid_size):
423
+ y = int(i * cell_h)
424
+ draw.line([(0, y), (img_w, y)], fill=(230, 25, 25, 216), width=line_width)
425
+
426
+ font_size = max(8, min(int(cell_min / 6), 60))
427
+ font = load_bold_font(font_size)
428
+
429
+ for row in range(grid_size):
430
+ for col in range(grid_size):
431
+ cx = int(col * cell_w + cell_w / 2)
432
+ cy = int(row * cell_h + cell_h / 2)
433
+ label = f"R{row + 1}C{col + 1}"
434
+ bbox = draw.textbbox((0, 0), label, font=font)
435
+ tw = bbox[2] - bbox[0]
436
+ th = bbox[3] - bbox[1]
437
+ draw.text(
438
+ (cx - tw // 2, cy - th // 2),
439
+ label,
440
+ fill=(230, 25, 25, 255),
441
+ font=font,
442
+ )
443
+
444
+ result = Image.alpha_composite(img, overlay).convert("RGB")
445
+ result.info = img.info
446
+ result.save(screenshot_path, "PNG")
447
+
448
+
449
+ # ---------------------------------------------------------------------------
450
+ # Screenshot annotation (red numbered badges, matching macOS style)
451
+ # ---------------------------------------------------------------------------
452
+
453
+ def annotate_screenshot(screenshot_path, elements):
454
+ if not elements:
455
+ return
456
+
457
+ try:
458
+ from PIL import Image, ImageDraw
459
+ except ImportError:
460
+ exit_with_error(
461
+ "Pillow not installed. Install: pip3 install Pillow (or sudo apt install python3-pil)"
462
+ )
463
+
464
+ img = Image.open(screenshot_path).convert("RGBA")
465
+ draw = ImageDraw.Draw(img, "RGBA")
466
+ img_w, img_h = img.size
467
+ screen_w, screen_h = get_screen_size()
468
+ scale_x = img_w / screen_w
469
+ scale_y = img_h / screen_h
470
+
471
+ font = load_bold_font(12)
472
+ font_small = load_bold_font(10)
473
+
474
+ for el in elements:
475
+ num_str = str(el["number"])
476
+ badge_size = 30 if len(num_str) > 2 else 24
477
+ active_font = font_small if len(num_str) > 2 else font
478
+
479
+ lx = el["x"] * scale_x - 2
480
+ ly = el["y"] * scale_y - 2
481
+
482
+ if lx < 0:
483
+ lx = 0
484
+ if ly < 0:
485
+ ly = 0
486
+ if lx + badge_size > img_w:
487
+ lx = img_w - badge_size
488
+ if ly + badge_size > img_h:
489
+ ly = img_h - badge_size
490
+
491
+ cx = lx + badge_size / 2
492
+ cy = ly + badge_size / 2
493
+ r = badge_size / 2
494
+
495
+ draw.ellipse(
496
+ [cx - r, cy - r, cx + r, cy + r],
497
+ fill=(230, 25, 25, 242),
498
+ outline=(255, 255, 255, 255),
499
+ width=2,
500
+ )
501
+
502
+ bbox = draw.textbbox((0, 0), num_str, font=active_font)
503
+ tw = bbox[2] - bbox[0]
504
+ th = bbox[3] - bbox[1]
505
+ tx = cx - tw / 2
506
+ ty = cy - th / 2
507
+ draw.text((tx, ty), num_str, fill=(255, 255, 255, 255), font=active_font)
508
+
509
+ img = img.convert("RGB")
510
+ img.save(screenshot_path, "PNG")
511
+
512
+
513
+ # ---------------------------------------------------------------------------
514
+ # Input simulation via xdotool
515
+ # ---------------------------------------------------------------------------
516
+
517
+ # xdotool mousemove --sync blocks until the X server confirms the pointer
518
+ # reached the target. Under display load - rapid screenshots + grid overlays and
519
+ # heavy page rendering during a CAPTCHA-solving loop - that confirmation can
520
+ # stall for many seconds and blow the 15s command timeout, failing the click
521
+ # outright. Bound the sync wait; if it stalls (or errors), fall back to a
522
+ # non-sync move so the cursor still lands and the click proceeds - a
523
+ # slightly-less-synchronous move instead of a hard failure.
524
+ _MOUSEMOVE_SYNC_TIMEOUT = 3
525
+
526
+ def mouse_move(x, y):
527
+ ix, iy = str(int(x)), str(int(y))
528
+ try:
529
+ subprocess.run(
530
+ ["xdotool", "mousemove", "--sync", ix, iy],
531
+ capture_output=True,
532
+ text=True,
533
+ timeout=_MOUSEMOVE_SYNC_TIMEOUT,
534
+ check=True,
535
+ )
536
+ time.sleep(0.025)
537
+ return
538
+ except FileNotFoundError:
539
+ exit_with_error("Required command not found: xdotool")
540
+ except (subprocess.TimeoutExpired, subprocess.CalledProcessError):
541
+ pass
542
+ # Sync confirmation stalled - re-issue without --sync (returns immediately)
543
+ # and give the async motion a slightly longer settle before the caller
544
+ # clicks, since a non-sync move carries no server arrival confirmation.
545
+ _run(["xdotool", "mousemove", ix, iy])
546
+ time.sleep(0.1)
547
+
548
+ def mouse_click(x, y):
549
+ mouse_move(x, y)
550
+ _run(["xdotool", "click", "1"])
551
+
552
+ def mouse_double_click(x, y):
553
+ mouse_move(x, y)
554
+ _run(["xdotool", "click", "--repeat", "2", "--delay", "50", "1"])
555
+
556
+ def mouse_right_click(x, y):
557
+ mouse_move(x, y)
558
+ _run(["xdotool", "click", "3"])
559
+
560
+ def type_text(text):
561
+ _run(["xdotool", "type", "--clearmodifiers", "--delay", "5", text])
562
+
563
+ # Mapping from our key-spec names to xdotool key names
564
+ _KEY_MAP = {
565
+ "return": "Return", "enter": "Return", "tab": "Tab", "space": "space",
566
+ "delete": "BackSpace", "backspace": "BackSpace", "forwarddelete": "Delete",
567
+ "escape": "Escape", "esc": "Escape",
568
+ "up": "Up", "down": "Down", "left": "Left", "right": "Right",
569
+ "home": "Home", "end": "End", "pageup": "Prior", "pagedown": "Next",
570
+ "f1": "F1", "f2": "F2", "f3": "F3", "f4": "F4", "f5": "F5", "f6": "F6",
571
+ "f7": "F7", "f8": "F8", "f9": "F9", "f10": "F10", "f11": "F11", "f12": "F12",
572
+ }
573
+
574
+ _MOD_MAP = {
575
+ "cmd": "super", "command": "super", "super": "super",
576
+ "shift": "shift",
577
+ "alt": "alt", "option": "alt", "opt": "alt",
578
+ "ctrl": "ctrl", "control": "ctrl",
579
+ }
580
+
581
+ def press_key(key_spec):
582
+ parts = key_spec.lower().split("+")
583
+ mods = []
584
+ key_name = ""
585
+
586
+ for part in parts:
587
+ if part in _MOD_MAP:
588
+ mods.append(_MOD_MAP[part])
589
+ else:
590
+ key_name = part
591
+
592
+ xdot_key = _KEY_MAP.get(key_name, key_name)
593
+
594
+ combo = "+".join(mods + [xdot_key]) if mods else xdot_key
595
+ _run(["xdotool", "key", "--clearmodifiers", combo])
596
+
597
+ def _interpolate_points(from_x, from_y, to_x, to_y, steps):
598
+ """Generate linearly interpolated points between two coordinates."""
599
+ points = []
600
+ for i in range(1, steps + 1):
601
+ t = i / steps
602
+ x = from_x + (to_x - from_x) * t
603
+ y = from_y + (to_y - from_y) * t
604
+ points.append((x, y))
605
+ return points
606
+
607
+
608
+ def mouse_drag(from_x, from_y, to_x, to_y, steps=10, waypoints=None):
609
+ """Perform a smooth drag from one position to another.
610
+
611
+ Uses xdotool mousedown/mousemove/mouseup with --sync to ensure
612
+ Chrome recognizes the drag gesture. The 16ms sleep between moves
613
+ matches 60fps frame timing.
614
+
615
+ When waypoints are provided the last waypoint is the destination, so
616
+ (to_x, to_y) is not appended to the segment list. The button is
617
+ released in a finally clause so a failed mousemove can't leave the
618
+ cursor stuck in the down state.
619
+ """
620
+ mouse_move(from_x, from_y)
621
+ _run(["xdotool", "mousedown", "1"])
622
+ try:
623
+ if waypoints:
624
+ segments = [(from_x, from_y)] + list(waypoints)
625
+ else:
626
+ segments = [(from_x, from_y), (to_x, to_y)]
627
+ for seg_idx in range(len(segments) - 1):
628
+ sx, sy = segments[seg_idx]
629
+ ex, ey = segments[seg_idx + 1]
630
+ for px, py in _interpolate_points(sx, sy, ex, ey, steps):
631
+ _run(["xdotool", "mousemove", "--sync", str(int(px)), str(int(py))])
632
+ time.sleep(0.016)
633
+ finally:
634
+ _run(["xdotool", "mouseup", "1"], check=False)
635
+
636
+
637
+ def do_scroll(direction, steps=3, at_x=None, at_y=None):
638
+ if at_x is not None and at_y is not None:
639
+ mouse_move(at_x, at_y)
640
+ time.sleep(0.05)
641
+ else:
642
+ sw, sh = get_screen_size()
643
+ mouse_move(sw // 2, sh // 2)
644
+ time.sleep(0.05)
645
+
646
+ button = "5" if direction == "down" else "4"
647
+ for _ in range(steps):
648
+ _run(["xdotool", "click", "--repeat", "3", button])
649
+ time.sleep(0.016)
650
+
651
+
652
+ # ---------------------------------------------------------------------------
653
+ # Application activation
654
+ # ---------------------------------------------------------------------------
655
+
656
+ _DESKTOP_DIRS = [
657
+ "/usr/share/applications",
658
+ "/usr/local/share/applications",
659
+ os.path.expanduser("~/.local/share/applications"),
660
+ ]
661
+
662
+
663
+ def _resolve_from_desktop_files(name):
664
+ """Search .desktop files for an app matching *name*.
665
+
666
+ Checks the Name= and GenericName= fields (case-insensitive), plus the
667
+ desktop file basename (e.g. "chromium" matches "chromium-browser.desktop").
668
+ Returns (binaries, desktop_ids) - binary commands from Exec= lines and
669
+ matching .desktop filenames for gtk-launch.
670
+ """
671
+ name_lower = name.lower()
672
+ binaries = []
673
+ desktop_ids = []
674
+
675
+ for desktop_dir in _DESKTOP_DIRS:
676
+ for filepath in glob.glob(os.path.join(desktop_dir, "*.desktop")):
677
+ try:
678
+ with open(filepath) as f:
679
+ content = f.read()
680
+ except Exception:
681
+ continue
682
+
683
+ basename = os.path.splitext(os.path.basename(filepath))[0].lower()
684
+ matched = name_lower in basename
685
+
686
+ if not matched:
687
+ for line in content.splitlines():
688
+ if line.startswith("Name=") or line.startswith("GenericName="):
689
+ value = line.split("=", 1)[1].strip().lower()
690
+ if value == name_lower:
691
+ matched = True
692
+ break
693
+
694
+ if not matched:
695
+ continue
696
+
697
+ desktop_ids.append(os.path.basename(filepath))
698
+
699
+ for line in content.splitlines():
700
+ if line.startswith("Exec="):
701
+ exec_cmd = line.split("=", 1)[1].strip()
702
+ exec_cmd = re.sub(r"\s+%[fFuUdDnNickvm]", "", exec_cmd)
703
+ binary = exec_cmd.split()[0]
704
+ if binary not in binaries:
705
+ binaries.append(binary)
706
+ break
707
+
708
+ return binaries, desktop_ids
709
+
710
+
711
+ def activate_app(name):
712
+ def find_windows(query):
713
+ try:
714
+ r = subprocess.run(
715
+ ["xdotool", "search", "--name", query],
716
+ capture_output=True, text=True, timeout=5,
717
+ )
718
+ return [wid for wid in r.stdout.strip().split("\n") if wid]
719
+ except Exception:
720
+ return []
721
+
722
+ def activate_existing_window():
723
+ for query in (name, name.lower()):
724
+ wids = find_windows(query)
725
+ if wids:
726
+ # No --sync: `windowactivate --sync` blocks until the WM confirms
727
+ # activation, which never arrives for a borderless/maximized
728
+ # window under fluxbox/xfwm and hangs until the subprocess
729
+ # timeout. A plain activate plus a short settle is enough; treat a
730
+ # non-zero exit as best-effort so it can't abort labeling.
731
+ _run(["xdotool", "windowactivate", wids[0]], check=False)
732
+ time.sleep(0.2)
733
+ return True
734
+ return False
735
+
736
+ def wait_for_window(timeout_s=4.0):
737
+ deadline = time.time() + timeout_s
738
+ while time.time() < deadline:
739
+ if activate_existing_window():
740
+ return True
741
+ time.sleep(0.2)
742
+ return False
743
+
744
+ if activate_existing_window():
745
+ return
746
+
747
+ desktop_binaries, desktop_ids = _resolve_from_desktop_files(name)
748
+
749
+ candidates = list(desktop_binaries)
750
+ direct = name.lower()
751
+ if direct not in candidates:
752
+ candidates.append(direct)
753
+ if name != direct and name not in candidates:
754
+ candidates.append(name)
755
+
756
+ launch_attempted = False
757
+ for command in candidates:
758
+ executable = shutil.which(command)
759
+ if executable is None:
760
+ continue
761
+
762
+ launch_attempted = True
763
+ try:
764
+ subprocess.Popen(
765
+ [executable],
766
+ stdout=subprocess.DEVNULL,
767
+ stderr=subprocess.DEVNULL,
768
+ )
769
+ if wait_for_window():
770
+ return
771
+ except Exception:
772
+ pass
773
+
774
+ if not launch_attempted and desktop_ids:
775
+ gtk_launch = shutil.which("gtk-launch")
776
+ if gtk_launch:
777
+ for desktop_id in desktop_ids:
778
+ try:
779
+ subprocess.Popen(
780
+ [gtk_launch, desktop_id],
781
+ stdout=subprocess.DEVNULL,
782
+ stderr=subprocess.DEVNULL,
783
+ )
784
+ if wait_for_window():
785
+ return
786
+ except Exception:
787
+ pass
788
+
789
+ try:
790
+ active_pid = _get_active_window_pid()
791
+ except Exception:
792
+ active_pid = None
793
+
794
+ raise RuntimeError(
795
+ f"Could not open or focus application '{name}'. "
796
+ f"Active window pid remains {active_pid if active_pid is not None else 'unknown'}."
797
+ )
798
+
799
+
800
+ # ---------------------------------------------------------------------------
801
+ # Main CLI
802
+ # ---------------------------------------------------------------------------
803
+
804
+ def main():
805
+ args = sys.argv[1:]
806
+ if not args:
807
+ exit_with_error("Usage: linux-driver.py <command> [args...]")
808
+
809
+ command = args[0]
810
+
811
+ if command == "observe":
812
+ pid = None
813
+ screenshot_path = "/tmp/octavus-screenshot.png"
814
+ i = 1
815
+ while i < len(args):
816
+ if args[i] == "--pid" and i + 1 < len(args):
817
+ pid = int(args[i + 1])
818
+ i += 2
819
+ elif args[i] == "--screenshot" and i + 1 < len(args):
820
+ screenshot_path = args[i + 1]
821
+ i += 2
822
+ else:
823
+ i += 1
824
+
825
+ elements = enumerate_elements(pid=pid)
826
+ capture_screenshot(screenshot_path)
827
+ annotate_screenshot(screenshot_path, elements)
828
+
829
+ result = {
830
+ "elements": elements,
831
+ "screenshotPath": screenshot_path,
832
+ }
833
+ print(json.dumps(result, indent=2))
834
+
835
+ elif command == "click":
836
+ if len(args) < 3:
837
+ exit_with_error("Usage: linux-driver.py click <x> <y>")
838
+ mouse_click(float(args[1]), float(args[2]))
839
+ print_success()
840
+
841
+ elif command == "double-click":
842
+ if len(args) < 3:
843
+ exit_with_error("Usage: linux-driver.py double-click <x> <y>")
844
+ mouse_double_click(float(args[1]), float(args[2]))
845
+ print_success()
846
+
847
+ elif command == "right-click":
848
+ if len(args) < 3:
849
+ exit_with_error("Usage: linux-driver.py right-click <x> <y>")
850
+ mouse_right_click(float(args[1]), float(args[2]))
851
+ print_success()
852
+
853
+ elif command == "type":
854
+ if len(args) < 2:
855
+ exit_with_error("Usage: linux-driver.py type <text>")
856
+ text = " ".join(args[1:])
857
+ type_text(text)
858
+ print_success()
859
+
860
+ elif command == "key":
861
+ if len(args) < 2:
862
+ exit_with_error("Usage: linux-driver.py key <key-spec>")
863
+ press_key(args[1])
864
+ print_success()
865
+
866
+ elif command == "scroll":
867
+ if len(args) < 2:
868
+ exit_with_error("Usage: linux-driver.py scroll <direction> [steps] [--at x y]")
869
+ direction = args[1]
870
+ steps = 3
871
+ at_x = None
872
+ at_y = None
873
+ si = 2
874
+ while si < len(args):
875
+ if args[si] == "--at" and si + 2 < len(args):
876
+ at_x = float(args[si + 1])
877
+ at_y = float(args[si + 2])
878
+ si += 3
879
+ else:
880
+ try:
881
+ steps = int(args[si])
882
+ except ValueError:
883
+ pass
884
+ si += 1
885
+ do_scroll(direction, steps, at_x, at_y)
886
+ print_success()
887
+
888
+ elif command == "drag":
889
+ if len(args) < 5:
890
+ exit_with_error(
891
+ "Usage: linux-driver.py drag <from_x> <from_y> <to_x> <to_y> "
892
+ "[--steps N] [--path x1,y1 ...]"
893
+ )
894
+ from_x, from_y = float(args[1]), float(args[2])
895
+ to_x, to_y = float(args[3]), float(args[4])
896
+ steps = 10
897
+ waypoints = None
898
+ di = 5
899
+ while di < len(args):
900
+ if args[di] == "--steps" and di + 1 < len(args):
901
+ steps = int(args[di + 1])
902
+ di += 2
903
+ elif args[di] == "--path":
904
+ waypoints = []
905
+ di += 1
906
+ while di < len(args) and not args[di].startswith("--"):
907
+ parts = args[di].split(",")
908
+ waypoints.append((float(parts[0]), float(parts[1])))
909
+ di += 1
910
+ else:
911
+ di += 1
912
+ mouse_drag(from_x, from_y, to_x, to_y, steps, waypoints)
913
+ print_success()
914
+
915
+ elif command == "screenshot":
916
+ if len(args) < 2:
917
+ exit_with_error("Usage: linux-driver.py screenshot <path>")
918
+ path = args[1]
919
+ capture_screenshot(path)
920
+ print_success(path=path)
921
+
922
+ elif command == "screenshot-grid":
923
+ if len(args) < 2:
924
+ exit_with_error(
925
+ "Usage: linux-driver.py screenshot-grid <path> --region x y w h --grid <2|3>"
926
+ )
927
+ path = args[1]
928
+ region = None
929
+ grid_size = 3
930
+ si = 2
931
+ while si < len(args):
932
+ if args[si] == "--region" and si + 4 < len(args):
933
+ region = {
934
+ "x": float(args[si + 1]), "y": float(args[si + 2]),
935
+ "width": float(args[si + 3]), "height": float(args[si + 4]),
936
+ }
937
+ si += 5
938
+ elif args[si] == "--grid" and si + 1 < len(args):
939
+ grid_size = int(args[si + 1])
940
+ si += 2
941
+ else:
942
+ si += 1
943
+ if not region:
944
+ exit_with_error("screenshot-grid requires --region x y w h")
945
+ capture_screenshot(path)
946
+ crop_screenshot(path, region)
947
+ draw_grid_overlay(path, grid_size)
948
+ print_success(path=path)
949
+
950
+ elif command == "activate":
951
+ if len(args) < 2:
952
+ exit_with_error("Usage: linux-driver.py activate <app-name>")
953
+ app_name = " ".join(args[1:])
954
+ try:
955
+ activate_app(app_name)
956
+ except RuntimeError as e:
957
+ exit_with_error(str(e))
958
+ time.sleep(0.5)
959
+ print_success()
960
+
961
+ else:
962
+ exit_with_error(f"Unknown command: '{command}'")
963
+
964
+
965
+ if __name__ == "__main__":
966
+ main()