@octavus/agent 1.0.3 → 1.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -127,68 +127,98 @@ def _has_actionable(node, Atspi):
127
127
  return False
128
128
 
129
129
 
130
+ def _overlaps_screen(x, y, w, h, screen_w, screen_h):
131
+ """Whether a screen-coordinate box intersects the visible screen."""
132
+ return x < screen_w and y < screen_h and x + w > 0 and y + h > 0
133
+
134
+
130
135
  def _walk_atspi(node, elements, Atspi, screen_w, screen_h, depth=0, max_depth=20):
131
136
  if depth > max_depth:
132
137
  return
138
+
139
+ visible = False
140
+ showing = False
133
141
  try:
134
142
  state_set = node.get_state_set()
135
- if not state_set.contains(Atspi.StateType.VISIBLE):
136
- return
137
- if not state_set.contains(Atspi.StateType.SHOWING):
138
- return
143
+ visible = state_set.contains(Atspi.StateType.VISIBLE)
144
+ showing = state_set.contains(Atspi.StateType.SHOWING)
139
145
  except Exception:
140
146
  pass
141
147
 
148
+ rect = None
142
149
  try:
143
150
  comp = node.get_component_iface()
144
151
  if comp is not None:
145
- rect = comp.get_extents(Atspi.CoordType.SCREEN)
146
- x, y, w, h = rect.x, rect.y, rect.width, rect.height
147
- if w > 0 and h > 0:
148
- role_name = node.get_role_name().lower()
149
- is_click = role_name in CLICKABLE_ROLES or _has_actionable(node, Atspi)
150
-
151
- if is_click:
152
- area = w * h
153
- screen_area = screen_w * screen_h
154
- if role_name in CONTAINER_ROLES and area > screen_area * 0.25:
152
+ extents = comp.get_extents(Atspi.CoordType.SCREEN)
153
+ rect = (extents.x, extents.y, extents.width, extents.height)
154
+ except Exception:
155
+ rect = None
156
+
157
+ on_screen = (
158
+ rect is not None
159
+ and rect[2] > 0
160
+ and rect[3] > 0
161
+ and _overlaps_screen(rect[0], rect[1], rect[2], rect[3], screen_w, screen_h)
162
+ )
163
+
164
+ # Include this node as an actionable element when it is a visible, on-screen
165
+ # clickable widget. Inclusion keys on real screen geometry, not the AT-SPI
166
+ # SHOWING flag: some toolkits (notably GTK4, e.g. gnome-text-editor) leave
167
+ # SHOWING unset on every on-screen descendant below the top frame, so gating
168
+ # on SHOWING would drop the entire app's controls. Off-screen and zero-area
169
+ # nodes fail the on_screen test instead.
170
+ if rect is not None and on_screen and visible:
171
+ try:
172
+ role_name = node.get_role_name().lower()
173
+ except Exception:
174
+ role_name = ""
175
+ if role_name and (role_name in CLICKABLE_ROLES or _has_actionable(node, Atspi)):
176
+ x, y, w, h = rect
177
+ area = w * h
178
+ screen_area = screen_w * screen_h
179
+ too_big = (
180
+ role_name in CONTAINER_ROLES and area > screen_area * 0.25
181
+ ) or area > screen_area * 0.5
182
+ if not too_big:
183
+ title = ""
184
+ try:
185
+ title = node.get_name() or ""
186
+ except Exception:
187
+ pass
188
+ if not title:
189
+ try:
190
+ title = node.get_description() or ""
191
+ except Exception:
155
192
  pass
156
- elif area > screen_area * 0.5:
193
+ if not title:
194
+ try:
195
+ text_iface = node.get_text_iface()
196
+ if text_iface:
197
+ val = text_iface.get_text(0, min(80, text_iface.get_character_count()))
198
+ if val:
199
+ title = val
200
+ except Exception:
157
201
  pass
158
- else:
159
- title = ""
160
- try:
161
- title = node.get_name() or ""
162
- except Exception:
163
- pass
164
- if not title:
165
- try:
166
- title = node.get_description() or ""
167
- except Exception:
168
- pass
169
- if not title:
170
- try:
171
- text_iface = node.get_text_iface()
172
- if text_iface:
173
- val = text_iface.get_text(0, min(80, text_iface.get_character_count()))
174
- if val:
175
- title = val
176
- except Exception:
177
- pass
178
- if len(title) > 60:
179
- title = title[:60] + "..."
180
-
181
- state = _get_element_state(node, role_name, Atspi)
182
-
183
- elements.append({
184
- "role": role_name,
185
- "title": title,
186
- "state": state,
187
- "x": x, "y": y,
188
- "width": w, "height": h,
189
- })
190
- except Exception:
191
- pass
202
+ if len(title) > 60:
203
+ title = title[:60] + "..."
204
+
205
+ elements.append({
206
+ "role": role_name,
207
+ "title": title,
208
+ "state": _get_element_state(node, role_name, Atspi),
209
+ "x": x, "y": y,
210
+ "width": w, "height": h,
211
+ })
212
+
213
+ # Descend into children unless this node is provably off-screen. Keep walking
214
+ # through nodes with no geometry (structural containers such as the
215
+ # `application` node, which reports VISIBLE=False and no extents) and through
216
+ # on-screen containers that merely lack SHOWING - prune only a node that has a
217
+ # real box that is off-screen or zero-area and is also not SHOWING. That
218
+ # bounds the walk over large web trees (off-viewport content) without severing
219
+ # a native app's subtree at a SHOWING-less container.
220
+ if rect is not None and not on_screen and not showing:
221
+ return
192
222
 
193
223
  try:
194
224
  n_children = node.get_child_count()
@@ -272,28 +302,122 @@ def _get_active_window_pid():
272
302
  return None
273
303
 
274
304
 
305
+ def _get_active_window_geometry():
306
+ """Screen rect of the focused window as {x, y, width, height}, or None.
307
+
308
+ Used to clip a recovered element set to the window the agent is actually
309
+ looking at (one desktop, what-you-see-is-what-you-get).
310
+ """
311
+ try:
312
+ out = subprocess.check_output(
313
+ ["xdotool", "getactivewindow", "getwindowgeometry", "--shell"],
314
+ text=True,
315
+ timeout=5,
316
+ )
317
+ except Exception:
318
+ return None
319
+
320
+ vals = {}
321
+ for line in out.splitlines():
322
+ key, sep, value = line.partition("=")
323
+ if sep:
324
+ vals[key.strip()] = value.strip()
325
+ try:
326
+ return {
327
+ "x": int(vals["X"]),
328
+ "y": int(vals["Y"]),
329
+ "width": int(vals["WIDTH"]),
330
+ "height": int(vals["HEIGHT"]),
331
+ }
332
+ except (KeyError, ValueError):
333
+ return None
334
+
335
+
336
+ def _active_window_is_chromium(pid):
337
+ """Whether the foreground PID is a Chrome/Chromium process.
338
+
339
+ Read from /proc rather than the window class: the cloud xdotool build has no
340
+ getwindowclassname and xprop is not installed, but the browser owns the
341
+ focused window's _NET_WM_PID, so its process name is the reliable signal.
342
+ """
343
+ if pid is None:
344
+ return False
345
+ for proc_file in (f"/proc/{pid}/comm", f"/proc/{pid}/cmdline"):
346
+ try:
347
+ with open(proc_file, "rb") as f:
348
+ data = f.read().replace(b"\x00", b" ").decode("utf-8", "ignore").lower()
349
+ except Exception:
350
+ continue
351
+ if "chrome" in data or "chromium" in data:
352
+ return True
353
+ return False
354
+
355
+
275
356
  def _collect_elements(desktop, target_pid, Atspi, screen_w, screen_h):
276
- elements = []
277
- matched_target = False
357
+ """Collect actionable elements from the AT-SPI app(s) matching target_pid.
278
358
 
359
+ Strictly PID-scoped: a None target_pid (foreground PID unavailable) matches no
360
+ app and returns empty, so labeling never falls back to walking the whole
361
+ desktop, which would overlay the always-open browser's labels onto whatever
362
+ native app is in front.
363
+ """
364
+ elements = []
279
365
  n_apps = desktop.get_child_count()
280
366
  for i in range(n_apps):
281
367
  app = desktop.get_child_at_index(i)
282
368
  if app is None:
283
369
  continue
284
-
285
- if target_pid is not None:
286
- try:
287
- app_pid = app.get_process_id()
288
- if app_pid != target_pid:
289
- continue
290
- matched_target = True
291
- except Exception:
370
+ try:
371
+ if app.get_process_id() != target_pid:
292
372
  continue
293
-
373
+ except Exception:
374
+ continue
294
375
  _walk_atspi(app, elements, Atspi, screen_w, screen_h)
376
+ return elements
295
377
 
296
- return elements, matched_target
378
+
379
+ def _collect_chromium(desktop, Atspi, screen_w, screen_h):
380
+ """Walk only the Chrome/Chromium AT-SPI app node(s), matched by name.
381
+
382
+ Recovery for when the foreground window is Chromium but the PID-scoped walk
383
+ came back empty (a lazily-exported page tree, or an active-window PID that
384
+ disagrees with the AT-SPI app PID). Scoping by name keeps the recovery on the
385
+ browser alone, so a native app in front is never overlaid with the always-open
386
+ browser's labels.
387
+ """
388
+ elements = []
389
+ try:
390
+ n_apps = desktop.get_child_count()
391
+ except Exception:
392
+ return elements
393
+ for i in range(n_apps):
394
+ try:
395
+ app = desktop.get_child_at_index(i)
396
+ except Exception:
397
+ continue
398
+ if app is None:
399
+ continue
400
+ try:
401
+ name = (app.get_name() or "").lower()
402
+ except Exception:
403
+ name = ""
404
+ if "chrome" in name or "chromium" in name:
405
+ _walk_atspi(app, elements, Atspi, screen_w, screen_h)
406
+ return elements
407
+
408
+
409
+ def _within_rect(elements, rect):
410
+ """Keep only elements whose center lies inside the given screen rect."""
411
+ if not rect:
412
+ return elements
413
+ rx, ry, rw, rh = rect["x"], rect["y"], rect["width"], rect["height"]
414
+ kept = []
415
+ for el in elements:
416
+ cx = el["x"] + el["width"] / 2
417
+ cy = el["y"] + el["height"] / 2
418
+ if rx <= cx <= rx + rw and ry <= cy <= ry + rh:
419
+ kept.append(el)
420
+ return kept
297
421
 
298
422
 
299
423
  def enumerate_elements(pid=None):
@@ -307,24 +431,29 @@ def enumerate_elements(pid=None):
307
431
  target_pid = pid or _get_active_window_pid()
308
432
 
309
433
  desktop = Atspi.get_desktop(0)
310
- elements, matched_target = _collect_elements(
311
- desktop, target_pid, Atspi, screen_w, screen_h
312
- )
313
-
314
- # Linux/X11 accessibility can disagree with xdotool's active-window PID,
315
- # especially for Chromium/Electron renderers. If the focused PID yields
316
- # nothing, retry against the full AT-SPI desktop tree.
317
- if target_pid is not None and (not matched_target or not elements):
318
- elements, _ = _collect_elements(desktop, None, Atspi, screen_w, screen_h)
434
+ elements = _collect_elements(desktop, target_pid, Atspi, screen_w, screen_h)
435
+
436
+ # The foreground app exported its own actionable tree - use it verbatim. It is
437
+ # already scoped to the single window the agent sees (one desktop, WYSIWYG).
438
+ if not elements and _active_window_is_chromium(target_pid):
439
+ # Chromium publishes its accessibility tree lazily, and its active-window
440
+ # PID can disagree with the AT-SPI app PID, so the PID-scoped walk above
441
+ # can miss it. Recover by walking only the Chrome app node(s), retrying
442
+ # briefly for the lazy export, then clip to the focused window. We never
443
+ # fall back to the whole desktop, which would overlay the always-open
444
+ # browser's labels on top of whatever native app is in front.
445
+ active_rect = _get_active_window_geometry()
446
+ for _ in range(2):
447
+ time.sleep(0.6)
448
+ desktop = Atspi.get_desktop(0)
449
+ elements = _collect_chromium(desktop, Atspi, screen_w, screen_h)
450
+ if elements:
451
+ break
452
+ elements = _within_rect(elements, active_rect)
319
453
 
320
- # Chromium publishes its accessibility tree to AT-SPI lazily, so a label fired
321
- # right after navigation can beat that export and see an empty desktop. The
322
- # a11y bus is already enabled by the runtime at startup, so wait briefly and
323
- # re-walk the full tree once before giving up.
324
- if not elements:
325
- time.sleep(0.6)
326
- desktop = Atspi.get_desktop(0)
327
- elements, _ = _collect_elements(desktop, None, Atspi, screen_w, screen_h)
454
+ # A native (non-Chromium) foreground app that exports nothing yields an empty
455
+ # result on purpose: the agent falls back to screenshot_grid coordinates
456
+ # rather than being handed the background browser's elements.
328
457
 
329
458
  deduped = _dedup_elements(elements, screen_w, screen_h)
330
459