@octavus/agent 1.0.4 → 1.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/computer-mcp/linux/linux-driver.py +207 -78
- package/dist/index.js +38 -38
- package/dist/screen-publisher/index.js +2 -2
- package/package.json +1 -1
|
@@ -127,68 +127,98 @@ def _has_actionable(node, Atspi):
|
|
|
127
127
|
return False
|
|
128
128
|
|
|
129
129
|
|
|
130
|
+
def _overlaps_screen(x, y, w, h, screen_w, screen_h):
|
|
131
|
+
"""Whether a screen-coordinate box intersects the visible screen."""
|
|
132
|
+
return x < screen_w and y < screen_h and x + w > 0 and y + h > 0
|
|
133
|
+
|
|
134
|
+
|
|
130
135
|
def _walk_atspi(node, elements, Atspi, screen_w, screen_h, depth=0, max_depth=20):
|
|
131
136
|
if depth > max_depth:
|
|
132
137
|
return
|
|
138
|
+
|
|
139
|
+
visible = False
|
|
140
|
+
showing = False
|
|
133
141
|
try:
|
|
134
142
|
state_set = node.get_state_set()
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
if not state_set.contains(Atspi.StateType.SHOWING):
|
|
138
|
-
return
|
|
143
|
+
visible = state_set.contains(Atspi.StateType.VISIBLE)
|
|
144
|
+
showing = state_set.contains(Atspi.StateType.SHOWING)
|
|
139
145
|
except Exception:
|
|
140
146
|
pass
|
|
141
147
|
|
|
148
|
+
rect = None
|
|
142
149
|
try:
|
|
143
150
|
comp = node.get_component_iface()
|
|
144
151
|
if comp is not None:
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
152
|
+
extents = comp.get_extents(Atspi.CoordType.SCREEN)
|
|
153
|
+
rect = (extents.x, extents.y, extents.width, extents.height)
|
|
154
|
+
except Exception:
|
|
155
|
+
rect = None
|
|
156
|
+
|
|
157
|
+
on_screen = (
|
|
158
|
+
rect is not None
|
|
159
|
+
and rect[2] > 0
|
|
160
|
+
and rect[3] > 0
|
|
161
|
+
and _overlaps_screen(rect[0], rect[1], rect[2], rect[3], screen_w, screen_h)
|
|
162
|
+
)
|
|
163
|
+
|
|
164
|
+
# Include this node as an actionable element when it is a visible, on-screen
|
|
165
|
+
# clickable widget. Inclusion keys on real screen geometry, not the AT-SPI
|
|
166
|
+
# SHOWING flag: some toolkits (notably GTK4, e.g. gnome-text-editor) leave
|
|
167
|
+
# SHOWING unset on every on-screen descendant below the top frame, so gating
|
|
168
|
+
# on SHOWING would drop the entire app's controls. Off-screen and zero-area
|
|
169
|
+
# nodes fail the on_screen test instead.
|
|
170
|
+
if rect is not None and on_screen and visible:
|
|
171
|
+
try:
|
|
172
|
+
role_name = node.get_role_name().lower()
|
|
173
|
+
except Exception:
|
|
174
|
+
role_name = ""
|
|
175
|
+
if role_name and (role_name in CLICKABLE_ROLES or _has_actionable(node, Atspi)):
|
|
176
|
+
x, y, w, h = rect
|
|
177
|
+
area = w * h
|
|
178
|
+
screen_area = screen_w * screen_h
|
|
179
|
+
too_big = (
|
|
180
|
+
role_name in CONTAINER_ROLES and area > screen_area * 0.25
|
|
181
|
+
) or area > screen_area * 0.5
|
|
182
|
+
if not too_big:
|
|
183
|
+
title = ""
|
|
184
|
+
try:
|
|
185
|
+
title = node.get_name() or ""
|
|
186
|
+
except Exception:
|
|
187
|
+
pass
|
|
188
|
+
if not title:
|
|
189
|
+
try:
|
|
190
|
+
title = node.get_description() or ""
|
|
191
|
+
except Exception:
|
|
155
192
|
pass
|
|
156
|
-
|
|
193
|
+
if not title:
|
|
194
|
+
try:
|
|
195
|
+
text_iface = node.get_text_iface()
|
|
196
|
+
if text_iface:
|
|
197
|
+
val = text_iface.get_text(0, min(80, text_iface.get_character_count()))
|
|
198
|
+
if val:
|
|
199
|
+
title = val
|
|
200
|
+
except Exception:
|
|
157
201
|
pass
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
if len(title) > 60:
|
|
179
|
-
title = title[:60] + "..."
|
|
180
|
-
|
|
181
|
-
state = _get_element_state(node, role_name, Atspi)
|
|
182
|
-
|
|
183
|
-
elements.append({
|
|
184
|
-
"role": role_name,
|
|
185
|
-
"title": title,
|
|
186
|
-
"state": state,
|
|
187
|
-
"x": x, "y": y,
|
|
188
|
-
"width": w, "height": h,
|
|
189
|
-
})
|
|
190
|
-
except Exception:
|
|
191
|
-
pass
|
|
202
|
+
if len(title) > 60:
|
|
203
|
+
title = title[:60] + "..."
|
|
204
|
+
|
|
205
|
+
elements.append({
|
|
206
|
+
"role": role_name,
|
|
207
|
+
"title": title,
|
|
208
|
+
"state": _get_element_state(node, role_name, Atspi),
|
|
209
|
+
"x": x, "y": y,
|
|
210
|
+
"width": w, "height": h,
|
|
211
|
+
})
|
|
212
|
+
|
|
213
|
+
# Descend into children unless this node is provably off-screen. Keep walking
|
|
214
|
+
# through nodes with no geometry (structural containers such as the
|
|
215
|
+
# `application` node, which reports VISIBLE=False and no extents) and through
|
|
216
|
+
# on-screen containers that merely lack SHOWING - prune only a node that has a
|
|
217
|
+
# real box that is off-screen or zero-area and is also not SHOWING. That
|
|
218
|
+
# bounds the walk over large web trees (off-viewport content) without severing
|
|
219
|
+
# a native app's subtree at a SHOWING-less container.
|
|
220
|
+
if rect is not None and not on_screen and not showing:
|
|
221
|
+
return
|
|
192
222
|
|
|
193
223
|
try:
|
|
194
224
|
n_children = node.get_child_count()
|
|
@@ -272,28 +302,122 @@ def _get_active_window_pid():
|
|
|
272
302
|
return None
|
|
273
303
|
|
|
274
304
|
|
|
305
|
+
def _get_active_window_geometry():
|
|
306
|
+
"""Screen rect of the focused window as {x, y, width, height}, or None.
|
|
307
|
+
|
|
308
|
+
Used to clip a recovered element set to the window the agent is actually
|
|
309
|
+
looking at (one desktop, what-you-see-is-what-you-get).
|
|
310
|
+
"""
|
|
311
|
+
try:
|
|
312
|
+
out = subprocess.check_output(
|
|
313
|
+
["xdotool", "getactivewindow", "getwindowgeometry", "--shell"],
|
|
314
|
+
text=True,
|
|
315
|
+
timeout=5,
|
|
316
|
+
)
|
|
317
|
+
except Exception:
|
|
318
|
+
return None
|
|
319
|
+
|
|
320
|
+
vals = {}
|
|
321
|
+
for line in out.splitlines():
|
|
322
|
+
key, sep, value = line.partition("=")
|
|
323
|
+
if sep:
|
|
324
|
+
vals[key.strip()] = value.strip()
|
|
325
|
+
try:
|
|
326
|
+
return {
|
|
327
|
+
"x": int(vals["X"]),
|
|
328
|
+
"y": int(vals["Y"]),
|
|
329
|
+
"width": int(vals["WIDTH"]),
|
|
330
|
+
"height": int(vals["HEIGHT"]),
|
|
331
|
+
}
|
|
332
|
+
except (KeyError, ValueError):
|
|
333
|
+
return None
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
def _active_window_is_chromium(pid):
|
|
337
|
+
"""Whether the foreground PID is a Chrome/Chromium process.
|
|
338
|
+
|
|
339
|
+
Read from /proc rather than the window class: the cloud xdotool build has no
|
|
340
|
+
getwindowclassname and xprop is not installed, but the browser owns the
|
|
341
|
+
focused window's _NET_WM_PID, so its process name is the reliable signal.
|
|
342
|
+
"""
|
|
343
|
+
if pid is None:
|
|
344
|
+
return False
|
|
345
|
+
for proc_file in (f"/proc/{pid}/comm", f"/proc/{pid}/cmdline"):
|
|
346
|
+
try:
|
|
347
|
+
with open(proc_file, "rb") as f:
|
|
348
|
+
data = f.read().replace(b"\x00", b" ").decode("utf-8", "ignore").lower()
|
|
349
|
+
except Exception:
|
|
350
|
+
continue
|
|
351
|
+
if "chrome" in data or "chromium" in data:
|
|
352
|
+
return True
|
|
353
|
+
return False
|
|
354
|
+
|
|
355
|
+
|
|
275
356
|
def _collect_elements(desktop, target_pid, Atspi, screen_w, screen_h):
|
|
276
|
-
elements
|
|
277
|
-
matched_target = False
|
|
357
|
+
"""Collect actionable elements from the AT-SPI app(s) matching target_pid.
|
|
278
358
|
|
|
359
|
+
Strictly PID-scoped: a None target_pid (foreground PID unavailable) matches no
|
|
360
|
+
app and returns empty, so labeling never falls back to walking the whole
|
|
361
|
+
desktop, which would overlay the always-open browser's labels onto whatever
|
|
362
|
+
native app is in front.
|
|
363
|
+
"""
|
|
364
|
+
elements = []
|
|
279
365
|
n_apps = desktop.get_child_count()
|
|
280
366
|
for i in range(n_apps):
|
|
281
367
|
app = desktop.get_child_at_index(i)
|
|
282
368
|
if app is None:
|
|
283
369
|
continue
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
try:
|
|
287
|
-
app_pid = app.get_process_id()
|
|
288
|
-
if app_pid != target_pid:
|
|
289
|
-
continue
|
|
290
|
-
matched_target = True
|
|
291
|
-
except Exception:
|
|
370
|
+
try:
|
|
371
|
+
if app.get_process_id() != target_pid:
|
|
292
372
|
continue
|
|
293
|
-
|
|
373
|
+
except Exception:
|
|
374
|
+
continue
|
|
294
375
|
_walk_atspi(app, elements, Atspi, screen_w, screen_h)
|
|
376
|
+
return elements
|
|
295
377
|
|
|
296
|
-
|
|
378
|
+
|
|
379
|
+
def _collect_chromium(desktop, Atspi, screen_w, screen_h):
|
|
380
|
+
"""Walk only the Chrome/Chromium AT-SPI app node(s), matched by name.
|
|
381
|
+
|
|
382
|
+
Recovery for when the foreground window is Chromium but the PID-scoped walk
|
|
383
|
+
came back empty (a lazily-exported page tree, or an active-window PID that
|
|
384
|
+
disagrees with the AT-SPI app PID). Scoping by name keeps the recovery on the
|
|
385
|
+
browser alone, so a native app in front is never overlaid with the always-open
|
|
386
|
+
browser's labels.
|
|
387
|
+
"""
|
|
388
|
+
elements = []
|
|
389
|
+
try:
|
|
390
|
+
n_apps = desktop.get_child_count()
|
|
391
|
+
except Exception:
|
|
392
|
+
return elements
|
|
393
|
+
for i in range(n_apps):
|
|
394
|
+
try:
|
|
395
|
+
app = desktop.get_child_at_index(i)
|
|
396
|
+
except Exception:
|
|
397
|
+
continue
|
|
398
|
+
if app is None:
|
|
399
|
+
continue
|
|
400
|
+
try:
|
|
401
|
+
name = (app.get_name() or "").lower()
|
|
402
|
+
except Exception:
|
|
403
|
+
name = ""
|
|
404
|
+
if "chrome" in name or "chromium" in name:
|
|
405
|
+
_walk_atspi(app, elements, Atspi, screen_w, screen_h)
|
|
406
|
+
return elements
|
|
407
|
+
|
|
408
|
+
|
|
409
|
+
def _within_rect(elements, rect):
|
|
410
|
+
"""Keep only elements whose center lies inside the given screen rect."""
|
|
411
|
+
if not rect:
|
|
412
|
+
return elements
|
|
413
|
+
rx, ry, rw, rh = rect["x"], rect["y"], rect["width"], rect["height"]
|
|
414
|
+
kept = []
|
|
415
|
+
for el in elements:
|
|
416
|
+
cx = el["x"] + el["width"] / 2
|
|
417
|
+
cy = el["y"] + el["height"] / 2
|
|
418
|
+
if rx <= cx <= rx + rw and ry <= cy <= ry + rh:
|
|
419
|
+
kept.append(el)
|
|
420
|
+
return kept
|
|
297
421
|
|
|
298
422
|
|
|
299
423
|
def enumerate_elements(pid=None):
|
|
@@ -307,24 +431,29 @@ def enumerate_elements(pid=None):
|
|
|
307
431
|
target_pid = pid or _get_active_window_pid()
|
|
308
432
|
|
|
309
433
|
desktop = Atspi.get_desktop(0)
|
|
310
|
-
elements
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
434
|
+
elements = _collect_elements(desktop, target_pid, Atspi, screen_w, screen_h)
|
|
435
|
+
|
|
436
|
+
# The foreground app exported its own actionable tree - use it verbatim. It is
|
|
437
|
+
# already scoped to the single window the agent sees (one desktop, WYSIWYG).
|
|
438
|
+
if not elements and _active_window_is_chromium(target_pid):
|
|
439
|
+
# Chromium publishes its accessibility tree lazily, and its active-window
|
|
440
|
+
# PID can disagree with the AT-SPI app PID, so the PID-scoped walk above
|
|
441
|
+
# can miss it. Recover by walking only the Chrome app node(s), retrying
|
|
442
|
+
# briefly for the lazy export, then clip to the focused window. We never
|
|
443
|
+
# fall back to the whole desktop, which would overlay the always-open
|
|
444
|
+
# browser's labels on top of whatever native app is in front.
|
|
445
|
+
active_rect = _get_active_window_geometry()
|
|
446
|
+
for _ in range(2):
|
|
447
|
+
time.sleep(0.6)
|
|
448
|
+
desktop = Atspi.get_desktop(0)
|
|
449
|
+
elements = _collect_chromium(desktop, Atspi, screen_w, screen_h)
|
|
450
|
+
if elements:
|
|
451
|
+
break
|
|
452
|
+
elements = _within_rect(elements, active_rect)
|
|
319
453
|
|
|
320
|
-
#
|
|
321
|
-
#
|
|
322
|
-
#
|
|
323
|
-
# re-walk the full tree once before giving up.
|
|
324
|
-
if not elements:
|
|
325
|
-
time.sleep(0.6)
|
|
326
|
-
desktop = Atspi.get_desktop(0)
|
|
327
|
-
elements, _ = _collect_elements(desktop, None, Atspi, screen_w, screen_h)
|
|
454
|
+
# A native (non-Chromium) foreground app that exports nothing yields an empty
|
|
455
|
+
# result on purpose: the agent falls back to screenshot_grid coordinates
|
|
456
|
+
# rather than being handed the background browser's elements.
|
|
328
457
|
|
|
329
458
|
deduped = _dedup_elements(elements, screen_w, screen_h)
|
|
330
459
|
|