@octavus/agent 1.0.8 → 1.0.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/computer-mcp/linux/linux-driver.py +30 -14
- package/dist/index.js +1 -1
- package/package.json +1 -1
|
@@ -132,7 +132,9 @@ def _overlaps_screen(x, y, w, h, screen_w, screen_h):
|
|
|
132
132
|
return x < screen_w and y < screen_h and x + w > 0 and y + h > 0
|
|
133
133
|
|
|
134
134
|
|
|
135
|
-
def _walk_atspi(
|
|
135
|
+
def _walk_atspi(
|
|
136
|
+
node, elements, Atspi, screen_w, screen_h, depth=0, max_depth=20, trust_geometry=False
|
|
137
|
+
):
|
|
136
138
|
if depth > max_depth:
|
|
137
139
|
return
|
|
138
140
|
|
|
@@ -161,13 +163,17 @@ def _walk_atspi(node, elements, Atspi, screen_w, screen_h, depth=0, max_depth=20
|
|
|
161
163
|
and _overlaps_screen(rect[0], rect[1], rect[2], rect[3], screen_w, screen_h)
|
|
162
164
|
)
|
|
163
165
|
|
|
164
|
-
# Include this node as an actionable element when it is
|
|
165
|
-
#
|
|
166
|
-
#
|
|
167
|
-
#
|
|
168
|
-
#
|
|
169
|
-
#
|
|
170
|
-
|
|
166
|
+
# Include this node as an actionable element when it is an on-screen clickable
|
|
167
|
+
# widget. Inclusion keys on real screen geometry, not the AT-SPI SHOWING flag:
|
|
168
|
+
# some toolkits (notably GTK4, e.g. gnome-text-editor) leave SHOWING unset on
|
|
169
|
+
# every on-screen descendant below the top frame, so gating on SHOWING would
|
|
170
|
+
# drop the entire app's controls. `trust_geometry` extends this to the VISIBLE
|
|
171
|
+
# flag, which GTK4 likewise leaves unset on those descendants: for a native
|
|
172
|
+
# foreground app, geometry + role decide inclusion; for Chromium (whose web
|
|
173
|
+
# a11y flags are reliable) VISIBLE stays required, so web labeling never
|
|
174
|
+
# surfaces hidden-but-positioned elements. Off-screen and zero-area nodes fail
|
|
175
|
+
# the on_screen test regardless.
|
|
176
|
+
if rect is not None and on_screen and (trust_geometry or visible):
|
|
171
177
|
try:
|
|
172
178
|
role_name = node.get_role_name().lower()
|
|
173
179
|
except Exception:
|
|
@@ -225,7 +231,9 @@ def _walk_atspi(node, elements, Atspi, screen_w, screen_h, depth=0, max_depth=20
|
|
|
225
231
|
for i in range(n_children):
|
|
226
232
|
child = node.get_child_at_index(i)
|
|
227
233
|
if child is not None:
|
|
228
|
-
_walk_atspi(
|
|
234
|
+
_walk_atspi(
|
|
235
|
+
child, elements, Atspi, screen_w, screen_h, depth + 1, max_depth, trust_geometry
|
|
236
|
+
)
|
|
229
237
|
except Exception:
|
|
230
238
|
pass
|
|
231
239
|
|
|
@@ -353,13 +361,15 @@ def _active_window_is_chromium(pid):
|
|
|
353
361
|
return False
|
|
354
362
|
|
|
355
363
|
|
|
356
|
-
def _collect_elements(desktop, target_pid, Atspi, screen_w, screen_h):
|
|
364
|
+
def _collect_elements(desktop, target_pid, Atspi, screen_w, screen_h, trust_geometry=False):
|
|
357
365
|
"""Collect actionable elements from the AT-SPI app(s) matching target_pid.
|
|
358
366
|
|
|
359
367
|
Strictly PID-scoped: a None target_pid (foreground PID unavailable) matches no
|
|
360
368
|
app and returns empty, so labeling never falls back to walking the whole
|
|
361
369
|
desktop, which would overlay the always-open browser's labels onto whatever
|
|
362
|
-
native app is in front.
|
|
370
|
+
native app is in front. `trust_geometry` is forwarded to the walk so a native
|
|
371
|
+
foreground app's on-screen controls are included from geometry even when its
|
|
372
|
+
toolkit leaves the AT-SPI VISIBLE/SHOWING flags unset (GTK4).
|
|
363
373
|
"""
|
|
364
374
|
elements = []
|
|
365
375
|
n_apps = desktop.get_child_count()
|
|
@@ -372,7 +382,7 @@ def _collect_elements(desktop, target_pid, Atspi, screen_w, screen_h):
|
|
|
372
382
|
continue
|
|
373
383
|
except Exception:
|
|
374
384
|
continue
|
|
375
|
-
_walk_atspi(app, elements, Atspi, screen_w, screen_h)
|
|
385
|
+
_walk_atspi(app, elements, Atspi, screen_w, screen_h, trust_geometry=trust_geometry)
|
|
376
386
|
return elements
|
|
377
387
|
|
|
378
388
|
|
|
@@ -429,13 +439,19 @@ def enumerate_elements(pid=None):
|
|
|
429
439
|
|
|
430
440
|
screen_w, screen_h = get_screen_size()
|
|
431
441
|
target_pid = pid or _get_active_window_pid()
|
|
442
|
+
# A native foreground app (GTK4 etc.) leaves the AT-SPI VISIBLE/SHOWING flags
|
|
443
|
+
# unset on its on-screen descendants, so trust geometry for it; Chromium's web
|
|
444
|
+
# a11y flags are reliable, so keep requiring VISIBLE there.
|
|
445
|
+
foreground_is_chromium = _active_window_is_chromium(target_pid)
|
|
432
446
|
|
|
433
447
|
desktop = Atspi.get_desktop(0)
|
|
434
|
-
elements = _collect_elements(
|
|
448
|
+
elements = _collect_elements(
|
|
449
|
+
desktop, target_pid, Atspi, screen_w, screen_h, trust_geometry=not foreground_is_chromium
|
|
450
|
+
)
|
|
435
451
|
|
|
436
452
|
# The foreground app exported its own actionable tree - use it verbatim. It is
|
|
437
453
|
# already scoped to the single window the agent sees (one desktop, WYSIWYG).
|
|
438
|
-
if not elements and
|
|
454
|
+
if not elements and foreground_is_chromium:
|
|
439
455
|
# Chromium publishes its accessibility tree lazily, and its active-window
|
|
440
456
|
# PID can disagree with the AT-SPI app PID, so the PID-scoped walk above
|
|
441
457
|
# can miss it. Recover by walking only the Chrome app node(s), retrying
|
package/dist/index.js
CHANGED
|
@@ -323,7 +323,7 @@ for w in $(xdotool search --class "$CLASS_RE" 2>/dev/null); do
|
|
|
323
323
|
done
|
|
324
324
|
done
|
|
325
325
|
exit 0
|
|
326
|
-
`,rF=!1;function nF(e){if(Tue()!=="linux"||!e.display)return;if(zue("which",["xdotool"],{stdio:"ignore"}).status!==0){rF||(e.log.warn("xdotool not installed; the browser window may be blank in recordings",{hint:"Install xdotool (e.g. apt-get install -y xdotool) so the browser window is raised on the shared display."}),rF=!0);return}let t=e.dimensions?Math.floor(e.dimensions.width*.9):0,r=e.dimensions?Math.floor(e.dimensions.height*.9):0,o=Eue("bash",["-c",Iue],{stdio:"ignore",detached:!0,env:{...process.env,DISPLAY:e.display,RAISE_MIN_W:String(t),RAISE_MIN_H:String(r)}});o.on("error",n=>{e.log.warn("Failed to raise the browser window",{error:String(n)})}),o.unref()}var gE=Pue(import.meta.url);function oF(e){return iF(gE.resolve(`${e}/package.json`))}function Cue(){let e=iF(Rue(import.meta.url)),t=Sa(e,"browser-extension-mcp.js");return Oue(t)?{browserMcpEntry:t,computerUseMcpEntry:Sa(e,"computer-mcp","dist","index.js"),filesystemMcpEntry:Sa(e,"filesystem-mcp.mjs"),extensionDir:Sa(e,"browser-extension")}:{browserMcpEntry:gE.resolve("@octavus/browser-extension-mcp"),computerUseMcpEntry:gE.resolve("@octavus/computer-use-mcp"),filesystemMcpEntry:Sa(oF("@modelcontextprotocol/server-filesystem"),"dist/index.js"),extensionDir:Sa(oF("browser-extension"),"dist")}}var vE=class extends si{constructor(r,o){super(r);this.raise=o}async ensureReady(){let r=await super.ensureReady();return r.recovered?.includes("browser-extension")&&nF(this.raise),r}};function sF(e){let{browserMcpEntry:t,computerUseMcpEntry:r,filesystemMcpEntry:o,extensionDir:n}=Cue(),i={};e.display&&(i.DISPLAY=e.display),process.env.DBUS_SESSION_BUS_ADDRESS&&(i.DBUS_SESSION_BUS_ADDRESS=process.env.DBUS_SESSION_BUS_ADDRESS);let s=[t,`--extension-dir=${n}`,`--profile-dir=${e.browserProfileDir}`,`--download-dir=${e.downloadsDir}`,...e.chromePath?[`--chrome-path=${e.chromePath}`]:[],"--durable-profile","--reset-window-placement",...e.fillWindowSize?["--extra-flag=--window-position=0,0",`--extra-flag=--window-size=${e.fillWindowSize.width},${e.fillWindowSize.height}`]:[]],a={mcpServers:{filesystem:si.stdio("node",[o,e.workspaceDir,e.skillsDir]),shell:si.shell({cwd:e.workspaceDir,mode:eF}),"computer-use":si.stdio("node",[r],{env:i}),"browser-extension":si.stdio("node",s,{env:i})},deferredEntries:["browser-extension","computer-use"]};return e.raiseBrowserWindowOnReady?new vE(a,{display:e.display,dimensions:e.dimensions,log:e.log}):new si(a)}import{spawn as aF,spawnSync as yl}from"child_process";import{existsSync as Aue,mkdtempSync as Nue,rmSync as Due,writeFileSync as _E}from"fs";import{platform as jue,tmpdir as Uue}from"os";import{join as gl}from"path";var vl={width:1920,height:1080};function _l(e){return yl("which",[e],{stdio:"ignore"}).status===0}function cF(e){return yl("xdpyinfo",["-display",e],{stdio:"ignore"}).status===0}function Lue(e){let t=yl("xdpyinfo",["-display",e],{encoding:"utf8"}),r=/dimensions:\s+(\d+)x(\d+)\s+pixels/.exec(t.stdout??"");return r?{width:Number(r[1]),height:Number(r[2])}:void 0}function Mue(){for(let e=99;e<120;e++)if(!Aue(`/tmp/.X11-unix/X${e}`))return`:${e}`;return":99"}function yE(e){return new Promise(t=>{setTimeout(t,e)})}async function Zue(e,t){let r=Date.now()+t;for(;Date.now()<r;){if(cF(e))return!0;await yE(200)}return!1}function uF(e){if(!_l("xprop"))return;let t=yl("xprop",["-display",e,"-root","_NET_SUPPORTING_WM_CHECK"],{encoding:"utf8"});if(t.error||t.status!==0)return;let r=t.stdout??"";if(/0x[0-9a-f]+/i.test(r))return!0;if(/not found/i.test(r))return!1}async function Fue(e,t,r){if(!_l("fluxbox"))return r.warn("No window manager (fluxbox) found; computer-use labeling will be degraded",{hint:"Install fluxbox (e.g. apt-get install -y fluxbox) so on-screen element labeling works."}),!1;let o;try{o=Nue(gl(Uue(),"octavus-fluxbox-"));let i=gl(o,"apps"),s=gl(o,"overlay");_E(gl(o,"init"),["session.screen0.toolbar.visible: false","session.screen0.defaultDeco: NONE","session.screen0.rootCommand: fbsetroot -solid rgb:1f/24/30",`session.styleOverlay: ${s}`,`session.appsFile: ${i}`,""].join(`
|
|
326
|
+
`,rF=!1;function nF(e){if(Tue()!=="linux"||!e.display)return;if(zue("which",["xdotool"],{stdio:"ignore"}).status!==0){rF||(e.log.warn("xdotool not installed; the browser window may be blank in recordings",{hint:"Install xdotool (e.g. apt-get install -y xdotool) so the browser window is raised on the shared display."}),rF=!0);return}let t=e.dimensions?Math.floor(e.dimensions.width*.9):0,r=e.dimensions?Math.floor(e.dimensions.height*.9):0,o=Eue("bash",["-c",Iue],{stdio:"ignore",detached:!0,env:{...process.env,DISPLAY:e.display,RAISE_MIN_W:String(t),RAISE_MIN_H:String(r)}});o.on("error",n=>{e.log.warn("Failed to raise the browser window",{error:String(n)})}),o.unref()}var gE=Pue(import.meta.url);function oF(e){return iF(gE.resolve(`${e}/package.json`))}function Cue(){let e=iF(Rue(import.meta.url)),t=Sa(e,"browser-extension-mcp.js");return Oue(t)?{browserMcpEntry:t,computerUseMcpEntry:Sa(e,"computer-mcp","dist","index.js"),filesystemMcpEntry:Sa(e,"filesystem-mcp.mjs"),extensionDir:Sa(e,"browser-extension")}:{browserMcpEntry:gE.resolve("@octavus/browser-extension-mcp"),computerUseMcpEntry:gE.resolve("@octavus/computer-use-mcp"),filesystemMcpEntry:Sa(oF("@modelcontextprotocol/server-filesystem"),"dist/index.js"),extensionDir:Sa(oF("browser-extension"),"dist")}}var vE=class extends si{constructor(r,o){super(r);this.raise=o}async ensureReady(){let r=await super.ensureReady();return r.recovered?.includes("browser-extension")&&nF(this.raise),r}};function sF(e){let{browserMcpEntry:t,computerUseMcpEntry:r,filesystemMcpEntry:o,extensionDir:n}=Cue(),i={};e.display&&(i.DISPLAY=e.display),process.env.DBUS_SESSION_BUS_ADDRESS&&(i.DBUS_SESSION_BUS_ADDRESS=process.env.DBUS_SESSION_BUS_ADDRESS);let s=[t,`--extension-dir=${n}`,`--profile-dir=${e.browserProfileDir}`,`--download-dir=${e.downloadsDir}`,...e.chromePath?[`--chrome-path=${e.chromePath}`]:[],"--durable-profile","--reset-window-placement","--enable-recording",...e.fillWindowSize?["--extra-flag=--window-position=0,0",`--extra-flag=--window-size=${e.fillWindowSize.width},${e.fillWindowSize.height}`]:[]],a={mcpServers:{filesystem:si.stdio("node",[o,e.workspaceDir,e.skillsDir]),shell:si.shell({cwd:e.workspaceDir,mode:eF}),"computer-use":si.stdio("node",[r],{env:i}),"browser-extension":si.stdio("node",s,{env:i})},deferredEntries:["browser-extension","computer-use"]};return e.raiseBrowserWindowOnReady?new vE(a,{display:e.display,dimensions:e.dimensions,log:e.log}):new si(a)}import{spawn as aF,spawnSync as yl}from"child_process";import{existsSync as Aue,mkdtempSync as Nue,rmSync as Due,writeFileSync as _E}from"fs";import{platform as jue,tmpdir as Uue}from"os";import{join as gl}from"path";var vl={width:1920,height:1080};function _l(e){return yl("which",[e],{stdio:"ignore"}).status===0}function cF(e){return yl("xdpyinfo",["-display",e],{stdio:"ignore"}).status===0}function Lue(e){let t=yl("xdpyinfo",["-display",e],{encoding:"utf8"}),r=/dimensions:\s+(\d+)x(\d+)\s+pixels/.exec(t.stdout??"");return r?{width:Number(r[1]),height:Number(r[2])}:void 0}function Mue(){for(let e=99;e<120;e++)if(!Aue(`/tmp/.X11-unix/X${e}`))return`:${e}`;return":99"}function yE(e){return new Promise(t=>{setTimeout(t,e)})}async function Zue(e,t){let r=Date.now()+t;for(;Date.now()<r;){if(cF(e))return!0;await yE(200)}return!1}function uF(e){if(!_l("xprop"))return;let t=yl("xprop",["-display",e,"-root","_NET_SUPPORTING_WM_CHECK"],{encoding:"utf8"});if(t.error||t.status!==0)return;let r=t.stdout??"";if(/0x[0-9a-f]+/i.test(r))return!0;if(/not found/i.test(r))return!1}async function Fue(e,t,r){if(!_l("fluxbox"))return r.warn("No window manager (fluxbox) found; computer-use labeling will be degraded",{hint:"Install fluxbox (e.g. apt-get install -y fluxbox) so on-screen element labeling works."}),!1;let o;try{o=Nue(gl(Uue(),"octavus-fluxbox-"));let i=gl(o,"apps"),s=gl(o,"overlay");_E(gl(o,"init"),["session.screen0.toolbar.visible: false","session.screen0.defaultDeco: NONE","session.screen0.rootCommand: fbsetroot -solid rgb:1f/24/30",`session.styleOverlay: ${s}`,`session.appsFile: ${i}`,""].join(`
|
|
327
327
|
`)),_E(s,`background: unset
|
|
328
328
|
`),_E(i,["chromium","Chromium-browser","google-chrome","Google-chrome","chrome"].map(a=>`[app] (name=${a})
|
|
329
329
|
[Deco] {NONE}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@octavus/agent",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.9",
|
|
4
4
|
"description": "Octavus Agent CLI - run one of your cloud Octavus Agents on a machine you control, using that machine as the agent's computer. The agent lives in the cloud (fully configured and observable); the local machine only supplies the computer and the trigger.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"octavus",
|