@agent-sh/computer-use-linux 0.7.43 → 0.7.44

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -20,6 +20,7 @@ import {
20
20
  readFileSync,
21
21
  statSync,
22
22
  } from "node:fs";
23
+ import { spawn } from "node:child_process";
23
24
  import { createRequire } from "node:module";
24
25
  import { homedir } from "node:os";
25
26
  import { delimiter, isAbsolute, join } from "node:path";
@@ -95,8 +96,11 @@ interface BinaryLaunch {
95
96
  }
96
97
 
97
98
  interface ExtensionDependencies {
99
+ /** Replaces the whole lookup, the first-use download included. */
98
100
  findBinary?: () => BinaryLaunch | null;
99
101
  loadClientModule?: () => NativeMcpClientModule;
102
+ /** Directory holding npm/bin and npm/install.js. Defaults to this package. */
103
+ packageRoot?: string;
100
104
  }
101
105
 
102
106
  const ORIGINAL_TOOL_NAMES = AVAILABLE_MCP_TOOLS.map((tool) => tool.name);
@@ -191,57 +195,72 @@ export function findExecutableOnPath(
191
195
  return null;
192
196
  }
193
197
 
198
+ /**
199
+ * Variables the server child receives, plus every LC_ variable. Everything
200
+ * else in Pi's environment stays out of the desktop process.
201
+ * scripts/check_env_forwarding.py fails CI when the server reads a variable
202
+ * missing here. COMPUTER_USE_LINUX_NOTIFY_ON_COMPLETE is left out: it adds a
203
+ * tool, and the client refuses a catalog that differs from the generated one.
204
+ */
205
+ const FORWARDED_ENVIRONMENT = [
206
+ "PATH",
207
+ "HOME",
208
+ "USER",
209
+ "LOGNAME",
210
+ "SHELL",
211
+ "LANG",
212
+ "LANGUAGE",
213
+ "TERM",
214
+ "TMPDIR",
215
+ "DISPLAY",
216
+ "WAYLAND_DISPLAY",
217
+ "XAUTHORITY",
218
+ "DBUS_SESSION_BUS_ADDRESS",
219
+ "XDG_RUNTIME_DIR",
220
+ "XDG_CURRENT_DESKTOP",
221
+ "XDG_SESSION_TYPE",
222
+ "XDG_SESSION_DESKTOP",
223
+ "XDG_DATA_DIRS",
224
+ "XDG_CONFIG_HOME",
225
+ "XDG_CACHE_HOME",
226
+ "XDG_STATE_HOME",
227
+ "DESKTOP_SESSION",
228
+ "GDMSESSION",
229
+ "HYPRLAND_INSTANCE_SIGNATURE",
230
+ "NIRI_SOCKET",
231
+ "I3SOCK",
232
+ "SWAYSOCK",
233
+ "YDOTOOL_SOCKET",
234
+ "GSETTINGS_SCHEMA_DIR",
235
+ "GIO_EXTRA_MODULES",
236
+ "GI_TYPELIB_PATH",
237
+ "LD_LIBRARY_PATH",
238
+ "NIX_LD",
239
+ "NIX_LD_LIBRARY_PATH",
240
+ "RUST_LOG",
241
+ "COMPUTER_USE_LINUX_AGENT_NAME",
242
+ "COMPUTER_USE_LINUX_COSMIC_HELPER",
243
+ "COMPUTER_USE_LINUX_ENABLE_SHELL",
244
+ "COMPUTER_USE_LINUX_FORCE_PORTAL_KEYBOARD",
245
+ "COMPUTER_USE_LINUX_FORCE_PORTAL_POINTER",
246
+ "COMPUTER_USE_LINUX_FORCE_XDOTOOL_KEYBOARD",
247
+ "COMPUTER_USE_LINUX_FORCE_YDOTOOL_KEYBOARD",
248
+ "COMPUTER_USE_LINUX_FORCE_YDOTOOL_POINTER",
249
+ "COMPUTER_USE_LINUX_HYPRLAND_BACKGROUND",
250
+ "COMPUTER_USE_LINUX_INDICATOR",
251
+ "COMPUTER_USE_LINUX_INDICATOR_BIN",
252
+ "COMPUTER_USE_LINUX_INDICATOR_HIDE_TEXT",
253
+ "COMPUTER_USE_LINUX_PERSIST_REMOTE_DESKTOP",
254
+ "COMPUTER_USE_LINUX_PORTAL_SCROLL_INVERT",
255
+ "COMPUTER_USE_LINUX_SCREENSHOT_BACKEND",
256
+ "COMPUTER_USE_LINUX_XDOTOOL_TYPE_DELAY_MS",
257
+ "CU_DISABLE_ABS_POINTER",
258
+ "PIXELFLUX_CU",
259
+ "PIXELFLUX_CU_TOKEN",
260
+ ] as const;
261
+
194
262
  function runtimeEnvironment(): Record<string, string> {
195
- const allowed = new Set([
196
- "PATH",
197
- "HOME",
198
- "USER",
199
- "LOGNAME",
200
- "SHELL",
201
- "LANG",
202
- "LANGUAGE",
203
- "TERM",
204
- "TMPDIR",
205
- "DISPLAY",
206
- "WAYLAND_DISPLAY",
207
- "XAUTHORITY",
208
- "DBUS_SESSION_BUS_ADDRESS",
209
- "XDG_RUNTIME_DIR",
210
- "XDG_CURRENT_DESKTOP",
211
- "XDG_SESSION_TYPE",
212
- "XDG_SESSION_DESKTOP",
213
- "XDG_DATA_DIRS",
214
- "XDG_CONFIG_HOME",
215
- "XDG_CACHE_HOME",
216
- "XDG_STATE_HOME",
217
- "DESKTOP_SESSION",
218
- "GDMSESSION",
219
- "HYPRLAND_INSTANCE_SIGNATURE",
220
- "I3SOCK",
221
- "SWAYSOCK",
222
- "YDOTOOL_SOCKET",
223
- "GSETTINGS_SCHEMA_DIR",
224
- "GIO_EXTRA_MODULES",
225
- "GI_TYPELIB_PATH",
226
- "LD_LIBRARY_PATH",
227
- "NIX_LD",
228
- "NIX_LD_LIBRARY_PATH",
229
- "RUST_LOG",
230
- "COMPUTER_USE_LINUX_COSMIC_HELPER",
231
- "COMPUTER_USE_LINUX_AGENT_NAME",
232
- "COMPUTER_USE_LINUX_INDICATOR",
233
- "COMPUTER_USE_LINUX_INDICATOR_BIN",
234
- "COMPUTER_USE_LINUX_INDICATOR_HIDE_TEXT",
235
- "COMPUTER_USE_LINUX_ENABLE_SHELL",
236
- "COMPUTER_USE_LINUX_FORCE_PORTAL_KEYBOARD",
237
- "COMPUTER_USE_LINUX_FORCE_PORTAL_POINTER",
238
- "COMPUTER_USE_LINUX_FORCE_XDOTOOL_KEYBOARD",
239
- "COMPUTER_USE_LINUX_FORCE_YDOTOOL_KEYBOARD",
240
- "COMPUTER_USE_LINUX_FORCE_YDOTOOL_POINTER",
241
- "COMPUTER_USE_LINUX_PORTAL_SCROLL_INVERT",
242
- "COMPUTER_USE_LINUX_SCREENSHOT_BACKEND",
243
- "CU_DISABLE_ABS_POINTER",
244
- ]);
263
+ const allowed = new Set<string>(FORWARDED_ENVIRONMENT);
245
264
  const env = Object.fromEntries(
246
265
  Object.entries(process.env).filter(
247
266
  (entry): entry is [string, string] =>
@@ -254,7 +273,16 @@ function runtimeEnvironment(): Record<string, string> {
254
273
  return env;
255
274
  }
256
275
 
257
- function defaultFindBinary(): BinaryLaunch | null {
276
+ /** Directory holding npm/bin and npm/install.js. */
277
+ const PACKAGE_ROOT = join(__dirname, "..", "..");
278
+ const INSTALL_OUTPUT_CHARS = 4000;
279
+
280
+ function installerPath(packageRoot: string): string {
281
+ return join(packageRoot, "npm", "install.js");
282
+ }
283
+
284
+ /** COMPUTER_USE_LINUX_BIN, then the package's own binary. */
285
+ function localLaunch(packageRoot: string): BinaryLaunch | null {
258
286
  const env = runtimeEnvironment();
259
287
  const override = process.env.COMPUTER_USE_LINUX_BIN?.trim();
260
288
  // Relative overrides would be resolved from Pi's potentially untrusted
@@ -263,50 +291,93 @@ function defaultFindBinary(): BinaryLaunch | null {
263
291
  return { binaryPath: override, env };
264
292
  }
265
293
 
294
+ const binDir = join(packageRoot, "npm", "bin");
266
295
  const bundledBinary = join(
267
- __dirname,
268
- "..",
269
- "..",
270
- "npm",
271
- "bin",
296
+ binDir,
272
297
  `computer-use-linux-${process.platform}-${process.arch}`,
273
298
  );
274
- if (executable(bundledBinary)) {
275
- const cosmicHelper = join(
276
- __dirname,
277
- "..",
278
- "..",
279
- "npm",
280
- "bin",
281
- "computer-use-linux-cosmic",
282
- );
283
- if (!env.COMPUTER_USE_LINUX_COSMIC_HELPER && executable(cosmicHelper)) {
284
- env.COMPUTER_USE_LINUX_COSMIC_HELPER = cosmicHelper;
285
- }
286
- const indicator = join(
287
- __dirname,
288
- "..",
289
- "..",
290
- "npm",
291
- "bin",
292
- "computer-use-linux-indicator",
293
- );
294
- if (!env.COMPUTER_USE_LINUX_INDICATOR_BIN && executable(indicator)) {
295
- env.COMPUTER_USE_LINUX_INDICATOR_BIN = indicator;
296
- }
297
- return { binaryPath: bundledBinary, env };
299
+ if (!executable(bundledBinary)) return null;
300
+ const cosmicHelper = join(binDir, "computer-use-linux-cosmic");
301
+ if (!env.COMPUTER_USE_LINUX_COSMIC_HELPER && executable(cosmicHelper)) {
302
+ env.COMPUTER_USE_LINUX_COSMIC_HELPER = cosmicHelper;
303
+ }
304
+ const indicator = join(binDir, "computer-use-linux-indicator");
305
+ if (!env.COMPUTER_USE_LINUX_INDICATOR_BIN && executable(indicator)) {
306
+ env.COMPUTER_USE_LINUX_INDICATOR_BIN = indicator;
298
307
  }
308
+ return { binaryPath: bundledBinary, env };
309
+ }
299
310
 
300
- // A temporary extension (`pi -e npm:@agent-sh/computer-use-linux`) is staged
301
- // without the postinstall-downloaded platform binary, so fall back to a
302
- // global npm or cargo install. The MCP client still checks the server
303
- // version and tool catalog, so a mismatched install fails with that reason.
311
+ /**
312
+ * A global npm or cargo install. The MCP client still checks the server
313
+ * version and tool catalog, so a mismatched install fails with that reason.
314
+ */
315
+ function pathLaunch(): BinaryLaunch | null {
304
316
  const onPath = findExecutableOnPath(process.env.PATH, "computer-use-linux");
305
- if (onPath) {
306
- return { binaryPath: onPath, env };
317
+ return onPath ? { binaryPath: onPath, env: runtimeEnvironment() } : null;
318
+ }
319
+
320
+ /**
321
+ * Runs npm/install.js, the first-use download the CLI wrapper also runs.
322
+ * npm 12 skips install scripts it was not told to allow, so `pi install` can
323
+ * stage the package without its binary. The installer checks each download's
324
+ * sha256 and moves the main binary into place last. Its output is kept for
325
+ * the error and never reaches Pi's terminal.
326
+ */
327
+ function runInstaller(packageRoot: string): Promise<void> {
328
+ const installer = installerPath(packageRoot);
329
+ // Under Bun, execPath is the Bun or Pi executable, not node.
330
+ const node = process.versions.bun
331
+ ? findExecutableOnPath(process.env.PATH, "node")
332
+ : process.execPath;
333
+ if (!node) {
334
+ return Promise.reject(
335
+ new Error(`node was not found on PATH to run ${installer}`),
336
+ );
307
337
  }
338
+ return new Promise((resolve, reject) => {
339
+ let output = "";
340
+ const keep = (chunk: Buffer) => {
341
+ output = (output + chunk.toString()).slice(-INSTALL_OUTPUT_CHARS);
342
+ };
343
+ const child = spawn(node, [installer], {
344
+ cwd: packageRoot,
345
+ stdio: ["ignore", "pipe", "pipe"],
346
+ });
347
+ child.stdout.on("data", keep);
348
+ child.stderr.on("data", keep);
349
+ child.on("error", reject);
350
+ child.on("close", (code, signal) => {
351
+ if (code === 0) {
352
+ resolve();
353
+ return;
354
+ }
355
+ reject(
356
+ new Error(
357
+ `${installer} exited with ${signal ?? `status ${code}`}: ${output.trim()}`,
358
+ ),
359
+ );
360
+ });
361
+ });
362
+ }
308
363
 
309
- return null;
364
+ /** Settles with `promise`, or rejects with the signal's reason once it aborts. */
365
+ function untilAborted<T>(promise: Promise<T>, signal?: AbortSignal): Promise<T> {
366
+ if (!signal) return promise;
367
+ return new Promise((resolve, reject) => {
368
+ const onAbort = () => reject(signal.reason);
369
+ signal.addEventListener("abort", onAbort, { once: true });
370
+ promise.then(
371
+ (value) => {
372
+ signal.removeEventListener("abort", onAbort);
373
+ resolve(value);
374
+ },
375
+ (error) => {
376
+ signal.removeEventListener("abort", onAbort);
377
+ reject(error);
378
+ },
379
+ );
380
+ });
310
381
  }
311
382
 
312
383
  function defaultLoadClientModule(): NativeMcpClientModule {
@@ -581,7 +652,12 @@ function restoredNativeTools(ctx: ExtensionContext): string[] {
581
652
  export function createComputerUseLinuxExtension(
582
653
  dependencies: ExtensionDependencies = {},
583
654
  ) {
584
- const findBinary = dependencies.findBinary ?? defaultFindBinary;
655
+ const packageRoot = dependencies.packageRoot ?? PACKAGE_ROOT;
656
+ const findLocal =
657
+ dependencies.findBinary ?? (() => localLaunch(packageRoot));
658
+ const findOnPath = dependencies.findBinary ? () => null : pathLaunch;
659
+ const canInstall = () =>
660
+ !dependencies.findBinary && existsSync(installerPath(packageRoot));
585
661
  const loadClientModule =
586
662
  dependencies.loadClientModule ?? defaultLoadClientModule;
587
663
 
@@ -589,21 +665,64 @@ export function createComputerUseLinuxExtension(
589
665
  let launch: BinaryLaunch | undefined;
590
666
  let client: NativeMcpClient | undefined;
591
667
  let enabledNativeTools = new Set<string>();
592
-
593
- const resolveLaunch = () => {
668
+ let install: Promise<string | undefined> | undefined;
669
+ let installFailure: string | undefined;
670
+ // True while the launch is a PATH binary taken because the package
671
+ // binary could not be downloaded. It may not match this extension.
672
+ let launchIsFallback = false;
673
+ // Bumped by session_shutdown, so a download that outlives the session
674
+ // never starts a server nothing would close.
675
+ let session = 0;
676
+
677
+ // The package's binary comes before PATH, as after an install whose
678
+ // script ran: it matches this extension's catalog.
679
+ const resolveLaunch = async () => {
594
680
  if (launch) return launch;
595
- const resolved = findBinary();
681
+ let resolved = findLocal();
682
+ if (!resolved && canInstall()) {
683
+ install ??= runInstaller(packageRoot).then(
684
+ () => undefined,
685
+ (error: Error) => error.message,
686
+ );
687
+ const pending = install;
688
+ installFailure = await pending;
689
+ if (install === pending) install = undefined;
690
+ resolved = findLocal();
691
+ }
692
+ launchIsFallback = false;
693
+ if (!resolved) {
694
+ resolved = findOnPath();
695
+ launchIsFallback = resolved !== null && installFailure !== undefined;
696
+ }
596
697
  if (resolved) launch = resolved;
597
698
  return resolved;
598
699
  };
599
700
 
600
- const getClient = () => {
701
+ const dropClient = async () => {
702
+ const current = client;
703
+ client = undefined;
704
+ launch = undefined;
705
+ launchIsFallback = false;
706
+ await current?.close();
707
+ };
708
+
709
+ const getClient = async () => {
710
+ const started = session;
711
+ // The package binary appeared since the fallback was chosen.
712
+ if (launchIsFallback && findLocal()) await dropClient();
713
+ if (client) return client;
714
+ const resolved = await resolveLaunch();
715
+ if (session !== started) {
716
+ throw new Error("Pi session ended while computer-use-linux was being installed");
717
+ }
601
718
  if (client) return client;
602
- const resolved = resolveLaunch();
603
719
  if (!resolved) {
720
+ const failure = installFailure
721
+ ? ` Downloading it failed: ${installFailure}`
722
+ : "";
604
723
  throw new Error(
605
- "computer-use-linux binary was not found. Reinstall " +
606
- `${PACKAGE_NAME}, install computer-use-linux on PATH, or set COMPUTER_USE_LINUX_BIN.`,
724
+ `computer-use-linux binary was not found.${failure} Reinstall ` +
725
+ `${PACKAGE_NAME} with its install script allowed, install computer-use-linux on PATH, or set COMPUTER_USE_LINUX_BIN.`,
607
726
  );
608
727
  }
609
728
  const { ComputerUseMcpClient } = loadClientModule();
@@ -620,6 +739,32 @@ export function createComputerUseLinuxExtension(
620
739
  return client;
621
740
  };
622
741
 
742
+ const callTool = async (
743
+ name: string,
744
+ args: Record<string, unknown>,
745
+ signal: AbortSignal | undefined,
746
+ ) => {
747
+ // A cancelled call never starts a download. A call cancelled while
748
+ // one runs stops waiting; the download goes on for the next call.
749
+ signal?.throwIfAborted();
750
+ const active = await untilAborted(getClient(), signal);
751
+ try {
752
+ return await active.callTool(name, args, signal);
753
+ } catch (error) {
754
+ // A PATH binary taken after a failed download can be the wrong
755
+ // version. Drop it so the next call tries the download again.
756
+ if (signal?.aborted || !launchIsFallback || client !== active) throw error;
757
+ const failure = installFailure;
758
+ await dropClient();
759
+ throw new Error(
760
+ `${(error instanceof Error ? error.message : String(error)).replace(/\.$/, "")}. ` +
761
+ `This computer-use-linux came from PATH because downloading the package binary failed: ${failure}. ` +
762
+ "The next call tries the download again.",
763
+ { cause: error },
764
+ );
765
+ }
766
+ };
767
+
623
768
  for (const tool of AVAILABLE_MCP_TOOLS) {
624
769
  const nativeName = `${TOOL_PREFIX}${tool.name}`;
625
770
  pi.registerTool({
@@ -630,7 +775,7 @@ export function createComputerUseLinuxExtension(
630
775
  outputSchema: NativeToolOutputSchema,
631
776
  executionMode: "sequential",
632
777
  async execute(_toolCallId, params, signal) {
633
- const result = await getClient().callTool(
778
+ const result = await callTool(
634
779
  tool.name,
635
780
  params as Record<string, unknown>,
636
781
  signal,
@@ -722,7 +867,9 @@ export function createComputerUseLinuxExtension(
722
867
  }
723
868
  pi.setActiveTools(active);
724
869
 
725
- if (!resolveLaunch() && ctx.hasUI) {
870
+ // Nothing is downloaded here; a missing package binary is fetched on
871
+ // the first tool call.
872
+ if (!findLocal() && !canInstall() && !findOnPath() && ctx.hasUI) {
726
873
  ctx.ui.notify(
727
874
  `${PACKAGE_NAME}: binary not found; reinstall the package, install computer-use-linux on PATH, or set COMPUTER_USE_LINUX_BIN.`,
728
875
  "warning",
@@ -746,6 +893,7 @@ export function createComputerUseLinuxExtension(
746
893
  });
747
894
 
748
895
  pi.on("session_shutdown", async () => {
896
+ session += 1;
749
897
  const current = client;
750
898
  client = undefined;
751
899
  await current?.close();
@@ -11,7 +11,7 @@ compatibility: "Native Pi tools require Pi 0.84.4+ and Node.js 22.19+; the stand
11
11
 
12
12
  Use `computer-use-linux` when an agent needs to observe or operate a local Linux desktop: inspect the accessibility tree, list/focus windows, take screenshots, click, scroll, type, press keys, or invoke AT-SPI actions.
13
13
 
14
- On labwc, window targeting uses the Selkies IPC patch's `labwc.sock` when it exists. Stock labwc and other wlroots compositors that advertise wlr-foreign-toplevel-management use that protocol instead, which reports no pids or bounds. Activation requires a unique app-id/title match. Window IDs include the title and must be refreshed after it changes. Ambiguous windows are rejected. Window bounds and relative coordinate targeting are unavailable until output scaling can be verified.
14
+ On labwc, window targeting uses the Selkies IPC patch's `labwc.sock` when it exists. Stock labwc and other wlroots compositors that advertise wlr-foreign-toplevel-management use that protocol instead, which reports no pids or bounds. Activation requires a unique app-id/title match. Window IDs include the title and must be refreshed after it changes. Ambiguous windows are rejected. Window bounds and relative coordinate targeting are unavailable until output scaling can be verified. Window-targeted screenshots, including `screenshot_after: "window"`, fail for the same reason; capture the full screen instead.
15
15
 
16
16
  Native X11 clicks, scrolls and drags use xdotool/XTEST on `DISPLAY`, ahead of uinput and portal input. Install xdotool for this route. Missing or failed commands return an error without a system-wide fallback. Drags send paced intermediate motion and always attempt release after a failed press or motion; verify the drop target afterward.
17
17
 
@@ -81,9 +81,11 @@ computer-use-linux setup-window-targeting
81
81
  computer-use-linux doctor | jq .readiness
82
82
  ```
83
83
 
84
- If `doctor` selects ydotool as the input backend, also enable its per-user daemon with `systemctl --user enable --now ydotoold`. Direct uinput, X11 xdotool, and RemoteDesktop portal input do not require `ydotoold`.
84
+ If `doctor` selects ydotool as the input backend, also enable its per-user daemon. `./install.sh` creates `ydotoold.service`; Ubuntu's ydotool package ships `ydotool.service` instead (`systemctl --user enable --now ydotool`). Direct uinput, X11 xdotool, and RemoteDesktop portal input do not require `ydotoold`, but a Wayland scroll at a point needs ydotool for the wheel, or the portal.
85
85
 
86
- On niri, export the session's `NIRI_SOCKET` to the server. Window listing and
86
+ On niri, the server reads `NIRI_SOCKET` from its parent processes or the systemd
87
+ user environment when the host does not pass it; without it, discovery needs an
88
+ unambiguous socket matching `WAYLAND_DISPLAY`. Window listing and
87
89
  exact focus use `niri msg` with a direct IPC fallback. Missing positions or
88
90
  unknown/mixed output scaling leave window-relative coordinates unavailable;
89
91
  focus the target and inspect a fresh full-screen screenshot before using
@@ -139,15 +141,15 @@ Pi native tools skip this MCP `command` config; see [Pi setup](references/pi-set
139
141
  ## Procedure
140
142
 
141
143
  1. In Pi, call `computer_use_linux_tools` with the exact tools or capability you need. Enabled tools use the `computer_use_linux_<name>` prefix, appear starting on the next model turn, and remain active for the session.
142
- 2. Begin every desktop-control turn with `get_app_state`, scoped to the app you are working in: pass `app_name_or_bundle_identifier` or a window target (`window_id`, `pid`, `app_id`, `wm_class`, `title`). An `app_id` may be a full desktop ID or its final dot-separated segment; matching is case-insensitive, ambiguous aliases are rejected, and an unresolved explicit `app_id` returns an empty scoped tree rather than the desktop. Without a target the result is the whole desktop AT-SPI tree, `tree_scoped` is `false`, and `message` warns; that can flood context. Use `include_screenshot: false` when the accessibility tree is sufficient. If `accessibility_tree_truncated` is `true`, the tree is incomplete: scope to a narrower target and raise `max_nodes` or `max_depth` (hard caps 10000 and 64) rather than lowering them. A tree too large to return inline comes back as its leading nodes plus `accessibility_tree_spill.path`, a private JSON file with the whole tree and the same `element_index` values: search it (for a role, name or text) instead of requesting the tree again, or page through it with `get_tree_page` when you cannot read files. Nodes are compact: null, empty, zero and false fields are left out and `actions` lists action names; `verbose: true` returns every field. To find controls in a large tree, pass `interactive_only: true`: it keeps nodes with an action, editable text or a value, plus headings, status and live regions, windows and dialogs, collapses containers, and keeps the same `element_index` values. A `title` or `window_id` target narrows a Chromium-style app with several windows under one AT-SPI root to that window, keeping the app's popups and dialogs. The compact readiness block identifies missing setup.
144
+ 2. Begin every desktop-control turn with `get_app_state`, scoped to the app you are working in: pass `app_name_or_bundle_identifier` or a window target (`window_id`, `pid`, `app_id`, `wm_class`, `title`). An `app_id` may be a full desktop ID or its final dot-separated segment; matching is case-insensitive, ambiguous aliases are rejected, and an unresolved explicit `app_id` returns an empty scoped tree rather than the desktop. Without a target the result is the whole desktop AT-SPI tree, `tree_scoped` is `false`, and `message` warns; that can flood context. Use `include_screenshot: false` when the accessibility tree is sufficient. If `accessibility_tree_truncated` is `true`, the tree is incomplete: scope to a narrower target and raise `max_nodes` or `max_depth` (hard caps 10000 and 64) rather than lowering them. A tree too large to return inline comes back as its leading nodes plus `accessibility_tree_spill.path`, a private JSON file with the whole tree and the same `element_index` values: search it (for a role, name or text) instead of requesting the tree again, or page through it with `get_tree_page` when you cannot read files. Nodes are compact: null, empty, zero and false fields are left out and `actions` lists action names; `verbose: true` returns every field. To find controls in a large tree, pass `interactive_only: true`: it keeps nodes with an action, editable text or a value, plus headings, status and live regions, windows and dialogs, collapses containers, and keeps the same `element_index` values. A `title` or `window_id` target narrows a Chromium-style app with several windows under one AT-SPI root to that window, keeping the app's popups and dialogs. A window-targeted screenshot needs that window visible and already focused (Hyprland background mode excepted); otherwise `screenshot_error` is set and only the tree returns, so use `screenshot` to raise and capture it. The compact readiness block identifies missing setup.
143
145
  3. Use `doctor` only when you need the full diagnostic report.
144
146
  4. If `can_build_accessibility_tree` is false, run `setup_accessibility` and restart the target app.
145
- 5. If `can_query_windows` is false on GNOME Wayland, run `setup_window_targeting` and ask the user to log out and back in if setup says the shell extension needs a reload.
147
+ 5. If `can_query_windows` or `can_focus_windows` is false on GNOME Wayland, run `setup_window_targeting` and ask the user to log out and back in if setup says the shell extension needs a reload.
146
148
  6. Before targeted input, call `list_windows` or `focused_window` and verify the intended window by title, app id, pid, or wm class.
147
- 7. Prefer semantic targeting from `get_app_state`: use element indices or role/name/text/states selectors. Automatic recovery of a dead semantic target requires verified PID/window ownership and invalidates cached indices; call `get_app_state` again before further index-based actions. When several apps are in play, pass the app target (`pid`, `window_id`, `app_id`, `wm_class`, `window_title`) to `click`, `scroll`, `perform_action`, and `set_value`: an index or selector from another app's snapshot is then refused instead of acting on that app. Where the window backend reports no pid (i3 without `xprop`, COSMIC), call `get_app_state` for the same window first or pass `pid`, or the call is refused.
149
+ 7. Prefer semantic targeting from `get_app_state`: use element indices or role/name/text/states selectors. Automatic recovery of a dead semantic target requires verified PID/window ownership and invalidates cached indices; call `get_app_state` again before further index-based actions. When several apps are in play, pass the app target (`pid`, `window_id`, `app_id`, `wm_class`, `window_title`) to `click`, `scroll`, `perform_action`, and `set_value`: an index or selector from another app's snapshot is then refused instead of acting on that app. On `click` and `scroll` the target also raises and focuses that window, and nothing is sent when focus cannot be verified; on GNOME Wayland, raising a window needs the Shell extension. The title field there is `window_title`. Where the window backend reports no pid (i3 without `xprop`, COSMIC, and wlr-foreign-toplevel on stock labwc and other wlroots compositors), call `get_app_state` for the same window first or pass `pid`, or the call is refused.
148
150
  8. Use coordinates only when the UI surface has no useful accessibility tree.
149
151
  9. For text input, prefer `type_text` with a target selector (`window_id`, `pid`, `app_id`, `wm_class`, `title`, `tty`, `terminal_pid`, `terminal_command`, or `terminal_cwd`) rather than relying on current focus. A `window_id` is valid only as returned by a recent `list_windows`; if a call rejects it as outside 0 to 2^53-1 or matches no window, re-run `list_windows`.
150
- 10. After mutating actions (`click`, `drag`, `scroll`, `press_key`, `type_text`, `perform_action`, `set_value`, `activate_window`, `move_window`, `resize_window`), either pass `screenshot_after: true | "screen" | "window"` (with optional `format`, `quality`, `scale`, `max_width`, `max_height`, `max_bytes`) and/or `include_tree: true` on the action call itself to observe the result in the same tool call, or re-check state with `get_app_state`, `focused_window`, or an app-specific readback.
152
+ 10. After mutating actions (`click`, `drag`, `scroll`, `press_key`, `type_text`, `perform_action`, `set_value`, `activate_window`, `move_window`, `resize_window`), either pass `screenshot_after: true | "screen" | "window"` (with optional `format`, `quality`, `scale`, `max_width`, `max_height`, `max_bytes`) and/or `include_tree: true` on the action call itself to observe the result in the same tool call, or re-check state with `get_app_state`, `focused_window`, or an app-specific readback. A failed action (`ok: false`) returns no post-action screenshot or tree, so call `get_app_state`; `set_value` still observes when `verified` is `false`.
151
153
  11. For `set_value`, `verified: true` is the only exact-content confirmation. `verified: false` requires fresh observation; password fields are deliberately unverifiable. The field is omitted for numeric values and early failures.
152
154
 
153
155
  On Hyprland, `type_text` passes one literal wtype argument after `--` for a
@@ -183,9 +185,9 @@ only a few thousand nodes before the deadline. A snapshot stops reading at 10 se
183
185
  marked truncated; if the editor is still missing, hide the embedded browser pane
184
186
  for the snapshot.
185
187
 
186
- Plain left element/index/selector `click` prefers native AT-SPI `click`,
187
- `press`, or `toggle` over toolkit bounds, avoiding coordinate
188
- conversion when available. This preference does not replace a coordinate click
188
+ Plain left element/index/selector `click` invokes the element's first AT-SPI
189
+ action instead of using toolkit bounds when that action is `click`, `press`, or
190
+ `toggle`, avoiding coordinate conversion. This preference does not replace a coordinate click
189
191
  with an arbitrary action name. Explicit `x`/`y`, right clicks, and double/multiple
190
192
  clicks retain pointer semantics. Explicit coordinates default to raw screenshot
191
193
  pixels (`coordinate_space: "capture"`). On KDE Wayland, pass
@@ -200,7 +202,15 @@ A coordinate `click`, `scroll` or `drag` result warns when the point is off
200
202
  the screenshot or between monitors, where no monitor shows that part of the
201
203
  screenshot. A `drag` warning names its start or end coordinate. The result
202
204
  says whether the input was sent, and the warning says where it may have
203
- landed; pick a point on a monitor and check the screen before retrying.
205
+ landed; pick a point on a monitor and check the screen before retrying. A KWin
206
+ element click, including one with `coordinate_space: "atspi"`, gets no such
207
+ warning, since KWin maps its point onto an output. On the portal route, a
208
+ refused point also closes the portal session; the next pointer call starts a
209
+ new one, which can show the consent dialog again.
210
+
211
+ The uinput pointer is sized to the latest full screenshot. After the desktop
212
+ size changes, take a new screenshot before coordinate input. If replacing the
213
+ device fails, nothing is sent and the result says where a retry goes.
204
214
 
205
215
  ### Screenshot-relative coordinates
206
216
 
@@ -218,9 +228,9 @@ directly to a window-relative click.
218
228
 
219
229
  - Already-running GTK, Qt, and Electron apps may need a restart after AT-SPI is enabled.
220
230
  - GNOME may show a portal prompt on the first screenshot or `get_app_state` call with screenshots enabled.
221
- - Inside a Selkies container, `PIXELFLUX_CU` plus `PIXELFLUX_CU_TOKEN` sends screenshots and input to pixelflux on loopback, with no portal prompt. A closed port falls through. A server error does not, so do not retry that call through another backend.
231
+ - Inside a Selkies container, `PIXELFLUX_CU` plus `PIXELFLUX_CU_TOKEN` sends screenshots and input to pixelflux on loopback, with no portal prompt. A closed port falls through. A server error does not, so do not retry that call through another backend. pixelflux takes left, right and middle clicks with counts 1 to 3, and multi-clicks only with the left button; other clicks go to the next backend. Its results carry no off-screen or monitor-gap warning.
222
232
  - On GNOME Wayland, the remote-control portal asks on each new `computer-use-linux` process. `COMPUTER_USE_LINUX_PERSIST_REMOTE_DESKTOP=1` in the server environment asks the portal to remember the grant (GNOME starts with the remember box checked). Later processes reuse a single-use restore token stored mode 0600 in the user state directory. Unset is the default and prompts every time. The flag does nothing when ydotool, not the portal, is the input backend.
223
- - On Hyprland, `COMPUTER_USE_LINUX_HYPRLAND_BACKGROUND=1` lets targeted `type_text`, `press_key`, `get_app_state` and `screenshot` work on a window without focusing it, including one on a hidden workspace. Pass `window_id` (or another target) on every call. An AT-SPI action can move focus inside the app, so re-focus a field with its `activate` action before typing into it. Text must be printable US-layout ASCII, tab or newline. An unfocused Chromium ignores browser accelerators such as Ctrl+R, and Chromium joins AT-SPI only when started with `ACCESSIBILITY_ENABLED=1` if the accessibility bus reports itself disabled.
233
+ - On Hyprland, `COMPUTER_USE_LINUX_HYPRLAND_BACKGROUND=1` lets targeted `type_text`, `press_key`, `get_app_state` and `screenshot` work on a window without focusing it, including one on a hidden workspace. Pass `window_id` (or another target) on every call. An AT-SPI action can move focus inside the app, so re-focus a field with its `activate` action before typing into it. The active Hyprland keymap must be US, or background `type_text` and `press_key` calls are refused; background `get_app_state` and `screenshot` still capture. Text must be printable ASCII, tab or newline. An unfocused Chromium ignores browser accelerators such as Ctrl+R, and Chromium joins AT-SPI only when started with `ACCESSIBILITY_ENABLED=1` if the accessibility bus reports itself disabled.
224
234
  - Desktop input is stateful. Avoid concurrent tool calls against this MCP server.
225
235
  - Pi serializes the native Computer Use tools and keeps one process for the session. If that process exits, do not replay an ambiguous mutating call; obtain a fresh `get_app_state` before another element-based action.
226
236
  - `click`, `drag`, `press_key`, `type_text`, `perform_action`, and `set_value` can change real application state.
@@ -15,17 +15,20 @@ hermes mcp configure computer-use-linux
15
15
 
16
16
  `configure` opens Hermes' tool-selection UI for this MCP server.
17
17
 
18
- The generated config should look like this:
18
+ The generated config holds `command`, `args` and `enabled`, plus
19
+ `tools.include` when you pick a subset of tools:
19
20
 
20
21
  ```yaml
21
22
  mcp_servers:
22
23
  computer-use-linux:
23
24
  command: computer-use-linux
24
25
  args: ["mcp"]
25
- timeout: 120
26
- connect_timeout: 30
26
+ enabled: true
27
27
  ```
28
28
 
29
+ Hermes defaults to a 300-second tool-call timeout and a 60-second connect
30
+ timeout. Add `timeout` or `connect_timeout` under the server to change them.
31
+
29
32
  If the binary is not on `PATH`, pass the absolute path to `--command`.
30
33
 
31
34
  Hermes registers tools using the `mcp_<server>_<tool>` pattern. With this config, tool names are prefixed as `mcp_computer_use_linux_`, for example:
@@ -46,6 +46,9 @@ Or search by capability:
46
46
  computer_use_linux_tools({ query: "inspect a window and type text" })
47
47
  ```
48
48
 
49
+ Called with no arguments, the loader enables `doctor`, `list_windows`,
50
+ `focused_window`, `get_app_state` and `get_tree_page`.
51
+
49
52
  The selected native tools appear starting on the next model turn with their
50
53
  full upstream schemas and remain active for the session:
51
54
 
@@ -87,6 +90,9 @@ for (const block of result.images) image(block);
87
90
  Native Computer Use tools execute sequentially. Cancellation is forwarded to
88
91
  the MCP request. If the server process exits, the failed call is never replayed
89
92
  automatically; call `get_app_state` again before another element-based action.
93
+ Each call times out after 60 seconds. Set `COMPUTER_USE_LINUX_TIMEOUT_MS` in
94
+ Pi's environment to change it, for example for `run_shell` calls with
95
+ `timeout_seconds` above 60.
90
96
 
91
97
  ## Migration from the adapter-based package
92
98
 
@@ -110,14 +116,18 @@ legacy entry.
110
116
 
111
117
  The extension looks for `computer-use-linux` in this order:
112
118
 
113
- 1. `COMPUTER_USE_LINUX_BIN`
114
- 2. The binary downloaded inside the installed npm package
115
- 3. `computer-use-linux` on `PATH`, from `npm install -g` or `cargo install`
116
-
117
- A temporary extension (`pi -e npm:@agent-sh/computer-use-linux`) is staged
118
- without the downloaded binary, so it relies on the `PATH` step. The extension
119
- still checks that the server version and tool catalog match, so a mismatched
120
- install fails with that reason instead of starting.
119
+ 1. `COMPUTER_USE_LINUX_BIN`, when it is an absolute path to an executable. A
120
+ relative or non-executable value is ignored and the next step is used.
121
+ 2. The binary downloaded inside the installed npm package. When it is missing,
122
+ as after npm 12 skipped the install script, the first real tool call runs the
123
+ package's `npm/install.js` to download it.
124
+ 3. `computer-use-linux` on `PATH`, from `npm install -g` or `cargo install`,
125
+ when that download fails
126
+
127
+ The extension still checks that the server version and tool catalog match, so
128
+ a mismatched install fails with that reason instead of starting. Set
129
+ `COMPUTER_USE_LINUX_BIN` to run a `PATH` or development build instead of the
130
+ download.
121
131
 
122
132
  Reinstall the package if the bundled binary is missing:
123
133